/*
 * The AEGIS-128 Authenticated-Encryption Algorithm
 *   Glue for AES-NI + SSE2 implementation
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define AEGIS128_BLOCK_ALIGN 16
#define AEGIS128_BLOCK_SIZE 16
#define AEGIS128_NONCE_SIZE 16
#define AEGIS128_STATE_BLOCKS 5
#define AEGIS128_KEY_SIZE 16
#define AEGIS128_MIN_AUTH_SIZE 8
#define AEGIS128_MAX_AUTH_SIZE 16

asmlinkage void crypto_aegis128_aesni_init(void *state, void *key, void *iv);

asmlinkage void crypto_aegis128_aesni_ad(
                void *state, unsigned int length, const void *data);

asmlinkage void crypto_aegis128_aesni_enc(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_dec(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_enc_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_dec_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_final(
                void *state, void *tag_xor, unsigned int cryptlen,
                unsigned int assoclen);

struct aegis_block {
        u8 bytes[AEGIS128_BLOCK_SIZE] __aligned(AEGIS128_BLOCK_ALIGN);
};

struct aegis_state {
        struct aegis_block blocks[AEGIS128_STATE_BLOCKS];
};

struct aegis_ctx {
        struct aegis_block key;
};

struct aegis_crypt_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, unsigned int length, const void *src,
                             void *dst);
        void (*crypt_tail)(void *state, unsigned int length, const void *src,
                           void *dst);
};

static void crypto_aegis128_aesni_process_ad(
                struct aegis_state *state, struct scatterlist *sg_src,
                unsigned int assoclen)
{
        struct scatter_walk walk;
        struct aegis_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) { /*covered*/
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= AEGIS128_BLOCK_SIZE) { /*covered*/
                        if (pos > 0) { /*covered*/
                                unsigned int fill = AEGIS128_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                crypto_aegis128_aesni_ad(state,
                                                         AEGIS128_BLOCK_SIZE,
                                                         buf.bytes);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        crypto_aegis128_aesni_ad(state, left, src); /*covered*/

                        src += left & ~(AEGIS128_BLOCK_SIZE - 1);
                        left &= AEGIS128_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/
                pos += left;
                assoclen -= size;

                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos); /*covered*/
                crypto_aegis128_aesni_ad(state, AEGIS128_BLOCK_SIZE, buf.bytes); /*covered*/
        }
}

static void crypto_aegis128_aesni_process_crypt(
                struct aegis_state *state, struct skcipher_walk *walk,
                const struct aegis_crypt_ops *ops)
{
        while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
                ops->crypt_blocks(state, /*covered*/
                                  round_down(walk->nbytes, AEGIS128_BLOCK_SIZE),
                                  walk->src.virt.addr, walk->dst.virt.addr);
                skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE);
        }

        if (walk->nbytes) { /*covered*/
                ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr, /*covered*/
                                walk->dst.virt.addr);
                skcipher_walk_done(walk, 0);
        }
}

static struct aegis_ctx *crypto_aegis128_aesni_ctx(struct crypto_aead *aead)
{
        u8 *ctx = crypto_aead_ctx(aead); /*covered*/
        ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
        return (void *)ctx;
}

static int crypto_aegis128_aesni_setkey(struct crypto_aead *aead, const u8 *key,
                                        unsigned int keylen)
{
        struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(aead); /*covered*/

        if (keylen != AEGIS128_KEY_SIZE) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);

        return 0; /*covered*/
}

static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm,
                                                unsigned int authsize)
{
        if (authsize > AEGIS128_MAX_AUTH_SIZE) /*covered*/
                return -EINVAL;
        if (authsize < AEGIS128_MIN_AUTH_SIZE) /*covered*/
                return -EINVAL;
        return 0; /*covered*/
}

static void crypto_aegis128_aesni_crypt(struct aead_request *req,
                                        struct aegis_block *tag_xor,
                                        unsigned int cryptlen,
                                        const struct aegis_crypt_ops *ops)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
        struct skcipher_walk walk;
        struct aegis_state state;

        ops->skcipher_walk_init(&walk, req, true);

        kernel_fpu_begin();

        crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
        crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen); /*covered*/
        crypto_aegis128_aesni_process_crypt(&state, &walk, ops); /*covered*/
        crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen); /*covered*/

        kernel_fpu_end();
}

static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
{
        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = crypto_aegis128_aesni_enc,
                .crypt_tail = crypto_aegis128_aesni_enc_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}

static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
{
        static const struct aegis_block zeros = {};

        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = crypto_aegis128_aesni_dec,
                .crypt_tail = crypto_aegis128_aesni_dec_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);

        return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0; /*covered*/
}

static int crypto_aegis128_aesni_init_tfm(struct crypto_aead *aead)
{
        return 0; /*covered*/
}

static void crypto_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
{
} /*covered*/

static int cryptd_aegis128_aesni_setkey(struct crypto_aead *aead,
                                        const u8 *key, unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}

static int cryptd_aegis128_aesni_setauthsize(struct crypto_aead *aead,
                                             unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int cryptd_aegis128_aesni_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int cryptd_aegis128_aesni_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}

static int cryptd_aegis128_aesni_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__aegis128-aesni", CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void cryptd_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}

static struct aead_alg crypto_aegis128_aesni_alg[] = {
        {
                .setkey = crypto_aegis128_aesni_setkey,
                .setauthsize = crypto_aegis128_aesni_setauthsize,
                .encrypt = crypto_aegis128_aesni_encrypt,
                .decrypt = crypto_aegis128_aesni_decrypt,
                .init = crypto_aegis128_aesni_init_tfm,
                .exit = crypto_aegis128_aesni_exit_tfm,

                .ivsize = AEGIS128_NONCE_SIZE,
                .maxauthsize = AEGIS128_MAX_AUTH_SIZE,
                .chunksize = AEGIS128_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct aegis_ctx) +
                                __alignof__(struct aegis_ctx),
                        .cra_alignmask = 0,

                        .cra_name = "__aegis128",
                        .cra_driver_name = "__aegis128-aesni",

                        .cra_module = THIS_MODULE,
                }
        }, {
                .setkey = cryptd_aegis128_aesni_setkey,
                .setauthsize = cryptd_aegis128_aesni_setauthsize,
                .encrypt = cryptd_aegis128_aesni_encrypt,
                .decrypt = cryptd_aegis128_aesni_decrypt,
                .init = cryptd_aegis128_aesni_init_tfm,
                .exit = cryptd_aegis128_aesni_exit_tfm,

                .ivsize = AEGIS128_NONCE_SIZE,
                .maxauthsize = AEGIS128_MAX_AUTH_SIZE,
                .chunksize = AEGIS128_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_ASYNC,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct cryptd_aead *),
                        .cra_alignmask = 0,

                        .cra_priority = 400,

                        .cra_name = "aegis128",
                        .cra_driver_name = "aegis128-aesni",

                        .cra_module = THIS_MODULE,
                }
        }
};

static int __init crypto_aegis128_aesni_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_aegis128_aesni_alg,
                                     ARRAY_SIZE(crypto_aegis128_aesni_alg));
}

static void __exit crypto_aegis128_aesni_module_exit(void)
{
        crypto_unregister_aeads(crypto_aegis128_aesni_alg,
                                ARRAY_SIZE(crypto_aegis128_aesni_alg));
}

module_init(crypto_aegis128_aesni_module_init);
module_exit(crypto_aegis128_aesni_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE2 implementation");
MODULE_ALIAS_CRYPTO("aegis128");
MODULE_ALIAS_CRYPTO("aegis128-aesni");

/*
 * The AEGIS-128L Authenticated-Encryption Algorithm
 *   Glue for AES-NI + SSE2 implementation
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define AEGIS128L_BLOCK_ALIGN 16
#define AEGIS128L_BLOCK_SIZE 32
#define AEGIS128L_NONCE_SIZE 16
#define AEGIS128L_STATE_BLOCKS 8
#define AEGIS128L_KEY_SIZE 16
#define AEGIS128L_MIN_AUTH_SIZE 8
#define AEGIS128L_MAX_AUTH_SIZE 16

asmlinkage void crypto_aegis128l_aesni_init(void *state, void *key, void *iv);

asmlinkage void crypto_aegis128l_aesni_ad(
                void *state, unsigned int length, const void *data);

asmlinkage void crypto_aegis128l_aesni_enc(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_dec(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_enc_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_dec_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_final(
                void *state, void *tag_xor, unsigned int cryptlen,
                unsigned int assoclen);

struct aegis_block {
        u8 bytes[AEGIS128L_BLOCK_SIZE] __aligned(AEGIS128L_BLOCK_ALIGN);
};

struct aegis_state {
        struct aegis_block blocks[AEGIS128L_STATE_BLOCKS];
};

struct aegis_ctx {
        struct aegis_block key;
};

struct aegis_crypt_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, unsigned int length, const void *src,
                             void *dst);
        void (*crypt_tail)(void *state, unsigned int length, const void *src,
                           void *dst);
};

static void crypto_aegis128l_aesni_process_ad(
                struct aegis_state *state, struct scatterlist *sg_src,
                unsigned int assoclen)
{
        struct scatter_walk walk;
        struct aegis_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) { /*covered*/
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= AEGIS128L_BLOCK_SIZE) { /*covered*/
                        if (pos > 0) { /*covered*/
                                unsigned int fill = AEGIS128L_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                crypto_aegis128l_aesni_ad(state,
                                                          AEGIS128L_BLOCK_SIZE,
                                                          buf.bytes);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        crypto_aegis128l_aesni_ad(state, left, src); /*covered*/

                        src += left & ~(AEGIS128L_BLOCK_SIZE - 1);
                        left &= AEGIS128L_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/
                pos += left;
                assoclen -= size;

                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, AEGIS128L_BLOCK_SIZE - pos); /*covered*/
                crypto_aegis128l_aesni_ad(state, AEGIS128L_BLOCK_SIZE, buf.bytes); /*covered*/
        }
}

static void crypto_aegis128l_aesni_process_crypt(
                struct aegis_state *state, struct skcipher_walk *walk,
                const struct aegis_crypt_ops *ops)
{
        while (walk->nbytes >= AEGIS128L_BLOCK_SIZE) {
                ops->crypt_blocks(state, round_down(walk->nbytes, /*covered*/
                                                    AEGIS128L_BLOCK_SIZE),
                                  walk->src.virt.addr, walk->dst.virt.addr);
                skcipher_walk_done(walk, walk->nbytes % AEGIS128L_BLOCK_SIZE);
        }

        if (walk->nbytes) { /*covered*/
                ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr, /*covered*/
                                walk->dst.virt.addr);
                skcipher_walk_done(walk, 0);
        }
}

static struct aegis_ctx *crypto_aegis128l_aesni_ctx(struct crypto_aead *aead)
{
        u8 *ctx = crypto_aead_ctx(aead); /*covered*/
        ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
        return (void *)ctx;
}

static int crypto_aegis128l_aesni_setkey(struct crypto_aead *aead,
                                         const u8 *key, unsigned int keylen)
{
        struct aegis_ctx *ctx = crypto_aegis128l_aesni_ctx(aead); /*covered*/

        if (keylen != AEGIS128L_KEY_SIZE) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key.bytes, key, AEGIS128L_KEY_SIZE);

        return 0; /*covered*/
}

static int crypto_aegis128l_aesni_setauthsize(struct crypto_aead *tfm,
                                              unsigned int authsize)
{
        if (authsize > AEGIS128L_MAX_AUTH_SIZE) /*covered*/
                return -EINVAL;
        if (authsize < AEGIS128L_MIN_AUTH_SIZE) /*covered*/
                return -EINVAL;
        return 0; /*covered*/
}

static void crypto_aegis128l_aesni_crypt(struct aead_request *req,
                                         struct aegis_block *tag_xor,
                                         unsigned int cryptlen,
                                         const struct aegis_crypt_ops *ops)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_ctx *ctx = crypto_aegis128l_aesni_ctx(tfm);
        struct skcipher_walk walk;
        struct aegis_state state;

        ops->skcipher_walk_init(&walk, req, true);

        kernel_fpu_begin();

        crypto_aegis128l_aesni_init(&state, ctx->key.bytes, req->iv);
        crypto_aegis128l_aesni_process_ad(&state, req->src, req->assoclen); /*covered*/
        crypto_aegis128l_aesni_process_crypt(&state, &walk, ops); /*covered*/
        crypto_aegis128l_aesni_final(&state, tag_xor, req->assoclen, cryptlen); /*covered*/

        kernel_fpu_end();
}

static int crypto_aegis128l_aesni_encrypt(struct aead_request *req)
{
        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = crypto_aegis128l_aesni_enc,
                .crypt_tail = crypto_aegis128l_aesni_enc_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_aegis128l_aesni_crypt(req, &tag, cryptlen, &OPS);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}

static int crypto_aegis128l_aesni_decrypt(struct aead_request *req)
{
        static const struct aegis_block zeros = {};

        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = crypto_aegis128l_aesni_dec,
                .crypt_tail = crypto_aegis128l_aesni_dec_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_aegis128l_aesni_crypt(req, &tag, cryptlen, &OPS);

        return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0; /*covered*/
}

static int crypto_aegis128l_aesni_init_tfm(struct crypto_aead *aead)
{
        return 0; /*covered*/
}

static void crypto_aegis128l_aesni_exit_tfm(struct crypto_aead *aead)
{
} /*covered*/

static int cryptd_aegis128l_aesni_setkey(struct crypto_aead *aead,
                                         const u8 *key, unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}

static int cryptd_aegis128l_aesni_setauthsize(struct crypto_aead *aead,
                                              unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int cryptd_aegis128l_aesni_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int cryptd_aegis128l_aesni_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}

static int cryptd_aegis128l_aesni_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__aegis128l-aesni", CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void cryptd_aegis128l_aesni_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}

static struct aead_alg crypto_aegis128l_aesni_alg[] = {
        {
                .setkey = crypto_aegis128l_aesni_setkey,
                .setauthsize = crypto_aegis128l_aesni_setauthsize,
                .encrypt = crypto_aegis128l_aesni_encrypt,
                .decrypt = crypto_aegis128l_aesni_decrypt,
                .init = crypto_aegis128l_aesni_init_tfm,
                .exit = crypto_aegis128l_aesni_exit_tfm,

                .ivsize = AEGIS128L_NONCE_SIZE,
                .maxauthsize = AEGIS128L_MAX_AUTH_SIZE,
                .chunksize = AEGIS128L_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct aegis_ctx) +
                                __alignof__(struct aegis_ctx),
                        .cra_alignmask = 0,

                        .cra_name = "__aegis128l",
                        .cra_driver_name = "__aegis128l-aesni",

                        .cra_module = THIS_MODULE,
                }
        }, {
                .setkey = cryptd_aegis128l_aesni_setkey,
                .setauthsize = cryptd_aegis128l_aesni_setauthsize,
                .encrypt = cryptd_aegis128l_aesni_encrypt,
                .decrypt = cryptd_aegis128l_aesni_decrypt,
                .init = cryptd_aegis128l_aesni_init_tfm,
                .exit = cryptd_aegis128l_aesni_exit_tfm,

                .ivsize = AEGIS128L_NONCE_SIZE,
                .maxauthsize = AEGIS128L_MAX_AUTH_SIZE,
                .chunksize = AEGIS128L_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_ASYNC,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct cryptd_aead *),
                        .cra_alignmask = 0,

                        .cra_priority = 400,

                        .cra_name = "aegis128l",
                        .cra_driver_name = "aegis128l-aesni",

                        .cra_module = THIS_MODULE,
                }
        }
};

static int __init crypto_aegis128l_aesni_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_aegis128l_aesni_alg,
                                     ARRAY_SIZE(crypto_aegis128l_aesni_alg));
}

static void __exit crypto_aegis128l_aesni_module_exit(void)
{
        crypto_unregister_aeads(crypto_aegis128l_aesni_alg,
                                ARRAY_SIZE(crypto_aegis128l_aesni_alg));
}

module_init(crypto_aegis128l_aesni_module_init);
module_exit(crypto_aegis128l_aesni_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("AEGIS-128L AEAD algorithm -- AESNI+SSE2 implementation");
MODULE_ALIAS_CRYPTO("aegis128l");
MODULE_ALIAS_CRYPTO("aegis128l-aesni");

/*
 * The AEGIS-256 Authenticated-Encryption Algorithm
 *   Glue for AES-NI + SSE2 implementation
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define AEGIS256_BLOCK_ALIGN 16
#define AEGIS256_BLOCK_SIZE 16
#define AEGIS256_NONCE_SIZE 32
#define AEGIS256_STATE_BLOCKS 6
#define AEGIS256_KEY_SIZE 32
#define AEGIS256_MIN_AUTH_SIZE 8
#define AEGIS256_MAX_AUTH_SIZE 16

asmlinkage void crypto_aegis256_aesni_init(void *state, void *key, void *iv);

asmlinkage void crypto_aegis256_aesni_ad(
                void *state, unsigned int length, const void *data);

asmlinkage void crypto_aegis256_aesni_enc(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_dec(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_enc_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_dec_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_final(
                void *state, void *tag_xor, unsigned int cryptlen,
                unsigned int assoclen);

struct aegis_block {
        u8 bytes[AEGIS256_BLOCK_SIZE] __aligned(AEGIS256_BLOCK_ALIGN);
};

struct aegis_state {
        struct aegis_block blocks[AEGIS256_STATE_BLOCKS];
};

struct aegis_ctx {
        struct aegis_block key[AEGIS256_KEY_SIZE / AEGIS256_BLOCK_SIZE];
};

struct aegis_crypt_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, unsigned int length, const void *src,
                             void *dst);
        void (*crypt_tail)(void *state, unsigned int length, const void *src,
                           void *dst);
};

static void crypto_aegis256_aesni_process_ad(
                struct aegis_state *state, struct scatterlist *sg_src,
                unsigned int assoclen)
{
        struct scatter_walk walk;
        struct aegis_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) { /*covered*/
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= AEGIS256_BLOCK_SIZE) { /*covered*/
                        if (pos > 0) { /*covered*/
                                unsigned int fill = AEGIS256_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                crypto_aegis256_aesni_ad(state,
                                                         AEGIS256_BLOCK_SIZE,
                                                         buf.bytes);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        crypto_aegis256_aesni_ad(state, left, src); /*covered*/

                        src += left & ~(AEGIS256_BLOCK_SIZE - 1);
                        left &= AEGIS256_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/
                pos += left;
                assoclen -= size;

                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, AEGIS256_BLOCK_SIZE - pos); /*covered*/
                crypto_aegis256_aesni_ad(state, AEGIS256_BLOCK_SIZE, buf.bytes); /*covered*/
        }
}

static void crypto_aegis256_aesni_process_crypt(
                struct aegis_state *state, struct skcipher_walk *walk,
                const struct aegis_crypt_ops *ops)
{
        while (walk->nbytes >= AEGIS256_BLOCK_SIZE) {
                ops->crypt_blocks(state, /*covered*/
                                  round_down(walk->nbytes, AEGIS256_BLOCK_SIZE),
                                  walk->src.virt.addr, walk->dst.virt.addr);
                skcipher_walk_done(walk, walk->nbytes % AEGIS256_BLOCK_SIZE);
        }

        if (walk->nbytes) { /*covered*/
                ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr, /*covered*/
                                walk->dst.virt.addr);
                skcipher_walk_done(walk, 0);
        }
}

static struct aegis_ctx *crypto_aegis256_aesni_ctx(struct crypto_aead *aead)
{
        u8 *ctx = crypto_aead_ctx(aead); /*covered*/
        ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
        return (void *)ctx;
}

static int crypto_aegis256_aesni_setkey(struct crypto_aead *aead, const u8 *key,
                                        unsigned int keylen)
{
        struct aegis_ctx *ctx = crypto_aegis256_aesni_ctx(aead); /*covered*/

        if (keylen != AEGIS256_KEY_SIZE) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key, key, AEGIS256_KEY_SIZE); /*covered*/

        return 0; /*covered*/
}

static int crypto_aegis256_aesni_setauthsize(struct crypto_aead *tfm,
                                                unsigned int authsize)
{
        if (authsize > AEGIS256_MAX_AUTH_SIZE) /*covered*/
                return -EINVAL;
        if (authsize < AEGIS256_MIN_AUTH_SIZE) /*covered*/
                return -EINVAL;
        return 0; /*covered*/
}

static void crypto_aegis256_aesni_crypt(struct aead_request *req,
                                        struct aegis_block *tag_xor,
                                        unsigned int cryptlen,
                                        const struct aegis_crypt_ops *ops)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_ctx *ctx = crypto_aegis256_aesni_ctx(tfm);
        struct skcipher_walk walk;
        struct aegis_state state;

        ops->skcipher_walk_init(&walk, req, true);

        kernel_fpu_begin();

        crypto_aegis256_aesni_init(&state, ctx->key, req->iv);
        crypto_aegis256_aesni_process_ad(&state, req->src, req->assoclen); /*covered*/
        crypto_aegis256_aesni_process_crypt(&state, &walk, ops); /*covered*/
        crypto_aegis256_aesni_final(&state, tag_xor, req->assoclen, cryptlen); /*covered*/

        kernel_fpu_end();
}

static int crypto_aegis256_aesni_encrypt(struct aead_request *req)
{
        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = crypto_aegis256_aesni_enc,
                .crypt_tail = crypto_aegis256_aesni_enc_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}

static int crypto_aegis256_aesni_decrypt(struct aead_request *req)
{
        static const struct aegis_block zeros = {};

        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = crypto_aegis256_aesni_dec,
                .crypt_tail = crypto_aegis256_aesni_dec_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);

        return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0; /*covered*/
}

static int crypto_aegis256_aesni_init_tfm(struct crypto_aead *aead)
{
        return 0; /*covered*/
}

static void crypto_aegis256_aesni_exit_tfm(struct crypto_aead *aead)
{
} /*covered*/

static int cryptd_aegis256_aesni_setkey(struct crypto_aead *aead,
                                        const u8 *key, unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}

static int cryptd_aegis256_aesni_setauthsize(struct crypto_aead *aead,
                                             unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int cryptd_aegis256_aesni_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int cryptd_aegis256_aesni_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}

static int cryptd_aegis256_aesni_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__aegis256-aesni", CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void cryptd_aegis256_aesni_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}

static struct aead_alg crypto_aegis256_aesni_alg[] = {
        {
                .setkey = crypto_aegis256_aesni_setkey,
                .setauthsize = crypto_aegis256_aesni_setauthsize,
                .encrypt = crypto_aegis256_aesni_encrypt,
                .decrypt = crypto_aegis256_aesni_decrypt,
                .init = crypto_aegis256_aesni_init_tfm,
                .exit = crypto_aegis256_aesni_exit_tfm,

                .ivsize = AEGIS256_NONCE_SIZE,
                .maxauthsize = AEGIS256_MAX_AUTH_SIZE,
                .chunksize = AEGIS256_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct aegis_ctx) +
                                __alignof__(struct aegis_ctx),
                        .cra_alignmask = 0,

                        .cra_name = "__aegis256",
                        .cra_driver_name = "__aegis256-aesni",

                        .cra_module = THIS_MODULE,
                }
        }, {
                .setkey = cryptd_aegis256_aesni_setkey,
                .setauthsize = cryptd_aegis256_aesni_setauthsize,
                .encrypt = cryptd_aegis256_aesni_encrypt,
                .decrypt = cryptd_aegis256_aesni_decrypt,
                .init = cryptd_aegis256_aesni_init_tfm,
                .exit = cryptd_aegis256_aesni_exit_tfm,

                .ivsize = AEGIS256_NONCE_SIZE,
                .maxauthsize = AEGIS256_MAX_AUTH_SIZE,
                .chunksize = AEGIS256_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_ASYNC,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct cryptd_aead *),
                        .cra_alignmask = 0,

                        .cra_priority = 400,

                        .cra_name = "aegis256",
                        .cra_driver_name = "aegis256-aesni",

                        .cra_module = THIS_MODULE,
                }
        }
};

static int __init crypto_aegis256_aesni_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_aegis256_aesni_alg,
                                    ARRAY_SIZE(crypto_aegis256_aesni_alg));
}

static void __exit crypto_aegis256_aesni_module_exit(void)
{
        crypto_unregister_aeads(crypto_aegis256_aesni_alg,
                                ARRAY_SIZE(crypto_aegis256_aesni_alg));
}

module_init(crypto_aegis256_aesni_module_init);
module_exit(crypto_aegis256_aesni_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("AEGIS-256 AEAD algorithm -- AESNI+SSE2 implementation");
MODULE_ALIAS_CRYPTO("aegis256");
MODULE_ALIAS_CRYPTO("aegis256-aesni");

/*
 * Support for Intel AES-NI instructions. This file contains glue
 * code, the real AES implementation is in intel-aes_asm.S.
 *
 * Copyright (C) 2008, Intel Corp.
 *    Author: Huang Ying <ying.huang@intel.com>
 *
 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
 * interface for 64-bit kernels.
 *    Authors: Adrian Hoban <adrian.hoban@intel.com>
 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
 *             Tadeusz Struk (tadeusz.struk@intel.com)
 *             Aidan O'Mahony (aidan.o.mahony@intel.com)
 *    Copyright (c) 2010, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <linux/hardirq.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/cryptd.h>
#include <crypto/ctr.h>
#include <crypto/b128ops.h>
#include <crypto/gcm.h>
#include <crypto/xts.h>
#include <asm/cpu_device_id.h>
#include <asm/fpu/api.h>
#include <asm/crypto/aes.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
#ifdef CONFIG_X86_64
#include <asm/crypto/glue_helper.h>
#endif


#define AESNI_ALIGN        16
#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
#define AES_BLOCK_MASK        (~(AES_BLOCK_SIZE - 1))
#define RFC4106_HASH_SUBKEY_SIZE 16
#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)

/* This data is stored at the end of the crypto_tfm struct.
 * It's a type of per "session" data storage location.
 * This needs to be 16 byte aligned.
 */
struct aesni_rfc4106_gcm_ctx {
        u8 hash_subkey[16] AESNI_ALIGN_ATTR;
        struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
        u8 nonce[4];
};

struct generic_gcmaes_ctx {
        u8 hash_subkey[16] AESNI_ALIGN_ATTR;
        struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
};

struct aesni_xts_ctx {
        u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
        u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
};

#define GCM_BLOCK_LEN 16

struct gcm_context_data {
        /* init, update and finalize context data */
        u8 aad_hash[GCM_BLOCK_LEN];
        u64 aad_length;
        u64 in_length;
        u8 partial_block_enc_key[GCM_BLOCK_LEN];
        u8 orig_IV[GCM_BLOCK_LEN];
        u8 current_counter[GCM_BLOCK_LEN];
        u64 partial_block_len;
        u64 unused;
        u8 hash_keys[GCM_BLOCK_LEN * 16];
};

asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
                             unsigned int key_len);
asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
                          const u8 *in);
asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
                          const u8 *in);
asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len);
asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len);
asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);
asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);

#define AVX_GEN2_OPTSIZE 640
#define AVX_GEN4_OPTSIZE 4096

#ifdef CONFIG_X86_64

static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);
asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);

asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
                                 const u8 *in, bool enc, u8 *iv);

/* asmlinkage void aesni_gcm_enc()
 * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 * struct gcm_context_data.  May be uninitialized.
 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 * const u8 *in, Plaintext input
 * unsigned long plaintext_len, Length of data in bytes for encryption.
 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 *         16-byte aligned pointer.
 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 * const u8 *aad, Additional Authentication Data (AAD)
 * unsigned long aad_len, Length of AAD in bytes.
 * u8 *auth_tag, Authenticated Tag output.
 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 *          Valid values are 16 (most likely), 12 or 8.
 */
asmlinkage void aesni_gcm_enc(void *ctx,
                        struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

/* asmlinkage void aesni_gcm_dec()
 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 * struct gcm_context_data.  May be uninitialized.
 * u8 *out, Plaintext output. Decrypt in-place is allowed.
 * const u8 *in, Ciphertext input
 * unsigned long ciphertext_len, Length of data in bytes for decryption.
 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 *         16-byte aligned pointer.
 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 * const u8 *aad, Additional Authentication Data (AAD)
 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 * to be 8 or 12 bytes
 * u8 *auth_tag, Authenticated Tag output.
 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 * Valid values are 16 (most likely), 12 or 8.
 */
asmlinkage void aesni_gcm_dec(void *ctx,
                        struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

/* Scatter / Gather routines, with args similar to above */
asmlinkage void aesni_gcm_init(void *ctx,
                               struct gcm_context_data *gdata,
                               u8 *iv,
                               u8 *hash_subkey, const u8 *aad,
                               unsigned long aad_len);
asmlinkage void aesni_gcm_enc_update(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in, unsigned long plaintext_len);
asmlinkage void aesni_gcm_dec_update(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in,
                                     unsigned long ciphertext_len);
asmlinkage void aesni_gcm_finalize(void *ctx,
                                   struct gcm_context_data *gdata,
                                   u8 *auth_tag, unsigned long auth_tag_len);

static const struct aesni_gcm_tfm_s {
        void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv,
                     u8 *hash_subkey, const u8 *aad, unsigned long aad_len);
        void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
                           const u8 *in, unsigned long plaintext_len);
        void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out,
                           const u8 *in, unsigned long ciphertext_len);
        void (*finalize)(void *ctx, struct gcm_context_data *gdata,
                         u8 *auth_tag, unsigned long auth_tag_len);
} *aesni_gcm_tfm;

static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = {
        .init = &aesni_gcm_init,
        .enc_update = &aesni_gcm_enc_update,
        .dec_update = &aesni_gcm_dec_update,
        .finalize = &aesni_gcm_finalize,
};

#ifdef CONFIG_AS_AVX
asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
                void *keys, u8 *out, unsigned int num_bytes);
asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
                void *keys, u8 *out, unsigned int num_bytes);
asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
                void *keys, u8 *out, unsigned int num_bytes);
/*
 * asmlinkage void aesni_gcm_init_avx_gen2()
 * gcm_data *my_ctx_data, context data
 * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 */
asmlinkage void aesni_gcm_init_avx_gen2(void *my_ctx_data,
                                        struct gcm_context_data *gdata,
                                        u8 *iv,
                                        u8 *hash_subkey,
                                        const u8 *aad,
                                        unsigned long aad_len);

asmlinkage void aesni_gcm_enc_update_avx_gen2(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in, unsigned long plaintext_len);
asmlinkage void aesni_gcm_dec_update_avx_gen2(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in,
                                     unsigned long ciphertext_len);
asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx,
                                   struct gcm_context_data *gdata,
                                   u8 *auth_tag, unsigned long auth_tag_len);

asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx,
                                struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx,
                                struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = {
        .init = &aesni_gcm_init_avx_gen2,
        .enc_update = &aesni_gcm_enc_update_avx_gen2,
        .dec_update = &aesni_gcm_dec_update_avx_gen2,
        .finalize = &aesni_gcm_finalize_avx_gen2,
};

#endif

#ifdef CONFIG_AS_AVX2
/*
 * asmlinkage void aesni_gcm_init_avx_gen4()
 * gcm_data *my_ctx_data, context data
 * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 */
asmlinkage void aesni_gcm_init_avx_gen4(void *my_ctx_data,
                                        struct gcm_context_data *gdata,
                                        u8 *iv,
                                        u8 *hash_subkey,
                                        const u8 *aad,
                                        unsigned long aad_len);

asmlinkage void aesni_gcm_enc_update_avx_gen4(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in, unsigned long plaintext_len);
asmlinkage void aesni_gcm_dec_update_avx_gen4(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in,
                                     unsigned long ciphertext_len);
asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx,
                                   struct gcm_context_data *gdata,
                                   u8 *auth_tag, unsigned long auth_tag_len);

asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx,
                                struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx,
                                struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = {
        .init = &aesni_gcm_init_avx_gen4,
        .enc_update = &aesni_gcm_enc_update_avx_gen4,
        .dec_update = &aesni_gcm_dec_update_avx_gen4,
        .finalize = &aesni_gcm_finalize_avx_gen4,
};

#endif

static inline struct
aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
{
        unsigned long align = AESNI_ALIGN;

        if (align <= crypto_tfm_ctx_alignment())
                align = 1;
        return PTR_ALIGN(crypto_aead_ctx(tfm), align); /*covered*/
}

static inline struct
generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
{
        unsigned long align = AESNI_ALIGN;

        if (align <= crypto_tfm_ctx_alignment())
                align = 1;
        return PTR_ALIGN(crypto_aead_ctx(tfm), align); /*covered*/
}
#endif

static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
{
        unsigned long addr = (unsigned long)raw_ctx; /*covered*/
        unsigned long align = AESNI_ALIGN;

        if (align <= crypto_tfm_ctx_alignment())
                align = 1;
        return (struct crypto_aes_ctx *)ALIGN(addr, align);
}

static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
                              const u8 *in_key, unsigned int key_len)
{
        struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); /*covered*/
        u32 *flags = &tfm->crt_flags;
        int err;

        if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && /*covered*/
            key_len != AES_KEYSIZE_256) {
                *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; /*covered*/
                return -EINVAL;
        }

        if (!irq_fpu_usable()) /*covered*/
                err = crypto_aes_expand_key(ctx, in_key, key_len);
        else {
                kernel_fpu_begin(); /*covered*/
                err = aesni_set_key(ctx, in_key, key_len);
                kernel_fpu_end(); /*covered*/
        }

        return err;
}

static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
                       unsigned int key_len)
{
        return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); /*covered*/
}

static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); /*covered*/

        if (!irq_fpu_usable())
                crypto_aes_encrypt_x86(ctx, dst, src);
        else {
                kernel_fpu_begin(); /*covered*/
                aesni_enc(ctx, dst, src);
                kernel_fpu_end();
        }
} /*covered*/

static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); /*covered*/

        if (!irq_fpu_usable())
                crypto_aes_decrypt_x86(ctx, dst, src);
        else {
                kernel_fpu_begin(); /*covered*/
                aesni_dec(ctx, dst, src);
                kernel_fpu_end();
        }
} /*covered*/

static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));

        aesni_enc(ctx, dst, src);
}

static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));

        aesni_dec(ctx, dst, src);
}

static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
                                 unsigned int len)
{
        return aes_set_key_common(crypto_skcipher_tfm(tfm), /*covered*/
                                  crypto_skcipher_ctx(tfm), key, len);
}

static int ecb_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
                              nbytes & AES_BLOCK_MASK);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end(); /*covered*/

        return err;
}

static int ecb_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, /*covered*/
                              nbytes & AES_BLOCK_MASK);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end(); /*covered*/

        return err;
}

static int cbc_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
                              nbytes & AES_BLOCK_MASK, walk.iv);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end();

        return err;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
                              nbytes & AES_BLOCK_MASK, walk.iv); /*covered*/
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end(); /*covered*/

        return err;
}

#ifdef CONFIG_X86_64
static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
                            struct skcipher_walk *walk)
{
        u8 *ctrblk = walk->iv; /*covered*/
        u8 keystream[AES_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        aesni_enc(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, AES_BLOCK_SIZE);
}

#ifdef CONFIG_AS_AVX
static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv)
{
        /*
         * based on key length, override with the by8 version
         * of ctr mode encryption/decryption for improved performance
         * aes_set_key_common() ensures that key length is one of
         * {128,192,256}
         */
        if (ctx->key_length == AES_KEYSIZE_128) /*covered*/
                aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); /*covered*/
        else if (ctx->key_length == AES_KEYSIZE_192) /*covered*/
                aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); /*covered*/
        else
                aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); /*covered*/
} /*covered*/
#endif

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
                aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, /*covered*/
                                      nbytes & AES_BLOCK_MASK, walk.iv);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        if (walk.nbytes) { /*covered*/
                ctr_crypt_final(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, 0);
        }
        kernel_fpu_end(); /*covered*/

        return err;
}

static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, /*covered*/
                            unsigned int keylen)
{
        struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        keylen /= 2;

        /* first half of xts-key is for crypt */
        err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, /*covered*/
                                 key, keylen);
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, /*covered*/
                                  key + keylen, keylen);
}


static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
{
        aesni_enc(ctx, out, in); /*covered*/
}

static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc));
}

static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); /*covered*/
}

static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv);
}

static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); /*covered*/
}

static const struct common_glue_ctx aesni_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = 1,

        .funcs = { {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
        } }
};

static const struct common_glue_ctx aesni_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = 1,

        .funcs = { {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
        } }
};

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&aesni_enc_xts, req,
                                   XTS_TWEAK_CAST(aesni_xts_tweak),
                                   aes_ctx(ctx->raw_tweak_ctx),
                                   aes_ctx(ctx->raw_crypt_ctx));
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&aesni_dec_xts, req,
                                   XTS_TWEAK_CAST(aesni_xts_tweak),
                                   aes_ctx(ctx->raw_tweak_ctx),
                                   aes_ctx(ctx->raw_crypt_ctx));
}

static int rfc4106_init(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", /*covered*/
                                       CRYPTO_ALG_INTERNAL,
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void rfc4106_exit(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}

static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
        struct crypto_cipher *tfm;
        int ret;

        tfm = crypto_alloc_cipher("aes", 0, 0); /*covered*/
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        ret = crypto_cipher_setkey(tfm, key, key_len); /*covered*/
        if (ret)
                goto out_free_cipher;

        /* Clear the data in the hash sub key container to zero.*/
        /* We want to cipher all zeros to create the hash sub key. */
        memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); /*covered*/

        crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);

out_free_cipher:
        crypto_free_cipher(tfm); /*covered*/
        return ret; /*covered*/
}

static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
                                  unsigned int key_len)
{
        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); /*covered*/

        if (key_len < 4) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }
        /*Account for 4 byte nonce at the end.*/
        key_len -= 4; /*covered*/

        memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));

        return aes_set_key_common(crypto_aead_tfm(aead),
                                  &ctx->aes_key_expanded, key, key_len) ?: /*covered*/
               rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); /*covered*/
}

static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
                                  unsigned int key_len)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(parent);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
}

static int common_rfc4106_set_authsize(struct crypto_aead *aead,
                                       unsigned int authsize)
{
        switch (authsize) { /*covered*/
        case 8:
        case 12:
        case 16:
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

/* This is the Integrity Check Value (aka the authentication tag length and can
 * be 8, 12 or 16 bytes long. */
static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
                                       unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(parent);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
                                       unsigned int authsize)
{
        switch (authsize) { /*covered*/
        case 4:
        case 8:
        case 12:
        case 13:
        case 14:
        case 15:
        case 16:
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
                              unsigned int assoclen, u8 *hash_subkey,
                              u8 *iv, void *aes_ctx)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        unsigned long auth_tag_len = crypto_aead_authsize(tfm);
        const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm;
        struct gcm_context_data data AESNI_ALIGN_ATTR;
        struct scatter_walk dst_sg_walk = {};
        unsigned long left = req->cryptlen;
        unsigned long len, srclen, dstlen;
        struct scatter_walk assoc_sg_walk;
        struct scatter_walk src_sg_walk;
        struct scatterlist src_start[2];
        struct scatterlist dst_start[2];
        struct scatterlist *src_sg;
        struct scatterlist *dst_sg;
        u8 *src, *dst, *assoc;
        u8 *assocmem = NULL;
        u8 authTag[16];

        if (!enc)
                left -= auth_tag_len; /*covered*/

#ifdef CONFIG_AS_AVX2
        if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4) /*covered*/
                gcm_tfm = &aesni_gcm_tfm_avx_gen2;
#endif
#ifdef CONFIG_AS_AVX
        if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2) /*covered*/
                gcm_tfm = &aesni_gcm_tfm_sse;
#endif

        /* Linearize assoc, if not already linear */
        if (req->src->length >= assoclen && req->src->length && /*covered*/
                (!PageHighMem(sg_page(req->src)) || /*covered*/
                        req->src->offset + req->src->length <= PAGE_SIZE)) {
                scatterwalk_start(&assoc_sg_walk, req->src); /*covered*/
                assoc = scatterwalk_map(&assoc_sg_walk);
        } else {
                /* assoc can be any length, so must be on heap */
                assocmem = kmalloc(assoclen, GFP_ATOMIC); /*covered*/
                if (unlikely(!assocmem))
                        return -ENOMEM;
                assoc = assocmem;

                scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); /*covered*/
        }

        if (left) { /*covered*/
                src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); /*covered*/
                scatterwalk_start(&src_sg_walk, src_sg);
                if (req->src != req->dst) {
                        dst_sg = scatterwalk_ffwd(dst_start, req->dst, /*covered*/
                                                  req->assoclen);
                        scatterwalk_start(&dst_sg_walk, dst_sg);
                }
        }

        kernel_fpu_begin(); /*covered*/
        gcm_tfm->init(aes_ctx, &data, iv,
                hash_subkey, assoc, assoclen);
        if (req->src != req->dst) {
                while (left) { /*covered*/
                        src = scatterwalk_map(&src_sg_walk); /*covered*/
                        dst = scatterwalk_map(&dst_sg_walk); /*covered*/
                        srclen = scatterwalk_clamp(&src_sg_walk, left);
                        dstlen = scatterwalk_clamp(&dst_sg_walk, left);
                        len = min(srclen, dstlen);
                        if (len) { /*covered*/
                                if (enc)
                                        gcm_tfm->enc_update(aes_ctx, &data, /*covered*/
                                                             dst, src, len);
                                else
                                        gcm_tfm->dec_update(aes_ctx, &data,
                                                             dst, src, len);
                        }
                        left -= len; /*covered*/

                        scatterwalk_unmap(src);
                        scatterwalk_unmap(dst); /*covered*/
                        scatterwalk_advance(&src_sg_walk, len); /*covered*/
                        scatterwalk_advance(&dst_sg_walk, len);
                        scatterwalk_done(&src_sg_walk, 0, left); /*covered*/
                        scatterwalk_done(&dst_sg_walk, 1, left); /*covered*/
                }
        } else {
                while (left) { /*covered*/
                        dst = src = scatterwalk_map(&src_sg_walk); /*covered*/
                        len = scatterwalk_clamp(&src_sg_walk, left);
                        if (len) { /*covered*/
                                if (enc)
                                        gcm_tfm->enc_update(aes_ctx, &data, /*covered*/
                                                             src, src, len);
                                else
                                        gcm_tfm->dec_update(aes_ctx, &data, /*covered*/
                                                             src, src, len);
                        }
                        left -= len; /*covered*/
                        scatterwalk_unmap(src);
                        scatterwalk_advance(&src_sg_walk, len); /*covered*/
                        scatterwalk_done(&src_sg_walk, 1, left); /*covered*/
                }
        }
        gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len); /*covered*/
        kernel_fpu_end();

        if (!assocmem)
                scatterwalk_unmap(assoc); /*covered*/
        else
                kfree(assocmem); /*covered*/

        if (!enc) { /*covered*/
                u8 authTagMsg[16];

                /* Copy out original authTag */
                scatterwalk_map_and_copy(authTagMsg, req->src, /*covered*/
                                         req->assoclen + req->cryptlen -
                                         auth_tag_len,
                                         auth_tag_len, 0);

                /* Compare generated tag with passed in tag. */
                return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
                        -EBADMSG : 0; /*covered*/
        }

        /* Copy in the authTag */
        scatterwalk_map_and_copy(authTag, req->dst, /*covered*/
                                 req->assoclen + req->cryptlen,
                                 auth_tag_len, 1);

        return 0; /*covered*/
}

static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
                          u8 *hash_subkey, u8 *iv, void *aes_ctx)
{
        return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv,
                                aes_ctx);
}

static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
                          u8 *hash_subkey, u8 *iv, void *aes_ctx)
{
        return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, /*covered*/
                                aes_ctx);
}

static int helper_rfc4106_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
        unsigned int i;
        __be32 counter = cpu_to_be32(1);

        /* Assuming we are supporting rfc4106 64-bit extended */
        /* sequence numbers We need to have the AAD length equal */
        /* to 16 or 20 bytes */
        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
                return -EINVAL;

        /* IV below built */
        for (i = 0; i < 4; i++)
                *(iv+i) = ctx->nonce[i];
        for (i = 0; i < 8; i++)
                *(iv+4+i) = req->iv[i];
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int helper_rfc4106_decrypt(struct aead_request *req)
{
        __be32 counter = cpu_to_be32(1);
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
        unsigned int i;

        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
                return -EINVAL;

        /* Assuming we are supporting rfc4106 64-bit extended */
        /* sequence numbers We need to have the AAD length */
        /* equal to 16 or 20 bytes */

        /* IV below built */
        for (i = 0; i < 4; i++)
                *(iv+i) = ctx->nonce[i];
        for (i = 0; i < 8; i++)
                *(iv+4+i) = req->iv[i];
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int gcmaes_wrapper_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
        struct cryptd_aead *cryptd_tfm = *ctx;

        tfm = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                tfm = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, tfm); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int gcmaes_wrapper_decrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
        struct cryptd_aead *cryptd_tfm = *ctx;

        tfm = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                tfm = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, tfm); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}
#endif

static struct crypto_alg aesni_algs[] = { {
        .cra_name                = "aes",
        .cra_driver_name        = "aes-aesni",
        .cra_priority                = 300,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = AES_BLOCK_SIZE,
        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
        .cra_module                = THIS_MODULE,
        .cra_u        = {
                .cipher        = {
                        .cia_min_keysize        = AES_MIN_KEY_SIZE,
                        .cia_max_keysize        = AES_MAX_KEY_SIZE,
                        .cia_setkey                = aes_set_key,
                        .cia_encrypt                = aes_encrypt,
                        .cia_decrypt                = aes_decrypt
                }
        }
}, {
        .cra_name                = "__aes",
        .cra_driver_name        = "__aes-aesni",
        .cra_priority                = 300,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
        .cra_blocksize                = AES_BLOCK_SIZE,
        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
        .cra_module                = THIS_MODULE,
        .cra_u        = {
                .cipher        = {
                        .cia_min_keysize        = AES_MIN_KEY_SIZE,
                        .cia_max_keysize        = AES_MAX_KEY_SIZE,
                        .cia_setkey                = aes_set_key,
                        .cia_encrypt                = __aes_encrypt,
                        .cia_decrypt                = __aes_decrypt
                }
        }
} };

static struct skcipher_alg aesni_skciphers[] = {
        {
                .base = {
                        .cra_name                = "__ecb(aes)",
                        .cra_driver_name        = "__ecb-aes-aesni",
                        .cra_priority                = 400,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = AES_BLOCK_SIZE,
                        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = AES_MIN_KEY_SIZE,
                .max_keysize        = AES_MAX_KEY_SIZE,
                .setkey                = aesni_skcipher_setkey,
                .encrypt        = ecb_encrypt,
                .decrypt        = ecb_decrypt,
        }, {
                .base = {
                        .cra_name                = "__cbc(aes)",
                        .cra_driver_name        = "__cbc-aes-aesni",
                        .cra_priority                = 400,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = AES_BLOCK_SIZE,
                        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = AES_MIN_KEY_SIZE,
                .max_keysize        = AES_MAX_KEY_SIZE,
                .ivsize                = AES_BLOCK_SIZE,
                .setkey                = aesni_skcipher_setkey,
                .encrypt        = cbc_encrypt,
                .decrypt        = cbc_decrypt,
#ifdef CONFIG_X86_64
        }, {
                .base = {
                        .cra_name                = "__ctr(aes)",
                        .cra_driver_name        = "__ctr-aes-aesni",
                        .cra_priority                = 400,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = 1,
                        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = AES_MIN_KEY_SIZE,
                .max_keysize        = AES_MAX_KEY_SIZE,
                .ivsize                = AES_BLOCK_SIZE,
                .chunksize        = AES_BLOCK_SIZE,
                .setkey                = aesni_skcipher_setkey,
                .encrypt        = ctr_crypt,
                .decrypt        = ctr_crypt,
        }, {
                .base = {
                        .cra_name                = "__xts(aes)",
                        .cra_driver_name        = "__xts-aes-aesni",
                        .cra_priority                = 401,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = AES_BLOCK_SIZE,
                        .cra_ctxsize                = XTS_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = 2 * AES_MIN_KEY_SIZE,
                .max_keysize        = 2 * AES_MAX_KEY_SIZE,
                .ivsize                = AES_BLOCK_SIZE,
                .setkey                = xts_aesni_setkey,
                .encrypt        = xts_encrypt,
                .decrypt        = xts_decrypt,
#endif
        }
};

static
struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];

#ifdef CONFIG_X86_64
static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
                                  unsigned int key_len)
{
        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); /*covered*/

        return aes_set_key_common(crypto_aead_tfm(aead),
                                  &ctx->aes_key_expanded, key, key_len) ?: /*covered*/
               rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); /*covered*/
}

static int generic_gcmaes_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
        __be32 counter = cpu_to_be32(1);

        memcpy(iv, req->iv, 12);
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int generic_gcmaes_decrypt(struct aead_request *req)
{
        __be32 counter = cpu_to_be32(1);
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));

        memcpy(iv, req->iv, 12);
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int generic_gcmaes_init(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni", /*covered*/
                                       CRYPTO_ALG_INTERNAL,
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));

        return 0; /*covered*/
}

static void generic_gcmaes_exit(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx);
}

static struct aead_alg aesni_aead_algs[] = { {
        .setkey                        = common_rfc4106_set_key,
        .setauthsize                = common_rfc4106_set_authsize,
        .encrypt                = helper_rfc4106_encrypt,
        .decrypt                = helper_rfc4106_decrypt,
        .ivsize                        = GCM_RFC4106_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "__gcm-aes-aesni",
                .cra_driver_name        = "__driver-gcm-aes-aesni",
                .cra_flags                = CRYPTO_ALG_INTERNAL,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct aesni_rfc4106_gcm_ctx),
                .cra_alignmask                = AESNI_ALIGN - 1,
                .cra_module                = THIS_MODULE,
        },
}, {
        .init                        = rfc4106_init,
        .exit                        = rfc4106_exit,
        .setkey                        = gcmaes_wrapper_set_key,
        .setauthsize                = gcmaes_wrapper_set_authsize,
        .encrypt                = gcmaes_wrapper_encrypt,
        .decrypt                = gcmaes_wrapper_decrypt,
        .ivsize                        = GCM_RFC4106_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "rfc4106(gcm(aes))",
                .cra_driver_name        = "rfc4106-gcm-aesni",
                .cra_priority                = 400,
                .cra_flags                = CRYPTO_ALG_ASYNC,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct cryptd_aead *),
                .cra_module                = THIS_MODULE,
        },
}, {
        .setkey                        = generic_gcmaes_set_key,
        .setauthsize                = generic_gcmaes_set_authsize,
        .encrypt                = generic_gcmaes_encrypt,
        .decrypt                = generic_gcmaes_decrypt,
        .ivsize                        = GCM_AES_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "__generic-gcm-aes-aesni",
                .cra_driver_name        = "__driver-generic-gcm-aes-aesni",
                .cra_priority                = 0,
                .cra_flags                = CRYPTO_ALG_INTERNAL,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct generic_gcmaes_ctx),
                .cra_alignmask                = AESNI_ALIGN - 1,
                .cra_module                = THIS_MODULE,
        },
}, {
        .init                        = generic_gcmaes_init,
        .exit                        = generic_gcmaes_exit,
        .setkey                        = gcmaes_wrapper_set_key,
        .setauthsize                = gcmaes_wrapper_set_authsize,
        .encrypt                = gcmaes_wrapper_encrypt,
        .decrypt                = gcmaes_wrapper_decrypt,
        .ivsize                        = GCM_AES_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "gcm(aes)",
                .cra_driver_name        = "generic-gcm-aesni",
                .cra_priority                = 400,
                .cra_flags                = CRYPTO_ALG_ASYNC,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct cryptd_aead *),
                .cra_module                = THIS_MODULE,
        },
} };
#else
static struct aead_alg aesni_aead_algs[0];
#endif


static const struct x86_cpu_id aesni_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_AES),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);

static void aesni_free_simds(void)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
                    aesni_simd_skciphers[i]; i++)
                simd_skcipher_free(aesni_simd_skciphers[i]);
}

static int __init aesni_init(void)
{
        struct simd_skcipher_alg *simd;
        const char *basename;
        const char *algname;
        const char *drvname;
        int err;
        int i;

        if (!x86_match_cpu(aesni_cpu_id))
                return -ENODEV;
#ifdef CONFIG_X86_64
#ifdef CONFIG_AS_AVX2
        if (boot_cpu_has(X86_FEATURE_AVX2)) {
                pr_info("AVX2 version of gcm_enc/dec engaged.\n");
                aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4;
        } else
#endif
#ifdef CONFIG_AS_AVX
        if (boot_cpu_has(X86_FEATURE_AVX)) {
                pr_info("AVX version of gcm_enc/dec engaged.\n");
                aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2;
        } else
#endif
        {
                pr_info("SSE version of gcm_enc/dec engaged.\n");
                aesni_gcm_tfm = &aesni_gcm_tfm_sse;
        }
        aesni_ctr_enc_tfm = aesni_ctr_enc;
#ifdef CONFIG_AS_AVX
        if (boot_cpu_has(X86_FEATURE_AVX)) {
                /* optimize performance of ctr mode encryption transform */
                aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
                pr_info("AES CTR mode by8 optimization enabled\n");
        }
#endif
#endif

        err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
        if (err)
                return err;

        err = crypto_register_skciphers(aesni_skciphers,
                                        ARRAY_SIZE(aesni_skciphers));
        if (err)
                goto unregister_algs;

        err = crypto_register_aeads(aesni_aead_algs,
                                    ARRAY_SIZE(aesni_aead_algs));
        if (err)
                goto unregister_skciphers;

        for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
                algname = aesni_skciphers[i].base.cra_name + 2;
                drvname = aesni_skciphers[i].base.cra_driver_name + 2;
                basename = aesni_skciphers[i].base.cra_driver_name;
                simd = simd_skcipher_create_compat(algname, drvname, basename);
                err = PTR_ERR(simd);
                if (IS_ERR(simd))
                        goto unregister_simds;

                aesni_simd_skciphers[i] = simd;
        }

        return 0;

unregister_simds:
        aesni_free_simds();
        crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
unregister_skciphers:
        crypto_unregister_skciphers(aesni_skciphers,
                                    ARRAY_SIZE(aesni_skciphers));
unregister_algs:
        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
        return err;
}

static void __exit aesni_exit(void)
{
        aesni_free_simds();
        crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
        crypto_unregister_skciphers(aesni_skciphers,
                                    ARRAY_SIZE(aesni_skciphers));
        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
}

late_initcall(aesni_init);
module_exit(aesni_exit);

MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");

/*
 * Glue Code for assembler optimized version of Blowfish
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <crypto/algapi.h>
#include <crypto/blowfish.h>
#include <crypto/internal/skcipher.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

/* regular block cipher functions */
asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
                                   bool xor);
asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);

/* 4-way parallel cipher functions */
asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                        const u8 *src, bool xor);
asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                      const u8 *src);

static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
{
        __blowfish_enc_blk(ctx, dst, src, false); /*covered*/
}

static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __blowfish_enc_blk(ctx, dst, src, true);
}

static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        __blowfish_enc_blk_4way(ctx, dst, src, false); /*covered*/
}

static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
                                      const u8 *src)
{
        __blowfish_enc_blk_4way(ctx, dst, src, true);
}

static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
}

static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static int blowfish_setkey_skcipher(struct crypto_skcipher *tfm,
                                    const u8 *key, unsigned int keylen)
{
        return blowfish_setkey(&tfm->base, key, keylen); /*covered*/
}

static int ecb_crypt(struct skcipher_request *req,
                     void (*fn)(struct bf_ctx *, u8 *, const u8 *),
                     void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
{
        unsigned int bsize = BF_BLOCK_SIZE;
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                u8 *wsrc = walk.src.virt.addr; /*covered*/
                u8 *wdst = walk.dst.virt.addr;

                /* Process four block batch */
                if (nbytes >= bsize * 4) {
                        do {
                                fn_4way(ctx, wdst, wsrc); /*covered*/

                                wsrc += bsize * 4;
                                wdst += bsize * 4;
                                nbytes -= bsize * 4;
                        } while (nbytes >= bsize * 4);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }

                /* Handle leftovers */
                do {
                        fn(ctx, wdst, wsrc); /*covered*/

                        wsrc += bsize;
                        wdst += bsize;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

done:
                err = skcipher_walk_done(&walk, nbytes); /*covered*/
        }

        return err; /*covered*/
}

static int ecb_encrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, blowfish_enc_blk, blowfish_enc_blk_4way); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, blowfish_dec_blk, blowfish_dec_blk_4way); /*covered*/
}

static unsigned int __cbc_encrypt(struct bf_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = BF_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 *iv = (u64 *)walk->iv;

        do {
                *dst = *src ^ *iv; /*covered*/
                blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
                iv = dst;

                src += 1;
                dst += 1;
                nbytes -= bsize;
        } while (nbytes >= bsize);

        *(u64 *)walk->iv = *iv; /*covered*/
        return nbytes;
}

static int cbc_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                nbytes = __cbc_encrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static unsigned int __cbc_decrypt(struct bf_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = BF_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 ivs[4 - 1];
        u64 last_iv;

        /* Start of the last block. */
        src += nbytes / bsize - 1;
        dst += nbytes / bsize - 1;

        last_iv = *src;

        /* Process four block batch */
        if (nbytes >= bsize * 4) {
                do {
                        nbytes -= bsize * 4 - bsize;
                        src -= 4 - 1; /*covered*/
                        dst -= 4 - 1;

                        ivs[0] = src[0];
                        ivs[1] = src[1];
                        ivs[2] = src[2];

                        blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);

                        dst[1] ^= ivs[0];
                        dst[2] ^= ivs[1];
                        dst[3] ^= ivs[2];

                        nbytes -= bsize;
                        if (nbytes < bsize)
                                goto done;

                        *dst ^= *(src - 1); /*covered*/
                        src -= 1;
                        dst -= 1;
                } while (nbytes >= bsize * 4);
        }

        /* Handle leftovers */
        for (;;) {
                blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src); /*covered*/

                nbytes -= bsize;
                if (nbytes < bsize)
                        break;

                *dst ^= *(src - 1); /*covered*/
                src -= 1;
                dst -= 1;
        }

done:
        *dst ^= *(u64 *)walk->iv; /*covered*/
        *(u64 *)walk->iv = last_iv;

        return nbytes;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                nbytes = __cbc_decrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static void ctr_crypt_final(struct bf_ctx *ctx, struct skcipher_walk *walk)
{
        u8 *ctrblk = walk->iv; /*covered*/
        u8 keystream[BF_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        blowfish_enc_blk(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, BF_BLOCK_SIZE);
}

static unsigned int __ctr_crypt(struct bf_ctx *ctx, struct skcipher_walk *walk)
{
        unsigned int bsize = BF_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
        __be64 ctrblocks[4];

        /* Process four block batch */
        if (nbytes >= bsize * 4) {
                do {
                        if (dst != src) { /*covered*/
                                dst[0] = src[0]; /*covered*/
                                dst[1] = src[1];
                                dst[2] = src[2];
                                dst[3] = src[3];
                        }

                        /* create ctrblks for parallel encrypt */
                        ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/
                        ctrblocks[1] = cpu_to_be64(ctrblk++);
                        ctrblocks[2] = cpu_to_be64(ctrblk++);
                        ctrblocks[3] = cpu_to_be64(ctrblk++);

                        blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
                                                  (u8 *)ctrblocks);

                        src += 4;
                        dst += 4;
                } while ((nbytes -= bsize * 4) >= bsize * 4);

                if (nbytes < bsize) /*covered*/
                        goto done;
        }

        /* Handle leftovers */
        do {
                if (dst != src) /*covered*/
                        *dst = *src; /*covered*/

                ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/

                blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);

                src += 1;
                dst += 1;
        } while ((nbytes -= bsize) >= bsize);

done:
        *(__be64 *)walk->iv = cpu_to_be64(ctrblk); /*covered*/
        return nbytes;
}

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
                nbytes = __ctr_crypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        if (nbytes) { /*covered*/
                ctr_crypt_final(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, 0);
        }

        return err; /*covered*/
}

static struct crypto_alg bf_cipher_alg = {
        .cra_name                = "blowfish",
        .cra_driver_name        = "blowfish-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = BF_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct bf_ctx),
        .cra_alignmask                = 0,
        .cra_module                = THIS_MODULE,
        .cra_u = {
                .cipher = {
                        .cia_min_keysize        = BF_MIN_KEY_SIZE,
                        .cia_max_keysize        = BF_MAX_KEY_SIZE,
                        .cia_setkey                = blowfish_setkey,
                        .cia_encrypt                = blowfish_encrypt,
                        .cia_decrypt                = blowfish_decrypt,
                }
        }
};

static struct skcipher_alg bf_skcipher_algs[] = {
        {
                .base.cra_name                = "ecb(blowfish)",
                .base.cra_driver_name        = "ecb-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = BF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(blowfish)",
                .base.cra_driver_name        = "cbc-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = BF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .ivsize                        = BF_BLOCK_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(blowfish)",
                .base.cra_driver_name        = "ctr-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .ivsize                        = BF_BLOCK_SIZE,
                .chunksize                = BF_BLOCK_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        },
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, blowfish-x86_64 is slower than generic C
                 * implementation because use of 64bit rotates (which are really
                 * slow on P4). Therefore blacklist P4s.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "blowfish-x86_64: performance on this CPU "
                        "would be suboptimal: disabling "
                        "blowfish-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&bf_cipher_alg);
        if (err)
                return err;

        err = crypto_register_skciphers(bf_skcipher_algs,
                                        ARRAY_SIZE(bf_skcipher_algs));
        if (err)
                crypto_unregister_alg(&bf_cipher_alg);

        return err;
}

static void __exit fini(void)
{
        crypto_unregister_alg(&bf_cipher_alg);
        crypto_unregister_skciphers(bf_skcipher_algs,
                                    ARRAY_SIZE(bf_skcipher_algs));
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("blowfish");
MODULE_ALIAS_CRYPTO("blowfish-asm");

/*
 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 */

#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/xts.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/types.h>

#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
#define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32

/* 32-way AVX2/AES-NI parallel cipher functions */
asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);

asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);

asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);
asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);

static const struct common_glue_ctx camellia_enc = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
        } }
};

static const struct common_glue_ctx camellia_ctr = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
        } }
};

static const struct common_glue_ctx camellia_enc_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
        } }
};

static const struct common_glue_ctx camellia_dec = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_cbc = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
        } }
};

static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
                           unsigned int keylen)
{
        return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen, /*covered*/
                                 &tfm->base.crt_flags);
}

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&camellia_ctr, req); /*covered*/
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&camellia_enc_xts, req,
                                   XTS_TWEAK_CAST(camellia_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&camellia_dec_xts, req,
                                   XTS_TWEAK_CAST(camellia_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg camellia_algs[] = {
        {
                .base.cra_name                = "__ecb(camellia)",
                .base.cra_driver_name        = "__ecb-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(camellia)",
                .base.cra_driver_name        = "__cbc-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(camellia)",
                .base.cra_driver_name        = "__ctr-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .chunksize                = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(camellia)",
                .base.cra_driver_name        = "__xts-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = 2 * CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = xts_camellia_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];

static int __init camellia_aesni_init(void)
{
        const char *feature_name;

        if (!boot_cpu_has(X86_FEATURE_AVX) ||
            !boot_cpu_has(X86_FEATURE_AVX2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
                pr_info("AVX2 or AES-NI instructions are not detected.\n");
                return -ENODEV;
        }

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(camellia_algs,
                                              ARRAY_SIZE(camellia_algs),
                                              camellia_simd_algs);
}

static void __exit camellia_aesni_fini(void)
{
        simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
                                  camellia_simd_algs);
}

module_init(camellia_aesni_init);
module_exit(camellia_aesni_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-asm");

/*
 * Glue Code for assembler optimized version of Camellia
 *
 * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * Camellia parts based on code by:
 *  Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <asm/unaligned.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>
#include <crypto/algapi.h>
#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>

/* regular block cipher functions */
asmlinkage void __camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, bool xor);
EXPORT_SYMBOL_GPL(__camellia_enc_blk);
asmlinkage void camellia_dec_blk(struct camellia_ctx *ctx, u8 *dst,
                                 const u8 *src);
EXPORT_SYMBOL_GPL(camellia_dec_blk);

/* 2-way parallel cipher functions */
asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                        const u8 *src, bool xor);
EXPORT_SYMBOL_GPL(__camellia_enc_blk_2way);
asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                      const u8 *src);
EXPORT_SYMBOL_GPL(camellia_dec_blk_2way);

static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        camellia_enc_blk(crypto_tfm_ctx(tfm), dst, src); /*covered*/
}

static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        camellia_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

/* camellia sboxes */
__visible const u64 camellia_sp10011110[256] = {
        0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL,
        0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL,
        0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL,
        0x8500008585858500ULL, 0x5700005757575700ULL, 0x3500003535353500ULL,
        0xea0000eaeaeaea00ULL, 0x0c00000c0c0c0c00ULL, 0xae0000aeaeaeae00ULL,
        0x4100004141414100ULL, 0x2300002323232300ULL, 0xef0000efefefef00ULL,
        0x6b00006b6b6b6b00ULL, 0x9300009393939300ULL, 0x4500004545454500ULL,
        0x1900001919191900ULL, 0xa50000a5a5a5a500ULL, 0x2100002121212100ULL,
        0xed0000edededed00ULL, 0x0e00000e0e0e0e00ULL, 0x4f00004f4f4f4f00ULL,
        0x4e00004e4e4e4e00ULL, 0x1d00001d1d1d1d00ULL, 0x6500006565656500ULL,
        0x9200009292929200ULL, 0xbd0000bdbdbdbd00ULL, 0x8600008686868600ULL,
        0xb80000b8b8b8b800ULL, 0xaf0000afafafaf00ULL, 0x8f00008f8f8f8f00ULL,
        0x7c00007c7c7c7c00ULL, 0xeb0000ebebebeb00ULL, 0x1f00001f1f1f1f00ULL,
        0xce0000cececece00ULL, 0x3e00003e3e3e3e00ULL, 0x3000003030303000ULL,
        0xdc0000dcdcdcdc00ULL, 0x5f00005f5f5f5f00ULL, 0x5e00005e5e5e5e00ULL,
        0xc50000c5c5c5c500ULL, 0x0b00000b0b0b0b00ULL, 0x1a00001a1a1a1a00ULL,
        0xa60000a6a6a6a600ULL, 0xe10000e1e1e1e100ULL, 0x3900003939393900ULL,
        0xca0000cacacaca00ULL, 0xd50000d5d5d5d500ULL, 0x4700004747474700ULL,
        0x5d00005d5d5d5d00ULL, 0x3d00003d3d3d3d00ULL, 0xd90000d9d9d9d900ULL,
        0x0100000101010100ULL, 0x5a00005a5a5a5a00ULL, 0xd60000d6d6d6d600ULL,
        0x5100005151515100ULL, 0x5600005656565600ULL, 0x6c00006c6c6c6c00ULL,
        0x4d00004d4d4d4d00ULL, 0x8b00008b8b8b8b00ULL, 0x0d00000d0d0d0d00ULL,
        0x9a00009a9a9a9a00ULL, 0x6600006666666600ULL, 0xfb0000fbfbfbfb00ULL,
        0xcc0000cccccccc00ULL, 0xb00000b0b0b0b000ULL, 0x2d00002d2d2d2d00ULL,
        0x7400007474747400ULL, 0x1200001212121200ULL, 0x2b00002b2b2b2b00ULL,
        0x2000002020202000ULL, 0xf00000f0f0f0f000ULL, 0xb10000b1b1b1b100ULL,
        0x8400008484848400ULL, 0x9900009999999900ULL, 0xdf0000dfdfdfdf00ULL,
        0x4c00004c4c4c4c00ULL, 0xcb0000cbcbcbcb00ULL, 0xc20000c2c2c2c200ULL,
        0x3400003434343400ULL, 0x7e00007e7e7e7e00ULL, 0x7600007676767600ULL,
        0x0500000505050500ULL, 0x6d00006d6d6d6d00ULL, 0xb70000b7b7b7b700ULL,
        0xa90000a9a9a9a900ULL, 0x3100003131313100ULL, 0xd10000d1d1d1d100ULL,
        0x1700001717171700ULL, 0x0400000404040400ULL, 0xd70000d7d7d7d700ULL,
        0x1400001414141400ULL, 0x5800005858585800ULL, 0x3a00003a3a3a3a00ULL,
        0x6100006161616100ULL, 0xde0000dededede00ULL, 0x1b00001b1b1b1b00ULL,
        0x1100001111111100ULL, 0x1c00001c1c1c1c00ULL, 0x3200003232323200ULL,
        0x0f00000f0f0f0f00ULL, 0x9c00009c9c9c9c00ULL, 0x1600001616161600ULL,
        0x5300005353535300ULL, 0x1800001818181800ULL, 0xf20000f2f2f2f200ULL,
        0x2200002222222200ULL, 0xfe0000fefefefe00ULL, 0x4400004444444400ULL,
        0xcf0000cfcfcfcf00ULL, 0xb20000b2b2b2b200ULL, 0xc30000c3c3c3c300ULL,
        0xb50000b5b5b5b500ULL, 0x7a00007a7a7a7a00ULL, 0x9100009191919100ULL,
        0x2400002424242400ULL, 0x0800000808080800ULL, 0xe80000e8e8e8e800ULL,
        0xa80000a8a8a8a800ULL, 0x6000006060606000ULL, 0xfc0000fcfcfcfc00ULL,
        0x6900006969696900ULL, 0x5000005050505000ULL, 0xaa0000aaaaaaaa00ULL,
        0xd00000d0d0d0d000ULL, 0xa00000a0a0a0a000ULL, 0x7d00007d7d7d7d00ULL,
        0xa10000a1a1a1a100ULL, 0x8900008989898900ULL, 0x6200006262626200ULL,
        0x9700009797979700ULL, 0x5400005454545400ULL, 0x5b00005b5b5b5b00ULL,
        0x1e00001e1e1e1e00ULL, 0x9500009595959500ULL, 0xe00000e0e0e0e000ULL,
        0xff0000ffffffff00ULL, 0x6400006464646400ULL, 0xd20000d2d2d2d200ULL,
        0x1000001010101000ULL, 0xc40000c4c4c4c400ULL, 0x0000000000000000ULL,
        0x4800004848484800ULL, 0xa30000a3a3a3a300ULL, 0xf70000f7f7f7f700ULL,
        0x7500007575757500ULL, 0xdb0000dbdbdbdb00ULL, 0x8a00008a8a8a8a00ULL,
        0x0300000303030300ULL, 0xe60000e6e6e6e600ULL, 0xda0000dadadada00ULL,
        0x0900000909090900ULL, 0x3f00003f3f3f3f00ULL, 0xdd0000dddddddd00ULL,
        0x9400009494949400ULL, 0x8700008787878700ULL, 0x5c00005c5c5c5c00ULL,
        0x8300008383838300ULL, 0x0200000202020200ULL, 0xcd0000cdcdcdcd00ULL,
        0x4a00004a4a4a4a00ULL, 0x9000009090909000ULL, 0x3300003333333300ULL,
        0x7300007373737300ULL, 0x6700006767676700ULL, 0xf60000f6f6f6f600ULL,
        0xf30000f3f3f3f300ULL, 0x9d00009d9d9d9d00ULL, 0x7f00007f7f7f7f00ULL,
        0xbf0000bfbfbfbf00ULL, 0xe20000e2e2e2e200ULL, 0x5200005252525200ULL,
        0x9b00009b9b9b9b00ULL, 0xd80000d8d8d8d800ULL, 0x2600002626262600ULL,
        0xc80000c8c8c8c800ULL, 0x3700003737373700ULL, 0xc60000c6c6c6c600ULL,
        0x3b00003b3b3b3b00ULL, 0x8100008181818100ULL, 0x9600009696969600ULL,
        0x6f00006f6f6f6f00ULL, 0x4b00004b4b4b4b00ULL, 0x1300001313131300ULL,
        0xbe0000bebebebe00ULL, 0x6300006363636300ULL, 0x2e00002e2e2e2e00ULL,
        0xe90000e9e9e9e900ULL, 0x7900007979797900ULL, 0xa70000a7a7a7a700ULL,
        0x8c00008c8c8c8c00ULL, 0x9f00009f9f9f9f00ULL, 0x6e00006e6e6e6e00ULL,
        0xbc0000bcbcbcbc00ULL, 0x8e00008e8e8e8e00ULL, 0x2900002929292900ULL,
        0xf50000f5f5f5f500ULL, 0xf90000f9f9f9f900ULL, 0xb60000b6b6b6b600ULL,
        0x2f00002f2f2f2f00ULL, 0xfd0000fdfdfdfd00ULL, 0xb40000b4b4b4b400ULL,
        0x5900005959595900ULL, 0x7800007878787800ULL, 0x9800009898989800ULL,
        0x0600000606060600ULL, 0x6a00006a6a6a6a00ULL, 0xe70000e7e7e7e700ULL,
        0x4600004646464600ULL, 0x7100007171717100ULL, 0xba0000babababa00ULL,
        0xd40000d4d4d4d400ULL, 0x2500002525252500ULL, 0xab0000abababab00ULL,
        0x4200004242424200ULL, 0x8800008888888800ULL, 0xa20000a2a2a2a200ULL,
        0x8d00008d8d8d8d00ULL, 0xfa0000fafafafa00ULL, 0x7200007272727200ULL,
        0x0700000707070700ULL, 0xb90000b9b9b9b900ULL, 0x5500005555555500ULL,
        0xf80000f8f8f8f800ULL, 0xee0000eeeeeeee00ULL, 0xac0000acacacac00ULL,
        0x0a00000a0a0a0a00ULL, 0x3600003636363600ULL, 0x4900004949494900ULL,
        0x2a00002a2a2a2a00ULL, 0x6800006868686800ULL, 0x3c00003c3c3c3c00ULL,
        0x3800003838383800ULL, 0xf10000f1f1f1f100ULL, 0xa40000a4a4a4a400ULL,
        0x4000004040404000ULL, 0x2800002828282800ULL, 0xd30000d3d3d3d300ULL,
        0x7b00007b7b7b7b00ULL, 0xbb0000bbbbbbbb00ULL, 0xc90000c9c9c9c900ULL,
        0x4300004343434300ULL, 0xc10000c1c1c1c100ULL, 0x1500001515151500ULL,
        0xe30000e3e3e3e300ULL, 0xad0000adadadad00ULL, 0xf40000f4f4f4f400ULL,
        0x7700007777777700ULL, 0xc70000c7c7c7c700ULL, 0x8000008080808000ULL,
        0x9e00009e9e9e9e00ULL,
};

__visible const u64 camellia_sp22000222[256] = {
        0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL,
        0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL,
        0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL,
        0x0b0b0000000b0b0bULL, 0xaeae000000aeaeaeULL, 0x6a6a0000006a6a6aULL,
        0xd5d5000000d5d5d5ULL, 0x1818000000181818ULL, 0x5d5d0000005d5d5dULL,
        0x8282000000828282ULL, 0x4646000000464646ULL, 0xdfdf000000dfdfdfULL,
        0xd6d6000000d6d6d6ULL, 0x2727000000272727ULL, 0x8a8a0000008a8a8aULL,
        0x3232000000323232ULL, 0x4b4b0000004b4b4bULL, 0x4242000000424242ULL,
        0xdbdb000000dbdbdbULL, 0x1c1c0000001c1c1cULL, 0x9e9e0000009e9e9eULL,
        0x9c9c0000009c9c9cULL, 0x3a3a0000003a3a3aULL, 0xcaca000000cacacaULL,
        0x2525000000252525ULL, 0x7b7b0000007b7b7bULL, 0x0d0d0000000d0d0dULL,
        0x7171000000717171ULL, 0x5f5f0000005f5f5fULL, 0x1f1f0000001f1f1fULL,
        0xf8f8000000f8f8f8ULL, 0xd7d7000000d7d7d7ULL, 0x3e3e0000003e3e3eULL,
        0x9d9d0000009d9d9dULL, 0x7c7c0000007c7c7cULL, 0x6060000000606060ULL,
        0xb9b9000000b9b9b9ULL, 0xbebe000000bebebeULL, 0xbcbc000000bcbcbcULL,
        0x8b8b0000008b8b8bULL, 0x1616000000161616ULL, 0x3434000000343434ULL,
        0x4d4d0000004d4d4dULL, 0xc3c3000000c3c3c3ULL, 0x7272000000727272ULL,
        0x9595000000959595ULL, 0xabab000000abababULL, 0x8e8e0000008e8e8eULL,
        0xbaba000000bababaULL, 0x7a7a0000007a7a7aULL, 0xb3b3000000b3b3b3ULL,
        0x0202000000020202ULL, 0xb4b4000000b4b4b4ULL, 0xadad000000adadadULL,
        0xa2a2000000a2a2a2ULL, 0xacac000000acacacULL, 0xd8d8000000d8d8d8ULL,
        0x9a9a0000009a9a9aULL, 0x1717000000171717ULL, 0x1a1a0000001a1a1aULL,
        0x3535000000353535ULL, 0xcccc000000ccccccULL, 0xf7f7000000f7f7f7ULL,
        0x9999000000999999ULL, 0x6161000000616161ULL, 0x5a5a0000005a5a5aULL,
        0xe8e8000000e8e8e8ULL, 0x2424000000242424ULL, 0x5656000000565656ULL,
        0x4040000000404040ULL, 0xe1e1000000e1e1e1ULL, 0x6363000000636363ULL,
        0x0909000000090909ULL, 0x3333000000333333ULL, 0xbfbf000000bfbfbfULL,
        0x9898000000989898ULL, 0x9797000000979797ULL, 0x8585000000858585ULL,
        0x6868000000686868ULL, 0xfcfc000000fcfcfcULL, 0xecec000000ecececULL,
        0x0a0a0000000a0a0aULL, 0xdada000000dadadaULL, 0x6f6f0000006f6f6fULL,
        0x5353000000535353ULL, 0x6262000000626262ULL, 0xa3a3000000a3a3a3ULL,
        0x2e2e0000002e2e2eULL, 0x0808000000080808ULL, 0xafaf000000afafafULL,
        0x2828000000282828ULL, 0xb0b0000000b0b0b0ULL, 0x7474000000747474ULL,
        0xc2c2000000c2c2c2ULL, 0xbdbd000000bdbdbdULL, 0x3636000000363636ULL,
        0x2222000000222222ULL, 0x3838000000383838ULL, 0x6464000000646464ULL,
        0x1e1e0000001e1e1eULL, 0x3939000000393939ULL, 0x2c2c0000002c2c2cULL,
        0xa6a6000000a6a6a6ULL, 0x3030000000303030ULL, 0xe5e5000000e5e5e5ULL,
        0x4444000000444444ULL, 0xfdfd000000fdfdfdULL, 0x8888000000888888ULL,
        0x9f9f0000009f9f9fULL, 0x6565000000656565ULL, 0x8787000000878787ULL,
        0x6b6b0000006b6b6bULL, 0xf4f4000000f4f4f4ULL, 0x2323000000232323ULL,
        0x4848000000484848ULL, 0x1010000000101010ULL, 0xd1d1000000d1d1d1ULL,
        0x5151000000515151ULL, 0xc0c0000000c0c0c0ULL, 0xf9f9000000f9f9f9ULL,
        0xd2d2000000d2d2d2ULL, 0xa0a0000000a0a0a0ULL, 0x5555000000555555ULL,
        0xa1a1000000a1a1a1ULL, 0x4141000000414141ULL, 0xfafa000000fafafaULL,
        0x4343000000434343ULL, 0x1313000000131313ULL, 0xc4c4000000c4c4c4ULL,
        0x2f2f0000002f2f2fULL, 0xa8a8000000a8a8a8ULL, 0xb6b6000000b6b6b6ULL,
        0x3c3c0000003c3c3cULL, 0x2b2b0000002b2b2bULL, 0xc1c1000000c1c1c1ULL,
        0xffff000000ffffffULL, 0xc8c8000000c8c8c8ULL, 0xa5a5000000a5a5a5ULL,
        0x2020000000202020ULL, 0x8989000000898989ULL, 0x0000000000000000ULL,
        0x9090000000909090ULL, 0x4747000000474747ULL, 0xefef000000efefefULL,
        0xeaea000000eaeaeaULL, 0xb7b7000000b7b7b7ULL, 0x1515000000151515ULL,
        0x0606000000060606ULL, 0xcdcd000000cdcdcdULL, 0xb5b5000000b5b5b5ULL,
        0x1212000000121212ULL, 0x7e7e0000007e7e7eULL, 0xbbbb000000bbbbbbULL,
        0x2929000000292929ULL, 0x0f0f0000000f0f0fULL, 0xb8b8000000b8b8b8ULL,
        0x0707000000070707ULL, 0x0404000000040404ULL, 0x9b9b0000009b9b9bULL,
        0x9494000000949494ULL, 0x2121000000212121ULL, 0x6666000000666666ULL,
        0xe6e6000000e6e6e6ULL, 0xcece000000cececeULL, 0xeded000000edededULL,
        0xe7e7000000e7e7e7ULL, 0x3b3b0000003b3b3bULL, 0xfefe000000fefefeULL,
        0x7f7f0000007f7f7fULL, 0xc5c5000000c5c5c5ULL, 0xa4a4000000a4a4a4ULL,
        0x3737000000373737ULL, 0xb1b1000000b1b1b1ULL, 0x4c4c0000004c4c4cULL,
        0x9191000000919191ULL, 0x6e6e0000006e6e6eULL, 0x8d8d0000008d8d8dULL,
        0x7676000000767676ULL, 0x0303000000030303ULL, 0x2d2d0000002d2d2dULL,
        0xdede000000dededeULL, 0x9696000000969696ULL, 0x2626000000262626ULL,
        0x7d7d0000007d7d7dULL, 0xc6c6000000c6c6c6ULL, 0x5c5c0000005c5c5cULL,
        0xd3d3000000d3d3d3ULL, 0xf2f2000000f2f2f2ULL, 0x4f4f0000004f4f4fULL,
        0x1919000000191919ULL, 0x3f3f0000003f3f3fULL, 0xdcdc000000dcdcdcULL,
        0x7979000000797979ULL, 0x1d1d0000001d1d1dULL, 0x5252000000525252ULL,
        0xebeb000000ebebebULL, 0xf3f3000000f3f3f3ULL, 0x6d6d0000006d6d6dULL,
        0x5e5e0000005e5e5eULL, 0xfbfb000000fbfbfbULL, 0x6969000000696969ULL,
        0xb2b2000000b2b2b2ULL, 0xf0f0000000f0f0f0ULL, 0x3131000000313131ULL,
        0x0c0c0000000c0c0cULL, 0xd4d4000000d4d4d4ULL, 0xcfcf000000cfcfcfULL,
        0x8c8c0000008c8c8cULL, 0xe2e2000000e2e2e2ULL, 0x7575000000757575ULL,
        0xa9a9000000a9a9a9ULL, 0x4a4a0000004a4a4aULL, 0x5757000000575757ULL,
        0x8484000000848484ULL, 0x1111000000111111ULL, 0x4545000000454545ULL,
        0x1b1b0000001b1b1bULL, 0xf5f5000000f5f5f5ULL, 0xe4e4000000e4e4e4ULL,
        0x0e0e0000000e0e0eULL, 0x7373000000737373ULL, 0xaaaa000000aaaaaaULL,
        0xf1f1000000f1f1f1ULL, 0xdddd000000ddddddULL, 0x5959000000595959ULL,
        0x1414000000141414ULL, 0x6c6c0000006c6c6cULL, 0x9292000000929292ULL,
        0x5454000000545454ULL, 0xd0d0000000d0d0d0ULL, 0x7878000000787878ULL,
        0x7070000000707070ULL, 0xe3e3000000e3e3e3ULL, 0x4949000000494949ULL,
        0x8080000000808080ULL, 0x5050000000505050ULL, 0xa7a7000000a7a7a7ULL,
        0xf6f6000000f6f6f6ULL, 0x7777000000777777ULL, 0x9393000000939393ULL,
        0x8686000000868686ULL, 0x8383000000838383ULL, 0x2a2a0000002a2a2aULL,
        0xc7c7000000c7c7c7ULL, 0x5b5b0000005b5b5bULL, 0xe9e9000000e9e9e9ULL,
        0xeeee000000eeeeeeULL, 0x8f8f0000008f8f8fULL, 0x0101000000010101ULL,
        0x3d3d0000003d3d3dULL,
};

__visible const u64 camellia_sp03303033[256] = {
        0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL,
        0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL,
        0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL,
        0x00c2c200c200c2c2ULL, 0x00abab00ab00ababULL, 0x009a9a009a009a9aULL,
        0x0075750075007575ULL, 0x0006060006000606ULL, 0x0057570057005757ULL,
        0x00a0a000a000a0a0ULL, 0x0091910091009191ULL, 0x00f7f700f700f7f7ULL,
        0x00b5b500b500b5b5ULL, 0x00c9c900c900c9c9ULL, 0x00a2a200a200a2a2ULL,
        0x008c8c008c008c8cULL, 0x00d2d200d200d2d2ULL, 0x0090900090009090ULL,
        0x00f6f600f600f6f6ULL, 0x0007070007000707ULL, 0x00a7a700a700a7a7ULL,
        0x0027270027002727ULL, 0x008e8e008e008e8eULL, 0x00b2b200b200b2b2ULL,
        0x0049490049004949ULL, 0x00dede00de00dedeULL, 0x0043430043004343ULL,
        0x005c5c005c005c5cULL, 0x00d7d700d700d7d7ULL, 0x00c7c700c700c7c7ULL,
        0x003e3e003e003e3eULL, 0x00f5f500f500f5f5ULL, 0x008f8f008f008f8fULL,
        0x0067670067006767ULL, 0x001f1f001f001f1fULL, 0x0018180018001818ULL,
        0x006e6e006e006e6eULL, 0x00afaf00af00afafULL, 0x002f2f002f002f2fULL,
        0x00e2e200e200e2e2ULL, 0x0085850085008585ULL, 0x000d0d000d000d0dULL,
        0x0053530053005353ULL, 0x00f0f000f000f0f0ULL, 0x009c9c009c009c9cULL,
        0x0065650065006565ULL, 0x00eaea00ea00eaeaULL, 0x00a3a300a300a3a3ULL,
        0x00aeae00ae00aeaeULL, 0x009e9e009e009e9eULL, 0x00ecec00ec00ececULL,
        0x0080800080008080ULL, 0x002d2d002d002d2dULL, 0x006b6b006b006b6bULL,
        0x00a8a800a800a8a8ULL, 0x002b2b002b002b2bULL, 0x0036360036003636ULL,
        0x00a6a600a600a6a6ULL, 0x00c5c500c500c5c5ULL, 0x0086860086008686ULL,
        0x004d4d004d004d4dULL, 0x0033330033003333ULL, 0x00fdfd00fd00fdfdULL,
        0x0066660066006666ULL, 0x0058580058005858ULL, 0x0096960096009696ULL,
        0x003a3a003a003a3aULL, 0x0009090009000909ULL, 0x0095950095009595ULL,
        0x0010100010001010ULL, 0x0078780078007878ULL, 0x00d8d800d800d8d8ULL,
        0x0042420042004242ULL, 0x00cccc00cc00ccccULL, 0x00efef00ef00efefULL,
        0x0026260026002626ULL, 0x00e5e500e500e5e5ULL, 0x0061610061006161ULL,
        0x001a1a001a001a1aULL, 0x003f3f003f003f3fULL, 0x003b3b003b003b3bULL,
        0x0082820082008282ULL, 0x00b6b600b600b6b6ULL, 0x00dbdb00db00dbdbULL,
        0x00d4d400d400d4d4ULL, 0x0098980098009898ULL, 0x00e8e800e800e8e8ULL,
        0x008b8b008b008b8bULL, 0x0002020002000202ULL, 0x00ebeb00eb00ebebULL,
        0x000a0a000a000a0aULL, 0x002c2c002c002c2cULL, 0x001d1d001d001d1dULL,
        0x00b0b000b000b0b0ULL, 0x006f6f006f006f6fULL, 0x008d8d008d008d8dULL,
        0x0088880088008888ULL, 0x000e0e000e000e0eULL, 0x0019190019001919ULL,
        0x0087870087008787ULL, 0x004e4e004e004e4eULL, 0x000b0b000b000b0bULL,
        0x00a9a900a900a9a9ULL, 0x000c0c000c000c0cULL, 0x0079790079007979ULL,
        0x0011110011001111ULL, 0x007f7f007f007f7fULL, 0x0022220022002222ULL,
        0x00e7e700e700e7e7ULL, 0x0059590059005959ULL, 0x00e1e100e100e1e1ULL,
        0x00dada00da00dadaULL, 0x003d3d003d003d3dULL, 0x00c8c800c800c8c8ULL,
        0x0012120012001212ULL, 0x0004040004000404ULL, 0x0074740074007474ULL,
        0x0054540054005454ULL, 0x0030300030003030ULL, 0x007e7e007e007e7eULL,
        0x00b4b400b400b4b4ULL, 0x0028280028002828ULL, 0x0055550055005555ULL,
        0x0068680068006868ULL, 0x0050500050005050ULL, 0x00bebe00be00bebeULL,
        0x00d0d000d000d0d0ULL, 0x00c4c400c400c4c4ULL, 0x0031310031003131ULL,
        0x00cbcb00cb00cbcbULL, 0x002a2a002a002a2aULL, 0x00adad00ad00adadULL,
        0x000f0f000f000f0fULL, 0x00caca00ca00cacaULL, 0x0070700070007070ULL,
        0x00ffff00ff00ffffULL, 0x0032320032003232ULL, 0x0069690069006969ULL,
        0x0008080008000808ULL, 0x0062620062006262ULL, 0x0000000000000000ULL,
        0x0024240024002424ULL, 0x00d1d100d100d1d1ULL, 0x00fbfb00fb00fbfbULL,
        0x00baba00ba00babaULL, 0x00eded00ed00ededULL, 0x0045450045004545ULL,
        0x0081810081008181ULL, 0x0073730073007373ULL, 0x006d6d006d006d6dULL,
        0x0084840084008484ULL, 0x009f9f009f009f9fULL, 0x00eeee00ee00eeeeULL,
        0x004a4a004a004a4aULL, 0x00c3c300c300c3c3ULL, 0x002e2e002e002e2eULL,
        0x00c1c100c100c1c1ULL, 0x0001010001000101ULL, 0x00e6e600e600e6e6ULL,
        0x0025250025002525ULL, 0x0048480048004848ULL, 0x0099990099009999ULL,
        0x00b9b900b900b9b9ULL, 0x00b3b300b300b3b3ULL, 0x007b7b007b007b7bULL,
        0x00f9f900f900f9f9ULL, 0x00cece00ce00ceceULL, 0x00bfbf00bf00bfbfULL,
        0x00dfdf00df00dfdfULL, 0x0071710071007171ULL, 0x0029290029002929ULL,
        0x00cdcd00cd00cdcdULL, 0x006c6c006c006c6cULL, 0x0013130013001313ULL,
        0x0064640064006464ULL, 0x009b9b009b009b9bULL, 0x0063630063006363ULL,
        0x009d9d009d009d9dULL, 0x00c0c000c000c0c0ULL, 0x004b4b004b004b4bULL,
        0x00b7b700b700b7b7ULL, 0x00a5a500a500a5a5ULL, 0x0089890089008989ULL,
        0x005f5f005f005f5fULL, 0x00b1b100b100b1b1ULL, 0x0017170017001717ULL,
        0x00f4f400f400f4f4ULL, 0x00bcbc00bc00bcbcULL, 0x00d3d300d300d3d3ULL,
        0x0046460046004646ULL, 0x00cfcf00cf00cfcfULL, 0x0037370037003737ULL,
        0x005e5e005e005e5eULL, 0x0047470047004747ULL, 0x0094940094009494ULL,
        0x00fafa00fa00fafaULL, 0x00fcfc00fc00fcfcULL, 0x005b5b005b005b5bULL,
        0x0097970097009797ULL, 0x00fefe00fe00fefeULL, 0x005a5a005a005a5aULL,
        0x00acac00ac00acacULL, 0x003c3c003c003c3cULL, 0x004c4c004c004c4cULL,
        0x0003030003000303ULL, 0x0035350035003535ULL, 0x00f3f300f300f3f3ULL,
        0x0023230023002323ULL, 0x00b8b800b800b8b8ULL, 0x005d5d005d005d5dULL,
        0x006a6a006a006a6aULL, 0x0092920092009292ULL, 0x00d5d500d500d5d5ULL,
        0x0021210021002121ULL, 0x0044440044004444ULL, 0x0051510051005151ULL,
        0x00c6c600c600c6c6ULL, 0x007d7d007d007d7dULL, 0x0039390039003939ULL,
        0x0083830083008383ULL, 0x00dcdc00dc00dcdcULL, 0x00aaaa00aa00aaaaULL,
        0x007c7c007c007c7cULL, 0x0077770077007777ULL, 0x0056560056005656ULL,
        0x0005050005000505ULL, 0x001b1b001b001b1bULL, 0x00a4a400a400a4a4ULL,
        0x0015150015001515ULL, 0x0034340034003434ULL, 0x001e1e001e001e1eULL,
        0x001c1c001c001c1cULL, 0x00f8f800f800f8f8ULL, 0x0052520052005252ULL,
        0x0020200020002020ULL, 0x0014140014001414ULL, 0x00e9e900e900e9e9ULL,
        0x00bdbd00bd00bdbdULL, 0x00dddd00dd00ddddULL, 0x00e4e400e400e4e4ULL,
        0x00a1a100a100a1a1ULL, 0x00e0e000e000e0e0ULL, 0x008a8a008a008a8aULL,
        0x00f1f100f100f1f1ULL, 0x00d6d600d600d6d6ULL, 0x007a7a007a007a7aULL,
        0x00bbbb00bb00bbbbULL, 0x00e3e300e300e3e3ULL, 0x0040400040004040ULL,
        0x004f4f004f004f4fULL,
};

__visible const u64 camellia_sp00444404[256] = {
        0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL,
        0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL,
        0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL,
        0x00006b6b6b6b006bULL, 0x0000454545450045ULL, 0x0000a5a5a5a500a5ULL,
        0x0000edededed00edULL, 0x00004f4f4f4f004fULL, 0x00001d1d1d1d001dULL,
        0x0000929292920092ULL, 0x0000868686860086ULL, 0x0000afafafaf00afULL,
        0x00007c7c7c7c007cULL, 0x00001f1f1f1f001fULL, 0x00003e3e3e3e003eULL,
        0x0000dcdcdcdc00dcULL, 0x00005e5e5e5e005eULL, 0x00000b0b0b0b000bULL,
        0x0000a6a6a6a600a6ULL, 0x0000393939390039ULL, 0x0000d5d5d5d500d5ULL,
        0x00005d5d5d5d005dULL, 0x0000d9d9d9d900d9ULL, 0x00005a5a5a5a005aULL,
        0x0000515151510051ULL, 0x00006c6c6c6c006cULL, 0x00008b8b8b8b008bULL,
        0x00009a9a9a9a009aULL, 0x0000fbfbfbfb00fbULL, 0x0000b0b0b0b000b0ULL,
        0x0000747474740074ULL, 0x00002b2b2b2b002bULL, 0x0000f0f0f0f000f0ULL,
        0x0000848484840084ULL, 0x0000dfdfdfdf00dfULL, 0x0000cbcbcbcb00cbULL,
        0x0000343434340034ULL, 0x0000767676760076ULL, 0x00006d6d6d6d006dULL,
        0x0000a9a9a9a900a9ULL, 0x0000d1d1d1d100d1ULL, 0x0000040404040004ULL,
        0x0000141414140014ULL, 0x00003a3a3a3a003aULL, 0x0000dededede00deULL,
        0x0000111111110011ULL, 0x0000323232320032ULL, 0x00009c9c9c9c009cULL,
        0x0000535353530053ULL, 0x0000f2f2f2f200f2ULL, 0x0000fefefefe00feULL,
        0x0000cfcfcfcf00cfULL, 0x0000c3c3c3c300c3ULL, 0x00007a7a7a7a007aULL,
        0x0000242424240024ULL, 0x0000e8e8e8e800e8ULL, 0x0000606060600060ULL,
        0x0000696969690069ULL, 0x0000aaaaaaaa00aaULL, 0x0000a0a0a0a000a0ULL,
        0x0000a1a1a1a100a1ULL, 0x0000626262620062ULL, 0x0000545454540054ULL,
        0x00001e1e1e1e001eULL, 0x0000e0e0e0e000e0ULL, 0x0000646464640064ULL,
        0x0000101010100010ULL, 0x0000000000000000ULL, 0x0000a3a3a3a300a3ULL,
        0x0000757575750075ULL, 0x00008a8a8a8a008aULL, 0x0000e6e6e6e600e6ULL,
        0x0000090909090009ULL, 0x0000dddddddd00ddULL, 0x0000878787870087ULL,
        0x0000838383830083ULL, 0x0000cdcdcdcd00cdULL, 0x0000909090900090ULL,
        0x0000737373730073ULL, 0x0000f6f6f6f600f6ULL, 0x00009d9d9d9d009dULL,
        0x0000bfbfbfbf00bfULL, 0x0000525252520052ULL, 0x0000d8d8d8d800d8ULL,
        0x0000c8c8c8c800c8ULL, 0x0000c6c6c6c600c6ULL, 0x0000818181810081ULL,
        0x00006f6f6f6f006fULL, 0x0000131313130013ULL, 0x0000636363630063ULL,
        0x0000e9e9e9e900e9ULL, 0x0000a7a7a7a700a7ULL, 0x00009f9f9f9f009fULL,
        0x0000bcbcbcbc00bcULL, 0x0000292929290029ULL, 0x0000f9f9f9f900f9ULL,
        0x00002f2f2f2f002fULL, 0x0000b4b4b4b400b4ULL, 0x0000787878780078ULL,
        0x0000060606060006ULL, 0x0000e7e7e7e700e7ULL, 0x0000717171710071ULL,
        0x0000d4d4d4d400d4ULL, 0x0000abababab00abULL, 0x0000888888880088ULL,
        0x00008d8d8d8d008dULL, 0x0000727272720072ULL, 0x0000b9b9b9b900b9ULL,
        0x0000f8f8f8f800f8ULL, 0x0000acacacac00acULL, 0x0000363636360036ULL,
        0x00002a2a2a2a002aULL, 0x00003c3c3c3c003cULL, 0x0000f1f1f1f100f1ULL,
        0x0000404040400040ULL, 0x0000d3d3d3d300d3ULL, 0x0000bbbbbbbb00bbULL,
        0x0000434343430043ULL, 0x0000151515150015ULL, 0x0000adadadad00adULL,
        0x0000777777770077ULL, 0x0000808080800080ULL, 0x0000828282820082ULL,
        0x0000ecececec00ecULL, 0x0000272727270027ULL, 0x0000e5e5e5e500e5ULL,
        0x0000858585850085ULL, 0x0000353535350035ULL, 0x00000c0c0c0c000cULL,
        0x0000414141410041ULL, 0x0000efefefef00efULL, 0x0000939393930093ULL,
        0x0000191919190019ULL, 0x0000212121210021ULL, 0x00000e0e0e0e000eULL,
        0x00004e4e4e4e004eULL, 0x0000656565650065ULL, 0x0000bdbdbdbd00bdULL,
        0x0000b8b8b8b800b8ULL, 0x00008f8f8f8f008fULL, 0x0000ebebebeb00ebULL,
        0x0000cececece00ceULL, 0x0000303030300030ULL, 0x00005f5f5f5f005fULL,
        0x0000c5c5c5c500c5ULL, 0x00001a1a1a1a001aULL, 0x0000e1e1e1e100e1ULL,
        0x0000cacacaca00caULL, 0x0000474747470047ULL, 0x00003d3d3d3d003dULL,
        0x0000010101010001ULL, 0x0000d6d6d6d600d6ULL, 0x0000565656560056ULL,
        0x00004d4d4d4d004dULL, 0x00000d0d0d0d000dULL, 0x0000666666660066ULL,
        0x0000cccccccc00ccULL, 0x00002d2d2d2d002dULL, 0x0000121212120012ULL,
        0x0000202020200020ULL, 0x0000b1b1b1b100b1ULL, 0x0000999999990099ULL,
        0x00004c4c4c4c004cULL, 0x0000c2c2c2c200c2ULL, 0x00007e7e7e7e007eULL,
        0x0000050505050005ULL, 0x0000b7b7b7b700b7ULL, 0x0000313131310031ULL,
        0x0000171717170017ULL, 0x0000d7d7d7d700d7ULL, 0x0000585858580058ULL,
        0x0000616161610061ULL, 0x00001b1b1b1b001bULL, 0x00001c1c1c1c001cULL,
        0x00000f0f0f0f000fULL, 0x0000161616160016ULL, 0x0000181818180018ULL,
        0x0000222222220022ULL, 0x0000444444440044ULL, 0x0000b2b2b2b200b2ULL,
        0x0000b5b5b5b500b5ULL, 0x0000919191910091ULL, 0x0000080808080008ULL,
        0x0000a8a8a8a800a8ULL, 0x0000fcfcfcfc00fcULL, 0x0000505050500050ULL,
        0x0000d0d0d0d000d0ULL, 0x00007d7d7d7d007dULL, 0x0000898989890089ULL,
        0x0000979797970097ULL, 0x00005b5b5b5b005bULL, 0x0000959595950095ULL,
        0x0000ffffffff00ffULL, 0x0000d2d2d2d200d2ULL, 0x0000c4c4c4c400c4ULL,
        0x0000484848480048ULL, 0x0000f7f7f7f700f7ULL, 0x0000dbdbdbdb00dbULL,
        0x0000030303030003ULL, 0x0000dadadada00daULL, 0x00003f3f3f3f003fULL,
        0x0000949494940094ULL, 0x00005c5c5c5c005cULL, 0x0000020202020002ULL,
        0x00004a4a4a4a004aULL, 0x0000333333330033ULL, 0x0000676767670067ULL,
        0x0000f3f3f3f300f3ULL, 0x00007f7f7f7f007fULL, 0x0000e2e2e2e200e2ULL,
        0x00009b9b9b9b009bULL, 0x0000262626260026ULL, 0x0000373737370037ULL,
        0x00003b3b3b3b003bULL, 0x0000969696960096ULL, 0x00004b4b4b4b004bULL,
        0x0000bebebebe00beULL, 0x00002e2e2e2e002eULL, 0x0000797979790079ULL,
        0x00008c8c8c8c008cULL, 0x00006e6e6e6e006eULL, 0x00008e8e8e8e008eULL,
        0x0000f5f5f5f500f5ULL, 0x0000b6b6b6b600b6ULL, 0x0000fdfdfdfd00fdULL,
        0x0000595959590059ULL, 0x0000989898980098ULL, 0x00006a6a6a6a006aULL,
        0x0000464646460046ULL, 0x0000babababa00baULL, 0x0000252525250025ULL,
        0x0000424242420042ULL, 0x0000a2a2a2a200a2ULL, 0x0000fafafafa00faULL,
        0x0000070707070007ULL, 0x0000555555550055ULL, 0x0000eeeeeeee00eeULL,
        0x00000a0a0a0a000aULL, 0x0000494949490049ULL, 0x0000686868680068ULL,
        0x0000383838380038ULL, 0x0000a4a4a4a400a4ULL, 0x0000282828280028ULL,
        0x00007b7b7b7b007bULL, 0x0000c9c9c9c900c9ULL, 0x0000c1c1c1c100c1ULL,
        0x0000e3e3e3e300e3ULL, 0x0000f4f4f4f400f4ULL, 0x0000c7c7c7c700c7ULL,
        0x00009e9e9e9e009eULL,
};

__visible const u64 camellia_sp02220222[256] = {
        0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL,
        0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL,
        0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL,
        0x000b0b0b000b0b0bULL, 0x00aeaeae00aeaeaeULL, 0x006a6a6a006a6a6aULL,
        0x00d5d5d500d5d5d5ULL, 0x0018181800181818ULL, 0x005d5d5d005d5d5dULL,
        0x0082828200828282ULL, 0x0046464600464646ULL, 0x00dfdfdf00dfdfdfULL,
        0x00d6d6d600d6d6d6ULL, 0x0027272700272727ULL, 0x008a8a8a008a8a8aULL,
        0x0032323200323232ULL, 0x004b4b4b004b4b4bULL, 0x0042424200424242ULL,
        0x00dbdbdb00dbdbdbULL, 0x001c1c1c001c1c1cULL, 0x009e9e9e009e9e9eULL,
        0x009c9c9c009c9c9cULL, 0x003a3a3a003a3a3aULL, 0x00cacaca00cacacaULL,
        0x0025252500252525ULL, 0x007b7b7b007b7b7bULL, 0x000d0d0d000d0d0dULL,
        0x0071717100717171ULL, 0x005f5f5f005f5f5fULL, 0x001f1f1f001f1f1fULL,
        0x00f8f8f800f8f8f8ULL, 0x00d7d7d700d7d7d7ULL, 0x003e3e3e003e3e3eULL,
        0x009d9d9d009d9d9dULL, 0x007c7c7c007c7c7cULL, 0x0060606000606060ULL,
        0x00b9b9b900b9b9b9ULL, 0x00bebebe00bebebeULL, 0x00bcbcbc00bcbcbcULL,
        0x008b8b8b008b8b8bULL, 0x0016161600161616ULL, 0x0034343400343434ULL,
        0x004d4d4d004d4d4dULL, 0x00c3c3c300c3c3c3ULL, 0x0072727200727272ULL,
        0x0095959500959595ULL, 0x00ababab00abababULL, 0x008e8e8e008e8e8eULL,
        0x00bababa00bababaULL, 0x007a7a7a007a7a7aULL, 0x00b3b3b300b3b3b3ULL,
        0x0002020200020202ULL, 0x00b4b4b400b4b4b4ULL, 0x00adadad00adadadULL,
        0x00a2a2a200a2a2a2ULL, 0x00acacac00acacacULL, 0x00d8d8d800d8d8d8ULL,
        0x009a9a9a009a9a9aULL, 0x0017171700171717ULL, 0x001a1a1a001a1a1aULL,
        0x0035353500353535ULL, 0x00cccccc00ccccccULL, 0x00f7f7f700f7f7f7ULL,
        0x0099999900999999ULL, 0x0061616100616161ULL, 0x005a5a5a005a5a5aULL,
        0x00e8e8e800e8e8e8ULL, 0x0024242400242424ULL, 0x0056565600565656ULL,
        0x0040404000404040ULL, 0x00e1e1e100e1e1e1ULL, 0x0063636300636363ULL,
        0x0009090900090909ULL, 0x0033333300333333ULL, 0x00bfbfbf00bfbfbfULL,
        0x0098989800989898ULL, 0x0097979700979797ULL, 0x0085858500858585ULL,
        0x0068686800686868ULL, 0x00fcfcfc00fcfcfcULL, 0x00ececec00ecececULL,
        0x000a0a0a000a0a0aULL, 0x00dadada00dadadaULL, 0x006f6f6f006f6f6fULL,
        0x0053535300535353ULL, 0x0062626200626262ULL, 0x00a3a3a300a3a3a3ULL,
        0x002e2e2e002e2e2eULL, 0x0008080800080808ULL, 0x00afafaf00afafafULL,
        0x0028282800282828ULL, 0x00b0b0b000b0b0b0ULL, 0x0074747400747474ULL,
        0x00c2c2c200c2c2c2ULL, 0x00bdbdbd00bdbdbdULL, 0x0036363600363636ULL,
        0x0022222200222222ULL, 0x0038383800383838ULL, 0x0064646400646464ULL,
        0x001e1e1e001e1e1eULL, 0x0039393900393939ULL, 0x002c2c2c002c2c2cULL,
        0x00a6a6a600a6a6a6ULL, 0x0030303000303030ULL, 0x00e5e5e500e5e5e5ULL,
        0x0044444400444444ULL, 0x00fdfdfd00fdfdfdULL, 0x0088888800888888ULL,
        0x009f9f9f009f9f9fULL, 0x0065656500656565ULL, 0x0087878700878787ULL,
        0x006b6b6b006b6b6bULL, 0x00f4f4f400f4f4f4ULL, 0x0023232300232323ULL,
        0x0048484800484848ULL, 0x0010101000101010ULL, 0x00d1d1d100d1d1d1ULL,
        0x0051515100515151ULL, 0x00c0c0c000c0c0c0ULL, 0x00f9f9f900f9f9f9ULL,
        0x00d2d2d200d2d2d2ULL, 0x00a0a0a000a0a0a0ULL, 0x0055555500555555ULL,
        0x00a1a1a100a1a1a1ULL, 0x0041414100414141ULL, 0x00fafafa00fafafaULL,
        0x0043434300434343ULL, 0x0013131300131313ULL, 0x00c4c4c400c4c4c4ULL,
        0x002f2f2f002f2f2fULL, 0x00a8a8a800a8a8a8ULL, 0x00b6b6b600b6b6b6ULL,
        0x003c3c3c003c3c3cULL, 0x002b2b2b002b2b2bULL, 0x00c1c1c100c1c1c1ULL,
        0x00ffffff00ffffffULL, 0x00c8c8c800c8c8c8ULL, 0x00a5a5a500a5a5a5ULL,
        0x0020202000202020ULL, 0x0089898900898989ULL, 0x0000000000000000ULL,
        0x0090909000909090ULL, 0x0047474700474747ULL, 0x00efefef00efefefULL,
        0x00eaeaea00eaeaeaULL, 0x00b7b7b700b7b7b7ULL, 0x0015151500151515ULL,
        0x0006060600060606ULL, 0x00cdcdcd00cdcdcdULL, 0x00b5b5b500b5b5b5ULL,
        0x0012121200121212ULL, 0x007e7e7e007e7e7eULL, 0x00bbbbbb00bbbbbbULL,
        0x0029292900292929ULL, 0x000f0f0f000f0f0fULL, 0x00b8b8b800b8b8b8ULL,
        0x0007070700070707ULL, 0x0004040400040404ULL, 0x009b9b9b009b9b9bULL,
        0x0094949400949494ULL, 0x0021212100212121ULL, 0x0066666600666666ULL,
        0x00e6e6e600e6e6e6ULL, 0x00cecece00cececeULL, 0x00ededed00edededULL,
        0x00e7e7e700e7e7e7ULL, 0x003b3b3b003b3b3bULL, 0x00fefefe00fefefeULL,
        0x007f7f7f007f7f7fULL, 0x00c5c5c500c5c5c5ULL, 0x00a4a4a400a4a4a4ULL,
        0x0037373700373737ULL, 0x00b1b1b100b1b1b1ULL, 0x004c4c4c004c4c4cULL,
        0x0091919100919191ULL, 0x006e6e6e006e6e6eULL, 0x008d8d8d008d8d8dULL,
        0x0076767600767676ULL, 0x0003030300030303ULL, 0x002d2d2d002d2d2dULL,
        0x00dedede00dededeULL, 0x0096969600969696ULL, 0x0026262600262626ULL,
        0x007d7d7d007d7d7dULL, 0x00c6c6c600c6c6c6ULL, 0x005c5c5c005c5c5cULL,
        0x00d3d3d300d3d3d3ULL, 0x00f2f2f200f2f2f2ULL, 0x004f4f4f004f4f4fULL,
        0x0019191900191919ULL, 0x003f3f3f003f3f3fULL, 0x00dcdcdc00dcdcdcULL,
        0x0079797900797979ULL, 0x001d1d1d001d1d1dULL, 0x0052525200525252ULL,
        0x00ebebeb00ebebebULL, 0x00f3f3f300f3f3f3ULL, 0x006d6d6d006d6d6dULL,
        0x005e5e5e005e5e5eULL, 0x00fbfbfb00fbfbfbULL, 0x0069696900696969ULL,
        0x00b2b2b200b2b2b2ULL, 0x00f0f0f000f0f0f0ULL, 0x0031313100313131ULL,
        0x000c0c0c000c0c0cULL, 0x00d4d4d400d4d4d4ULL, 0x00cfcfcf00cfcfcfULL,
        0x008c8c8c008c8c8cULL, 0x00e2e2e200e2e2e2ULL, 0x0075757500757575ULL,
        0x00a9a9a900a9a9a9ULL, 0x004a4a4a004a4a4aULL, 0x0057575700575757ULL,
        0x0084848400848484ULL, 0x0011111100111111ULL, 0x0045454500454545ULL,
        0x001b1b1b001b1b1bULL, 0x00f5f5f500f5f5f5ULL, 0x00e4e4e400e4e4e4ULL,
        0x000e0e0e000e0e0eULL, 0x0073737300737373ULL, 0x00aaaaaa00aaaaaaULL,
        0x00f1f1f100f1f1f1ULL, 0x00dddddd00ddddddULL, 0x0059595900595959ULL,
        0x0014141400141414ULL, 0x006c6c6c006c6c6cULL, 0x0092929200929292ULL,
        0x0054545400545454ULL, 0x00d0d0d000d0d0d0ULL, 0x0078787800787878ULL,
        0x0070707000707070ULL, 0x00e3e3e300e3e3e3ULL, 0x0049494900494949ULL,
        0x0080808000808080ULL, 0x0050505000505050ULL, 0x00a7a7a700a7a7a7ULL,
        0x00f6f6f600f6f6f6ULL, 0x0077777700777777ULL, 0x0093939300939393ULL,
        0x0086868600868686ULL, 0x0083838300838383ULL, 0x002a2a2a002a2a2aULL,
        0x00c7c7c700c7c7c7ULL, 0x005b5b5b005b5b5bULL, 0x00e9e9e900e9e9e9ULL,
        0x00eeeeee00eeeeeeULL, 0x008f8f8f008f8f8fULL, 0x0001010100010101ULL,
        0x003d3d3d003d3d3dULL,
};

__visible const u64 camellia_sp30333033[256] = {
        0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL,
        0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL,
        0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL,
        0xc200c2c2c200c2c2ULL, 0xab00ababab00ababULL, 0x9a009a9a9a009a9aULL,
        0x7500757575007575ULL, 0x0600060606000606ULL, 0x5700575757005757ULL,
        0xa000a0a0a000a0a0ULL, 0x9100919191009191ULL, 0xf700f7f7f700f7f7ULL,
        0xb500b5b5b500b5b5ULL, 0xc900c9c9c900c9c9ULL, 0xa200a2a2a200a2a2ULL,
        0x8c008c8c8c008c8cULL, 0xd200d2d2d200d2d2ULL, 0x9000909090009090ULL,
        0xf600f6f6f600f6f6ULL, 0x0700070707000707ULL, 0xa700a7a7a700a7a7ULL,
        0x2700272727002727ULL, 0x8e008e8e8e008e8eULL, 0xb200b2b2b200b2b2ULL,
        0x4900494949004949ULL, 0xde00dedede00dedeULL, 0x4300434343004343ULL,
        0x5c005c5c5c005c5cULL, 0xd700d7d7d700d7d7ULL, 0xc700c7c7c700c7c7ULL,
        0x3e003e3e3e003e3eULL, 0xf500f5f5f500f5f5ULL, 0x8f008f8f8f008f8fULL,
        0x6700676767006767ULL, 0x1f001f1f1f001f1fULL, 0x1800181818001818ULL,
        0x6e006e6e6e006e6eULL, 0xaf00afafaf00afafULL, 0x2f002f2f2f002f2fULL,
        0xe200e2e2e200e2e2ULL, 0x8500858585008585ULL, 0x0d000d0d0d000d0dULL,
        0x5300535353005353ULL, 0xf000f0f0f000f0f0ULL, 0x9c009c9c9c009c9cULL,
        0x6500656565006565ULL, 0xea00eaeaea00eaeaULL, 0xa300a3a3a300a3a3ULL,
        0xae00aeaeae00aeaeULL, 0x9e009e9e9e009e9eULL, 0xec00ececec00ececULL,
        0x8000808080008080ULL, 0x2d002d2d2d002d2dULL, 0x6b006b6b6b006b6bULL,
        0xa800a8a8a800a8a8ULL, 0x2b002b2b2b002b2bULL, 0x3600363636003636ULL,
        0xa600a6a6a600a6a6ULL, 0xc500c5c5c500c5c5ULL, 0x8600868686008686ULL,
        0x4d004d4d4d004d4dULL, 0x3300333333003333ULL, 0xfd00fdfdfd00fdfdULL,
        0x6600666666006666ULL, 0x5800585858005858ULL, 0x9600969696009696ULL,
        0x3a003a3a3a003a3aULL, 0x0900090909000909ULL, 0x9500959595009595ULL,
        0x1000101010001010ULL, 0x7800787878007878ULL, 0xd800d8d8d800d8d8ULL,
        0x4200424242004242ULL, 0xcc00cccccc00ccccULL, 0xef00efefef00efefULL,
        0x2600262626002626ULL, 0xe500e5e5e500e5e5ULL, 0x6100616161006161ULL,
        0x1a001a1a1a001a1aULL, 0x3f003f3f3f003f3fULL, 0x3b003b3b3b003b3bULL,
        0x8200828282008282ULL, 0xb600b6b6b600b6b6ULL, 0xdb00dbdbdb00dbdbULL,
        0xd400d4d4d400d4d4ULL, 0x9800989898009898ULL, 0xe800e8e8e800e8e8ULL,
        0x8b008b8b8b008b8bULL, 0x0200020202000202ULL, 0xeb00ebebeb00ebebULL,
        0x0a000a0a0a000a0aULL, 0x2c002c2c2c002c2cULL, 0x1d001d1d1d001d1dULL,
        0xb000b0b0b000b0b0ULL, 0x6f006f6f6f006f6fULL, 0x8d008d8d8d008d8dULL,
        0x8800888888008888ULL, 0x0e000e0e0e000e0eULL, 0x1900191919001919ULL,
        0x8700878787008787ULL, 0x4e004e4e4e004e4eULL, 0x0b000b0b0b000b0bULL,
        0xa900a9a9a900a9a9ULL, 0x0c000c0c0c000c0cULL, 0x7900797979007979ULL,
        0x1100111111001111ULL, 0x7f007f7f7f007f7fULL, 0x2200222222002222ULL,
        0xe700e7e7e700e7e7ULL, 0x5900595959005959ULL, 0xe100e1e1e100e1e1ULL,
        0xda00dadada00dadaULL, 0x3d003d3d3d003d3dULL, 0xc800c8c8c800c8c8ULL,
        0x1200121212001212ULL, 0x0400040404000404ULL, 0x7400747474007474ULL,
        0x5400545454005454ULL, 0x3000303030003030ULL, 0x7e007e7e7e007e7eULL,
        0xb400b4b4b400b4b4ULL, 0x2800282828002828ULL, 0x5500555555005555ULL,
        0x6800686868006868ULL, 0x5000505050005050ULL, 0xbe00bebebe00bebeULL,
        0xd000d0d0d000d0d0ULL, 0xc400c4c4c400c4c4ULL, 0x3100313131003131ULL,
        0xcb00cbcbcb00cbcbULL, 0x2a002a2a2a002a2aULL, 0xad00adadad00adadULL,
        0x0f000f0f0f000f0fULL, 0xca00cacaca00cacaULL, 0x7000707070007070ULL,
        0xff00ffffff00ffffULL, 0x3200323232003232ULL, 0x6900696969006969ULL,
        0x0800080808000808ULL, 0x6200626262006262ULL, 0x0000000000000000ULL,
        0x2400242424002424ULL, 0xd100d1d1d100d1d1ULL, 0xfb00fbfbfb00fbfbULL,
        0xba00bababa00babaULL, 0xed00ededed00ededULL, 0x4500454545004545ULL,
        0x8100818181008181ULL, 0x7300737373007373ULL, 0x6d006d6d6d006d6dULL,
        0x8400848484008484ULL, 0x9f009f9f9f009f9fULL, 0xee00eeeeee00eeeeULL,
        0x4a004a4a4a004a4aULL, 0xc300c3c3c300c3c3ULL, 0x2e002e2e2e002e2eULL,
        0xc100c1c1c100c1c1ULL, 0x0100010101000101ULL, 0xe600e6e6e600e6e6ULL,
        0x2500252525002525ULL, 0x4800484848004848ULL, 0x9900999999009999ULL,
        0xb900b9b9b900b9b9ULL, 0xb300b3b3b300b3b3ULL, 0x7b007b7b7b007b7bULL,
        0xf900f9f9f900f9f9ULL, 0xce00cecece00ceceULL, 0xbf00bfbfbf00bfbfULL,
        0xdf00dfdfdf00dfdfULL, 0x7100717171007171ULL, 0x2900292929002929ULL,
        0xcd00cdcdcd00cdcdULL, 0x6c006c6c6c006c6cULL, 0x1300131313001313ULL,
        0x6400646464006464ULL, 0x9b009b9b9b009b9bULL, 0x6300636363006363ULL,
        0x9d009d9d9d009d9dULL, 0xc000c0c0c000c0c0ULL, 0x4b004b4b4b004b4bULL,
        0xb700b7b7b700b7b7ULL, 0xa500a5a5a500a5a5ULL, 0x8900898989008989ULL,
        0x5f005f5f5f005f5fULL, 0xb100b1b1b100b1b1ULL, 0x1700171717001717ULL,
        0xf400f4f4f400f4f4ULL, 0xbc00bcbcbc00bcbcULL, 0xd300d3d3d300d3d3ULL,
        0x4600464646004646ULL, 0xcf00cfcfcf00cfcfULL, 0x3700373737003737ULL,
        0x5e005e5e5e005e5eULL, 0x4700474747004747ULL, 0x9400949494009494ULL,
        0xfa00fafafa00fafaULL, 0xfc00fcfcfc00fcfcULL, 0x5b005b5b5b005b5bULL,
        0x9700979797009797ULL, 0xfe00fefefe00fefeULL, 0x5a005a5a5a005a5aULL,
        0xac00acacac00acacULL, 0x3c003c3c3c003c3cULL, 0x4c004c4c4c004c4cULL,
        0x0300030303000303ULL, 0x3500353535003535ULL, 0xf300f3f3f300f3f3ULL,
        0x2300232323002323ULL, 0xb800b8b8b800b8b8ULL, 0x5d005d5d5d005d5dULL,
        0x6a006a6a6a006a6aULL, 0x9200929292009292ULL, 0xd500d5d5d500d5d5ULL,
        0x2100212121002121ULL, 0x4400444444004444ULL, 0x5100515151005151ULL,
        0xc600c6c6c600c6c6ULL, 0x7d007d7d7d007d7dULL, 0x3900393939003939ULL,
        0x8300838383008383ULL, 0xdc00dcdcdc00dcdcULL, 0xaa00aaaaaa00aaaaULL,
        0x7c007c7c7c007c7cULL, 0x7700777777007777ULL, 0x5600565656005656ULL,
        0x0500050505000505ULL, 0x1b001b1b1b001b1bULL, 0xa400a4a4a400a4a4ULL,
        0x1500151515001515ULL, 0x3400343434003434ULL, 0x1e001e1e1e001e1eULL,
        0x1c001c1c1c001c1cULL, 0xf800f8f8f800f8f8ULL, 0x5200525252005252ULL,
        0x2000202020002020ULL, 0x1400141414001414ULL, 0xe900e9e9e900e9e9ULL,
        0xbd00bdbdbd00bdbdULL, 0xdd00dddddd00ddddULL, 0xe400e4e4e400e4e4ULL,
        0xa100a1a1a100a1a1ULL, 0xe000e0e0e000e0e0ULL, 0x8a008a8a8a008a8aULL,
        0xf100f1f1f100f1f1ULL, 0xd600d6d6d600d6d6ULL, 0x7a007a7a7a007a7aULL,
        0xbb00bbbbbb00bbbbULL, 0xe300e3e3e300e3e3ULL, 0x4000404040004040ULL,
        0x4f004f4f4f004f4fULL,
};

__visible const u64 camellia_sp44044404[256] = {
        0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL,
        0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL,
        0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL,
        0x6b6b006b6b6b006bULL, 0x4545004545450045ULL, 0xa5a500a5a5a500a5ULL,
        0xeded00ededed00edULL, 0x4f4f004f4f4f004fULL, 0x1d1d001d1d1d001dULL,
        0x9292009292920092ULL, 0x8686008686860086ULL, 0xafaf00afafaf00afULL,
        0x7c7c007c7c7c007cULL, 0x1f1f001f1f1f001fULL, 0x3e3e003e3e3e003eULL,
        0xdcdc00dcdcdc00dcULL, 0x5e5e005e5e5e005eULL, 0x0b0b000b0b0b000bULL,
        0xa6a600a6a6a600a6ULL, 0x3939003939390039ULL, 0xd5d500d5d5d500d5ULL,
        0x5d5d005d5d5d005dULL, 0xd9d900d9d9d900d9ULL, 0x5a5a005a5a5a005aULL,
        0x5151005151510051ULL, 0x6c6c006c6c6c006cULL, 0x8b8b008b8b8b008bULL,
        0x9a9a009a9a9a009aULL, 0xfbfb00fbfbfb00fbULL, 0xb0b000b0b0b000b0ULL,
        0x7474007474740074ULL, 0x2b2b002b2b2b002bULL, 0xf0f000f0f0f000f0ULL,
        0x8484008484840084ULL, 0xdfdf00dfdfdf00dfULL, 0xcbcb00cbcbcb00cbULL,
        0x3434003434340034ULL, 0x7676007676760076ULL, 0x6d6d006d6d6d006dULL,
        0xa9a900a9a9a900a9ULL, 0xd1d100d1d1d100d1ULL, 0x0404000404040004ULL,
        0x1414001414140014ULL, 0x3a3a003a3a3a003aULL, 0xdede00dedede00deULL,
        0x1111001111110011ULL, 0x3232003232320032ULL, 0x9c9c009c9c9c009cULL,
        0x5353005353530053ULL, 0xf2f200f2f2f200f2ULL, 0xfefe00fefefe00feULL,
        0xcfcf00cfcfcf00cfULL, 0xc3c300c3c3c300c3ULL, 0x7a7a007a7a7a007aULL,
        0x2424002424240024ULL, 0xe8e800e8e8e800e8ULL, 0x6060006060600060ULL,
        0x6969006969690069ULL, 0xaaaa00aaaaaa00aaULL, 0xa0a000a0a0a000a0ULL,
        0xa1a100a1a1a100a1ULL, 0x6262006262620062ULL, 0x5454005454540054ULL,
        0x1e1e001e1e1e001eULL, 0xe0e000e0e0e000e0ULL, 0x6464006464640064ULL,
        0x1010001010100010ULL, 0x0000000000000000ULL, 0xa3a300a3a3a300a3ULL,
        0x7575007575750075ULL, 0x8a8a008a8a8a008aULL, 0xe6e600e6e6e600e6ULL,
        0x0909000909090009ULL, 0xdddd00dddddd00ddULL, 0x8787008787870087ULL,
        0x8383008383830083ULL, 0xcdcd00cdcdcd00cdULL, 0x9090009090900090ULL,
        0x7373007373730073ULL, 0xf6f600f6f6f600f6ULL, 0x9d9d009d9d9d009dULL,
        0xbfbf00bfbfbf00bfULL, 0x5252005252520052ULL, 0xd8d800d8d8d800d8ULL,
        0xc8c800c8c8c800c8ULL, 0xc6c600c6c6c600c6ULL, 0x8181008181810081ULL,
        0x6f6f006f6f6f006fULL, 0x1313001313130013ULL, 0x6363006363630063ULL,
        0xe9e900e9e9e900e9ULL, 0xa7a700a7a7a700a7ULL, 0x9f9f009f9f9f009fULL,
        0xbcbc00bcbcbc00bcULL, 0x2929002929290029ULL, 0xf9f900f9f9f900f9ULL,
        0x2f2f002f2f2f002fULL, 0xb4b400b4b4b400b4ULL, 0x7878007878780078ULL,
        0x0606000606060006ULL, 0xe7e700e7e7e700e7ULL, 0x7171007171710071ULL,
        0xd4d400d4d4d400d4ULL, 0xabab00ababab00abULL, 0x8888008888880088ULL,
        0x8d8d008d8d8d008dULL, 0x7272007272720072ULL, 0xb9b900b9b9b900b9ULL,
        0xf8f800f8f8f800f8ULL, 0xacac00acacac00acULL, 0x3636003636360036ULL,
        0x2a2a002a2a2a002aULL, 0x3c3c003c3c3c003cULL, 0xf1f100f1f1f100f1ULL,
        0x4040004040400040ULL, 0xd3d300d3d3d300d3ULL, 0xbbbb00bbbbbb00bbULL,
        0x4343004343430043ULL, 0x1515001515150015ULL, 0xadad00adadad00adULL,
        0x7777007777770077ULL, 0x8080008080800080ULL, 0x8282008282820082ULL,
        0xecec00ececec00ecULL, 0x2727002727270027ULL, 0xe5e500e5e5e500e5ULL,
        0x8585008585850085ULL, 0x3535003535350035ULL, 0x0c0c000c0c0c000cULL,
        0x4141004141410041ULL, 0xefef00efefef00efULL, 0x9393009393930093ULL,
        0x1919001919190019ULL, 0x2121002121210021ULL, 0x0e0e000e0e0e000eULL,
        0x4e4e004e4e4e004eULL, 0x6565006565650065ULL, 0xbdbd00bdbdbd00bdULL,
        0xb8b800b8b8b800b8ULL, 0x8f8f008f8f8f008fULL, 0xebeb00ebebeb00ebULL,
        0xcece00cecece00ceULL, 0x3030003030300030ULL, 0x5f5f005f5f5f005fULL,
        0xc5c500c5c5c500c5ULL, 0x1a1a001a1a1a001aULL, 0xe1e100e1e1e100e1ULL,
        0xcaca00cacaca00caULL, 0x4747004747470047ULL, 0x3d3d003d3d3d003dULL,
        0x0101000101010001ULL, 0xd6d600d6d6d600d6ULL, 0x5656005656560056ULL,
        0x4d4d004d4d4d004dULL, 0x0d0d000d0d0d000dULL, 0x6666006666660066ULL,
        0xcccc00cccccc00ccULL, 0x2d2d002d2d2d002dULL, 0x1212001212120012ULL,
        0x2020002020200020ULL, 0xb1b100b1b1b100b1ULL, 0x9999009999990099ULL,
        0x4c4c004c4c4c004cULL, 0xc2c200c2c2c200c2ULL, 0x7e7e007e7e7e007eULL,
        0x0505000505050005ULL, 0xb7b700b7b7b700b7ULL, 0x3131003131310031ULL,
        0x1717001717170017ULL, 0xd7d700d7d7d700d7ULL, 0x5858005858580058ULL,
        0x6161006161610061ULL, 0x1b1b001b1b1b001bULL, 0x1c1c001c1c1c001cULL,
        0x0f0f000f0f0f000fULL, 0x1616001616160016ULL, 0x1818001818180018ULL,
        0x2222002222220022ULL, 0x4444004444440044ULL, 0xb2b200b2b2b200b2ULL,
        0xb5b500b5b5b500b5ULL, 0x9191009191910091ULL, 0x0808000808080008ULL,
        0xa8a800a8a8a800a8ULL, 0xfcfc00fcfcfc00fcULL, 0x5050005050500050ULL,
        0xd0d000d0d0d000d0ULL, 0x7d7d007d7d7d007dULL, 0x8989008989890089ULL,
        0x9797009797970097ULL, 0x5b5b005b5b5b005bULL, 0x9595009595950095ULL,
        0xffff00ffffff00ffULL, 0xd2d200d2d2d200d2ULL, 0xc4c400c4c4c400c4ULL,
        0x4848004848480048ULL, 0xf7f700f7f7f700f7ULL, 0xdbdb00dbdbdb00dbULL,
        0x0303000303030003ULL, 0xdada00dadada00daULL, 0x3f3f003f3f3f003fULL,
        0x9494009494940094ULL, 0x5c5c005c5c5c005cULL, 0x0202000202020002ULL,
        0x4a4a004a4a4a004aULL, 0x3333003333330033ULL, 0x6767006767670067ULL,
        0xf3f300f3f3f300f3ULL, 0x7f7f007f7f7f007fULL, 0xe2e200e2e2e200e2ULL,
        0x9b9b009b9b9b009bULL, 0x2626002626260026ULL, 0x3737003737370037ULL,
        0x3b3b003b3b3b003bULL, 0x9696009696960096ULL, 0x4b4b004b4b4b004bULL,
        0xbebe00bebebe00beULL, 0x2e2e002e2e2e002eULL, 0x7979007979790079ULL,
        0x8c8c008c8c8c008cULL, 0x6e6e006e6e6e006eULL, 0x8e8e008e8e8e008eULL,
        0xf5f500f5f5f500f5ULL, 0xb6b600b6b6b600b6ULL, 0xfdfd00fdfdfd00fdULL,
        0x5959005959590059ULL, 0x9898009898980098ULL, 0x6a6a006a6a6a006aULL,
        0x4646004646460046ULL, 0xbaba00bababa00baULL, 0x2525002525250025ULL,
        0x4242004242420042ULL, 0xa2a200a2a2a200a2ULL, 0xfafa00fafafa00faULL,
        0x0707000707070007ULL, 0x5555005555550055ULL, 0xeeee00eeeeee00eeULL,
        0x0a0a000a0a0a000aULL, 0x4949004949490049ULL, 0x6868006868680068ULL,
        0x3838003838380038ULL, 0xa4a400a4a4a400a4ULL, 0x2828002828280028ULL,
        0x7b7b007b7b7b007bULL, 0xc9c900c9c9c900c9ULL, 0xc1c100c1c1c100c1ULL,
        0xe3e300e3e3e300e3ULL, 0xf4f400f4f4f400f4ULL, 0xc7c700c7c7c700c7ULL,
        0x9e9e009e9e9e009eULL,
};

__visible const u64 camellia_sp11101110[256] = {
        0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL,
        0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL,
        0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL,
        0x8585850085858500ULL, 0x5757570057575700ULL, 0x3535350035353500ULL,
        0xeaeaea00eaeaea00ULL, 0x0c0c0c000c0c0c00ULL, 0xaeaeae00aeaeae00ULL,
        0x4141410041414100ULL, 0x2323230023232300ULL, 0xefefef00efefef00ULL,
        0x6b6b6b006b6b6b00ULL, 0x9393930093939300ULL, 0x4545450045454500ULL,
        0x1919190019191900ULL, 0xa5a5a500a5a5a500ULL, 0x2121210021212100ULL,
        0xededed00ededed00ULL, 0x0e0e0e000e0e0e00ULL, 0x4f4f4f004f4f4f00ULL,
        0x4e4e4e004e4e4e00ULL, 0x1d1d1d001d1d1d00ULL, 0x6565650065656500ULL,
        0x9292920092929200ULL, 0xbdbdbd00bdbdbd00ULL, 0x8686860086868600ULL,
        0xb8b8b800b8b8b800ULL, 0xafafaf00afafaf00ULL, 0x8f8f8f008f8f8f00ULL,
        0x7c7c7c007c7c7c00ULL, 0xebebeb00ebebeb00ULL, 0x1f1f1f001f1f1f00ULL,
        0xcecece00cecece00ULL, 0x3e3e3e003e3e3e00ULL, 0x3030300030303000ULL,
        0xdcdcdc00dcdcdc00ULL, 0x5f5f5f005f5f5f00ULL, 0x5e5e5e005e5e5e00ULL,
        0xc5c5c500c5c5c500ULL, 0x0b0b0b000b0b0b00ULL, 0x1a1a1a001a1a1a00ULL,
        0xa6a6a600a6a6a600ULL, 0xe1e1e100e1e1e100ULL, 0x3939390039393900ULL,
        0xcacaca00cacaca00ULL, 0xd5d5d500d5d5d500ULL, 0x4747470047474700ULL,
        0x5d5d5d005d5d5d00ULL, 0x3d3d3d003d3d3d00ULL, 0xd9d9d900d9d9d900ULL,
        0x0101010001010100ULL, 0x5a5a5a005a5a5a00ULL, 0xd6d6d600d6d6d600ULL,
        0x5151510051515100ULL, 0x5656560056565600ULL, 0x6c6c6c006c6c6c00ULL,
        0x4d4d4d004d4d4d00ULL, 0x8b8b8b008b8b8b00ULL, 0x0d0d0d000d0d0d00ULL,
        0x9a9a9a009a9a9a00ULL, 0x6666660066666600ULL, 0xfbfbfb00fbfbfb00ULL,
        0xcccccc00cccccc00ULL, 0xb0b0b000b0b0b000ULL, 0x2d2d2d002d2d2d00ULL,
        0x7474740074747400ULL, 0x1212120012121200ULL, 0x2b2b2b002b2b2b00ULL,
        0x2020200020202000ULL, 0xf0f0f000f0f0f000ULL, 0xb1b1b100b1b1b100ULL,
        0x8484840084848400ULL, 0x9999990099999900ULL, 0xdfdfdf00dfdfdf00ULL,
        0x4c4c4c004c4c4c00ULL, 0xcbcbcb00cbcbcb00ULL, 0xc2c2c200c2c2c200ULL,
        0x3434340034343400ULL, 0x7e7e7e007e7e7e00ULL, 0x7676760076767600ULL,
        0x0505050005050500ULL, 0x6d6d6d006d6d6d00ULL, 0xb7b7b700b7b7b700ULL,
        0xa9a9a900a9a9a900ULL, 0x3131310031313100ULL, 0xd1d1d100d1d1d100ULL,
        0x1717170017171700ULL, 0x0404040004040400ULL, 0xd7d7d700d7d7d700ULL,
        0x1414140014141400ULL, 0x5858580058585800ULL, 0x3a3a3a003a3a3a00ULL,
        0x6161610061616100ULL, 0xdedede00dedede00ULL, 0x1b1b1b001b1b1b00ULL,
        0x1111110011111100ULL, 0x1c1c1c001c1c1c00ULL, 0x3232320032323200ULL,
        0x0f0f0f000f0f0f00ULL, 0x9c9c9c009c9c9c00ULL, 0x1616160016161600ULL,
        0x5353530053535300ULL, 0x1818180018181800ULL, 0xf2f2f200f2f2f200ULL,
        0x2222220022222200ULL, 0xfefefe00fefefe00ULL, 0x4444440044444400ULL,
        0xcfcfcf00cfcfcf00ULL, 0xb2b2b200b2b2b200ULL, 0xc3c3c300c3c3c300ULL,
        0xb5b5b500b5b5b500ULL, 0x7a7a7a007a7a7a00ULL, 0x9191910091919100ULL,
        0x2424240024242400ULL, 0x0808080008080800ULL, 0xe8e8e800e8e8e800ULL,
        0xa8a8a800a8a8a800ULL, 0x6060600060606000ULL, 0xfcfcfc00fcfcfc00ULL,
        0x6969690069696900ULL, 0x5050500050505000ULL, 0xaaaaaa00aaaaaa00ULL,
        0xd0d0d000d0d0d000ULL, 0xa0a0a000a0a0a000ULL, 0x7d7d7d007d7d7d00ULL,
        0xa1a1a100a1a1a100ULL, 0x8989890089898900ULL, 0x6262620062626200ULL,
        0x9797970097979700ULL, 0x5454540054545400ULL, 0x5b5b5b005b5b5b00ULL,
        0x1e1e1e001e1e1e00ULL, 0x9595950095959500ULL, 0xe0e0e000e0e0e000ULL,
        0xffffff00ffffff00ULL, 0x6464640064646400ULL, 0xd2d2d200d2d2d200ULL,
        0x1010100010101000ULL, 0xc4c4c400c4c4c400ULL, 0x0000000000000000ULL,
        0x4848480048484800ULL, 0xa3a3a300a3a3a300ULL, 0xf7f7f700f7f7f700ULL,
        0x7575750075757500ULL, 0xdbdbdb00dbdbdb00ULL, 0x8a8a8a008a8a8a00ULL,
        0x0303030003030300ULL, 0xe6e6e600e6e6e600ULL, 0xdadada00dadada00ULL,
        0x0909090009090900ULL, 0x3f3f3f003f3f3f00ULL, 0xdddddd00dddddd00ULL,
        0x9494940094949400ULL, 0x8787870087878700ULL, 0x5c5c5c005c5c5c00ULL,
        0x8383830083838300ULL, 0x0202020002020200ULL, 0xcdcdcd00cdcdcd00ULL,
        0x4a4a4a004a4a4a00ULL, 0x9090900090909000ULL, 0x3333330033333300ULL,
        0x7373730073737300ULL, 0x6767670067676700ULL, 0xf6f6f600f6f6f600ULL,
        0xf3f3f300f3f3f300ULL, 0x9d9d9d009d9d9d00ULL, 0x7f7f7f007f7f7f00ULL,
        0xbfbfbf00bfbfbf00ULL, 0xe2e2e200e2e2e200ULL, 0x5252520052525200ULL,
        0x9b9b9b009b9b9b00ULL, 0xd8d8d800d8d8d800ULL, 0x2626260026262600ULL,
        0xc8c8c800c8c8c800ULL, 0x3737370037373700ULL, 0xc6c6c600c6c6c600ULL,
        0x3b3b3b003b3b3b00ULL, 0x8181810081818100ULL, 0x9696960096969600ULL,
        0x6f6f6f006f6f6f00ULL, 0x4b4b4b004b4b4b00ULL, 0x1313130013131300ULL,
        0xbebebe00bebebe00ULL, 0x6363630063636300ULL, 0x2e2e2e002e2e2e00ULL,
        0xe9e9e900e9e9e900ULL, 0x7979790079797900ULL, 0xa7a7a700a7a7a700ULL,
        0x8c8c8c008c8c8c00ULL, 0x9f9f9f009f9f9f00ULL, 0x6e6e6e006e6e6e00ULL,
        0xbcbcbc00bcbcbc00ULL, 0x8e8e8e008e8e8e00ULL, 0x2929290029292900ULL,
        0xf5f5f500f5f5f500ULL, 0xf9f9f900f9f9f900ULL, 0xb6b6b600b6b6b600ULL,
        0x2f2f2f002f2f2f00ULL, 0xfdfdfd00fdfdfd00ULL, 0xb4b4b400b4b4b400ULL,
        0x5959590059595900ULL, 0x7878780078787800ULL, 0x9898980098989800ULL,
        0x0606060006060600ULL, 0x6a6a6a006a6a6a00ULL, 0xe7e7e700e7e7e700ULL,
        0x4646460046464600ULL, 0x7171710071717100ULL, 0xbababa00bababa00ULL,
        0xd4d4d400d4d4d400ULL, 0x2525250025252500ULL, 0xababab00ababab00ULL,
        0x4242420042424200ULL, 0x8888880088888800ULL, 0xa2a2a200a2a2a200ULL,
        0x8d8d8d008d8d8d00ULL, 0xfafafa00fafafa00ULL, 0x7272720072727200ULL,
        0x0707070007070700ULL, 0xb9b9b900b9b9b900ULL, 0x5555550055555500ULL,
        0xf8f8f800f8f8f800ULL, 0xeeeeee00eeeeee00ULL, 0xacacac00acacac00ULL,
        0x0a0a0a000a0a0a00ULL, 0x3636360036363600ULL, 0x4949490049494900ULL,
        0x2a2a2a002a2a2a00ULL, 0x6868680068686800ULL, 0x3c3c3c003c3c3c00ULL,
        0x3838380038383800ULL, 0xf1f1f100f1f1f100ULL, 0xa4a4a400a4a4a400ULL,
        0x4040400040404000ULL, 0x2828280028282800ULL, 0xd3d3d300d3d3d300ULL,
        0x7b7b7b007b7b7b00ULL, 0xbbbbbb00bbbbbb00ULL, 0xc9c9c900c9c9c900ULL,
        0x4343430043434300ULL, 0xc1c1c100c1c1c100ULL, 0x1515150015151500ULL,
        0xe3e3e300e3e3e300ULL, 0xadadad00adadad00ULL, 0xf4f4f400f4f4f400ULL,
        0x7777770077777700ULL, 0xc7c7c700c7c7c700ULL, 0x8080800080808000ULL,
        0x9e9e9e009e9e9e00ULL,
};

/* key constants */
#define CAMELLIA_SIGMA1L (0xA09E667FL)
#define CAMELLIA_SIGMA1R (0x3BCC908BL)
#define CAMELLIA_SIGMA2L (0xB67AE858L)
#define CAMELLIA_SIGMA2R (0x4CAA73B2L)
#define CAMELLIA_SIGMA3L (0xC6EF372FL)
#define CAMELLIA_SIGMA3R (0xE94F82BEL)
#define CAMELLIA_SIGMA4L (0x54FF53A5L)
#define CAMELLIA_SIGMA4R (0xF1D36F1CL)
#define CAMELLIA_SIGMA5L (0x10E527FAL)
#define CAMELLIA_SIGMA5R (0xDE682D1DL)
#define CAMELLIA_SIGMA6L (0xB05688C2L)
#define CAMELLIA_SIGMA6R (0xB3E6C1FDL)

/* macros */
#define ROLDQ(l, r, bits) ({ \
        u64 t = l;                                        \
        l = (l << bits) | (r >> (64 - bits));                \
        r = (r << bits) | (t >> (64 - bits));                \
})

#define CAMELLIA_F(x, kl, kr, y) ({ \
        u64 ii = x ^ (((u64)kl << 32) | kr);                                \
        y = camellia_sp11101110[(uint8_t)ii];                                \
        y ^= camellia_sp44044404[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp30333033[(uint8_t)ii];                                \
        y ^= camellia_sp02220222[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp00444404[(uint8_t)ii];                                \
        y ^= camellia_sp03303033[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp22000222[(uint8_t)ii];                                \
        y ^= camellia_sp10011110[(uint8_t)(ii >> 8)];                        \
        y = ror64(y, 32);                                                \
})

#define SET_SUBKEY_LR(INDEX, sRL) (subkey[(INDEX)] = ror64((sRL), 32))

static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
{
        u64 kw4, tt;
        u32 dw, tl, tr;

        /* absorb kw2 to other subkeys */
        /* round 2 */
        subRL[3] ^= subRL[1]; /*covered*/
        /* round 4 */
        subRL[5] ^= subRL[1];
        /* round 6 */
        subRL[7] ^= subRL[1];

        subRL[1] ^= (subRL[1] & ~subRL[9]) << 32;
        /* modified for FLinv(kl2) */
        dw = (subRL[1] & subRL[9]) >> 32;
        subRL[1] ^= rol32(dw, 1);

        /* round 8 */
        subRL[11] ^= subRL[1];
        /* round 10 */
        subRL[13] ^= subRL[1];
        /* round 12 */
        subRL[15] ^= subRL[1];

        subRL[1] ^= (subRL[1] & ~subRL[17]) << 32;
        /* modified for FLinv(kl4) */
        dw = (subRL[1] & subRL[17]) >> 32;
        subRL[1] ^= rol32(dw, 1);

        /* round 14 */
        subRL[19] ^= subRL[1];
        /* round 16 */
        subRL[21] ^= subRL[1];
        /* round 18 */
        subRL[23] ^= subRL[1];

        if (max == 24) {
                /* kw3 */
                subRL[24] ^= subRL[1]; /*covered*/

                /* absorb kw4 to other subkeys */
                kw4 = subRL[25];
        } else {
                subRL[1] ^= (subRL[1] & ~subRL[25]) << 32; /*covered*/
                /* modified for FLinv(kl6) */
                dw = (subRL[1] & subRL[25]) >> 32;
                subRL[1] ^= rol32(dw, 1);

                /* round 20 */
                subRL[27] ^= subRL[1];
                /* round 22 */
                subRL[29] ^= subRL[1];
                /* round 24 */
                subRL[31] ^= subRL[1];
                /* kw3 */
                subRL[32] ^= subRL[1];

                /* absorb kw4 to other subkeys */
                kw4 = subRL[33];
                /* round 23 */
                subRL[30] ^= kw4;
                /* round 21 */
                subRL[28] ^= kw4;
                /* round 19 */
                subRL[26] ^= kw4;

                kw4 ^= (kw4 & ~subRL[24]) << 32;
                /* modified for FL(kl5) */
                dw = (kw4 & subRL[24]) >> 32;
                kw4 ^= rol32(dw, 1);
        }

        /* round 17 */
        subRL[22] ^= kw4; /*covered*/
        /* round 15 */
        subRL[20] ^= kw4;
        /* round 13 */
        subRL[18] ^= kw4;

        kw4 ^= (kw4 & ~subRL[16]) << 32;
        /* modified for FL(kl3) */
        dw = (kw4 & subRL[16]) >> 32;
        kw4 ^= rol32(dw, 1);

        /* round 11 */
        subRL[14] ^= kw4;
        /* round 9 */
        subRL[12] ^= kw4;
        /* round 7 */
        subRL[10] ^= kw4;

        kw4 ^= (kw4 & ~subRL[8]) << 32;
        /* modified for FL(kl1) */
        dw = (kw4 & subRL[8]) >> 32;
        kw4 ^= rol32(dw, 1);

        /* round 5 */
        subRL[6] ^= kw4;
        /* round 3 */
        subRL[4] ^= kw4;
        /* round 1 */
        subRL[2] ^= kw4;
        /* kw1 */
        subRL[0] ^= kw4;

        /* key XOR is end of F-function */
        SET_SUBKEY_LR(0, subRL[0] ^ subRL[2]);                        /* kw1 */
        SET_SUBKEY_LR(2, subRL[3]);                                /* round 1 */
        SET_SUBKEY_LR(3, subRL[2] ^ subRL[4]);                        /* round 2 */
        SET_SUBKEY_LR(4, subRL[3] ^ subRL[5]);                        /* round 3 */
        SET_SUBKEY_LR(5, subRL[4] ^ subRL[6]);                        /* round 4 */
        SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]);                        /* round 5 */

        tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]);
        dw = tl & (subRL[8] >> 32);                                /* FL(kl1) */
        tr = subRL[10] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(7, subRL[6] ^ tt);                        /* round 6 */
        SET_SUBKEY_LR(8, subRL[8]);                                /* FL(kl1) */
        SET_SUBKEY_LR(9, subRL[9]);                                /* FLinv(kl2) */

        tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]);
        dw = tl & (subRL[9] >> 32);                                /* FLinv(kl2) */
        tr = subRL[7] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(10, subRL[11] ^ tt);                        /* round 7 */
        SET_SUBKEY_LR(11, subRL[10] ^ subRL[12]);                /* round 8 */
        SET_SUBKEY_LR(12, subRL[11] ^ subRL[13]);                /* round 9 */
        SET_SUBKEY_LR(13, subRL[12] ^ subRL[14]);                /* round 10 */
        SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]);                /* round 11 */

        tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]);
        dw = tl & (subRL[16] >> 32);                                /* FL(kl3) */
        tr = subRL[18] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(15, subRL[14] ^ tt);                        /* round 12 */
        SET_SUBKEY_LR(16, subRL[16]);                                /* FL(kl3) */
        SET_SUBKEY_LR(17, subRL[17]);                                /* FLinv(kl4) */

        tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]);
        dw = tl & (subRL[17] >> 32);                                /* FLinv(kl4) */
        tr = subRL[15] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(18, subRL[19] ^ tt);                        /* round 13 */
        SET_SUBKEY_LR(19, subRL[18] ^ subRL[20]);                /* round 14 */
        SET_SUBKEY_LR(20, subRL[19] ^ subRL[21]);                /* round 15 */
        SET_SUBKEY_LR(21, subRL[20] ^ subRL[22]);                /* round 16 */
        SET_SUBKEY_LR(22, subRL[21] ^ subRL[23]);                /* round 17 */

        if (max == 24) {
                SET_SUBKEY_LR(23, subRL[22]);                        /* round 18 */ /*covered*/
                SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]);        /* kw3 */
        } else {
                tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]); /*covered*/
                dw = tl & (subRL[24] >> 32);                        /* FL(kl5) */
                tr = subRL[26] ^ rol32(dw, 1);
                tt = (tr | ((u64)tl << 32));

                SET_SUBKEY_LR(23, subRL[22] ^ tt);                /* round 18 */
                SET_SUBKEY_LR(24, subRL[24]);                        /* FL(kl5) */
                SET_SUBKEY_LR(25, subRL[25]);                        /* FLinv(kl6) */

                tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]);
                dw = tl & (subRL[25] >> 32);                        /* FLinv(kl6) */
                tr = subRL[23] ^ rol32(dw, 1);
                tt = (tr | ((u64)tl << 32));

                SET_SUBKEY_LR(26, subRL[27] ^ tt);                /* round 19 */
                SET_SUBKEY_LR(27, subRL[26] ^ subRL[28]);        /* round 20 */
                SET_SUBKEY_LR(28, subRL[27] ^ subRL[29]);        /* round 21 */
                SET_SUBKEY_LR(29, subRL[28] ^ subRL[30]);        /* round 22 */
                SET_SUBKEY_LR(30, subRL[29] ^ subRL[31]);        /* round 23 */
                SET_SUBKEY_LR(31, subRL[30]);                        /* round 24 */
                SET_SUBKEY_LR(32, subRL[32] ^ subRL[31]);        /* kw3 */
        }
} /*covered*/

static void camellia_setup128(const unsigned char *key, u64 *subkey)
{
        u64 kl, kr, ww;
        u64 subRL[26];

        /**
         *  k == kl || kr (|| is concatenation)
         */
        kl = get_unaligned_be64(key); /*covered*/
        kr = get_unaligned_be64(key + 8);

        /* generate KL dependent subkeys */
        /* kw1 */
        subRL[0] = kl;
        /* kw2 */
        subRL[1] = kr;

        /* rotation left shift 15bit */
        ROLDQ(kl, kr, 15);

        /* k3 */
        subRL[4] = kl;
        /* k4 */
        subRL[5] = kr;

        /* rotation left shift 15+30bit */
        ROLDQ(kl, kr, 30);

        /* k7 */
        subRL[10] = kl;
        /* k8 */
        subRL[11] = kr;

        /* rotation left shift 15+30+15bit */
        ROLDQ(kl, kr, 15);

        /* k10 */
        subRL[13] = kr;
        /* rotation left shift 15+30+15+17 bit */
        ROLDQ(kl, kr, 17);

        /* kl3 */
        subRL[16] = kl;
        /* kl4 */
        subRL[17] = kr;

        /* rotation left shift 15+30+15+17+17 bit */
        ROLDQ(kl, kr, 17);

        /* k13 */
        subRL[18] = kl;
        /* k14 */
        subRL[19] = kr;

        /* rotation left shift 15+30+15+17+17+17 bit */
        ROLDQ(kl, kr, 17);

        /* k17 */
        subRL[22] = kl;
        /* k18 */
        subRL[23] = kr;

        /* generate KA */
        kl = subRL[0];
        kr = subRL[1];
        CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl);

        /* current status == (kll, klr, w0, w1) */
        CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww);
        kl ^= ww;

        /* generate KA dependent subkeys */
        /* k1, k2 */
        subRL[2] = kl;
        subRL[3] = kr;
        ROLDQ(kl, kr, 15);
        /* k5,k6 */
        subRL[6] = kl;
        subRL[7] = kr;
        ROLDQ(kl, kr, 15);
        /* kl1, kl2 */
        subRL[8] = kl;
        subRL[9] = kr;
        ROLDQ(kl, kr, 15);
        /* k9 */
        subRL[12] = kl;
        ROLDQ(kl, kr, 15);
        /* k11, k12 */
        subRL[14] = kl;
        subRL[15] = kr;
        ROLDQ(kl, kr, 34);
        /* k15, k16 */
        subRL[20] = kl;
        subRL[21] = kr;
        ROLDQ(kl, kr, 17);
        /* kw3, kw4 */
        subRL[24] = kl;
        subRL[25] = kr;

        camellia_setup_tail(subkey, subRL, 24);
}

static void camellia_setup256(const unsigned char *key, u64 *subkey)
{
        u64 kl, kr;                        /* left half of key */
        u64 krl, krr;                        /* right half of key */
        u64 ww;                                /* temporary variables */
        u64 subRL[34];

        /**
         *  key = (kl || kr || krl || krr) (|| is concatenation)
         */
        kl = get_unaligned_be64(key); /*covered*/
        kr = get_unaligned_be64(key + 8);
        krl = get_unaligned_be64(key + 16);
        krr = get_unaligned_be64(key + 24);

        /* generate KL dependent subkeys */
        /* kw1 */
        subRL[0] = kl;
        /* kw2 */
        subRL[1] = kr;
        ROLDQ(kl, kr, 45);
        /* k9 */
        subRL[12] = kl;
        /* k10 */
        subRL[13] = kr;
        ROLDQ(kl, kr, 15);
        /* kl3 */
        subRL[16] = kl;
        /* kl4 */
        subRL[17] = kr;
        ROLDQ(kl, kr, 17);
        /* k17 */
        subRL[22] = kl;
        /* k18 */
        subRL[23] = kr;
        ROLDQ(kl, kr, 34);
        /* k23 */
        subRL[30] = kl;
        /* k24 */
        subRL[31] = kr;

        /* generate KR dependent subkeys */
        ROLDQ(krl, krr, 15);
        /* k3 */
        subRL[4] = krl;
        /* k4 */
        subRL[5] = krr;
        ROLDQ(krl, krr, 15);
        /* kl1 */
        subRL[8] = krl;
        /* kl2 */
        subRL[9] = krr;
        ROLDQ(krl, krr, 30);
        /* k13 */
        subRL[18] = krl;
        /* k14 */
        subRL[19] = krr;
        ROLDQ(krl, krr, 34);
        /* k19 */
        subRL[26] = krl;
        /* k20 */
        subRL[27] = krr;
        ROLDQ(krl, krr, 34);

        /* generate KA */
        kl = subRL[0] ^ krl;
        kr = subRL[1] ^ krr;

        CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl);
        kl ^= krl;
        CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr);
        kr ^= ww ^ krr;
        CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww);
        kl ^= ww;

        /* generate KB */
        krl ^= kl;
        krr ^= kr;
        CAMELLIA_F(krl, CAMELLIA_SIGMA5L, CAMELLIA_SIGMA5R, ww);
        krr ^= ww;
        CAMELLIA_F(krr, CAMELLIA_SIGMA6L, CAMELLIA_SIGMA6R, ww);
        krl ^= ww;

        /* generate KA dependent subkeys */
        ROLDQ(kl, kr, 15);
        /* k5 */
        subRL[6] = kl;
        /* k6 */
        subRL[7] = kr;
        ROLDQ(kl, kr, 30);
        /* k11 */
        subRL[14] = kl;
        /* k12 */
        subRL[15] = kr;
        /* rotation left shift 32bit */
        ROLDQ(kl, kr, 32);
        /* kl5 */
        subRL[24] = kl;
        /* kl6 */
        subRL[25] = kr;
        /* rotation left shift 17 from k11,k12 -> k21,k22 */
        ROLDQ(kl, kr, 17);
        /* k21 */
        subRL[28] = kl;
        /* k22 */
        subRL[29] = kr;

        /* generate KB dependent subkeys */
        /* k1 */
        subRL[2] = krl;
        /* k2 */
        subRL[3] = krr;
        ROLDQ(krl, krr, 30);
        /* k7 */
        subRL[10] = krl;
        /* k8 */
        subRL[11] = krr;
        ROLDQ(krl, krr, 30);
        /* k15 */
        subRL[20] = krl;
        /* k16 */
        subRL[21] = krr;
        ROLDQ(krl, krr, 51);
        /* kw3 */
        subRL[32] = krl;
        /* kw4 */
        subRL[33] = krr;

        camellia_setup_tail(subkey, subRL, 32);
}

static void camellia_setup192(const unsigned char *key, u64 *subkey)
{
        unsigned char kk[32];
        u64 krl, krr;

        memcpy(kk, key, 24);
        memcpy((unsigned char *)&krl, key+16, 8);
        krr = ~krl;
        memcpy(kk+24, (unsigned char *)&krr, 8);
        camellia_setup256(kk, subkey); /*covered*/
}

int __camellia_setkey(struct camellia_ctx *cctx, const unsigned char *key,
                      unsigned int key_len, u32 *flags)
{
        if (key_len != 16 && key_len != 24 && key_len != 32) { /*covered*/
                *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; /*covered*/
                return -EINVAL;
        }

        cctx->key_length = key_len; /*covered*/

        switch (key_len) {
        case 16:
                camellia_setup128(key, cctx->key_table);
                break;
        case 24:
                camellia_setup192(key, cctx->key_table); /*covered*/
                break;
        case 32:
                camellia_setup256(key, cctx->key_table);
                break;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(__camellia_setkey);

static int camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
                           unsigned int key_len)
{
        return __camellia_setkey(crypto_tfm_ctx(tfm), key, key_len, /*covered*/
                                 &tfm->crt_flags);
}

static int camellia_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
                                    unsigned int key_len)
{
        return camellia_setkey(&tfm->base, key, key_len); /*covered*/
}

void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
{
        u128 iv = *src; /*covered*/

        camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src);

        u128_xor(&dst[1], &dst[1], &iv);
}
EXPORT_SYMBOL_GPL(camellia_decrypt_cbc_2way);

void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk;

        if (dst != src) /*covered*/
                *dst = *src; /*covered*/

        le128_to_be128(&ctrblk, iv); /*covered*/
        le128_inc(iv); /*covered*/

        camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk);
}
EXPORT_SYMBOL_GPL(camellia_crypt_ctr);

void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblks[2];

        if (dst != src) { /*covered*/
                dst[0] = src[0]; /*covered*/
                dst[1] = src[1];
        }

        le128_to_be128(&ctrblks[0], iv); /*covered*/
        le128_inc(iv);
        le128_to_be128(&ctrblks[1], iv); /*covered*/
        le128_inc(iv); /*covered*/

        camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks);
}
EXPORT_SYMBOL_GPL(camellia_crypt_ctr_2way);

static const struct common_glue_ctx camellia_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
        } }
};

static const struct common_glue_ctx camellia_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
        } }
};

static const struct common_glue_ctx camellia_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&camellia_ctr, req); /*covered*/
}

static struct crypto_alg camellia_cipher_alg = {
        .cra_name                = "camellia",
        .cra_driver_name        = "camellia-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = CAMELLIA_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct camellia_ctx),
        .cra_alignmask                = 0,
        .cra_module                = THIS_MODULE,
        .cra_u                        = {
                .cipher = {
                        .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE,
                        .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE,
                        .cia_setkey         = camellia_setkey,
                        .cia_encrypt         = camellia_encrypt,
                        .cia_decrypt         = camellia_decrypt
                }
        }
};

static struct skcipher_alg camellia_skcipher_algs[] = {
        {
                .base.cra_name                = "ecb(camellia)",
                .base.cra_driver_name        = "ecb-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(camellia)",
                .base.cra_driver_name        = "cbc-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(camellia)",
                .base.cra_driver_name        = "ctr-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .chunksize                = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, camellia-asm is slower than original assembler
                 * implementation because excessive uses of 64bit rotate and
                 * left-shifts (which are really slow on P4) needed to store and
                 * handle 128bit block in two 64bit registers.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "camellia-x86_64: performance on this CPU "
                        "would be suboptimal: disabling "
                        "camellia-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&camellia_cipher_alg);
        if (err)
                return err;

        err = crypto_register_skciphers(camellia_skcipher_algs,
                                        ARRAY_SIZE(camellia_skcipher_algs));
        if (err)
                crypto_unregister_alg(&camellia_cipher_alg);

        return err;
}

static void __exit fini(void)
{
        crypto_unregister_alg(&camellia_cipher_alg);
        crypto_unregister_skciphers(camellia_skcipher_algs,
                                    ARRAY_SIZE(camellia_skcipher_algs));
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-asm");

/*
 * Glue Code for the AVX assembler implementation of the Cast5 Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <asm/crypto/glue_helper.h>
#include <crypto/algapi.h>
#include <crypto/cast5.h>
#include <crypto/internal/simd.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/types.h>

#define CAST5_PARALLEL_BLOCKS 16

asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst,
                                    const u8 *src);
asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst,
                                    const u8 *src);
asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst,
                                    const u8 *src);
asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src,
                                __be64 *iv);

static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
                                 unsigned int keylen)
{
        return cast5_setkey(&tfm->base, key, keylen); /*covered*/
}

static inline bool cast5_fpu_begin(bool fpu_enabled, struct skcipher_walk *walk, /*covered*/
                                   unsigned int nbytes)
{
        return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS, /*covered*/
                              walk, fpu_enabled, nbytes);
}

static inline void cast5_fpu_end(bool fpu_enabled)
{
        return glue_fpu_end(fpu_enabled); /*covered*/
}

static int ecb_crypt(struct skcipher_request *req, bool enc)
{
        bool fpu_enabled = false;
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        unsigned int nbytes;
        void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) { /*covered*/
                u8 *wsrc = walk.src.virt.addr; /*covered*/
                u8 *wdst = walk.dst.virt.addr;

                fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); /*covered*/

                /* Process multi-block batch */
                if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { /*covered*/
                        fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way;
                        do {
                                fn(ctx, wdst, wsrc); /*covered*/

                                wsrc += bsize * CAST5_PARALLEL_BLOCKS;
                                wdst += bsize * CAST5_PARALLEL_BLOCKS;
                                nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
                        } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }

                fn = (enc) ? __cast5_encrypt : __cast5_decrypt;

                /* Handle leftovers */
                do {
                        fn(ctx, wdst, wsrc); /*covered*/

                        wsrc += bsize;
                        wdst += bsize;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

done:
                err = skcipher_walk_done(&walk, nbytes); /*covered*/
        }

        cast5_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}

static int ecb_encrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, true); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, false); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                u64 *src = (u64 *)walk.src.virt.addr; /*covered*/
                u64 *dst = (u64 *)walk.dst.virt.addr;
                u64 *iv = (u64 *)walk.iv;

                do {
                        *dst = *src ^ *iv; /*covered*/
                        __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
                        iv = dst;
                        src++;
                        dst++;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

                *(u64 *)walk.iv = *iv; /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static unsigned int __cbc_decrypt(struct cast5_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes; /*covered*/
        u64 *src = (u64 *)walk->src.virt.addr;
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 last_iv;

        /* Start of the last block. */
        src += nbytes / bsize - 1;
        dst += nbytes / bsize - 1;

        last_iv = *src;

        /* Process multi-block batch */
        if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
                do {
                        nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
                        src -= CAST5_PARALLEL_BLOCKS - 1; /*covered*/
                        dst -= CAST5_PARALLEL_BLOCKS - 1;

                        cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src);

                        nbytes -= bsize;
                        if (nbytes < bsize)
                                goto done;

                        *dst ^= *(src - 1); /*covered*/
                        src -= 1;
                        dst -= 1;
                } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
        }

        /* Handle leftovers */
        for (;;) {
                __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src); /*covered*/

                nbytes -= bsize;
                if (nbytes < bsize)
                        break;

                *dst ^= *(src - 1); /*covered*/
                src -= 1;
                dst -= 1;
        }

done:
        *dst ^= *(u64 *)walk->iv; /*covered*/
        *(u64 *)walk->iv = last_iv;

        return nbytes;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        bool fpu_enabled = false;
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); /*covered*/
                nbytes = __cbc_decrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        cast5_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}

static void ctr_crypt_final(struct skcipher_walk *walk, struct cast5_ctx *ctx)
{
        u8 *ctrblk = walk->iv; /*covered*/
        u8 keystream[CAST5_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        __cast5_encrypt(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
}

static unsigned int __ctr_crypt(struct skcipher_walk *walk,
                                struct cast5_ctx *ctx)
{
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes; /*covered*/
        u64 *src = (u64 *)walk->src.virt.addr;
        u64 *dst = (u64 *)walk->dst.virt.addr;

        /* Process multi-block batch */
        if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
                do {
                        cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src,
                                        (__be64 *)walk->iv); /*covered*/

                        src += CAST5_PARALLEL_BLOCKS;
                        dst += CAST5_PARALLEL_BLOCKS;
                        nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
                } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);

                if (nbytes < bsize) /*covered*/
                        goto done;
        }

        /* Handle leftovers */
        do {
                u64 ctrblk;

                if (dst != src) /*covered*/
                        *dst = *src; /*covered*/

                ctrblk = *(u64 *)walk->iv; /*covered*/
                be64_add_cpu((__be64 *)walk->iv, 1);

                __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
                *dst ^= ctrblk;

                src += 1;
                dst += 1;
                nbytes -= bsize;
        } while (nbytes >= bsize);

done:
        return nbytes;
}

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        bool fpu_enabled = false;
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
                fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); /*covered*/
                nbytes = __ctr_crypt(&walk, ctx); /*covered*/
                err = skcipher_walk_done(&walk, nbytes); /*covered*/
        }

        cast5_fpu_end(fpu_enabled); /*covered*/

        if (walk.nbytes) { /*covered*/
                ctr_crypt_final(&walk, ctx); /*covered*/
                err = skcipher_walk_done(&walk, 0);
        }

        return err; /*covered*/
}

static struct skcipher_alg cast5_algs[] = {
        {
                .base.cra_name                = "__ecb(cast5)",
                .base.cra_driver_name        = "__ecb-cast5-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST5_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast5_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST5_MIN_KEY_SIZE,
                .max_keysize                = CAST5_MAX_KEY_SIZE,
                .setkey                        = cast5_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(cast5)",
                .base.cra_driver_name        = "__cbc-cast5-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST5_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast5_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST5_MIN_KEY_SIZE,
                .max_keysize                = CAST5_MAX_KEY_SIZE,
                .ivsize                        = CAST5_BLOCK_SIZE,
                .setkey                        = cast5_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(cast5)",
                .base.cra_driver_name        = "__ctr-cast5-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct cast5_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST5_MIN_KEY_SIZE,
                .max_keysize                = CAST5_MAX_KEY_SIZE,
                .ivsize                        = CAST5_BLOCK_SIZE,
                .chunksize                = CAST5_BLOCK_SIZE,
                .setkey                        = cast5_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }
};

static struct simd_skcipher_alg *cast5_simd_algs[ARRAY_SIZE(cast5_algs)];

static int __init cast5_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(cast5_algs,
                                              ARRAY_SIZE(cast5_algs),
                                              cast5_simd_algs);
}

static void __exit cast5_exit(void)
{
        simd_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs),
                                  cast5_simd_algs);
}

module_init(cast5_init);
module_exit(cast5_exit);

MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("cast5");

/*
 * Glue Code for the AVX assembler implementation of the Cast6 Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/cast6.h>
#include <crypto/internal/simd.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>

#define CAST6_PARALLEL_BLOCKS 8

asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src);
asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src);

asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src);
asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
                               le128 *iv);

asmlinkage void cast6_xts_enc_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);
asmlinkage void cast6_xts_dec_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);

static int cast6_setkey_skcipher(struct crypto_skcipher *tfm,
                                 const u8 *key, unsigned int keylen)
{
        return cast6_setkey(&tfm->base, key, keylen); /*covered*/
}

static void cast6_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(__cast6_encrypt));
}

static void cast6_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, /*covered*/
                                  GLUE_FUNC_CAST(__cast6_decrypt));
}

static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk;

        le128_to_be128(&ctrblk, iv); /*covered*/
        le128_inc(iv); /*covered*/

        __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
}

static const struct common_glue_ctx cast6_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
        } }
};

static const struct common_glue_ctx cast6_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
        } }
};

static const struct common_glue_ctx cast6_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
        } }
};

static const struct common_glue_ctx cast6_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
        } }
};

static const struct common_glue_ctx cast6_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
        } }
};

static const struct common_glue_ctx cast6_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&cast6_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&cast6_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__cast6_encrypt), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&cast6_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&cast6_ctr, req); /*covered*/
}

struct cast6_xts_ctx {
        struct cast6_ctx tweak_ctx;
        struct cast6_ctx crypt_ctx;
};

static int xts_cast6_setkey(struct crypto_skcipher *tfm, const u8 *key,
                            unsigned int keylen)
{
        struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        u32 *flags = &tfm->base.crt_flags; /*covered*/
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        /* first half of xts-key is for crypt */
        err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); /*covered*/
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, /*covered*/
                              flags);
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&cast6_enc_xts, req,
                                   XTS_TWEAK_CAST(__cast6_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&cast6_dec_xts, req,
                                   XTS_TWEAK_CAST(__cast6_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg cast6_algs[] = {
        {
                .base.cra_name                = "__ecb(cast6)",
                .base.cra_driver_name        = "__ecb-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST6_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast6_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST6_MIN_KEY_SIZE,
                .max_keysize                = CAST6_MAX_KEY_SIZE,
                .setkey                        = cast6_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(cast6)",
                .base.cra_driver_name        = "__cbc-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST6_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast6_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST6_MIN_KEY_SIZE,
                .max_keysize                = CAST6_MAX_KEY_SIZE,
                .ivsize                        = CAST6_BLOCK_SIZE,
                .setkey                        = cast6_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(cast6)",
                .base.cra_driver_name        = "__ctr-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct cast6_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST6_MIN_KEY_SIZE,
                .max_keysize                = CAST6_MAX_KEY_SIZE,
                .ivsize                        = CAST6_BLOCK_SIZE,
                .chunksize                = CAST6_BLOCK_SIZE,
                .setkey                        = cast6_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(cast6)",
                .base.cra_driver_name        = "__xts-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST6_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast6_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * CAST6_MIN_KEY_SIZE,
                .max_keysize                = 2 * CAST6_MAX_KEY_SIZE,
                .ivsize                        = CAST6_BLOCK_SIZE,
                .setkey                        = xts_cast6_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *cast6_simd_algs[ARRAY_SIZE(cast6_algs)];

static int __init cast6_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(cast6_algs,
                                              ARRAY_SIZE(cast6_algs),
                                              cast6_simd_algs);
}

static void __exit cast6_exit(void)
{
        simd_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs),
                                  cast6_simd_algs);
}

module_init(cast6_init);
module_exit(cast6_exit);

MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("cast6");

/*
 * x64 SIMD accelerated ChaCha and XChaCha stream ciphers,
 * including ChaCha20 (RFC7539)
 *
 * Copyright (C) 2015 Martin Willi
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <crypto/algapi.h>
#include <crypto/chacha.h>
#include <crypto/internal/skcipher.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/simd.h>

#define CHACHA_STATE_ALIGN 16

asmlinkage void chacha_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
                                       unsigned int len, int nrounds);
asmlinkage void chacha_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src,
                                        unsigned int len, int nrounds);
asmlinkage void hchacha_block_ssse3(const u32 *state, u32 *out, int nrounds);
#ifdef CONFIG_AS_AVX2
asmlinkage void chacha_2block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
                                       unsigned int len, int nrounds);
asmlinkage void chacha_4block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
                                       unsigned int len, int nrounds);
asmlinkage void chacha_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src,
                                       unsigned int len, int nrounds);
static bool chacha_use_avx2;
#ifdef CONFIG_AS_AVX512
asmlinkage void chacha_2block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
                                           unsigned int len, int nrounds);
asmlinkage void chacha_4block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
                                           unsigned int len, int nrounds);
asmlinkage void chacha_8block_xor_avx512vl(u32 *state, u8 *dst, const u8 *src,
                                           unsigned int len, int nrounds);
static bool chacha_use_avx512vl;
#endif
#endif

static unsigned int chacha_advance(unsigned int len, unsigned int maxblocks)
{
        len = min(len, maxblocks * CHACHA_BLOCK_SIZE);
        return round_up(len, CHACHA_BLOCK_SIZE) / CHACHA_BLOCK_SIZE;
}

static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src,
                          unsigned int bytes, int nrounds)
{
#ifdef CONFIG_AS_AVX2
#ifdef CONFIG_AS_AVX512
        if (chacha_use_avx512vl) {
                while (bytes >= CHACHA_BLOCK_SIZE * 8) {
                        chacha_8block_xor_avx512vl(state, dst, src, bytes,
                                                   nrounds);
                        bytes -= CHACHA_BLOCK_SIZE * 8;
                        src += CHACHA_BLOCK_SIZE * 8;
                        dst += CHACHA_BLOCK_SIZE * 8;
                        state[12] += 8;
                }
                if (bytes > CHACHA_BLOCK_SIZE * 4) {
                        chacha_8block_xor_avx512vl(state, dst, src, bytes,
                                                   nrounds);
                        state[12] += chacha_advance(bytes, 8);
                        return;
                }
                if (bytes > CHACHA_BLOCK_SIZE * 2) {
                        chacha_4block_xor_avx512vl(state, dst, src, bytes,
                                                   nrounds);
                        state[12] += chacha_advance(bytes, 4);
                        return;
                }
                if (bytes) {
                        chacha_2block_xor_avx512vl(state, dst, src, bytes,
                                                   nrounds);
                        state[12] += chacha_advance(bytes, 2);
                        return;
                }
        }
#endif
        if (chacha_use_avx2) { /*covered*/
                while (bytes >= CHACHA_BLOCK_SIZE * 8) { /*covered*/
                        chacha_8block_xor_avx2(state, dst, src, bytes, nrounds); /*covered*/
                        bytes -= CHACHA_BLOCK_SIZE * 8;
                        src += CHACHA_BLOCK_SIZE * 8;
                        dst += CHACHA_BLOCK_SIZE * 8;
                        state[12] += 8;
                }
                if (bytes > CHACHA_BLOCK_SIZE * 4) { /*covered*/
                        chacha_8block_xor_avx2(state, dst, src, bytes, nrounds); /*covered*/
                        state[12] += chacha_advance(bytes, 8);
                        return;
                }
                if (bytes > CHACHA_BLOCK_SIZE * 2) { /*covered*/
                        chacha_4block_xor_avx2(state, dst, src, bytes, nrounds); /*covered*/
                        state[12] += chacha_advance(bytes, 4);
                        return;
                }
                if (bytes > CHACHA_BLOCK_SIZE) { /*covered*/
                        chacha_2block_xor_avx2(state, dst, src, bytes, nrounds); /*covered*/
                        state[12] += chacha_advance(bytes, 2);
                        return;
                }
        }
#endif
        while (bytes >= CHACHA_BLOCK_SIZE * 4) {
                chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
                bytes -= CHACHA_BLOCK_SIZE * 4;
                src += CHACHA_BLOCK_SIZE * 4;
                dst += CHACHA_BLOCK_SIZE * 4;
                state[12] += 4;
        }
        if (bytes > CHACHA_BLOCK_SIZE) { /*covered*/
                chacha_4block_xor_ssse3(state, dst, src, bytes, nrounds);
                state[12] += chacha_advance(bytes, 4);
                return;
        }
        if (bytes) { /*covered*/
                chacha_block_xor_ssse3(state, dst, src, bytes, nrounds); /*covered*/
                state[12]++;
        }
}

static int chacha_simd_stream_xor(struct skcipher_walk *walk,
                                  struct chacha_ctx *ctx, u8 *iv)
{
        u32 *state, state_buf[16 + 2] __aligned(8);
        int next_yield = 4096; /* bytes until next FPU yield */
        int err = 0;

        BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
        state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN); /*covered*/

        crypto_chacha_init(state, ctx, iv);

        while (walk->nbytes > 0) {
                unsigned int nbytes = walk->nbytes;

                if (nbytes < walk->total) { /*covered*/
                        nbytes = round_down(nbytes, walk->stride); /*covered*/
                        next_yield -= nbytes;
                }

                chacha_dosimd(state, walk->dst.virt.addr, walk->src.virt.addr, /*covered*/
                              nbytes, ctx->nrounds);

                if (next_yield <= 0) { /*covered*/
                        /* temporarily allow preemption */
                        kernel_fpu_end(); /*covered*/
                        kernel_fpu_begin();
                        next_yield = 4096;
                }

                err = skcipher_walk_done(walk, walk->nbytes - nbytes); /*covered*/
        }

        return err; /*covered*/
}

static int chacha_simd(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        int err;

        if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable()) /*covered*/
                return crypto_chacha_crypt(req); /*covered*/

        err = skcipher_walk_virt(&walk, req, true); /*covered*/
        if (err)
                return err;

        kernel_fpu_begin(); /*covered*/
        err = chacha_simd_stream_xor(&walk, ctx, req->iv);
        kernel_fpu_end();
        return err;
}

static int xchacha_simd(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        struct chacha_ctx subctx;
        u32 *state, state_buf[16 + 2] __aligned(8);
        u8 real_iv[16];
        int err;

        if (req->cryptlen <= CHACHA_BLOCK_SIZE || !irq_fpu_usable()) /*covered*/
                return crypto_xchacha_crypt(req); /*covered*/

        err = skcipher_walk_virt(&walk, req, true); /*covered*/
        if (err)
                return err;

        BUILD_BUG_ON(CHACHA_STATE_ALIGN != 16);
        state = PTR_ALIGN(state_buf + 0, CHACHA_STATE_ALIGN); /*covered*/
        crypto_chacha_init(state, ctx, req->iv);

        kernel_fpu_begin();

        hchacha_block_ssse3(state, subctx.key, ctx->nrounds);
        subctx.nrounds = ctx->nrounds;

        memcpy(&real_iv[0], req->iv + 24, 8);
        memcpy(&real_iv[8], req->iv + 16, 8);
        err = chacha_simd_stream_xor(&walk, &subctx, real_iv);

        kernel_fpu_end();

        return err;
}

static struct skcipher_alg algs[] = {
        {
                .base.cra_name                = "chacha20",
                .base.cra_driver_name        = "chacha20-simd",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct chacha_ctx),
                .base.cra_module        = THIS_MODULE,

                .min_keysize                = CHACHA_KEY_SIZE,
                .max_keysize                = CHACHA_KEY_SIZE,
                .ivsize                        = CHACHA_IV_SIZE,
                .chunksize                = CHACHA_BLOCK_SIZE,
                .setkey                        = crypto_chacha20_setkey,
                .encrypt                = chacha_simd,
                .decrypt                = chacha_simd,
        }, {
                .base.cra_name                = "xchacha20",
                .base.cra_driver_name        = "xchacha20-simd",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct chacha_ctx),
                .base.cra_module        = THIS_MODULE,

                .min_keysize                = CHACHA_KEY_SIZE,
                .max_keysize                = CHACHA_KEY_SIZE,
                .ivsize                        = XCHACHA_IV_SIZE,
                .chunksize                = CHACHA_BLOCK_SIZE,
                .setkey                        = crypto_chacha20_setkey,
                .encrypt                = xchacha_simd,
                .decrypt                = xchacha_simd,
        }, {
                .base.cra_name                = "xchacha12",
                .base.cra_driver_name        = "xchacha12-simd",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct chacha_ctx),
                .base.cra_module        = THIS_MODULE,

                .min_keysize                = CHACHA_KEY_SIZE,
                .max_keysize                = CHACHA_KEY_SIZE,
                .ivsize                        = XCHACHA_IV_SIZE,
                .chunksize                = CHACHA_BLOCK_SIZE,
                .setkey                        = crypto_chacha12_setkey,
                .encrypt                = xchacha_simd,
                .decrypt                = xchacha_simd,
        },
};

static int __init chacha_simd_mod_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_SSSE3))
                return -ENODEV;

#ifdef CONFIG_AS_AVX2
        chacha_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) &&
                          boot_cpu_has(X86_FEATURE_AVX2) &&
                          cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
#ifdef CONFIG_AS_AVX512
        chacha_use_avx512vl = chacha_use_avx2 &&
                              boot_cpu_has(X86_FEATURE_AVX512VL) &&
                              boot_cpu_has(X86_FEATURE_AVX512BW); /* kmovq */
#endif
#endif
        return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
}

static void __exit chacha_simd_mod_fini(void)
{
        crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
}

module_init(chacha_simd_mod_init);
module_exit(chacha_simd_mod_fini);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (x64 SIMD accelerated)");
MODULE_ALIAS_CRYPTO("chacha20");
MODULE_ALIAS_CRYPTO("chacha20-simd");
MODULE_ALIAS_CRYPTO("xchacha20");
MODULE_ALIAS_CRYPTO("xchacha20-simd");
MODULE_ALIAS_CRYPTO("xchacha12");
MODULE_ALIAS_CRYPTO("xchacha12-simd");

/* GPL HEADER START
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 only,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License version 2 for more details (a copy is included
 * in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; If not, see http://www.gnu.org/licenses
 *
 * Please  visit http://www.xyratex.com/contact if you need additional
 * information or have any questions.
 *
 * GPL HEADER END
 */

/*
 * Copyright 2012 Xyratex Technology Limited
 *
 * Wrappers for kernel crypto shash api to pclmulqdq crc32 imlementation.
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/crc32.h>
#include <crypto/internal/hash.h>

#include <asm/cpufeatures.h>
#include <asm/cpu_device_id.h>
#include <asm/fpu/api.h>

#define CHKSUM_BLOCK_SIZE        1
#define CHKSUM_DIGEST_SIZE        4

#define PCLMUL_MIN_LEN                64L     /* minimum size of buffer
                                         * for crc32_pclmul_le_16 */
#define SCALE_F                        16L        /* size of xmm register */
#define SCALE_F_MASK                (SCALE_F - 1)

u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);

static u32 __attribute__((pure))
        crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len) /*covered*/
{
        unsigned int iquotient;
        unsigned int iremainder;
        unsigned int prealign;

        if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !irq_fpu_usable()) /*covered*/
                return crc32_le(crc, p, len); /*covered*/

        if ((long)p & SCALE_F_MASK) { /*covered*/
                /* align p to 16 byte */
                prealign = SCALE_F - ((long)p & SCALE_F_MASK); /*covered*/

                crc = crc32_le(crc, p, prealign);
                len -= prealign;
                p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
                                     ~SCALE_F_MASK);
        }
        iquotient = len & (~SCALE_F_MASK); /*covered*/
        iremainder = len & SCALE_F_MASK;

        kernel_fpu_begin();
        crc = crc32_pclmul_le_16(p, iquotient, crc);
        kernel_fpu_end();

        if (iremainder)
                crc = crc32_le(crc, p + iquotient, iremainder); /*covered*/

        return crc;
}

static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
{
        u32 *key = crypto_tfm_ctx(tfm);

        *key = 0; /*covered*/

        return 0;
}

static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
                        unsigned int keylen)
{
        u32 *mctx = crypto_shash_ctx(hash);

        if (keylen != sizeof(u32)) { /*covered*/
                crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }
        *mctx = le32_to_cpup((__le32 *)key); /*covered*/
        return 0; /*covered*/
}

static int crc32_pclmul_init(struct shash_desc *desc)
{
        u32 *mctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = *mctx;

        return 0;
}

static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
                               unsigned int len)
{
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = crc32_pclmul_le(*crcp, data, len); /*covered*/
        return 0;
}

/* No final XOR 0xFFFFFFFF, like crc32_le */
static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
                                u8 *out)
{
        *(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
        return 0;
}

static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out); /*covered*/
}

static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
{
        u32 *crcp = shash_desc_ctx(desc);

        *(__le32 *)out = cpu_to_le32p(crcp); /*covered*/
        return 0;
}

static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
                               unsigned int len, u8 *out)
{
        return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
                                    out);
}

static struct shash_alg alg = {
        .setkey                = crc32_pclmul_setkey,
        .init                = crc32_pclmul_init,
        .update                = crc32_pclmul_update,
        .final                = crc32_pclmul_final,
        .finup                = crc32_pclmul_finup,
        .digest                = crc32_pclmul_digest,
        .descsize        = sizeof(u32),
        .digestsize        = CHKSUM_DIGEST_SIZE,
        .base                = {
                        .cra_name                = "crc32",
                        .cra_driver_name        = "crc32-pclmul",
                        .cra_priority                = 200,
                        .cra_flags                = CRYPTO_ALG_OPTIONAL_KEY,
                        .cra_blocksize                = CHKSUM_BLOCK_SIZE,
                        .cra_ctxsize                = sizeof(u32),
                        .cra_module                = THIS_MODULE,
                        .cra_init                = crc32_pclmul_cra_init,
        }
};

static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);


static int __init crc32_pclmul_mod_init(void)
{

        if (!x86_match_cpu(crc32pclmul_cpu_id)) {
                pr_info("PCLMULQDQ-NI instructions are not detected.\n");
                return -ENODEV;
        }
        return crypto_register_shash(&alg);
}

static void __exit crc32_pclmul_mod_fini(void)
{
        crypto_unregister_shash(&alg);
}

module_init(crc32_pclmul_mod_init);
module_exit(crc32_pclmul_mod_fini);

MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
MODULE_LICENSE("GPL");

MODULE_ALIAS_CRYPTO("crc32");
MODULE_ALIAS_CRYPTO("crc32-pclmul");

/*
 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
 * http://www.intel.com/products/processor/manuals/
 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
 * Volume 2A: Instruction Set Reference, A-M
 *
 * Copyright (C) 2008 Intel Corporation
 * Authors: Austin Zhang <austin_zhang@linux.intel.com>
 *          Kent Liu <kent.liu@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <crypto/internal/hash.h>

#include <asm/cpufeatures.h>
#include <asm/cpu_device_id.h>
#include <asm/fpu/internal.h>

#define CHKSUM_BLOCK_SIZE        1
#define CHKSUM_DIGEST_SIZE        4

#define SCALE_F        sizeof(unsigned long)

#ifdef CONFIG_X86_64
#define REX_PRE "0x48, "
#else
#define REX_PRE
#endif

#ifdef CONFIG_X86_64
/*
 * use carryless multiply version of crc32c when buffer
 * size is >= 512 to account
 * for fpu state save/restore overhead.
 */
#define CRC32C_PCL_BREAKEVEN        512

asmlinkage unsigned int crc_pcl(const u8 *buffer, int len,
                                unsigned int crc_init);
#endif /* CONFIG_X86_64 */

static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
{
        while (length--) {
                __asm__ __volatile__(
                        ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
                        :"=S"(crc)
                        :"0"(crc), "c"(*data) /*covered*/
                );
                data++;
        }

        return crc;
}

static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len)
{
        unsigned int iquotient = len / SCALE_F;
        unsigned int iremainder = len % SCALE_F;
        unsigned long *ptmp = (unsigned long *)p;

        while (iquotient--) {
                __asm__ __volatile__(
                        ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
                        :"=S"(crc)
                        :"0"(crc), "c"(*ptmp) /*covered*/
                );
                ptmp++;
        }

        if (iremainder) /*covered*/
                crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp, /*covered*/
                                 iremainder);

        return crc;
}

/*
 * Setting the seed allows arbitrary accumulators and flexible XOR policy
 * If your algorithm starts with ~0, then XOR with ~0 before you set
 * the seed.
 */
static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key,
                        unsigned int keylen)
{
        u32 *mctx = crypto_shash_ctx(hash);

        if (keylen != sizeof(u32)) { /*covered*/
                crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }
        *mctx = le32_to_cpup((__le32 *)key); /*covered*/
        return 0; /*covered*/
}

static int crc32c_intel_init(struct shash_desc *desc)
{
        u32 *mctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = *mctx;

        return 0;
}

static int crc32c_intel_update(struct shash_desc *desc, const u8 *data,
                               unsigned int len)
{
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = crc32c_intel_le_hw(*crcp, data, len);
        return 0;
}

static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
                                u8 *out)
{
        *(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
        return 0;
}

static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out);
}

static int crc32c_intel_final(struct shash_desc *desc, u8 *out)
{
        u32 *crcp = shash_desc_ctx(desc);

        *(__le32 *)out = ~cpu_to_le32p(crcp); /*covered*/
        return 0;
}

static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data,
                               unsigned int len, u8 *out)
{
        return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
                                    out);
}

static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
{
        u32 *key = crypto_tfm_ctx(tfm);

        *key = ~0; /*covered*/

        return 0;
}

#ifdef CONFIG_X86_64
static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data, /*covered*/
                               unsigned int len)
{
        u32 *crcp = shash_desc_ctx(desc);

        /*
         * use faster PCL version if datasize is large enough to
         * overcome kernel fpu state save/restore overhead
         */
        if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                *crcp = crc_pcl(data, len, *crcp);
                kernel_fpu_end();
        } else
                *crcp = crc32c_intel_le_hw(*crcp, data, len); /*covered*/
        return 0; /*covered*/
}

static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len, /*covered*/
                                u8 *out)
{
        if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp));
                kernel_fpu_end();
        } else
                *(__le32 *)out =
                        ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); /*covered*/
        return 0; /*covered*/
}

static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out); /*covered*/
}

static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data,
                               unsigned int len, u8 *out)
{
        return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
                                    out);
}
#endif /* CONFIG_X86_64 */

static struct shash_alg alg = {
        .setkey                        =        crc32c_intel_setkey,
        .init                        =        crc32c_intel_init,
        .update                        =        crc32c_intel_update,
        .final                        =        crc32c_intel_final,
        .finup                        =        crc32c_intel_finup,
        .digest                        =        crc32c_intel_digest,
        .descsize                =        sizeof(u32),
        .digestsize                =        CHKSUM_DIGEST_SIZE,
        .base                        =        {
                .cra_name                =        "crc32c",
                .cra_driver_name        =        "crc32c-intel",
                .cra_priority                =        200,
                .cra_flags                =        CRYPTO_ALG_OPTIONAL_KEY,
                .cra_blocksize                =        CHKSUM_BLOCK_SIZE,
                .cra_ctxsize                =        sizeof(u32),
                .cra_module                =        THIS_MODULE,
                .cra_init                =        crc32c_intel_cra_init,
        }
};

static const struct x86_cpu_id crc32c_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_XMM4_2),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id);

static int __init crc32c_intel_mod_init(void)
{
        if (!x86_match_cpu(crc32c_cpu_id))
                return -ENODEV;
#ifdef CONFIG_X86_64
        if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
                alg.update = crc32c_pcl_intel_update;
                alg.finup = crc32c_pcl_intel_finup;
                alg.digest = crc32c_pcl_intel_digest;
        }
#endif
        return crypto_register_shash(&alg);
}

static void __exit crc32c_intel_mod_fini(void)
{
        crypto_unregister_shash(&alg);
}

module_init(crc32c_intel_mod_init);
module_exit(crc32c_intel_mod_fini);

MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
MODULE_LICENSE("GPL");

MODULE_ALIAS_CRYPTO("crc32c");
MODULE_ALIAS_CRYPTO("crc32c-intel");

/*
 * Cryptographic API.
 *
 * T10 Data Integrity Field CRC16 Crypto Transform using PCLMULQDQ Instructions
 *
 * Copyright (C) 2013 Intel Corporation
 * Author: Tim Chen <tim.c.chen@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-t10dif.h>
#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <asm/fpu/api.h>
#include <asm/cpufeatures.h>
#include <asm/cpu_device_id.h>

asmlinkage u16 crc_t10dif_pcl(u16 init_crc, const u8 *buf, size_t len);

struct chksum_desc_ctx {
        __u16 crc;
};

static int chksum_init(struct shash_desc *desc)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        ctx->crc = 0; /*covered*/

        return 0;
}

static int chksum_update(struct shash_desc *desc, const u8 *data, /*covered*/
                         unsigned int length)
{ /*covered*/
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        if (length >= 16 && irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
                kernel_fpu_end();
        } else
                ctx->crc = crc_t10dif_generic(ctx->crc, data, length); /*covered*/
        return 0; /*covered*/
}

static int chksum_final(struct shash_desc *desc, u8 *out)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        *(__u16 *)out = ctx->crc; /*covered*/
        return 0;
}

static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len, /*covered*/
                        u8 *out)
{ /*covered*/
        if (len >= 16 && irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len);
                kernel_fpu_end();
        } else
                *(__u16 *)out = crc_t10dif_generic(*crcp, data, len); /*covered*/
        return 0; /*covered*/
}

static int chksum_finup(struct shash_desc *desc, const u8 *data,
                        unsigned int len, u8 *out)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        return __chksum_finup(&ctx->crc, data, len, out); /*covered*/
}

static int chksum_digest(struct shash_desc *desc, const u8 *data,
                         unsigned int length, u8 *out)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        return __chksum_finup(&ctx->crc, data, length, out);
}

static struct shash_alg alg = {
        .digestsize                =        CRC_T10DIF_DIGEST_SIZE,
        .init                =        chksum_init,
        .update                =        chksum_update,
        .final                =        chksum_final,
        .finup                =        chksum_finup,
        .digest                =        chksum_digest,
        .descsize                =        sizeof(struct chksum_desc_ctx),
        .base                        =        {
                .cra_name                =        "crct10dif",
                .cra_driver_name        =        "crct10dif-pclmul",
                .cra_priority                =        200,
                .cra_blocksize                =        CRC_T10DIF_BLOCK_SIZE,
                .cra_module                =        THIS_MODULE,
        }
};

static const struct x86_cpu_id crct10dif_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id);

static int __init crct10dif_intel_mod_init(void)
{
        if (!x86_match_cpu(crct10dif_cpu_id))
                return -ENODEV;

        return crypto_register_shash(&alg);
}

static void __exit crct10dif_intel_mod_fini(void)
{
        crypto_unregister_shash(&alg);
}

module_init(crct10dif_intel_mod_init);
module_exit(crct10dif_intel_mod_fini);

MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with PCLMULQDQ.");
MODULE_LICENSE("GPL");

MODULE_ALIAS_CRYPTO("crct10dif");
MODULE_ALIAS_CRYPTO("crct10dif-pclmul");

/*
 * Glue Code for assembler optimized version of 3DES
 *
 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 */

#include <crypto/algapi.h>
#include <crypto/des.h>
#include <crypto/internal/skcipher.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

struct des3_ede_x86_ctx {
        u32 enc_expkey[DES3_EDE_EXPKEY_WORDS];
        u32 dec_expkey[DES3_EDE_EXPKEY_WORDS];
};

/* regular block cipher functions */
asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst,
                                          const u8 *src);

/* 3-way parallel cipher functions */
asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst,
                                               const u8 *src);

static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                    const u8 *src)
{
        u32 *enc_ctx = ctx->enc_expkey; /*covered*/

        des3_ede_x86_64_crypt_blk(enc_ctx, dst, src);
}

static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                    const u8 *src)
{
        u32 *dec_ctx = ctx->dec_expkey;

        des3_ede_x86_64_crypt_blk(dec_ctx, dst, src); /*covered*/
}

static inline void des3_ede_enc_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        u32 *enc_ctx = ctx->enc_expkey;

        des3_ede_x86_64_crypt_blk_3way(enc_ctx, dst, src);
}

static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        u32 *dec_ctx = ctx->dec_expkey;

        des3_ede_x86_64_crypt_blk_3way(dec_ctx, dst, src);
}

static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        des3_ede_enc_blk(crypto_tfm_ctx(tfm), dst, src); /*covered*/
}

static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        des3_ede_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static int ecb_crypt(struct skcipher_request *req, const u32 *expkey)
{
        const unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false); /*covered*/

        while ((nbytes = walk.nbytes)) {
                u8 *wsrc = walk.src.virt.addr; /*covered*/
                u8 *wdst = walk.dst.virt.addr;

                /* Process four block batch */
                if (nbytes >= bsize * 3) {
                        do {
                                des3_ede_x86_64_crypt_blk_3way(expkey, wdst, /*covered*/
                                                               wsrc);

                                wsrc += bsize * 3;
                                wdst += bsize * 3;
                                nbytes -= bsize * 3;
                        } while (nbytes >= bsize * 3);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }

                /* Handle leftovers */
                do {
                        des3_ede_x86_64_crypt_blk(expkey, wdst, wsrc); /*covered*/

                        wsrc += bsize;
                        wdst += bsize;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

done:
                err = skcipher_walk_done(&walk, nbytes); /*covered*/
        }

        return err; /*covered*/
}

static int ecb_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);

        return ecb_crypt(req, ctx->enc_expkey);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);

        return ecb_crypt(req, ctx->dec_expkey);
}

static unsigned int __cbc_encrypt(struct des3_ede_x86_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 *iv = (u64 *)walk->iv;

        do {
                *dst = *src ^ *iv; /*covered*/
                des3_ede_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
                iv = dst;

                src += 1;
                dst += 1;
                nbytes -= bsize;
        } while (nbytes >= bsize);

        *(u64 *)walk->iv = *iv; /*covered*/
        return nbytes;
}

static int cbc_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) { /*covered*/
                nbytes = __cbc_encrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static unsigned int __cbc_decrypt(struct des3_ede_x86_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 ivs[3 - 1];
        u64 last_iv;

        /* Start of the last block. */
        src += nbytes / bsize - 1;
        dst += nbytes / bsize - 1;

        last_iv = *src;

        /* Process four block batch */
        if (nbytes >= bsize * 3) {
                do {
                        nbytes -= bsize * 3 - bsize;
                        src -= 3 - 1; /*covered*/
                        dst -= 3 - 1;

                        ivs[0] = src[0];
                        ivs[1] = src[1];

                        des3_ede_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);

                        dst[1] ^= ivs[0];
                        dst[2] ^= ivs[1];

                        nbytes -= bsize;
                        if (nbytes < bsize)
                                goto done;

                        *dst ^= *(src - 1); /*covered*/
                        src -= 1;
                        dst -= 1;
                } while (nbytes >= bsize * 3);
        }

        /* Handle leftovers */
        for (;;) {
                des3_ede_dec_blk(ctx, (u8 *)dst, (u8 *)src); /*covered*/

                nbytes -= bsize;
                if (nbytes < bsize)
                        break;

                *dst ^= *(src - 1); /*covered*/
                src -= 1;
                dst -= 1;
        }

done:
        *dst ^= *(u64 *)walk->iv; /*covered*/
        *(u64 *)walk->iv = last_iv;

        return nbytes;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) { /*covered*/
                nbytes = __cbc_decrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static void ctr_crypt_final(struct des3_ede_x86_ctx *ctx,
                            struct skcipher_walk *walk)
{
        u8 *ctrblk = walk->iv; /*covered*/
        u8 keystream[DES3_EDE_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        des3_ede_enc_blk(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE);
}

static unsigned int __ctr_crypt(struct des3_ede_x86_ctx *ctx,
                                struct skcipher_walk *walk)
{
        unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        __be64 *src = (__be64 *)walk->src.virt.addr; /*covered*/
        __be64 *dst = (__be64 *)walk->dst.virt.addr;
        u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
        __be64 ctrblocks[3];

        /* Process four block batch */
        if (nbytes >= bsize * 3) {
                do {
                        /* create ctrblks for parallel encrypt */
                        ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/
                        ctrblocks[1] = cpu_to_be64(ctrblk++);
                        ctrblocks[2] = cpu_to_be64(ctrblk++);

                        des3_ede_enc_blk_3way(ctx, (u8 *)ctrblocks,
                                              (u8 *)ctrblocks);

                        dst[0] = src[0] ^ ctrblocks[0];
                        dst[1] = src[1] ^ ctrblocks[1];
                        dst[2] = src[2] ^ ctrblocks[2];

                        src += 3;
                        dst += 3;
                } while ((nbytes -= bsize * 3) >= bsize * 3);

                if (nbytes < bsize) /*covered*/
                        goto done;
        }

        /* Handle leftovers */
        do {
                ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/

                des3_ede_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);

                dst[0] = src[0] ^ ctrblocks[0];

                src += 1;
                dst += 1;
        } while ((nbytes -= bsize) >= bsize);

done:
        *(__be64 *)walk->iv = cpu_to_be64(ctrblk); /*covered*/
        return nbytes;
}

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= DES3_EDE_BLOCK_SIZE) { /*covered*/
                nbytes = __ctr_crypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        if (nbytes) { /*covered*/
                ctr_crypt_final(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, 0);
        }

        return err; /*covered*/
}

static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, /*covered*/
                               unsigned int keylen)
{
        struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm);
        u32 i, j, tmp;
        int err;

        /* Generate encryption context using generic implementation. */
        err = __des3_ede_setkey(ctx->enc_expkey, &tfm->crt_flags, key, keylen); /*covered*/
        if (err < 0)
                return err;

        /* Fix encryption context for this implementation and form decryption
         * context. */
        j = DES3_EDE_EXPKEY_WORDS - 2;
        for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) { /*covered*/
                tmp = ror32(ctx->enc_expkey[i + 1], 4); /*covered*/
                ctx->enc_expkey[i + 1] = tmp;

                ctx->dec_expkey[j + 0] = ctx->enc_expkey[i + 0];
                ctx->dec_expkey[j + 1] = tmp;
        }

        return 0;
}

static int des3_ede_x86_setkey_skcipher(struct crypto_skcipher *tfm,
                                        const u8 *key,
                                        unsigned int keylen)
{
        return des3_ede_x86_setkey(&tfm->base, key, keylen); /*covered*/
}

static struct crypto_alg des3_ede_cipher = {
        .cra_name                = "des3_ede",
        .cra_driver_name        = "des3_ede-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = DES3_EDE_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct des3_ede_x86_ctx),
        .cra_alignmask                = 0,
        .cra_module                = THIS_MODULE,
        .cra_u = {
                .cipher = {
                        .cia_min_keysize        = DES3_EDE_KEY_SIZE,
                        .cia_max_keysize        = DES3_EDE_KEY_SIZE,
                        .cia_setkey                = des3_ede_x86_setkey,
                        .cia_encrypt                = des3_ede_x86_encrypt,
                        .cia_decrypt                = des3_ede_x86_decrypt,
                }
        }
};

static struct skcipher_alg des3_ede_skciphers[] = {
        {
                .base.cra_name                = "ecb(des3_ede)",
                .base.cra_driver_name        = "ecb-des3_ede-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = DES3_EDE_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct des3_ede_x86_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = DES3_EDE_KEY_SIZE,
                .max_keysize                = DES3_EDE_KEY_SIZE,
                .setkey                        = des3_ede_x86_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(des3_ede)",
                .base.cra_driver_name        = "cbc-des3_ede-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = DES3_EDE_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct des3_ede_x86_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = DES3_EDE_KEY_SIZE,
                .max_keysize                = DES3_EDE_KEY_SIZE,
                .ivsize                        = DES3_EDE_BLOCK_SIZE,
                .setkey                        = des3_ede_x86_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(des3_ede)",
                .base.cra_driver_name        = "ctr-des3_ede-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct des3_ede_x86_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = DES3_EDE_KEY_SIZE,
                .max_keysize                = DES3_EDE_KEY_SIZE,
                .ivsize                        = DES3_EDE_BLOCK_SIZE,
                .chunksize                = DES3_EDE_BLOCK_SIZE,
                .setkey                        = des3_ede_x86_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, des3_ede-x86_64 is slower than generic C
                 * implementation because use of 64bit rotates (which are really
                 * slow on P4). Therefore blacklist P4s.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init des3_ede_x86_init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&des3_ede_cipher);
        if (err)
                return err;

        err = crypto_register_skciphers(des3_ede_skciphers,
                                        ARRAY_SIZE(des3_ede_skciphers));
        if (err)
                crypto_unregister_alg(&des3_ede_cipher);

        return err;
}

static void __exit des3_ede_x86_fini(void)
{
        crypto_unregister_alg(&des3_ede_cipher);
        crypto_unregister_skciphers(des3_ede_skciphers,
                                    ARRAY_SIZE(des3_ede_skciphers));
}

module_init(des3_ede_x86_init);
module_exit(des3_ede_x86_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("des3_ede");
MODULE_ALIAS_CRYPTO("des3_ede-asm");
MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");

/*
 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
 * instructions. This file contains glue code.
 *
 * Copyright (c) 2009 Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <linux/err.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/cryptd.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/hash.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define GHASH_BLOCK_SIZE        16
#define GHASH_DIGEST_SIZE        16

void clmul_ghash_mul(char *dst, const u128 *shash);

void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
                        const u128 *shash);

struct ghash_async_ctx {
        struct cryptd_ahash *cryptd_tfm;
};

struct ghash_ctx {
        u128 shash;
};

struct ghash_desc_ctx {
        u8 buffer[GHASH_BLOCK_SIZE];
        u32 bytes;
};

static int ghash_init(struct shash_desc *desc)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); /*covered*/

        memset(dctx, 0, sizeof(*dctx));

        return 0;
}

static int ghash_setkey(struct crypto_shash *tfm,
                        const u8 *key, unsigned int keylen)
{
        struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
        be128 *x = (be128 *)key;
        u64 a, b;

        if (keylen != GHASH_BLOCK_SIZE) { /*covered*/
                crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        /* perform multiplication by 'x' in GF(2^128) */
        a = be64_to_cpu(x->a); /*covered*/
        b = be64_to_cpu(x->b);

        ctx->shash.a = (b << 1) | (a >> 63);
        ctx->shash.b = (a << 1) | (b >> 63); /*covered*/

        if (a >> 63)
                ctx->shash.b ^= ((u64)0xc2) << 56; /*covered*/

        return 0; /*covered*/
}

static int ghash_update(struct shash_desc *desc,
                         const u8 *src, unsigned int srclen)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u8 *dst = dctx->buffer;

        kernel_fpu_begin();
        if (dctx->bytes) {
                int n = min(srclen, dctx->bytes); /*covered*/
                u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

                dctx->bytes -= n;
                srclen -= n;

                while (n--)
                        *pos++ ^= *src++; /*covered*/

                if (!dctx->bytes) /*covered*/
                        clmul_ghash_mul(dst, &ctx->shash); /*covered*/
        }

        clmul_ghash_update(dst, src, srclen, &ctx->shash); /*covered*/
        kernel_fpu_end();

        if (srclen & 0xf) {
                src += srclen - (srclen & 0xf); /*covered*/
                srclen &= 0xf;
                dctx->bytes = GHASH_BLOCK_SIZE - srclen;
                while (srclen--)
                        *dst++ ^= *src++; /*covered*/
        }

        return 0; /*covered*/
}

static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
{
        u8 *dst = dctx->buffer;

        if (dctx->bytes) { /*covered*/
                u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

                while (dctx->bytes--) /*covered*/
                        *tmp++ ^= 0;

                kernel_fpu_begin();
                clmul_ghash_mul(dst, &ctx->shash);
                kernel_fpu_end();
        }

        dctx->bytes = 0; /*covered*/
}

static int ghash_final(struct shash_desc *desc, u8 *dst)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u8 *buf = dctx->buffer;

        ghash_flush(ctx, dctx); /*covered*/
        memcpy(dst, buf, GHASH_BLOCK_SIZE);

        return 0;
}

static struct shash_alg ghash_alg = {
        .digestsize        = GHASH_DIGEST_SIZE,
        .init                = ghash_init,
        .update                = ghash_update,
        .final                = ghash_final,
        .setkey                = ghash_setkey,
        .descsize        = sizeof(struct ghash_desc_ctx),
        .base                = {
                .cra_name                = "__ghash",
                .cra_driver_name        = "__ghash-pclmulqdqni",
                .cra_priority                = 0,
                .cra_flags                = CRYPTO_ALG_INTERNAL,
                .cra_blocksize                = GHASH_BLOCK_SIZE,
                .cra_ctxsize                = sizeof(struct ghash_ctx),
                .cra_module                = THIS_MODULE,
        },
};

static int ghash_async_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *cryptd_req = ahash_request_ctx(req);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
        struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
        struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

        desc->tfm = child;
        desc->flags = req->base.flags;
        return crypto_shash_init(desc); /*covered*/
}

static int ghash_async_update(struct ahash_request *req)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

        if (!irq_fpu_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { /*covered*/
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
                return crypto_ahash_update(cryptd_req); /*covered*/
        } else {
                struct shash_desc *desc = cryptd_shash_desc(cryptd_req); /*covered*/
                return shash_ahash_update(req, desc);
        }
}

static int ghash_async_final(struct ahash_request *req)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

        if (!irq_fpu_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { /*covered*/
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
                return crypto_ahash_final(cryptd_req); /*covered*/
        } else {
                struct shash_desc *desc = cryptd_shash_desc(cryptd_req); /*covered*/
                return crypto_shash_final(desc, req->result);
        }
}

static int ghash_async_import(struct ahash_request *req, const void *in)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

        ghash_async_init(req);
        memcpy(dctx, in, sizeof(*dctx));
        return 0;

}

static int ghash_async_export(struct ahash_request *req, void *out)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

        memcpy(out, dctx, sizeof(*dctx));
        return 0;

}

static int ghash_async_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *cryptd_req = ahash_request_ctx(req);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

        if (!irq_fpu_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
                return crypto_ahash_digest(cryptd_req);
        } else {
                struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
                struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

                desc->tfm = child;
                desc->flags = req->base.flags;
                return shash_ahash_digest(req, desc);
        }
}

static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
                              unsigned int keylen)
{
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct crypto_ahash *child = &ctx->cryptd_tfm->base; /*covered*/
        int err;

        crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
        crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
                               & CRYPTO_TFM_REQ_MASK);
        err = crypto_ahash_setkey(child, key, keylen);
        crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
                               & CRYPTO_TFM_RES_MASK);

        return err;
}

static int ghash_async_init_tfm(struct crypto_tfm *tfm)
{
        struct cryptd_ahash *cryptd_tfm;
        struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

        cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", /*covered*/
                                        CRYPTO_ALG_INTERNAL,
                                        CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);
        ctx->cryptd_tfm = cryptd_tfm; /*covered*/
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
                                 sizeof(struct ahash_request) +
                                 crypto_ahash_reqsize(&cryptd_tfm->base));

        return 0; /*covered*/
}

static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
{
        struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

        cryptd_free_ahash(ctx->cryptd_tfm); /*covered*/
}

static struct ahash_alg ghash_async_alg = {
        .init                = ghash_async_init,
        .update                = ghash_async_update,
        .final                = ghash_async_final,
        .setkey                = ghash_async_setkey,
        .digest                = ghash_async_digest,
        .export                = ghash_async_export,
        .import                = ghash_async_import,
        .halg = {
                .digestsize        = GHASH_DIGEST_SIZE,
                .statesize = sizeof(struct ghash_desc_ctx),
                .base = {
                        .cra_name                = "ghash",
                        .cra_driver_name        = "ghash-clmulni",
                        .cra_priority                = 400,
                        .cra_ctxsize                = sizeof(struct ghash_async_ctx),
                        .cra_flags                = CRYPTO_ALG_ASYNC,
                        .cra_blocksize                = GHASH_BLOCK_SIZE,
                        .cra_module                = THIS_MODULE,
                        .cra_init                = ghash_async_init_tfm,
                        .cra_exit                = ghash_async_exit_tfm,
                },
        },
};

static const struct x86_cpu_id pcmul_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */
        {}
};
MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);

static int __init ghash_pclmulqdqni_mod_init(void)
{
        int err;

        if (!x86_match_cpu(pcmul_cpu_id))
                return -ENODEV;

        err = crypto_register_shash(&ghash_alg);
        if (err)
                goto err_out;
        err = crypto_register_ahash(&ghash_async_alg);
        if (err)
                goto err_shash;

        return 0;

err_shash:
        crypto_unregister_shash(&ghash_alg);
err_out:
        return err;
}

static void __exit ghash_pclmulqdqni_mod_exit(void)
{
        crypto_unregister_ahash(&ghash_async_alg);
        crypto_unregister_shash(&ghash_alg);
}

module_init(ghash_pclmulqdqni_mod_init);
module_exit(ghash_pclmulqdqni_mod_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
                   "accelerated by PCLMULQDQ-NI");
MODULE_ALIAS_CRYPTO("ghash");

/*
 * Shared glue code for 128bit block ciphers
 *
 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <crypto/b128ops.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/skcipher.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>

int glue_ecb_req_128bit(const struct common_glue_ctx *gctx,
                        struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                const u8 *src = walk.src.virt.addr; /*covered*/
                u8 *dst = walk.dst.virt.addr;
                unsigned int func_bytes;
                unsigned int i;

                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                             &walk, fpu_enabled, nbytes);
                for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                        func_bytes = bsize * gctx->funcs[i].num_blocks; /*covered*/

                        if (nbytes < func_bytes) /*covered*/
                                continue;

                        /* Process multi-block batch */
                        do {
                                gctx->funcs[i].fn_u.ecb(ctx, dst, src); /*covered*/
                                src += func_bytes;
                                dst += func_bytes;
                                nbytes -= func_bytes;
                        } while (nbytes >= func_bytes);

                        if (nbytes < bsize) /*covered*/
                                break;
                }
                err = skcipher_walk_done(&walk, nbytes); /*covered*/
        }

        glue_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_ecb_req_128bit);

int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn,
                                struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                const u128 *src = (u128 *)walk.src.virt.addr; /*covered*/
                u128 *dst = (u128 *)walk.dst.virt.addr;
                u128 *iv = (u128 *)walk.iv;

                do {
                        u128_xor(dst, src, iv); /*covered*/
                        fn(ctx, (u8 *)dst, (u8 *)dst);
                        iv = dst;
                        src++;
                        dst++;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

                *(u128 *)walk.iv = *iv; /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }
        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_cbc_encrypt_req_128bit);

int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx,
                                struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                const u128 *src = walk.src.virt.addr; /*covered*/
                u128 *dst = walk.dst.virt.addr;
                unsigned int func_bytes, num_blocks;
                unsigned int i;
                u128 last_iv;

                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                             &walk, fpu_enabled, nbytes);
                /* Start of the last block. */
                src += nbytes / bsize - 1; /*covered*/
                dst += nbytes / bsize - 1;

                last_iv = *src;

                for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                        num_blocks = gctx->funcs[i].num_blocks; /*covered*/
                        func_bytes = bsize * num_blocks;

                        if (nbytes < func_bytes)
                                continue;

                        /* Process multi-block batch */
                        do {
                                src -= num_blocks - 1; /*covered*/
                                dst -= num_blocks - 1;

                                gctx->funcs[i].fn_u.cbc(ctx, dst, src);

                                nbytes -= func_bytes;
                                if (nbytes < bsize)
                                        goto done;

                                u128_xor(dst, dst, --src); /*covered*/
                                dst--;
                        } while (nbytes >= func_bytes);
                }
done:
                u128_xor(dst, dst, (u128 *)walk.iv); /*covered*/
                *(u128 *)walk.iv = last_iv;
                err = skcipher_walk_done(&walk, nbytes);
        }

        glue_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_cbc_decrypt_req_128bit);

int glue_ctr_req_128bit(const struct common_glue_ctx *gctx,
                        struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= bsize) {
                const u128 *src = walk.src.virt.addr; /*covered*/
                u128 *dst = walk.dst.virt.addr;
                unsigned int func_bytes, num_blocks;
                unsigned int i;
                le128 ctrblk;

                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                             &walk, fpu_enabled, nbytes);

                be128_to_le128(&ctrblk, (be128 *)walk.iv); /*covered*/

                for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                        num_blocks = gctx->funcs[i].num_blocks; /*covered*/
                        func_bytes = bsize * num_blocks;

                        if (nbytes < func_bytes)
                                continue;

                        /* Process multi-block batch */
                        do {
                                gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk); /*covered*/
                                src += num_blocks; /*covered*/
                                dst += num_blocks;
                                nbytes -= func_bytes;
                        } while (nbytes >= func_bytes);

                        if (nbytes < bsize) /*covered*/
                                break;
                }

                le128_to_be128((be128 *)walk.iv, &ctrblk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        glue_fpu_end(fpu_enabled); /*covered*/

        if (nbytes) { /*covered*/
                le128 ctrblk;
                u128 tmp;

                be128_to_le128(&ctrblk, (be128 *)walk.iv); /*covered*/
                memcpy(&tmp, walk.src.virt.addr, nbytes);
                gctx->funcs[gctx->num_funcs - 1].fn_u.ctr(ctx, &tmp, &tmp,
                                                          &ctrblk);
                memcpy(walk.dst.virt.addr, &tmp, nbytes);
                le128_to_be128((be128 *)walk.iv, &ctrblk);

                err = skcipher_walk_done(&walk, 0);
        }

        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_ctr_req_128bit);

static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx,
                                          void *ctx,
                                          struct skcipher_walk *walk)
{
        const unsigned int bsize = 128 / 8;
        unsigned int nbytes = walk->nbytes;
        u128 *src = walk->src.virt.addr; /*covered*/
        u128 *dst = walk->dst.virt.addr;
        unsigned int num_blocks, func_bytes;
        unsigned int i;

        /* Process multi-block batch */
        for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                num_blocks = gctx->funcs[i].num_blocks; /*covered*/
                func_bytes = bsize * num_blocks;

                if (nbytes >= func_bytes) {
                        do {
                                gctx->funcs[i].fn_u.xts(ctx, dst, src, /*covered*/
                                                        walk->iv);

                                src += num_blocks; /*covered*/
                                dst += num_blocks;
                                nbytes -= func_bytes;
                        } while (nbytes >= func_bytes);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }
        }

done:
        return nbytes;
}

int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
                        struct skcipher_request *req,
                        common_glue_func_t tweak_fn, void *tweak_ctx,
                        void *crypt_ctx)
{
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false); /*covered*/
        nbytes = walk.nbytes;
        if (!nbytes)
                return err;

        /* set minimum length to bsize, for tweak_fn */
        fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                     &walk, fpu_enabled,
                                     nbytes < bsize ? bsize : nbytes);

        /* calculate first value of T */
        tweak_fn(tweak_ctx, walk.iv, walk.iv); /*covered*/

        while (nbytes) {
                nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk); /*covered*/

                err = skcipher_walk_done(&walk, nbytes); /*covered*/
                nbytes = walk.nbytes;
        }

        glue_fpu_end(fpu_enabled); /*covered*/

        return err;
}
EXPORT_SYMBOL_GPL(glue_xts_req_128bit);

void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
                               common_glue_func_t fn)
{
        le128 ivblk = *iv; /*covered*/

        /* generate next IV */
        gf128mul_x_ble(iv, &ivblk);

        /* CC <- T xor C */
        u128_xor(dst, src, (u128 *)&ivblk);

        /* PP <- D(Key2,CC) */
        fn(ctx, (u8 *)dst, (u8 *)dst);

        /* P <- T xor PP */
        u128_xor(dst, dst, (u128 *)&ivblk);
}
EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);

MODULE_LICENSE("GPL");

/*
 * The MORUS-1280 Authenticated-Encryption Algorithm
 *   Glue for AVX2 implementation
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/internal/aead.h>
#include <crypto/morus1280_glue.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

asmlinkage void crypto_morus1280_avx2_init(void *state, const void *key,
                                           const void *iv);
asmlinkage void crypto_morus1280_avx2_ad(void *state, const void *data,
                                         unsigned int length);

asmlinkage void crypto_morus1280_avx2_enc(void *state, const void *src,
                                          void *dst, unsigned int length);
asmlinkage void crypto_morus1280_avx2_dec(void *state, const void *src,
                                          void *dst, unsigned int length);

asmlinkage void crypto_morus1280_avx2_enc_tail(void *state, const void *src,
                                               void *dst, unsigned int length);
asmlinkage void crypto_morus1280_avx2_dec_tail(void *state, const void *src,
                                               void *dst, unsigned int length);

asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor,
                                            u64 assoclen, u64 cryptlen);

MORUS1280_DECLARE_ALGS(avx2, "morus1280-avx2", 400); /*covered*/

static int __init crypto_morus1280_avx2_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
            !boot_cpu_has(X86_FEATURE_OSXSAVE) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_morus1280_avx2_algs,
                                     ARRAY_SIZE(crypto_morus1280_avx2_algs));
}

static void __exit crypto_morus1280_avx2_module_exit(void)
{
        crypto_unregister_aeads(crypto_morus1280_avx2_algs,
                                ARRAY_SIZE(crypto_morus1280_avx2_algs));
}

module_init(crypto_morus1280_avx2_module_init);
module_exit(crypto_morus1280_avx2_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- AVX2 implementation");
MODULE_ALIAS_CRYPTO("morus1280");
MODULE_ALIAS_CRYPTO("morus1280-avx2");

/*
 * The MORUS-1280 Authenticated-Encryption Algorithm
 *   Glue for SSE2 implementation
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/internal/aead.h>
#include <crypto/morus1280_glue.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

asmlinkage void crypto_morus1280_sse2_init(void *state, const void *key,
                                           const void *iv);
asmlinkage void crypto_morus1280_sse2_ad(void *state, const void *data,
                                         unsigned int length);

asmlinkage void crypto_morus1280_sse2_enc(void *state, const void *src,
                                          void *dst, unsigned int length);
asmlinkage void crypto_morus1280_sse2_dec(void *state, const void *src,
                                          void *dst, unsigned int length);

asmlinkage void crypto_morus1280_sse2_enc_tail(void *state, const void *src,
                                               void *dst, unsigned int length);
asmlinkage void crypto_morus1280_sse2_dec_tail(void *state, const void *src,
                                               void *dst, unsigned int length);

asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor,
                                            u64 assoclen, u64 cryptlen);

MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350); /*covered*/

static int __init crypto_morus1280_sse2_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_morus1280_sse2_algs,
                                     ARRAY_SIZE(crypto_morus1280_sse2_algs));
}

static void __exit crypto_morus1280_sse2_module_exit(void)
{
        crypto_unregister_aeads(crypto_morus1280_sse2_algs,
                                ARRAY_SIZE(crypto_morus1280_sse2_algs));
}

module_init(crypto_morus1280_sse2_module_init);
module_exit(crypto_morus1280_sse2_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- SSE2 implementation");
MODULE_ALIAS_CRYPTO("morus1280");
MODULE_ALIAS_CRYPTO("morus1280-sse2");

/*
 * The MORUS-1280 Authenticated-Encryption Algorithm
 *   Common x86 SIMD glue skeleton
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/morus1280_glue.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <asm/fpu/api.h>

struct morus1280_state {
        struct morus1280_block s[MORUS_STATE_BLOCKS];
};

struct morus1280_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, const void *src, void *dst,
                             unsigned int length);
        void (*crypt_tail)(void *state, const void *src, void *dst,
                           unsigned int length);
};

static void crypto_morus1280_glue_process_ad(
                struct morus1280_state *state,
                const struct morus1280_glue_ops *ops,
                struct scatterlist *sg_src, unsigned int assoclen)
{
        struct scatter_walk walk;
        struct morus1280_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) { /*covered*/
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= MORUS1280_BLOCK_SIZE) { /*covered*/
                        if (pos > 0) { /*covered*/
                                unsigned int fill = MORUS1280_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        ops->ad(state, src, left); /*covered*/
                        src += left & ~(MORUS1280_BLOCK_SIZE - 1);
                        left &= MORUS1280_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/

                pos += left;
                assoclen -= size;
                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, MORUS1280_BLOCK_SIZE - pos); /*covered*/
                ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE); /*covered*/
        }
}

static void crypto_morus1280_glue_process_crypt(struct morus1280_state *state,
                                                struct morus1280_ops ops,
                                                struct skcipher_walk *walk)
{
        while (walk->nbytes >= MORUS1280_BLOCK_SIZE) {
                ops.crypt_blocks(state, walk->src.virt.addr, /*covered*/
                                 walk->dst.virt.addr,
                                 round_down(walk->nbytes,
                                            MORUS1280_BLOCK_SIZE));
                skcipher_walk_done(walk, walk->nbytes % MORUS1280_BLOCK_SIZE);
        }

        if (walk->nbytes) { /*covered*/
                ops.crypt_tail(state, walk->src.virt.addr, walk->dst.virt.addr, /*covered*/
                               walk->nbytes);
                skcipher_walk_done(walk, 0);
        }
}

int crypto_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                 unsigned int keylen)
{
        struct morus1280_ctx *ctx = crypto_aead_ctx(aead);

        if (keylen == MORUS1280_BLOCK_SIZE) { /*covered*/
                memcpy(ctx->key.bytes, key, MORUS1280_BLOCK_SIZE); /*covered*/
        } else if (keylen == MORUS1280_BLOCK_SIZE / 2) { /*covered*/
                memcpy(ctx->key.bytes, key, keylen);
                memcpy(ctx->key.bytes + keylen, key, keylen);
        } else {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL; /*covered*/
        }

        return 0;
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setkey);

int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm,
                                      unsigned int authsize)
{
        return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setauthsize);

static void crypto_morus1280_glue_crypt(struct aead_request *req,
                                        struct morus1280_ops ops,
                                        unsigned int cryptlen,
                                        struct morus1280_block *tag_xor)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus1280_state state;
        struct skcipher_walk walk;

        ops.skcipher_walk_init(&walk, req, true);

        kernel_fpu_begin();

        ctx->ops->init(&state, &ctx->key, req->iv);
        crypto_morus1280_glue_process_ad(&state, ctx->ops, req->src, req->assoclen); /*covered*/
        crypto_morus1280_glue_process_crypt(&state, ops, &walk); /*covered*/
        ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen); /*covered*/

        kernel_fpu_end();
}

int crypto_morus1280_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus1280_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = ctx->ops->enc,
                .crypt_tail = ctx->ops->enc_tail,
        };

        struct morus1280_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_encrypt);

int crypto_morus1280_glue_decrypt(struct aead_request *req)
{
        static const u8 zeros[MORUS1280_BLOCK_SIZE] = {};

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus1280_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = ctx->ops->dec,
                .crypt_tail = ctx->ops->dec_tail,
        };

        struct morus1280_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);

        return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_decrypt);

void crypto_morus1280_glue_init_ops(struct crypto_aead *aead,
                                    const struct morus1280_glue_ops *ops)
{
        struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
        ctx->ops = ops; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_init_ops);

int cryptd_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                 unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setkey);

int cryptd_morus1280_glue_setauthsize(struct crypto_aead *aead,
                                      unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setauthsize);

int cryptd_morus1280_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_encrypt);

int cryptd_morus1280_glue_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_decrypt);

int cryptd_morus1280_glue_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        const char *name = crypto_aead_alg(aead)->base.cra_driver_name; /*covered*/
        char internal_name[CRYPTO_MAX_ALG_NAME];

        if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name)
                        >= CRYPTO_MAX_ALG_NAME)
                return -ENAMETOOLONG;

        cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_init_tfm);

void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_exit_tfm);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-1280 AEAD mode -- glue for x86 optimizations");

/*
 * The MORUS-640 Authenticated-Encryption Algorithm
 *   Glue for SSE2 implementation
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/internal/aead.h>
#include <crypto/morus640_glue.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

asmlinkage void crypto_morus640_sse2_init(void *state, const void *key,
                                          const void *iv);
asmlinkage void crypto_morus640_sse2_ad(void *state, const void *data,
                                        unsigned int length);

asmlinkage void crypto_morus640_sse2_enc(void *state, const void *src,
                                         void *dst, unsigned int length);
asmlinkage void crypto_morus640_sse2_dec(void *state, const void *src,
                                         void *dst, unsigned int length);

asmlinkage void crypto_morus640_sse2_enc_tail(void *state, const void *src,
                                              void *dst, unsigned int length);
asmlinkage void crypto_morus640_sse2_dec_tail(void *state, const void *src,
                                              void *dst, unsigned int length);

asmlinkage void crypto_morus640_sse2_final(void *state, void *tag_xor,
                                           u64 assoclen, u64 cryptlen);

MORUS640_DECLARE_ALGS(sse2, "morus640-sse2", 400); /*covered*/

static int __init crypto_morus640_sse2_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_morus640_sse2_algs,
                                     ARRAY_SIZE(crypto_morus640_sse2_algs));
}

static void __exit crypto_morus640_sse2_module_exit(void)
{
        crypto_unregister_aeads(crypto_morus640_sse2_algs,
                                ARRAY_SIZE(crypto_morus640_sse2_algs));
}

module_init(crypto_morus640_sse2_module_init);
module_exit(crypto_morus640_sse2_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-640 AEAD algorithm -- SSE2 implementation");
MODULE_ALIAS_CRYPTO("morus640");
MODULE_ALIAS_CRYPTO("morus640-sse2");

/*
 * The MORUS-640 Authenticated-Encryption Algorithm
 *   Common x86 SIMD glue skeleton
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/morus640_glue.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <asm/fpu/api.h>

struct morus640_state {
        struct morus640_block s[MORUS_STATE_BLOCKS];
};

struct morus640_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, const void *src, void *dst,
                             unsigned int length);
        void (*crypt_tail)(void *state, const void *src, void *dst,
                           unsigned int length);
};

static void crypto_morus640_glue_process_ad(
                struct morus640_state *state,
                const struct morus640_glue_ops *ops,
                struct scatterlist *sg_src, unsigned int assoclen)
{
        struct scatter_walk walk;
        struct morus640_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) { /*covered*/
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= MORUS640_BLOCK_SIZE) { /*covered*/
                        if (pos > 0) { /*covered*/
                                unsigned int fill = MORUS640_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        ops->ad(state, src, left); /*covered*/
                        src += left & ~(MORUS640_BLOCK_SIZE - 1);
                        left &= MORUS640_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/

                pos += left;
                assoclen -= size;
                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, MORUS640_BLOCK_SIZE - pos); /*covered*/
                ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE); /*covered*/
        }
}

static void crypto_morus640_glue_process_crypt(struct morus640_state *state,
                                               struct morus640_ops ops,
                                               struct skcipher_walk *walk)
{
        while (walk->nbytes >= MORUS640_BLOCK_SIZE) {
                ops.crypt_blocks(state, walk->src.virt.addr, /*covered*/
                                 walk->dst.virt.addr,
                                 round_down(walk->nbytes, MORUS640_BLOCK_SIZE));
                skcipher_walk_done(walk, walk->nbytes % MORUS640_BLOCK_SIZE);
        }

        if (walk->nbytes) { /*covered*/
                ops.crypt_tail(state, walk->src.virt.addr, walk->dst.virt.addr, /*covered*/
                               walk->nbytes);
                skcipher_walk_done(walk, 0);
        }
}

int crypto_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                unsigned int keylen)
{
        struct morus640_ctx *ctx = crypto_aead_ctx(aead);

        if (keylen != MORUS640_BLOCK_SIZE) { /*covered*/
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key.bytes, key, MORUS640_BLOCK_SIZE);
        return 0; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_setkey);

int crypto_morus640_glue_setauthsize(struct crypto_aead *tfm,
                                     unsigned int authsize)
{
        return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_setauthsize);

static void crypto_morus640_glue_crypt(struct aead_request *req,
                                       struct morus640_ops ops,
                                       unsigned int cryptlen,
                                       struct morus640_block *tag_xor)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus640_state state;
        struct skcipher_walk walk;

        ops.skcipher_walk_init(&walk, req, true);

        kernel_fpu_begin();

        ctx->ops->init(&state, &ctx->key, req->iv);
        crypto_morus640_glue_process_ad(&state, ctx->ops, req->src, req->assoclen); /*covered*/
        crypto_morus640_glue_process_crypt(&state, ops, &walk); /*covered*/
        ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen); /*covered*/

        kernel_fpu_end();
}

int crypto_morus640_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus640_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = ctx->ops->enc,
                .crypt_tail = ctx->ops->enc_tail,
        };

        struct morus640_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_encrypt);

int crypto_morus640_glue_decrypt(struct aead_request *req)
{
        static const u8 zeros[MORUS640_BLOCK_SIZE] = {};

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus640_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = ctx->ops->dec,
                .crypt_tail = ctx->ops->dec_tail,
        };

        struct morus640_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);

        return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_decrypt);

void crypto_morus640_glue_init_ops(struct crypto_aead *aead,
                                   const struct morus640_glue_ops *ops)
{
        struct morus640_ctx *ctx = crypto_aead_ctx(aead);
        ctx->ops = ops; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_init_ops);

int cryptd_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setkey);

int cryptd_morus640_glue_setauthsize(struct crypto_aead *aead,
                                     unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setauthsize);

int cryptd_morus640_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_encrypt);

int cryptd_morus640_glue_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_decrypt);

int cryptd_morus640_glue_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        const char *name = crypto_aead_alg(aead)->base.cra_driver_name; /*covered*/
        char internal_name[CRYPTO_MAX_ALG_NAME];

        if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name)
                        >= CRYPTO_MAX_ALG_NAME)
                return -ENAMETOOLONG;

        cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_init_tfm);

void cryptd_morus640_glue_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_exit_tfm);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-640 AEAD mode -- glue for x86 optimizations");

/*
 * Poly1305 authenticator algorithm, RFC7539, SIMD glue code
 *
 * Copyright (C) 2015 Martin Willi
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/poly1305.h>
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/simd.h>

struct poly1305_simd_desc_ctx {
        struct poly1305_desc_ctx base;
        /* derived key u set? */
        bool uset;
#ifdef CONFIG_AS_AVX2
        /* derived keys r^3, r^4 set? */
        bool wset;
#endif
        /* derived Poly1305 key r^2 */
        u32 u[5];
        /* ... silently appended r^3 and r^4 when using AVX2 */
};

asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src,
                                    const u32 *r, unsigned int blocks);
asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r,
                                     unsigned int blocks, const u32 *u);
#ifdef CONFIG_AS_AVX2
asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
                                     unsigned int blocks, const u32 *u);
static bool poly1305_use_avx2;
#endif

static int poly1305_simd_init(struct shash_desc *desc)
{
        struct poly1305_simd_desc_ctx *sctx = shash_desc_ctx(desc);

        sctx->uset = false; /*covered*/
#ifdef CONFIG_AS_AVX2
        sctx->wset = false;
#endif

        return crypto_poly1305_init(desc);
}

static void poly1305_simd_mult(u32 *a, const u32 *b)
{
        u8 m[POLY1305_BLOCK_SIZE];

        memset(m, 0, sizeof(m)); /*covered*/
        /* The poly1305 block function adds a hi-bit to the accumulator which
         * we don't need for key multiplication; compensate for it. */
        a[4] -= 1 << 24;
        poly1305_block_sse2(a, m, b, 1);
}

static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
                                         const u8 *src, unsigned int srclen)
{
        struct poly1305_simd_desc_ctx *sctx;
        unsigned int blocks, datalen;

        BUILD_BUG_ON(offsetof(struct poly1305_simd_desc_ctx, base));
        sctx = container_of(dctx, struct poly1305_simd_desc_ctx, base);

        if (unlikely(!dctx->sset)) { /*covered*/
                datalen = crypto_poly1305_setdesckey(dctx, src, srclen); /*covered*/
                src += srclen - datalen;
                srclen = datalen;
        }

#ifdef CONFIG_AS_AVX2
        if (poly1305_use_avx2 && srclen >= POLY1305_BLOCK_SIZE * 4) { /*covered*/
                if (unlikely(!sctx->wset)) { /*covered*/
                        if (!sctx->uset) { /*covered*/
                                memcpy(sctx->u, dctx->r.r, sizeof(sctx->u)); /*covered*/
                                poly1305_simd_mult(sctx->u, dctx->r.r);
                                sctx->uset = true;
                        }
                        memcpy(sctx->u + 5, sctx->u, sizeof(sctx->u)); /*covered*/
                        poly1305_simd_mult(sctx->u + 5, dctx->r.r);
                        memcpy(sctx->u + 10, sctx->u + 5, sizeof(sctx->u));
                        poly1305_simd_mult(sctx->u + 10, dctx->r.r);
                        sctx->wset = true;
                }
                blocks = srclen / (POLY1305_BLOCK_SIZE * 4); /*covered*/
                poly1305_4block_avx2(dctx->h.h, src, dctx->r.r, blocks,
                                     sctx->u);
                src += POLY1305_BLOCK_SIZE * 4 * blocks;
                srclen -= POLY1305_BLOCK_SIZE * 4 * blocks;
        }
#endif
        if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) { /*covered*/
                if (unlikely(!sctx->uset)) { /*covered*/
                        memcpy(sctx->u, dctx->r.r, sizeof(sctx->u));
                        poly1305_simd_mult(sctx->u, dctx->r.r);
                        sctx->uset = true;
                }
                blocks = srclen / (POLY1305_BLOCK_SIZE * 2); /*covered*/
                poly1305_2block_sse2(dctx->h.h, src, dctx->r.r, blocks,
                                     sctx->u);
                src += POLY1305_BLOCK_SIZE * 2 * blocks;
                srclen -= POLY1305_BLOCK_SIZE * 2 * blocks;
        }
        if (srclen >= POLY1305_BLOCK_SIZE) { /*covered*/
                poly1305_block_sse2(dctx->h.h, src, dctx->r.r, 1); /*covered*/
                srclen -= POLY1305_BLOCK_SIZE;
        }
        return srclen; /*covered*/
}

static int poly1305_simd_update(struct shash_desc *desc,
                                const u8 *src, unsigned int srclen)
{
        struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); /*covered*/
        unsigned int bytes;

        /* kernel_fpu_begin/end is costly, use fallback for small updates */
        if (srclen <= 288 || !may_use_simd()) /*covered*/
                return crypto_poly1305_update(desc, src, srclen); /*covered*/

        kernel_fpu_begin();

        if (unlikely(dctx->buflen)) {
                bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); /*covered*/
                memcpy(dctx->buf + dctx->buflen, src, bytes);
                src += bytes;
                srclen -= bytes;
                dctx->buflen += bytes;

                if (dctx->buflen == POLY1305_BLOCK_SIZE) {
                        poly1305_simd_blocks(dctx, dctx->buf, /*covered*/
                                             POLY1305_BLOCK_SIZE);
                        dctx->buflen = 0;
                }
        }

        if (likely(srclen >= POLY1305_BLOCK_SIZE)) { /*covered*/
                bytes = poly1305_simd_blocks(dctx, src, srclen); /*covered*/
                src += srclen - bytes;
                srclen = bytes;
        }

        kernel_fpu_end(); /*covered*/

        if (unlikely(srclen)) {
                dctx->buflen = srclen; /*covered*/
                memcpy(dctx->buf, src, srclen);
        }

        return 0;
}

static struct shash_alg alg = {
        .digestsize        = POLY1305_DIGEST_SIZE,
        .init                = poly1305_simd_init,
        .update                = poly1305_simd_update,
        .final                = crypto_poly1305_final,
        .descsize        = sizeof(struct poly1305_simd_desc_ctx),
        .base                = {
                .cra_name                = "poly1305",
                .cra_driver_name        = "poly1305-simd",
                .cra_priority                = 300,
                .cra_blocksize                = POLY1305_BLOCK_SIZE,
                .cra_module                = THIS_MODULE,
        },
};

static int __init poly1305_simd_mod_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2))
                return -ENODEV;

#ifdef CONFIG_AS_AVX2
        poly1305_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) &&
                            boot_cpu_has(X86_FEATURE_AVX2) &&
                            cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
        alg.descsize = sizeof(struct poly1305_simd_desc_ctx);
        if (poly1305_use_avx2)
                alg.descsize += 10 * sizeof(u32);
#endif
        return crypto_register_shash(&alg);
}

static void __exit poly1305_simd_mod_exit(void)
{
        crypto_unregister_shash(&alg);
}

module_init(poly1305_simd_mod_init);
module_exit(poly1305_simd_mod_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
MODULE_DESCRIPTION("Poly1305 authenticator");
MODULE_ALIAS_CRYPTO("poly1305");
MODULE_ALIAS_CRYPTO("poly1305-simd");

/*
 * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
 *
 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/serpent.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <asm/crypto/serpent-avx.h>

#define SERPENT_AVX2_PARALLEL_BLOCKS 16

/* 16-way AVX2 parallel cipher functions */
asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src);
asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src);
asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);

asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
                                  le128 *iv);
asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src, le128 *iv);
asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src, le128 *iv);

static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); /*covered*/
}

static const struct common_glue_ctx serpent_enc = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
        } }
};

static const struct common_glue_ctx serpent_ctr = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
        },  {
                .num_blocks = 8,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
        } }
};

static const struct common_glue_ctx serpent_enc_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
        } }
};

static const struct common_glue_ctx serpent_dec = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_cbc = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&serpent_ctr, req); /*covered*/
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_enc_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_dec_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg serpent_algs[] = {
        {
                .base.cra_name                = "__ecb(serpent)",
                .base.cra_driver_name        = "__ecb-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(serpent)",
                .base.cra_driver_name        = "__cbc-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(serpent)",
                .base.cra_driver_name        = "__ctr-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .chunksize                = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(serpent)",
                .base.cra_driver_name        = "__xts-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * SERPENT_MIN_KEY_SIZE,
                .max_keysize                = 2 * SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = xts_serpent_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];

static int __init init(void)
{
        const char *feature_name;

        if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
                pr_info("AVX2 instructions are not detected.\n");
                return -ENODEV;
        }
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(serpent_algs,
                                              ARRAY_SIZE(serpent_algs),
                                              serpent_simd_algs);
}

static void __exit fini(void)
{
        simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
                                  serpent_simd_algs);
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
MODULE_ALIAS_CRYPTO("serpent");
MODULE_ALIAS_CRYPTO("serpent-asm");

/*
 * Glue Code for AVX assembler versions of Serpent Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2011-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/serpent.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <asm/crypto/serpent-avx.h>

/* 8-way parallel cipher functions */
asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src);
EXPORT_SYMBOL_GPL(serpent_ecb_enc_8way_avx);

asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src);
EXPORT_SYMBOL_GPL(serpent_ecb_dec_8way_avx);

asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src);
EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx);

asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                     const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx);

asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx);

asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx);

void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk;

        le128_to_be128(&ctrblk, iv); /*covered*/
        le128_inc(iv); /*covered*/

        __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
}
EXPORT_SYMBOL_GPL(__serpent_crypt_ctr);

void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(__serpent_encrypt));
}
EXPORT_SYMBOL_GPL(serpent_xts_enc);

void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, /*covered*/
                                  GLUE_FUNC_CAST(__serpent_decrypt));
}
EXPORT_SYMBOL_GPL(serpent_xts_dec);

static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); /*covered*/
}

int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key, /*covered*/
                       unsigned int keylen)
{
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        /* first half of xts-key is for crypt */
        err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); /*covered*/
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); /*covered*/
}
EXPORT_SYMBOL_GPL(xts_serpent_setkey);

static const struct common_glue_ctx serpent_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
        } }
};

static const struct common_glue_ctx serpent_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
        } }
};

static const struct common_glue_ctx serpent_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
        } }
};

static const struct common_glue_ctx serpent_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&serpent_ctr, req); /*covered*/
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_enc_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_dec_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg serpent_algs[] = {
        {
                .base.cra_name                = "__ecb(serpent)",
                .base.cra_driver_name        = "__ecb-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(serpent)",
                .base.cra_driver_name        = "__cbc-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(serpent)",
                .base.cra_driver_name        = "__ctr-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .chunksize                = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(serpent)",
                .base.cra_driver_name        = "__xts-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * SERPENT_MIN_KEY_SIZE,
                .max_keysize                = 2 * SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = xts_serpent_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];

static int __init serpent_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(serpent_algs,
                                              ARRAY_SIZE(serpent_algs),
                                              serpent_simd_algs);
}

static void __exit serpent_exit(void)
{
        simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
                                  serpent_simd_algs);
}

module_init(serpent_init);
module_exit(serpent_exit);

MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("serpent");

/*
 * Glue Code for SSE2 assembler versions of Serpent Cipher
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * Glue code based on aesni-intel_glue.c by:
 *  Copyright (C) 2008, Intel Corp.
 *    Author: Huang Ying <ying.huang@intel.com>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/b128ops.h>
#include <crypto/internal/simd.h>
#include <crypto/serpent.h>
#include <asm/crypto/serpent-sse2.h>
#include <asm/crypto/glue_helper.h>

static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); /*covered*/
}

static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
{
        u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
        unsigned int j;

        for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
                ivs[j] = src[j]; /*covered*/

        serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); /*covered*/

        for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) /*covered*/
                u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); /*covered*/
}

static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk;

        le128_to_be128(&ctrblk, iv);
        le128_inc(iv);

        __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
}

static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
                                   le128 *iv)
{
        be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
        unsigned int i;

        for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
                if (dst != src)
                        dst[i] = src[i];

                le128_to_be128(&ctrblks[i], iv);
                le128_inc(iv);
        }

        serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
}

static const struct common_glue_ctx serpent_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
        } }
};

static const struct common_glue_ctx serpent_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
        } }
};

static const struct common_glue_ctx serpent_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&serpent_ctr, req);
}

static struct skcipher_alg serpent_algs[] = {
        {
                .base.cra_name                = "__ecb(serpent)",
                .base.cra_driver_name        = "__ecb-serpent-sse2",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(serpent)",
                .base.cra_driver_name        = "__cbc-serpent-sse2",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(serpent)",
                .base.cra_driver_name        = "__ctr-serpent-sse2",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .chunksize                = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        },
};

static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];

static int __init serpent_sse2_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2)) {
                printk(KERN_INFO "SSE2 instructions are not detected.\n");
                return -ENODEV;
        }

        return simd_register_skciphers_compat(serpent_algs,
                                              ARRAY_SIZE(serpent_algs),
                                              serpent_simd_algs);
}

static void __exit serpent_sse2_exit(void)
{
        simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
                                  serpent_simd_algs);
}

module_init(serpent_sse2_init);
module_exit(serpent_sse2_exit);

MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("serpent");

/*
 * Cryptographic API.
 *
 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
 * Supplemental SSE3 instructions.
 *
 * This file is based on sha1_generic.c
 *
 * Copyright (c) Alan Smithee.
 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
 * Copyright (c) Mathias Krause <minipli@googlemail.com>
 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha1_base.h>
#include <asm/fpu/api.h>

typedef void (sha1_transform_fn)(u32 *digest, const char *data,
                                unsigned int rounds);

static int sha1_update(struct shash_desc *desc, const u8 *data, /*covered*/
                             unsigned int len, sha1_transform_fn *sha1_xform)
{
        struct sha1_state *sctx = shash_desc_ctx(desc);

        if (!irq_fpu_usable() || /*covered*/
            (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) /*covered*/
                return crypto_sha1_update(desc, data, len); /*covered*/

        /* make sure casting to sha1_block_fn() is safe */
        BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);

        kernel_fpu_begin(); /*covered*/
        sha1_base_do_update(desc, data, len, /*covered*/
                            (sha1_block_fn *)sha1_xform);
        kernel_fpu_end(); /*covered*/

        return 0;
}

static int sha1_finup(struct shash_desc *desc, const u8 *data, /*covered*/
                      unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
{
        if (!irq_fpu_usable()) /*covered*/
                return crypto_sha1_finup(desc, data, len, out);

        kernel_fpu_begin(); /*covered*/
        if (len)
                sha1_base_do_update(desc, data, len, /*covered*/
                                    (sha1_block_fn *)sha1_xform);
        sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform); /*covered*/
        kernel_fpu_end();

        return sha1_base_finish(desc, out); /*covered*/
}

asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
                                     unsigned int rounds);

static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len, /*covered*/
                        (sha1_transform_fn *) sha1_transform_ssse3);
}

static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out, /*covered*/
                        (sha1_transform_fn *) sha1_transform_ssse3);
}

/* Add padding and return the message digest. */
static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
{
        return sha1_ssse3_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha1_ssse3_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_ssse3_update,
        .final                =        sha1_ssse3_final,
        .finup                =        sha1_ssse3_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static int register_sha1_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                return crypto_register_shash(&sha1_ssse3_alg);
        return 0;
}

static void unregister_sha1_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                crypto_unregister_shash(&sha1_ssse3_alg);
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
                                   unsigned int rounds);

static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len, /*covered*/
                        (sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out, /*covered*/
                        (sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_final(struct shash_desc *desc, u8 *out)
{
        return sha1_avx_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha1_avx_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_avx_update,
        .final                =        sha1_avx_final,
        .finup                =        sha1_avx_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static bool avx_usable(void)
{
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
                if (boot_cpu_has(X86_FEATURE_AVX))
                        pr_info("AVX detected but unusable.\n");
                return false;
        }

        return true;
}

static int register_sha1_avx(void)
{
        if (avx_usable())
                return crypto_register_shash(&sha1_avx_alg);
        return 0;
}

static void unregister_sha1_avx(void)
{
        if (avx_usable())
                crypto_unregister_shash(&sha1_avx_alg);
}

#else  /* CONFIG_AS_AVX */
static inline int register_sha1_avx(void) { return 0; }
static inline void unregister_sha1_avx(void) { }
#endif /* CONFIG_AS_AVX */


#if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
#define SHA1_AVX2_BLOCK_OPTSIZE        4        /* optimal 4*64 bytes of SHA1 blocks */

asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
                                    unsigned int rounds);

static bool avx2_usable(void)
{
        if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
                && boot_cpu_has(X86_FEATURE_BMI1)
                && boot_cpu_has(X86_FEATURE_BMI2))
                return true;

        return false;
}

static void sha1_apply_transform_avx2(u32 *digest, const char *data,
                                unsigned int rounds)
{
        /* Select the optimal transform based on data block size */
        if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE) /*covered*/
                sha1_transform_avx2(digest, data, rounds); /*covered*/
        else
                sha1_transform_avx(digest, data, rounds); /*covered*/
} /*covered*/

static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len, /*covered*/
                (sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out, /*covered*/
                (sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
{
        return sha1_avx2_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha1_avx2_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_avx2_update,
        .final                =        sha1_avx2_final,
        .finup                =        sha1_avx2_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static int register_sha1_avx2(void)
{
        if (avx2_usable())
                return crypto_register_shash(&sha1_avx2_alg);
        return 0;
}

static void unregister_sha1_avx2(void)
{
        if (avx2_usable())
                crypto_unregister_shash(&sha1_avx2_alg);
}

#else
static inline int register_sha1_avx2(void) { return 0; }
static inline void unregister_sha1_avx2(void) { }
#endif

#ifdef CONFIG_AS_SHA1_NI
asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
                                   unsigned int rounds);

static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len,
                (sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out,
                (sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_final(struct shash_desc *desc, u8 *out)
{
        return sha1_ni_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_ni_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_ni_update,
        .final                =        sha1_ni_final,
        .finup                =        sha1_ni_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-ni",
                .cra_priority        =        250,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static int register_sha1_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                return crypto_register_shash(&sha1_ni_alg);
        return 0;
}

static void unregister_sha1_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                crypto_unregister_shash(&sha1_ni_alg);
}

#else
static inline int register_sha1_ni(void) { return 0; }
static inline void unregister_sha1_ni(void) { }
#endif

static int __init sha1_ssse3_mod_init(void)
{
        if (register_sha1_ssse3())
                goto fail;

        if (register_sha1_avx()) {
                unregister_sha1_ssse3();
                goto fail;
        }

        if (register_sha1_avx2()) {
                unregister_sha1_avx();
                unregister_sha1_ssse3();
                goto fail;
        }

        if (register_sha1_ni()) {
                unregister_sha1_avx2();
                unregister_sha1_avx();
                unregister_sha1_ssse3();
                goto fail;
        }

        return 0;
fail:
        return -ENODEV;
}

static void __exit sha1_ssse3_mod_fini(void)
{
        unregister_sha1_ni();
        unregister_sha1_avx2();
        unregister_sha1_avx();
        unregister_sha1_ssse3();
}

module_init(sha1_ssse3_mod_init);
module_exit(sha1_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha1");
MODULE_ALIAS_CRYPTO("sha1-ssse3");
MODULE_ALIAS_CRYPTO("sha1-avx");
MODULE_ALIAS_CRYPTO("sha1-avx2");
#ifdef CONFIG_AS_SHA1_NI
MODULE_ALIAS_CRYPTO("sha1-ni");
#endif

/*
 * Cryptographic API.
 *
 * Glue code for the SHA256 Secure Hash Algorithm assembler
 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
 *
 * This file is based on sha256_generic.c
 *
 * Copyright (C) 2013 Intel Corporation.
 *
 * Author:
 *     Tim Chen <tim.c.chen@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */


#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha256_base.h>
#include <asm/fpu/api.h>
#include <linux/string.h>

asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data,
                                       u64 rounds);
typedef void (sha256_transform_fn)(u32 *digest, const char *data, u64 rounds);

static int sha256_update(struct shash_desc *desc, const u8 *data, /*covered*/
                         unsigned int len, sha256_transform_fn *sha256_xform)
{
        struct sha256_state *sctx = shash_desc_ctx(desc);

        if (!irq_fpu_usable() || /*covered*/
            (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) /*covered*/
                return crypto_sha256_update(desc, data, len); /*covered*/

        /* make sure casting to sha256_block_fn() is safe */
        BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);

        kernel_fpu_begin(); /*covered*/
        sha256_base_do_update(desc, data, len, /*covered*/
                              (sha256_block_fn *)sha256_xform);
        kernel_fpu_end(); /*covered*/

        return 0;
}

static int sha256_finup(struct shash_desc *desc, const u8 *data, /*covered*/
              unsigned int len, u8 *out, sha256_transform_fn *sha256_xform)
{
        if (!irq_fpu_usable()) /*covered*/
                return crypto_sha256_finup(desc, data, len, out);

        kernel_fpu_begin(); /*covered*/
        if (len)
                sha256_base_do_update(desc, data, len, /*covered*/
                                      (sha256_block_fn *)sha256_xform);
        sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_xform); /*covered*/
        kernel_fpu_end();

        return sha256_base_finish(desc, out); /*covered*/
}

static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_transform_ssse3); /*covered*/
}

static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_transform_ssse3); /*covered*/
}

/* Add padding and return the message digest. */
static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
{
        return sha256_ssse3_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha256_ssse3_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_ssse3_update,
        .final                =        sha256_ssse3_final,
        .finup                =        sha256_ssse3_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_ssse3_update,
        .final                =        sha256_ssse3_final,
        .finup                =        sha256_ssse3_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha256_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                return crypto_register_shashes(sha256_ssse3_algs,
                                ARRAY_SIZE(sha256_ssse3_algs));
        return 0;
}

static void unregister_sha256_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                crypto_unregister_shashes(sha256_ssse3_algs,
                                ARRAY_SIZE(sha256_ssse3_algs));
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha256_transform_avx(u32 *digest, const char *data,
                                     u64 rounds);

static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_transform_avx); /*covered*/
}

static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_transform_avx); /*covered*/
}

static int sha256_avx_final(struct shash_desc *desc, u8 *out)
{
        return sha256_avx_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha256_avx_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_avx_update,
        .final                =        sha256_avx_final,
        .finup                =        sha256_avx_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_avx_update,
        .final                =        sha256_avx_final,
        .finup                =        sha256_avx_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static bool avx_usable(void)
{
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
                if (boot_cpu_has(X86_FEATURE_AVX))
                        pr_info("AVX detected but unusable.\n");
                return false;
        }

        return true;
}

static int register_sha256_avx(void)
{
        if (avx_usable())
                return crypto_register_shashes(sha256_avx_algs,
                                ARRAY_SIZE(sha256_avx_algs));
        return 0;
}

static void unregister_sha256_avx(void)
{
        if (avx_usable())
                crypto_unregister_shashes(sha256_avx_algs,
                                ARRAY_SIZE(sha256_avx_algs));
}

#else
static inline int register_sha256_avx(void) { return 0; }
static inline void unregister_sha256_avx(void) { }
#endif

#if defined(CONFIG_AS_AVX2) && defined(CONFIG_AS_AVX)
asmlinkage void sha256_transform_rorx(u32 *digest, const char *data,
                                      u64 rounds);

static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_transform_rorx); /*covered*/
}

static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_transform_rorx); /*covered*/
}

static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
{
        return sha256_avx2_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha256_avx2_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_avx2_update,
        .final                =        sha256_avx2_final,
        .finup                =        sha256_avx2_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_avx2_update,
        .final                =        sha256_avx2_final,
        .finup                =        sha256_avx2_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static bool avx2_usable(void)
{
        if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
                    boot_cpu_has(X86_FEATURE_BMI2))
                return true;

        return false;
}

static int register_sha256_avx2(void)
{
        if (avx2_usable())
                return crypto_register_shashes(sha256_avx2_algs,
                                ARRAY_SIZE(sha256_avx2_algs));
        return 0;
}

static void unregister_sha256_avx2(void)
{
        if (avx2_usable())
                crypto_unregister_shashes(sha256_avx2_algs,
                                ARRAY_SIZE(sha256_avx2_algs));
}

#else
static inline int register_sha256_avx2(void) { return 0; }
static inline void unregister_sha256_avx2(void) { }
#endif

#ifdef CONFIG_AS_SHA256_NI
asmlinkage void sha256_ni_transform(u32 *digest, const char *data,
                                   u64 rounds); /*unsigned int rounds);*/

static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_ni_transform);
}

static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_ni_transform);
}

static int sha256_ni_final(struct shash_desc *desc, u8 *out)
{
        return sha256_ni_finup(desc, NULL, 0, out);
}

static struct shash_alg sha256_ni_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_ni_update,
        .final                =        sha256_ni_final,
        .finup                =        sha256_ni_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-ni",
                .cra_priority        =        250,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_ni_update,
        .final                =        sha256_ni_final,
        .finup                =        sha256_ni_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-ni",
                .cra_priority        =        250,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha256_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                return crypto_register_shashes(sha256_ni_algs,
                                ARRAY_SIZE(sha256_ni_algs));
        return 0;
}

static void unregister_sha256_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                crypto_unregister_shashes(sha256_ni_algs,
                                ARRAY_SIZE(sha256_ni_algs));
}

#else
static inline int register_sha256_ni(void) { return 0; }
static inline void unregister_sha256_ni(void) { }
#endif

static int __init sha256_ssse3_mod_init(void)
{
        if (register_sha256_ssse3())
                goto fail;

        if (register_sha256_avx()) {
                unregister_sha256_ssse3();
                goto fail;
        }

        if (register_sha256_avx2()) {
                unregister_sha256_avx();
                unregister_sha256_ssse3();
                goto fail;
        }

        if (register_sha256_ni()) {
                unregister_sha256_avx2();
                unregister_sha256_avx();
                unregister_sha256_ssse3();
                goto fail;
        }

        return 0;
fail:
        return -ENODEV;
}

static void __exit sha256_ssse3_mod_fini(void)
{
        unregister_sha256_ni();
        unregister_sha256_avx2();
        unregister_sha256_avx();
        unregister_sha256_ssse3();
}

module_init(sha256_ssse3_mod_init);
module_exit(sha256_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha256");
MODULE_ALIAS_CRYPTO("sha256-ssse3");
MODULE_ALIAS_CRYPTO("sha256-avx");
MODULE_ALIAS_CRYPTO("sha256-avx2");
MODULE_ALIAS_CRYPTO("sha224");
MODULE_ALIAS_CRYPTO("sha224-ssse3");
MODULE_ALIAS_CRYPTO("sha224-avx");
MODULE_ALIAS_CRYPTO("sha224-avx2");
#ifdef CONFIG_AS_SHA256_NI
MODULE_ALIAS_CRYPTO("sha256-ni");
MODULE_ALIAS_CRYPTO("sha224-ni");
#endif

/*
 * Cryptographic API.
 *
 * Glue code for the SHA512 Secure Hash Algorithm assembler
 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
 *
 * This file is based on sha512_generic.c
 *
 * Copyright (C) 2013 Intel Corporation
 * Author: Tim Chen <tim.c.chen@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha512_base.h>
#include <asm/fpu/api.h>

#include <linux/string.h>

asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data,
                                       u64 rounds);

typedef void (sha512_transform_fn)(u64 *digest, const char *data, u64 rounds);

static int sha512_update(struct shash_desc *desc, const u8 *data, /*covered*/
                       unsigned int len, sha512_transform_fn *sha512_xform)
{
        struct sha512_state *sctx = shash_desc_ctx(desc);

        if (!irq_fpu_usable() || /*covered*/
            (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) /*covered*/
                return crypto_sha512_update(desc, data, len); /*covered*/

        /* make sure casting to sha512_block_fn() is safe */
        BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);

        kernel_fpu_begin(); /*covered*/
        sha512_base_do_update(desc, data, len, /*covered*/
                              (sha512_block_fn *)sha512_xform);
        kernel_fpu_end(); /*covered*/

        return 0;
}

static int sha512_finup(struct shash_desc *desc, const u8 *data, /*covered*/
              unsigned int len, u8 *out, sha512_transform_fn *sha512_xform)
{
        if (!irq_fpu_usable()) /*covered*/
                return crypto_sha512_finup(desc, data, len, out);

        kernel_fpu_begin(); /*covered*/
        if (len)
                sha512_base_do_update(desc, data, len, /*covered*/
                                      (sha512_block_fn *)sha512_xform);
        sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_xform); /*covered*/
        kernel_fpu_end();

        return sha512_base_finish(desc, out); /*covered*/
}

static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len)
{
        return sha512_update(desc, data, len, sha512_transform_ssse3); /*covered*/
}

static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha512_finup(desc, data, len, out, sha512_transform_ssse3); /*covered*/
}

/* Add padding and return the message digest. */
static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
{
        return sha512_ssse3_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha512_ssse3_algs[] = { {
        .digestsize        =        SHA512_DIGEST_SIZE,
        .init                =        sha512_base_init,
        .update                =        sha512_ssse3_update,
        .final                =        sha512_ssse3_final,
        .finup                =        sha512_ssse3_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha512",
                .cra_driver_name =        "sha512-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA512_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
},  {
        .digestsize        =        SHA384_DIGEST_SIZE,
        .init                =        sha384_base_init,
        .update                =        sha512_ssse3_update,
        .final                =        sha512_ssse3_final,
        .finup                =        sha512_ssse3_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha384",
                .cra_driver_name =        "sha384-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA384_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha512_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                return crypto_register_shashes(sha512_ssse3_algs,
                        ARRAY_SIZE(sha512_ssse3_algs));
        return 0;
}

static void unregister_sha512_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                crypto_unregister_shashes(sha512_ssse3_algs,
                        ARRAY_SIZE(sha512_ssse3_algs));
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha512_transform_avx(u64 *digest, const char *data,
                                     u64 rounds);
static bool avx_usable(void)
{
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
                if (boot_cpu_has(X86_FEATURE_AVX))
                        pr_info("AVX detected but unusable.\n");
                return false;
        }

        return true;
}

static int sha512_avx_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len)
{
        return sha512_update(desc, data, len, sha512_transform_avx); /*covered*/
}

static int sha512_avx_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha512_finup(desc, data, len, out, sha512_transform_avx); /*covered*/
}

/* Add padding and return the message digest. */
static int sha512_avx_final(struct shash_desc *desc, u8 *out)
{
        return sha512_avx_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha512_avx_algs[] = { {
        .digestsize        =        SHA512_DIGEST_SIZE,
        .init                =        sha512_base_init,
        .update                =        sha512_avx_update,
        .final                =        sha512_avx_final,
        .finup                =        sha512_avx_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha512",
                .cra_driver_name =        "sha512-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA512_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
},  {
        .digestsize        =        SHA384_DIGEST_SIZE,
        .init                =        sha384_base_init,
        .update                =        sha512_avx_update,
        .final                =        sha512_avx_final,
        .finup                =        sha512_avx_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha384",
                .cra_driver_name =        "sha384-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA384_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha512_avx(void)
{
        if (avx_usable())
                return crypto_register_shashes(sha512_avx_algs,
                        ARRAY_SIZE(sha512_avx_algs));
        return 0;
}

static void unregister_sha512_avx(void)
{
        if (avx_usable())
                crypto_unregister_shashes(sha512_avx_algs,
                        ARRAY_SIZE(sha512_avx_algs));
}
#else
static inline int register_sha512_avx(void) { return 0; }
static inline void unregister_sha512_avx(void) { }
#endif

#if defined(CONFIG_AS_AVX2) && defined(CONFIG_AS_AVX)
asmlinkage void sha512_transform_rorx(u64 *digest, const char *data,
                                      u64 rounds);

static int sha512_avx2_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len)
{
        return sha512_update(desc, data, len, sha512_transform_rorx); /*covered*/
}

static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha512_finup(desc, data, len, out, sha512_transform_rorx); /*covered*/
}

/* Add padding and return the message digest. */
static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
{
        return sha512_avx2_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha512_avx2_algs[] = { {
        .digestsize        =        SHA512_DIGEST_SIZE,
        .init                =        sha512_base_init,
        .update                =        sha512_avx2_update,
        .final                =        sha512_avx2_final,
        .finup                =        sha512_avx2_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha512",
                .cra_driver_name =        "sha512-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA512_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
},  {
        .digestsize        =        SHA384_DIGEST_SIZE,
        .init                =        sha384_base_init,
        .update                =        sha512_avx2_update,
        .final                =        sha512_avx2_final,
        .finup                =        sha512_avx2_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha384",
                .cra_driver_name =        "sha384-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA384_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static bool avx2_usable(void)
{
        if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
                    boot_cpu_has(X86_FEATURE_BMI2))
                return true;

        return false;
}

static int register_sha512_avx2(void)
{
        if (avx2_usable())
                return crypto_register_shashes(sha512_avx2_algs,
                        ARRAY_SIZE(sha512_avx2_algs));
        return 0;
}

static void unregister_sha512_avx2(void)
{
        if (avx2_usable())
                crypto_unregister_shashes(sha512_avx2_algs,
                        ARRAY_SIZE(sha512_avx2_algs));
}
#else
static inline int register_sha512_avx2(void) { return 0; }
static inline void unregister_sha512_avx2(void) { }
#endif

static int __init sha512_ssse3_mod_init(void)
{

        if (register_sha512_ssse3())
                goto fail;

        if (register_sha512_avx()) {
                unregister_sha512_ssse3();
                goto fail;
        }

        if (register_sha512_avx2()) {
                unregister_sha512_avx();
                unregister_sha512_ssse3();
                goto fail;
        }

        return 0;
fail:
        return -ENODEV;
}

static void __exit sha512_ssse3_mod_fini(void)
{
        unregister_sha512_avx2();
        unregister_sha512_avx();
        unregister_sha512_ssse3();
}

module_init(sha512_ssse3_mod_init);
module_exit(sha512_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha512");
MODULE_ALIAS_CRYPTO("sha512-ssse3");
MODULE_ALIAS_CRYPTO("sha512-avx");
MODULE_ALIAS_CRYPTO("sha512-avx2");
MODULE_ALIAS_CRYPTO("sha384");
MODULE_ALIAS_CRYPTO("sha384-ssse3");
MODULE_ALIAS_CRYPTO("sha384-avx");
MODULE_ALIAS_CRYPTO("sha384-avx2");

/*
 * Glue Code for AVX assembler version of Twofish Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/twofish.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <asm/crypto/twofish.h>

#define TWOFISH_PARALLEL_BLOCKS 8

/* 8-way parallel cipher functions */
asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src);
asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src);

asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src);
asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
                                 const u8 *src, le128 *iv);

asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src, le128 *iv);
asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src, le128 *iv);

static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return twofish_setkey(&tfm->base, key, keylen); /*covered*/
}

static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __twofish_enc_blk_3way(ctx, dst, src, false); /*covered*/
}

static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(twofish_enc_blk));
}

static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, /*covered*/
                                  GLUE_FUNC_CAST(twofish_dec_blk));
}

struct twofish_xts_ctx {
        struct twofish_ctx tweak_ctx;
        struct twofish_ctx crypt_ctx;
};

static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
                              unsigned int keylen)
{
        struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        u32 *flags = &tfm->base.crt_flags; /*covered*/
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        /* first half of xts-key is for crypt */
        err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); /*covered*/
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, /*covered*/
                                flags);
}

static const struct common_glue_ctx twofish_enc = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
        } }
};

static const struct common_glue_ctx twofish_ctr = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
        } }
};

static const struct common_glue_ctx twofish_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
        } }
};

static const struct common_glue_ctx twofish_dec = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
        } }
};

static const struct common_glue_ctx twofish_dec_cbc = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
        } }
};

static const struct common_glue_ctx twofish_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&twofish_ctr, req); /*covered*/
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&twofish_enc_xts, req,
                                   XTS_TWEAK_CAST(twofish_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&twofish_dec_xts, req,
                                   XTS_TWEAK_CAST(twofish_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg twofish_algs[] = {
        {
                .base.cra_name                = "__ecb(twofish)",
                .base.cra_driver_name        = "__ecb-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(twofish)",
                .base.cra_driver_name        = "__cbc-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(twofish)",
                .base.cra_driver_name        = "__ctr-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .chunksize                = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(twofish)",
                .base.cra_driver_name        = "__xts-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * TF_MIN_KEY_SIZE,
                .max_keysize                = 2 * TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .setkey                        = xts_twofish_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];

static int __init twofish_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(twofish_algs,
                                              ARRAY_SIZE(twofish_algs),
                                              twofish_simd_algs);
}

static void __exit twofish_exit(void)
{
        simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
                                  twofish_simd_algs);
}

module_init(twofish_init);
module_exit(twofish_exit);

MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("twofish");

/*
 * Glue Code for assembler optimized version of TWOFISH
 *
 * Originally Twofish for GPG
 * By Matthew Skala <mskala@ansuz.sooke.bc.ca>, July 26, 1998
 * 256-bit key length added March 20, 1999
 * Some modifications to reduce the text size by Werner Koch, April, 1998
 * Ported to the kerneli patch by Marc Mutz <Marc@Mutz.com>
 * Ported to CryptoAPI by Colin Slater <hoho@tacomeat.net>
 *
 * The original author has disclaimed all copyright interest in this
 * code and thus put it in the public domain. The subsequent authors
 * have put this under the GNU General Public License.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 * This code is a "clean room" implementation, written from the paper
 * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey,
 * Doug Whiting, David Wagner, Chris Hall, and Niels Ferguson, available
 * through http://www.counterpane.com/twofish.html
 *
 * For background information on multiplication in finite fields, used for
 * the matrix operations in the key schedule, see the book _Contemporary
 * Abstract Algebra_ by Joseph A. Gallian, especially chapter 22 in the
 * Third Edition.
 */

#include <crypto/twofish.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
                                const u8 *src);
EXPORT_SYMBOL_GPL(twofish_enc_blk);
asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
                                const u8 *src);
EXPORT_SYMBOL_GPL(twofish_dec_blk);

static void twofish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        twofish_enc_blk(crypto_tfm_ctx(tfm), dst, src); /*covered*/
}

static void twofish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        twofish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static struct crypto_alg alg = {
        .cra_name                =        "twofish",
        .cra_driver_name        =        "twofish-asm",
        .cra_priority                =        200,
        .cra_flags                =        CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                =        TF_BLOCK_SIZE,
        .cra_ctxsize                =        sizeof(struct twofish_ctx),
        .cra_alignmask                =        0,
        .cra_module                =        THIS_MODULE,
        .cra_u                        =        {
                .cipher = {
                        .cia_min_keysize        =        TF_MIN_KEY_SIZE,
                        .cia_max_keysize        =        TF_MAX_KEY_SIZE,
                        .cia_setkey                =        twofish_setkey,
                        .cia_encrypt                =        twofish_encrypt,
                        .cia_decrypt                =        twofish_decrypt
                }
        }
};

static int __init init(void)
{
        return crypto_register_alg(&alg);
}

static void __exit fini(void)
{
        crypto_unregister_alg(&alg);
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION ("Twofish Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("twofish");
MODULE_ALIAS_CRYPTO("twofish-asm");

/*
 * Glue Code for 3-way parallel assembler optimized version of Twofish
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <asm/crypto/glue_helper.h>
#include <asm/crypto/twofish.h>
#include <crypto/algapi.h>
#include <crypto/b128ops.h>
#include <crypto/internal/skcipher.h>
#include <crypto/twofish.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);

static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return twofish_setkey(&tfm->base, key, keylen); /*covered*/
}

static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __twofish_enc_blk_3way(ctx, dst, src, false); /*covered*/
}

static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
                                            const u8 *src)
{
        __twofish_enc_blk_3way(ctx, dst, src, true);
}

void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
{
        u128 ivs[2];

        ivs[0] = src[0]; /*covered*/
        ivs[1] = src[1];

        twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);

        u128_xor(&dst[1], &dst[1], &ivs[0]);
        u128_xor(&dst[2], &dst[2], &ivs[1]);
}
EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);

void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk;

        if (dst != src) /*covered*/
                *dst = *src; /*covered*/

        le128_to_be128(&ctrblk, iv); /*covered*/
        le128_inc(iv); /*covered*/

        twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, dst, (u128 *)&ctrblk);
}
EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);

void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
                              le128 *iv)
{
        be128 ctrblks[3];

        if (dst != src) { /*covered*/
                dst[0] = src[0]; /*covered*/
                dst[1] = src[1];
                dst[2] = src[2];
        }

        le128_to_be128(&ctrblks[0], iv); /*covered*/
        le128_inc(iv);
        le128_to_be128(&ctrblks[1], iv); /*covered*/
        le128_inc(iv);
        le128_to_be128(&ctrblks[2], iv); /*covered*/
        le128_inc(iv); /*covered*/

        twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
}
EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);

static const struct common_glue_ctx twofish_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
        } }
};

static const struct common_glue_ctx twofish_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
        } }
};

static const struct common_glue_ctx twofish_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
        } }
};

static const struct common_glue_ctx twofish_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&twofish_ctr, req); /*covered*/
}

static struct skcipher_alg tf_skciphers[] = {
        {
                .base.cra_name                = "ecb(twofish)",
                .base.cra_driver_name        = "ecb-twofish-3way",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(twofish)",
                .base.cra_driver_name        = "cbc-twofish-3way",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(twofish)",
                .base.cra_driver_name        = "ctr-twofish-3way",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .chunksize                = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        },
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x06 &&
                (boot_cpu_data.x86_model == 0x1c ||
                 boot_cpu_data.x86_model == 0x26 ||
                 boot_cpu_data.x86_model == 0x36)) {
                /*
                 * On Atom, twofish-3way is slower than original assembler
                 * implementation. Twofish-3way trades off some performance in
                 * storing blocks in 64bit registers to allow three blocks to
                 * be processed parallel. Parallel operation then allows gaining
                 * more performance than was trade off, on out-of-order CPUs.
                 * However Atom does not benefit from this parallellism and
                 * should be blacklisted.
                 */
                return true;
        }

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, twofish-3way is slower than original assembler
                 * implementation because excessive uses of 64bit rotate and
                 * left-shifts (which are really slow on P4) needed to store and
                 * handle 128bit block in two 64bit registers.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init init(void)
{
        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "twofish-x86_64-3way: performance on this CPU "
                        "would be suboptimal: disabling "
                        "twofish-x86_64-3way.\n");
                return -ENODEV;
        }

        return crypto_register_skciphers(tf_skciphers,
                                         ARRAY_SIZE(tf_skciphers));
}

static void __exit fini(void)
{
        crypto_unregister_skciphers(tf_skciphers, ARRAY_SIZE(tf_skciphers));
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
MODULE_ALIAS_CRYPTO("twofish");
MODULE_ALIAS_CRYPTO("twofish-asm");

/*
 * common.c - C code for kernel entry and exit
 * Copyright (c) 2015 Andrew Lutomirski
 * GPL v2
 *
 * Based on asm and ptrace code by many authors.  The code here originated
 * in ptrace.c and signal.c.
 */

#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/tracehook.h>
#include <linux/audit.h>
#include <linux/seccomp.h>
#include <linux/signal.h>
#include <linux/export.h>
#include <linux/context_tracking.h>
#include <linux/user-return-notifier.h>
#include <linux/nospec.h>
#include <linux/uprobes.h>
#include <linux/livepatch.h>
#include <linux/syscalls.h>

#include <asm/desc.h>
#include <asm/traps.h>
#include <asm/vdso.h>
#include <linux/uaccess.h>
#include <asm/cpufeature.h>

#define CREATE_TRACE_POINTS
#include <trace/events/syscalls.h>

#ifdef CONFIG_CONTEXT_TRACKING
/* Called on entry from user mode with IRQs off. */
__visible inline void enter_from_user_mode(void)
{
        CT_WARN_ON(ct_state() != CONTEXT_USER);
        user_exit_irqoff();
}
#else
static inline void enter_from_user_mode(void) {}
#endif

static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)
{
#ifdef CONFIG_X86_64
        if (arch == AUDIT_ARCH_X86_64) {
                audit_syscall_entry(regs->orig_ax, regs->di,
                                    regs->si, regs->dx, regs->r10);
        } else
#endif
        {
                audit_syscall_entry(regs->orig_ax, regs->bx,
                                    regs->cx, regs->dx, regs->si);
        }
}

/*
 * Returns the syscall nr to run (which should match regs->orig_ax) or -1
 * to skip the syscall.
 */
static long syscall_trace_enter(struct pt_regs *regs)
{
        u32 arch = in_ia32_syscall() ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;

        struct thread_info *ti = current_thread_info();
        unsigned long ret = 0;
        bool emulated = false;
        u32 work;

        if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
                BUG_ON(regs != task_pt_regs(current));

        work = READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY;

        if (unlikely(work & _TIF_SYSCALL_EMU))
                emulated = true;

        if ((emulated || (work & _TIF_SYSCALL_TRACE)) &&
            tracehook_report_syscall_entry(regs))
                return -1L;

        if (emulated)
                return -1L;

#ifdef CONFIG_SECCOMP
        /*
         * Do seccomp after ptrace, to catch any tracer changes.
         */
        if (work & _TIF_SECCOMP) {
                struct seccomp_data sd;

                sd.arch = arch;
                sd.nr = regs->orig_ax;
                sd.instruction_pointer = regs->ip;
#ifdef CONFIG_X86_64
                if (arch == AUDIT_ARCH_X86_64) {
                        sd.args[0] = regs->di;
                        sd.args[1] = regs->si;
                        sd.args[2] = regs->dx;
                        sd.args[3] = regs->r10;
                        sd.args[4] = regs->r8;
                        sd.args[5] = regs->r9;
                } else
#endif
                {
                        sd.args[0] = regs->bx;
                        sd.args[1] = regs->cx;
                        sd.args[2] = regs->dx;
                        sd.args[3] = regs->si;
                        sd.args[4] = regs->di;
                        sd.args[5] = regs->bp;
                }

                ret = __secure_computing(&sd);
                if (ret == -1)
                        return ret;
        }
#endif

        if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
                trace_sys_enter(regs, regs->orig_ax);

        do_audit_syscall_entry(regs, arch);

        return ret ?: regs->orig_ax;
}

#define EXIT_TO_USERMODE_LOOP_FLAGS                                \
        (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |        \
         _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY | _TIF_PATCH_PENDING)

static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
{
        /*
         * In order to return to user mode, we need to have IRQs off with
         * none of EXIT_TO_USERMODE_LOOP_FLAGS set.  Several of these flags
         * can be set at any time on preemptible kernels if we have IRQs on,
         * so we need to loop.  Disabling preemption wouldn't help: doing the
         * work to clear some of the flags can sleep.
         */
        while (true) {
                /* We have work to do. */
                local_irq_enable(); /*covered*/

                if (cached_flags & _TIF_NEED_RESCHED)
                        schedule(); /*covered*/

                if (cached_flags & _TIF_UPROBE) /*covered*/
                        uprobe_notify_resume(regs);

                if (cached_flags & _TIF_PATCH_PENDING)
                        klp_update_patch_state(current); /*covered*/

                /* deal with pending signal delivery */
                if (cached_flags & _TIF_SIGPENDING) /*covered*/
                        do_signal(regs); /*covered*/

                if (cached_flags & _TIF_NOTIFY_RESUME) { /*covered*/
                        clear_thread_flag(TIF_NOTIFY_RESUME); /*covered*/
                        tracehook_notify_resume(regs); /*covered*/
                        rseq_handle_notify_resume(NULL, regs);
                }

                if (cached_flags & _TIF_USER_RETURN_NOTIFY) /*covered*/
                        fire_user_return_notifiers();

                /* Disable IRQs and retry */
                local_irq_disable(); /*covered*/

                cached_flags = READ_ONCE(current_thread_info()->flags);

                if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
                        break;
        }
} /*covered*/

/* Called with IRQs disabled. */
__visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
{
        struct thread_info *ti = current_thread_info(); /*covered*/
        u32 cached_flags;

        addr_limit_user_check(); /*covered*/

        lockdep_assert_irqs_disabled(); /*covered*/
        lockdep_sys_exit(); /*covered*/

        cached_flags = READ_ONCE(ti->flags);

        if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
                exit_to_usermode_loop(regs, cached_flags); /*covered*/

#ifdef CONFIG_COMPAT
        /*
         * Compat syscalls set TS_COMPAT.  Make sure we clear it before
         * returning to user mode.  We need to clear it *after* signal
         * handling, because syscall restart has a fixup for compat
         * syscalls.  The fixup is exercised by the ptrace_syscall_32
         * selftest.
         *
         * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
         * special case only applies after poking regs and before the
         * very next return to user mode.
         */
        ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED); /*covered*/
#endif

        user_enter_irqoff();
}

#define SYSCALL_EXIT_WORK_FLAGS                                \
        (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT |        \
         _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)

static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
{
        bool step;

        audit_syscall_exit(regs);

        if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
                trace_sys_exit(regs, regs->ax);

        /*
         * If TIF_SYSCALL_EMU is set, we only get here because of
         * TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
         * We already reported this syscall instruction in
         * syscall_trace_enter().
         */
        step = unlikely(
                (cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
                == _TIF_SINGLESTEP);
        if (step || cached_flags & _TIF_SYSCALL_TRACE)
                tracehook_report_syscall_exit(regs, step);
}

/*
 * Called with IRQs on and fully valid regs.  Returns with IRQs off in a
 * state such that we can immediately switch to user mode.
 */
__visible inline void syscall_return_slowpath(struct pt_regs *regs)
{
        struct thread_info *ti = current_thread_info(); /*covered*/
        u32 cached_flags = READ_ONCE(ti->flags);

        CT_WARN_ON(ct_state() != CONTEXT_KERNEL);

        if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
            WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax)) /*covered*/
                local_irq_enable();

        rseq_syscall(regs);

        /*
         * First do one-time work.  If these work items are enabled, we
         * want to run them exactly once per syscall exit with IRQs on.
         */
        if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS)) /*covered*/
                syscall_slow_exit_work(regs, cached_flags);

        local_irq_disable(); /*covered*/
        prepare_exit_to_usermode(regs); /*covered*/
}

#ifdef CONFIG_X86_64
__visible void do_syscall_64(unsigned long nr, struct pt_regs *regs)
{
        struct thread_info *ti;

        enter_from_user_mode();
        local_irq_enable(); /*covered*/
        ti = current_thread_info();
        if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)
                nr = syscall_trace_enter(regs);

        /*
         * NB: Native and x32 syscalls are dispatched from the same
         * table.  The only functional difference is the x32 bit in
         * regs->orig_ax, which changes the behavior of some syscalls.
         */
        nr &= __SYSCALL_MASK; /*covered*/
        if (likely(nr < NR_syscalls)) {
                nr = array_index_nospec(nr, NR_syscalls); /*covered*/
                regs->ax = sys_call_table[nr](regs);
        }

        syscall_return_slowpath(regs); /*covered*/
}
#endif

#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
/*
 * Does a 32-bit syscall.  Called with IRQs on in CONTEXT_KERNEL.  Does
 * all entry and exit work and returns with IRQs off.  This function is
 * extremely hot in workloads that use it, and it's usually called from
 * do_fast_syscall_32, so forcibly inline it to improve performance.
 */
static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
{
        struct thread_info *ti = current_thread_info();
        unsigned int nr = (unsigned int)regs->orig_ax;

#ifdef CONFIG_IA32_EMULATION
        ti->status |= TS_COMPAT;
#endif

        if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
                /*
                 * Subtlety here: if ptrace pokes something larger than
                 * 2^32-1 into orig_ax, this truncates it.  This may or
                 * may not be necessary, but it matches the old asm
                 * behavior.
                 */
                nr = syscall_trace_enter(regs);
        }

        if (likely(nr < IA32_NR_syscalls)) {
                nr = array_index_nospec(nr, IA32_NR_syscalls);
#ifdef CONFIG_IA32_EMULATION
                regs->ax = ia32_sys_call_table[nr](regs);
#else
                /*
                 * It's possible that a 32-bit syscall implementation
                 * takes a 64-bit parameter but nonetheless assumes that
                 * the high bits are zero.  Make sure we zero-extend all
                 * of the args.
                 */
                regs->ax = ia32_sys_call_table[nr](
                        (unsigned int)regs->bx, (unsigned int)regs->cx,
                        (unsigned int)regs->dx, (unsigned int)regs->si,
                        (unsigned int)regs->di, (unsigned int)regs->bp);
#endif /* CONFIG_IA32_EMULATION */
        }

        syscall_return_slowpath(regs);
}

/* Handles int $0x80 */
__visible void do_int80_syscall_32(struct pt_regs *regs)
{
        enter_from_user_mode();
        local_irq_enable();
        do_syscall_32_irqs_on(regs);
}

/* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */
__visible long do_fast_syscall_32(struct pt_regs *regs)
{
        /*
         * Called using the internal vDSO SYSENTER/SYSCALL32 calling
         * convention.  Adjust regs so it looks like we entered using int80.
         */

        unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
                vdso_image_32.sym_int80_landing_pad;

        /*
         * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
         * so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
         * Fix it up.
         */
        regs->ip = landing_pad;

        enter_from_user_mode();

        local_irq_enable();

        /* Fetch EBP from where the vDSO stashed it. */
        if (
#ifdef CONFIG_X86_64
                /*
                 * Micro-optimization: the pointer we're following is explicitly
                 * 32 bits, so it can't be out of range.
                 */
                __get_user(*(u32 *)&regs->bp,
                            (u32 __user __force *)(unsigned long)(u32)regs->sp)
#else
                get_user(*(u32 *)&regs->bp,
                         (u32 __user __force *)(unsigned long)(u32)regs->sp)
#endif
                ) {

                /* User code screwed up. */
                local_irq_disable();
                regs->ax = -EFAULT;
                prepare_exit_to_usermode(regs);
                return 0;        /* Keep it simple: use IRET. */
        }

        /* Now this is just like a normal syscall. */
        do_syscall_32_irqs_on(regs);

#ifdef CONFIG_X86_64
        /*
         * Opportunistic SYSRETL: if possible, try to return using SYSRETL.
         * SYSRETL is available on all 64-bit CPUs, so we don't need to
         * bother with SYSEXIT.
         *
         * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
         * because the ECX fixup above will ensure that this is essentially
         * never the case.
         */
        return regs->cs == __USER32_CS && regs->ss == __USER_DS &&
                regs->ip == landing_pad &&
                (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)) == 0;
#else
        /*
         * Opportunistic SYSEXIT: if possible, try to return using SYSEXIT.
         *
         * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
         * because the ECX fixup above will ensure that this is essentially
         * never the case.
         *
         * We don't allow syscalls at all from VM86 mode, but we still
         * need to check VM, because we might be returning from sys_vm86.
         */
        return static_cpu_has(X86_FEATURE_SEP) &&
                regs->cs == __USER_CS && regs->ss == __USER_DS &&
                regs->ip == landing_pad &&
                (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)) == 0;
#endif
}
#endif

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2012-2014 Andy Lutomirski <luto@amacapital.net>
 *
 * Based on the original implementation which is:
 *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
 *  Copyright 2003 Andi Kleen, SuSE Labs.
 *
 *  Parts of the original code have been moved to arch/x86/vdso/vma.c
 *
 * This file implements vsyscall emulation.  vsyscalls are a legacy ABI:
 * Userspace can request certain kernel services by calling fixed
 * addresses.  This concept is problematic:
 *
 * - It interferes with ASLR.
 * - It's awkward to write code that lives in kernel addresses but is
 *   callable by userspace at fixed addresses.
 * - The whole concept is impossible for 32-bit compat userspace.
 * - UML cannot easily virtualize a vsyscall.
 *
 * As of mid-2014, I believe that there is no new userspace code that
 * will use a vsyscall if the vDSO is present.  I hope that there will
 * soon be no new userspace code that will ever use a vsyscall.
 *
 * The code in this file emulates vsyscalls when notified of a page
 * fault to a vsyscall address.
 */

#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/sched/signal.h>
#include <linux/mm_types.h>
#include <linux/syscalls.h>
#include <linux/ratelimit.h>

#include <asm/vsyscall.h>
#include <asm/unistd.h>
#include <asm/fixmap.h>
#include <asm/traps.h>
#include <asm/paravirt.h>

#define CREATE_TRACE_POINTS
#include "vsyscall_trace.h"

static enum { EMULATE, NONE } vsyscall_mode =
#ifdef CONFIG_LEGACY_VSYSCALL_NONE
        NONE;
#else
        EMULATE;
#endif

static int __init vsyscall_setup(char *str)
{
        if (str) {
                if (!strcmp("emulate", str))
                        vsyscall_mode = EMULATE;
                else if (!strcmp("none", str))
                        vsyscall_mode = NONE;
                else
                        return -EINVAL;

                return 0;
        }

        return -EINVAL;
}
early_param("vsyscall", vsyscall_setup);

static void warn_bad_vsyscall(const char *level, struct pt_regs *regs,
                              const char *message)
{
        if (!show_unhandled_signals)
                return;

        printk_ratelimited("%s%s[%d] %s ip:%lx cs:%lx sp:%lx ax:%lx si:%lx di:%lx\n",
                           level, current->comm, task_pid_nr(current),
                           message, regs->ip, regs->cs,
                           regs->sp, regs->ax, regs->si, regs->di);
}

static int addr_to_vsyscall_nr(unsigned long addr)
{
        int nr;

        if ((addr & ~0xC00UL) != VSYSCALL_ADDR)
                return -EINVAL;

        nr = (addr & 0xC00UL) >> 10;
        if (nr >= 3)
                return -EINVAL;

        return nr;
}

static bool write_ok_or_segv(unsigned long ptr, size_t size)
{
        /*
         * XXX: if access_ok, get_user, and put_user handled
         * sig_on_uaccess_err, this could go away.
         */

        if (!access_ok((void __user *)ptr, size)) {
                struct thread_struct *thread = &current->thread;

                thread->error_code        = X86_PF_USER | X86_PF_WRITE;
                thread->cr2                = ptr;
                thread->trap_nr                = X86_TRAP_PF;

                force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)ptr, current);
                return false;
        } else {
                return true;
        }
}

bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
{
        struct task_struct *tsk;
        unsigned long caller;
        int vsyscall_nr, syscall_nr, tmp;
        int prev_sig_on_uaccess_err;
        long ret;
        unsigned long orig_dx;

        /*
         * No point in checking CS -- the only way to get here is a user mode
         * trap to a high address, which means that we're in 64-bit user code.
         */

        WARN_ON_ONCE(address != regs->ip);

        if (vsyscall_mode == NONE) {
                warn_bad_vsyscall(KERN_INFO, regs,
                                  "vsyscall attempted with vsyscall=none");
                return false;
        }

        vsyscall_nr = addr_to_vsyscall_nr(address);

        trace_emulate_vsyscall(vsyscall_nr);

        if (vsyscall_nr < 0) {
                warn_bad_vsyscall(KERN_WARNING, regs,
                                  "misaligned vsyscall (exploit attempt or buggy program) -- look up the vsyscall kernel parameter if you need a workaround");
                goto sigsegv;
        }

        if (get_user(caller, (unsigned long __user *)regs->sp) != 0) {
                warn_bad_vsyscall(KERN_WARNING, regs,
                                  "vsyscall with bad stack (exploit attempt?)");
                goto sigsegv;
        }

        tsk = current;

        /*
         * Check for access_ok violations and find the syscall nr.
         *
         * NULL is a valid user pointer (in the access_ok sense) on 32-bit and
         * 64-bit, so we don't need to special-case it here.  For all the
         * vsyscalls, NULL means "don't write anything" not "write it at
         * address 0".
         */
        switch (vsyscall_nr) {
        case 0:
                if (!write_ok_or_segv(regs->di, sizeof(struct timeval)) ||
                    !write_ok_or_segv(regs->si, sizeof(struct timezone))) {
                        ret = -EFAULT;
                        goto check_fault;
                }

                syscall_nr = __NR_gettimeofday;
                break;

        case 1:
                if (!write_ok_or_segv(regs->di, sizeof(time_t))) {
                        ret = -EFAULT;
                        goto check_fault;
                }

                syscall_nr = __NR_time;
                break;

        case 2:
                if (!write_ok_or_segv(regs->di, sizeof(unsigned)) ||
                    !write_ok_or_segv(regs->si, sizeof(unsigned))) {
                        ret = -EFAULT;
                        goto check_fault;
                }

                syscall_nr = __NR_getcpu;
                break;
        }

        /*
         * Handle seccomp.  regs->ip must be the original value.
         * See seccomp_send_sigsys and Documentation/userspace-api/seccomp_filter.rst.
         *
         * We could optimize the seccomp disabled case, but performance
         * here doesn't matter.
         */
        regs->orig_ax = syscall_nr;
        regs->ax = -ENOSYS;
        tmp = secure_computing(NULL);
        if ((!tmp && regs->orig_ax != syscall_nr) || regs->ip != address) {
                warn_bad_vsyscall(KERN_DEBUG, regs,
                                  "seccomp tried to change syscall nr or ip");
                do_exit(SIGSYS);
        }
        regs->orig_ax = -1;
        if (tmp)
                goto do_ret;  /* skip requested */

        /*
         * With a real vsyscall, page faults cause SIGSEGV.  We want to
         * preserve that behavior to make writing exploits harder.
         */
        prev_sig_on_uaccess_err = current->thread.sig_on_uaccess_err;
        current->thread.sig_on_uaccess_err = 1;

        ret = -EFAULT;
        switch (vsyscall_nr) {
        case 0:
                /* this decodes regs->di and regs->si on its own */
                ret = __x64_sys_gettimeofday(regs);
                break;

        case 1:
                /* this decodes regs->di on its own */
                ret = __x64_sys_time(regs);
                break;

        case 2:
                /* while we could clobber regs->dx, we didn't in the past... */
                orig_dx = regs->dx;
                regs->dx = 0;
                /* this decodes regs->di, regs->si and regs->dx on its own */
                ret = __x64_sys_getcpu(regs);
                regs->dx = orig_dx;
                break;
        }

        current->thread.sig_on_uaccess_err = prev_sig_on_uaccess_err;

check_fault:
        if (ret == -EFAULT) {
                /* Bad news -- userspace fed a bad pointer to a vsyscall. */
                warn_bad_vsyscall(KERN_INFO, regs,
                                  "vsyscall fault (exploit attempt?)");

                /*
                 * If we failed to generate a signal for any reason,
                 * generate one here.  (This should be impossible.)
                 */
                if (WARN_ON_ONCE(!sigismember(&tsk->pending.signal, SIGBUS) &&
                                 !sigismember(&tsk->pending.signal, SIGSEGV)))
                        goto sigsegv;

                return true;  /* Don't emulate the ret. */
        }

        regs->ax = ret;

do_ret:
        /* Emulate a ret instruction. */
        regs->ip = caller;
        regs->sp += 8;
        return true;

sigsegv:
        force_sig(SIGSEGV, current);
        return true;
}

/*
 * A pseudo VMA to allow ptrace access for the vsyscall page.  This only
 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
 * not need special handling anymore:
 */
static const char *gate_vma_name(struct vm_area_struct *vma)
{
        return "[vsyscall]";
}
static const struct vm_operations_struct gate_vma_ops = {
        .name = gate_vma_name,
};
static struct vm_area_struct gate_vma = {
        .vm_start        = VSYSCALL_ADDR,
        .vm_end                = VSYSCALL_ADDR + PAGE_SIZE,
        .vm_page_prot        = PAGE_READONLY_EXEC,
        .vm_flags        = VM_READ | VM_EXEC,
        .vm_ops                = &gate_vma_ops,
};

struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
#ifdef CONFIG_COMPAT
        if (!mm || mm->context.ia32_compat) /*covered*/
                return NULL;
#endif
        if (vsyscall_mode == NONE) /*covered*/
                return NULL;
        return &gate_vma;
}

int in_gate_area(struct mm_struct *mm, unsigned long addr)
{
        struct vm_area_struct *vma = get_gate_vma(mm); /*covered*/

        if (!vma)
                return 0;

        return (addr >= vma->vm_start) && (addr < vma->vm_end); /*covered*/
}

/*
 * Use this when you have no reliable mm, typically from interrupt
 * context. It is less reliable than using a task's mm and may give
 * false positives.
 */
int in_gate_area_no_mm(unsigned long addr)
{
        return vsyscall_mode != NONE && (addr & PAGE_MASK) == VSYSCALL_ADDR;
}

/*
 * The VSYSCALL page is the only user-accessible page in the kernel address
 * range.  Normally, the kernel page tables can have _PAGE_USER clear, but
 * the tables covering VSYSCALL_ADDR need _PAGE_USER set if vsyscalls
 * are enabled.
 *
 * Some day we may create a "minimal" vsyscall mode in which we emulate
 * vsyscalls but leave the page not present.  If so, we skip calling
 * this.
 */
void __init set_vsyscall_pgtable_user_bits(pgd_t *root)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;

        pgd = pgd_offset_pgd(root, VSYSCALL_ADDR);
        set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER));
        p4d = p4d_offset(pgd, VSYSCALL_ADDR);
#if CONFIG_PGTABLE_LEVELS >= 5
        set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER));
#endif
        pud = pud_offset(p4d, VSYSCALL_ADDR);
        set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER));
        pmd = pmd_offset(pud, VSYSCALL_ADDR);
        set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_USER));
}

void __init map_vsyscall(void)
{
        extern char __vsyscall_page;
        unsigned long physaddr_vsyscall = __pa_symbol(&__vsyscall_page);

        if (vsyscall_mode != NONE) {
                __set_fixmap(VSYSCALL_PAGE, physaddr_vsyscall,
                             PAGE_KERNEL_VVAR);
                set_vsyscall_pgtable_user_bits(swapper_pg_dir);
        }

        BUILD_BUG_ON((unsigned long)__fix_to_virt(VSYSCALL_PAGE) !=
                     (unsigned long)VSYSCALL_ADDR);
}

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_HWEIGHT_H
#define _ASM_X86_HWEIGHT_H

#include <asm/cpufeatures.h>

#ifdef CONFIG_64BIT
#define REG_IN "D"
#define REG_OUT "a"
#else
#define REG_IN "a"
#define REG_OUT "a"
#endif

#define __HAVE_ARCH_SW_HWEIGHT

static __always_inline unsigned int __arch_hweight32(unsigned int w)
{
        unsigned int res;

        asm (ALTERNATIVE("call __sw_hweight32", "popcntl %1, %0", X86_FEATURE_POPCNT)
                         : "="REG_OUT (res)
                         : REG_IN (w));

        return res;
}

static inline unsigned int __arch_hweight16(unsigned int w)
{
        return __arch_hweight32(w & 0xffff);
}

static inline unsigned int __arch_hweight8(unsigned int w)
{
        return __arch_hweight32(w & 0xff);
}

#ifdef CONFIG_X86_32
static inline unsigned long __arch_hweight64(__u64 w)
{
        return  __arch_hweight32((u32)w) +
                __arch_hweight32((u32)(w >> 32));
}
#else
static __always_inline unsigned long __arch_hweight64(__u64 w)
{
        unsigned long res;

        asm (ALTERNATIVE("call __sw_hweight64", "popcntq %1, %0", X86_FEATURE_POPCNT) /*covered*/
                         : "="REG_OUT (res)
                         : REG_IN (w));

        return res;
}
#endif /* CONFIG_X86_32 */

#endif

/*
 * This file is part of the Linux kernel.
 *
 * Copyright (c) 2011-2014, Intel Corporation
 * Authors: Fenghua Yu <fenghua.yu@intel.com>,
 *          H. Peter Anvin <hpa@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */

#ifndef ASM_X86_ARCHRANDOM_H
#define ASM_X86_ARCHRANDOM_H

#include <asm/processor.h>
#include <asm/cpufeature.h>

#define RDRAND_RETRY_LOOPS        10

#define RDRAND_INT        ".byte 0x0f,0xc7,0xf0"
#define RDSEED_INT        ".byte 0x0f,0xc7,0xf8"
#ifdef CONFIG_X86_64
# define RDRAND_LONG        ".byte 0x48,0x0f,0xc7,0xf0"
# define RDSEED_LONG        ".byte 0x48,0x0f,0xc7,0xf8"
#else
# define RDRAND_LONG        RDRAND_INT
# define RDSEED_LONG        RDSEED_INT
#endif

/* Unconditional execution of RDRAND and RDSEED */

static inline bool rdrand_long(unsigned long *v)
{
        bool ok;
        unsigned int retry = RDRAND_RETRY_LOOPS;
        do {
                asm volatile(RDRAND_LONG /*covered*/
                             CC_SET(c)
                             : CC_OUT(c) (ok), "=a" (*v));
                if (ok)
                        return true;
        } while (--retry);
        return false;
}

static inline bool rdrand_int(unsigned int *v)
{
        bool ok;
        unsigned int retry = RDRAND_RETRY_LOOPS;
        do {
                asm volatile(RDRAND_INT /*covered*/
                             CC_SET(c)
                             : CC_OUT(c) (ok), "=a" (*v));
                if (ok)
                        return true;
        } while (--retry);
        return false;
}

static inline bool rdseed_long(unsigned long *v)
{
        bool ok;
        asm volatile(RDSEED_LONG
                     CC_SET(c)
                     : CC_OUT(c) (ok), "=a" (*v));
        return ok;
}

static inline bool rdseed_int(unsigned int *v)
{
        bool ok;
        asm volatile(RDSEED_INT
                     CC_SET(c)
                     : CC_OUT(c) (ok), "=a" (*v));
        return ok;
}

/* Conditional execution based on CPU type */
#define arch_has_random()        static_cpu_has(X86_FEATURE_RDRAND)
#define arch_has_random_seed()        static_cpu_has(X86_FEATURE_RDSEED)

/*
 * These are the generic interfaces; they must not be declared if the
 * stubs in <linux/random.h> are to be invoked,
 * i.e. CONFIG_ARCH_RANDOM is not defined.
 */
#ifdef CONFIG_ARCH_RANDOM

static inline bool arch_get_random_long(unsigned long *v) /*covered*/
{
        return arch_has_random() ? rdrand_long(v) : false; /*covered*/
}

static inline bool arch_get_random_int(unsigned int *v) /*covered*/
{
        return arch_has_random() ? rdrand_int(v) : false; /*covered*/
}

static inline bool arch_get_random_seed_long(unsigned long *v)
{
        return arch_has_random_seed() ? rdseed_long(v) : false;
}

static inline bool arch_get_random_seed_int(unsigned int *v)
{
        return arch_has_random_seed() ? rdseed_int(v) : false;
}

extern void x86_init_rdrand(struct cpuinfo_x86 *c);

#else  /* !CONFIG_ARCH_RANDOM */

static inline void x86_init_rdrand(struct cpuinfo_x86 *c) { }

#endif  /* !CONFIG_ARCH_RANDOM */

#endif /* ASM_X86_ARCHRANDOM_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_ATOMIC_H
#define _ASM_X86_ATOMIC_H

#include <linux/compiler.h>
#include <linux/types.h>
#include <asm/alternative.h>
#include <asm/cmpxchg.h>
#include <asm/rmwcc.h>
#include <asm/barrier.h>

/*
 * Atomic operations that C can't guarantee us.  Useful for
 * resource counting etc..
 */

#define ATOMIC_INIT(i)        { (i) }

/**
 * arch_atomic_read - read atomic variable
 * @v: pointer of type atomic_t
 *
 * Atomically reads the value of @v.
 */
static __always_inline int arch_atomic_read(const atomic_t *v)
{
        /*
         * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here,
         * it's non-inlined function that increases binary size and stack usage.
         */
        return READ_ONCE((v)->counter); /*covered*/
}

/**
 * arch_atomic_set - set atomic variable
 * @v: pointer of type atomic_t
 * @i: required value
 *
 * Atomically sets the value of @v to @i.
 */
static __always_inline void arch_atomic_set(atomic_t *v, int i)
{
        WRITE_ONCE(v->counter, i); /*covered*/
}

/**
 * arch_atomic_add - add integer to atomic variable
 * @i: integer value to add
 * @v: pointer of type atomic_t
 *
 * Atomically adds @i to @v.
 */
static __always_inline void arch_atomic_add(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "addl %1,%0"
                     : "+m" (v->counter)
                     : "ir" (i));
}

/**
 * arch_atomic_sub - subtract integer from atomic variable
 * @i: integer value to subtract
 * @v: pointer of type atomic_t
 *
 * Atomically subtracts @i from @v.
 */
static __always_inline void arch_atomic_sub(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "subl %1,%0"
                     : "+m" (v->counter)
                     : "ir" (i));
}

/**
 * arch_atomic_sub_and_test - subtract value from variable and test result
 * @i: integer value to subtract
 * @v: pointer of type atomic_t
 *
 * Atomically subtracts @i from @v and returns
 * true if the result is zero, or false for all
 * other cases.
 */
static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, e, "er", i);
}
#define arch_atomic_sub_and_test arch_atomic_sub_and_test

/**
 * arch_atomic_inc - increment atomic variable
 * @v: pointer of type atomic_t
 *
 * Atomically increments @v by 1.
 */
static __always_inline void arch_atomic_inc(atomic_t *v)
{
        asm volatile(LOCK_PREFIX "incl %0" /*covered*/
                     : "+m" (v->counter));
}
#define arch_atomic_inc arch_atomic_inc

/**
 * arch_atomic_dec - decrement atomic variable
 * @v: pointer of type atomic_t
 *
 * Atomically decrements @v by 1.
 */
static __always_inline void arch_atomic_dec(atomic_t *v)
{
        asm volatile(LOCK_PREFIX "decl %0" /*covered*/
                     : "+m" (v->counter));
}
#define arch_atomic_dec arch_atomic_dec

/**
 * arch_atomic_dec_and_test - decrement and test
 * @v: pointer of type atomic_t
 *
 * Atomically decrements @v by 1 and
 * returns true if the result is 0, or false for all other
 * cases.
 */
static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
{
        return GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, e);
}
#define arch_atomic_dec_and_test arch_atomic_dec_and_test

/**
 * arch_atomic_inc_and_test - increment and test
 * @v: pointer of type atomic_t
 *
 * Atomically increments @v by 1
 * and returns true if the result is zero, or false for all
 * other cases.
 */
static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
{
        return GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, e);
}
#define arch_atomic_inc_and_test arch_atomic_inc_and_test

/**
 * arch_atomic_add_negative - add and test if negative
 * @i: integer value to add
 * @v: pointer of type atomic_t
 *
 * Atomically adds @i to @v and returns true
 * if the result is negative, or false when
 * result is greater than or equal to zero.
 */
static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, s, "er", i);
}
#define arch_atomic_add_negative arch_atomic_add_negative

/**
 * arch_atomic_add_return - add integer and return
 * @i: integer value to add
 * @v: pointer of type atomic_t
 *
 * Atomically adds @i to @v and returns @i + @v
 */
static __always_inline int arch_atomic_add_return(int i, atomic_t *v)
{
        return i + xadd(&v->counter, i);
}

/**
 * arch_atomic_sub_return - subtract integer and return
 * @v: pointer of type atomic_t
 * @i: integer value to subtract
 *
 * Atomically subtracts @i from @v and returns @v - @i
 */
static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)
{
        return arch_atomic_add_return(-i, v);
}

static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)
{
        return xadd(&v->counter, i);
}

static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v)
{
        return xadd(&v->counter, -i);
}

static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
{
        return arch_cmpxchg(&v->counter, old, new);
}

#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{
        return try_cmpxchg(&v->counter, old, new); /*covered*/
}

static inline int arch_atomic_xchg(atomic_t *v, int new)
{
        return arch_xchg(&v->counter, new);
}

static inline void arch_atomic_and(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "andl %1,%0"
                        : "+m" (v->counter)
                        : "ir" (i)
                        : "memory");
}

static inline int arch_atomic_fetch_and(int i, atomic_t *v)
{
        int val = arch_atomic_read(v);

        do { } while (!arch_atomic_try_cmpxchg(v, &val, val & i));

        return val;
}

static inline void arch_atomic_or(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "orl %1,%0"
                        : "+m" (v->counter)
                        : "ir" (i)
                        : "memory");
}

static inline int arch_atomic_fetch_or(int i, atomic_t *v)
{
        int val = arch_atomic_read(v);

        do { } while (!arch_atomic_try_cmpxchg(v, &val, val | i));

        return val;
}

static inline void arch_atomic_xor(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "xorl %1,%0"
                        : "+m" (v->counter)
                        : "ir" (i)
                        : "memory");
}

static inline int arch_atomic_fetch_xor(int i, atomic_t *v)
{
        int val = arch_atomic_read(v);

        do { } while (!arch_atomic_try_cmpxchg(v, &val, val ^ i));

        return val;
}

#ifdef CONFIG_X86_32
# include <asm/atomic64_32.h>
#else
# include <asm/atomic64_64.h>
#endif

#include <asm-generic/atomic-instrumented.h>

#endif /* _ASM_X86_ATOMIC_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_ATOMIC64_64_H
#define _ASM_X86_ATOMIC64_64_H

#include <linux/types.h>
#include <asm/alternative.h>
#include <asm/cmpxchg.h>

/* The 64-bit atomic type */

#define ATOMIC64_INIT(i)        { (i) }

/**
 * arch_atomic64_read - read atomic64 variable
 * @v: pointer of type atomic64_t
 *
 * Atomically reads the value of @v.
 * Doesn't imply a read memory barrier.
 */
static inline long arch_atomic64_read(const atomic64_t *v)
{
        return READ_ONCE((v)->counter); /*covered*/
}

/**
 * arch_atomic64_set - set atomic64 variable
 * @v: pointer to type atomic64_t
 * @i: required value
 *
 * Atomically sets the value of @v to @i.
 */
static inline void arch_atomic64_set(atomic64_t *v, long i)
{
        WRITE_ONCE(v->counter, i);
}

/**
 * arch_atomic64_add - add integer to atomic64 variable
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v.
 */
static __always_inline void arch_atomic64_add(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "addq %1,%0" /*covered*/
                     : "=m" (v->counter)
                     : "er" (i), "m" (v->counter));
}

/**
 * arch_atomic64_sub - subtract the atomic64 variable
 * @i: integer value to subtract
 * @v: pointer to type atomic64_t
 *
 * Atomically subtracts @i from @v.
 */
static inline void arch_atomic64_sub(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "subq %1,%0"
                     : "=m" (v->counter)
                     : "er" (i), "m" (v->counter));
}

/**
 * arch_atomic64_sub_and_test - subtract value from variable and test result
 * @i: integer value to subtract
 * @v: pointer to type atomic64_t
 *
 * Atomically subtracts @i from @v and returns
 * true if the result is zero, or false for all
 * other cases.
 */
static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, e, "er", i);
}
#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test

/**
 * arch_atomic64_inc - increment atomic64 variable
 * @v: pointer to type atomic64_t
 *
 * Atomically increments @v by 1.
 */
static __always_inline void arch_atomic64_inc(atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "incq %0"
                     : "=m" (v->counter)
                     : "m" (v->counter));
}
#define arch_atomic64_inc arch_atomic64_inc

/**
 * arch_atomic64_dec - decrement atomic64 variable
 * @v: pointer to type atomic64_t
 *
 * Atomically decrements @v by 1.
 */
static __always_inline void arch_atomic64_dec(atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "decq %0"
                     : "=m" (v->counter)
                     : "m" (v->counter));
}
#define arch_atomic64_dec arch_atomic64_dec

/**
 * arch_atomic64_dec_and_test - decrement and test
 * @v: pointer to type atomic64_t
 *
 * Atomically decrements @v by 1 and
 * returns true if the result is 0, or false for all other
 * cases.
 */
static inline bool arch_atomic64_dec_and_test(atomic64_t *v)
{
        return GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, e);
}
#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test

/**
 * arch_atomic64_inc_and_test - increment and test
 * @v: pointer to type atomic64_t
 *
 * Atomically increments @v by 1
 * and returns true if the result is zero, or false for all
 * other cases.
 */
static inline bool arch_atomic64_inc_and_test(atomic64_t *v)
{
        return GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, e);
}
#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test

/**
 * arch_atomic64_add_negative - add and test if negative
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v and returns true
 * if the result is negative, or false when
 * result is greater than or equal to zero.
 */
static inline bool arch_atomic64_add_negative(long i, atomic64_t *v)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, s, "er", i);
}
#define arch_atomic64_add_negative arch_atomic64_add_negative

/**
 * arch_atomic64_add_return - add and return
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v and returns @i + @v
 */
static __always_inline long arch_atomic64_add_return(long i, atomic64_t *v)
{
        return i + xadd(&v->counter, i);
}

static inline long arch_atomic64_sub_return(long i, atomic64_t *v)
{
        return arch_atomic64_add_return(-i, v);
}

static inline long arch_atomic64_fetch_add(long i, atomic64_t *v)
{
        return xadd(&v->counter, i);
}

static inline long arch_atomic64_fetch_sub(long i, atomic64_t *v)
{
        return xadd(&v->counter, -i);
}

static inline long arch_atomic64_cmpxchg(atomic64_t *v, long old, long new)
{
        return arch_cmpxchg(&v->counter, old, new);
}

#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, long new)
{
        return try_cmpxchg(&v->counter, old, new); /*covered*/
}

static inline long arch_atomic64_xchg(atomic64_t *v, long new)
{
        return arch_xchg(&v->counter, new);
}

static inline void arch_atomic64_and(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "andq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static inline long arch_atomic64_fetch_and(long i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
        return val;
}

static inline void arch_atomic64_or(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "orq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static inline long arch_atomic64_fetch_or(long i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
        return val;
}

static inline void arch_atomic64_xor(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "xorq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static inline long arch_atomic64_fetch_xor(long i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
        return val;
}

#endif /* _ASM_X86_ATOMIC64_64_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_BARRIER_H
#define _ASM_X86_BARRIER_H

#include <asm/alternative.h>
#include <asm/nops.h>

/*
 * Force strict CPU ordering.
 * And yes, this might be required on UP too when we're talking
 * to devices.
 */

#ifdef CONFIG_X86_32
#define mb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "mfence", \
                                      X86_FEATURE_XMM2) ::: "memory", "cc")
#define rmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "lfence", \
                                       X86_FEATURE_XMM2) ::: "memory", "cc")
#define wmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "sfence", \
                                       X86_FEATURE_XMM2) ::: "memory", "cc")
#else
#define mb()         asm volatile("mfence":::"memory")
#define rmb()        asm volatile("lfence":::"memory")
#define wmb()        asm volatile("sfence" ::: "memory")
#endif

/**
 * array_index_mask_nospec() - generate a mask that is ~0UL when the
 *         bounds check succeeds and 0 otherwise
 * @index: array element index
 * @size: number of elements in array
 *
 * Returns:
 *     0 - (index < size)
 */
static inline unsigned long array_index_mask_nospec(unsigned long index,
                unsigned long size)
{
        unsigned long mask;

        asm volatile ("cmp %1,%2; sbb %0,%0;" /*covered*/
                        :"=r" (mask)
                        :"g"(size),"r" (index)
                        :"cc");
        return mask;
}

/* Override the default implementation from linux/nospec.h. */
#define array_index_mask_nospec array_index_mask_nospec

/* Prevent speculative execution past this barrier. */
#define barrier_nospec() alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC, \
                                           "lfence", X86_FEATURE_LFENCE_RDTSC)

#define dma_rmb()        barrier()
#define dma_wmb()        barrier()

#ifdef CONFIG_X86_32
#define __smp_mb()        asm volatile("lock; addl $0,-4(%%esp)" ::: "memory", "cc")
#else
#define __smp_mb()        asm volatile("lock; addl $0,-4(%%rsp)" ::: "memory", "cc")
#endif
#define __smp_rmb()        dma_rmb()
#define __smp_wmb()        barrier()
#define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)

#define __smp_store_release(p, v)                                        \
do {                                                                        \
        compiletime_assert_atomic_type(*p);                                \
        barrier();                                                        \
        WRITE_ONCE(*p, v);                                                \
} while (0)

#define __smp_load_acquire(p)                                                \
({                                                                        \
        typeof(*p) ___p1 = READ_ONCE(*p);                                \
        compiletime_assert_atomic_type(*p);                                \
        barrier();                                                        \
        ___p1;                                                                \
})

/* Atomic operations are already serializing on x86 */
#define __smp_mb__before_atomic()        barrier()
#define __smp_mb__after_atomic()        barrier()

#include <asm-generic/barrier.h>

#endif /* _ASM_X86_BARRIER_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_BITOPS_H
#define _ASM_X86_BITOPS_H

/*
 * Copyright 1992, Linus Torvalds.
 *
 * Note: inlines with more than a single statement should be marked
 * __always_inline to avoid problems with older gcc's inlining heuristics.
 */

#ifndef _LINUX_BITOPS_H
#error only <linux/bitops.h> can be included directly
#endif

#include <linux/compiler.h>
#include <asm/alternative.h>
#include <asm/rmwcc.h>
#include <asm/barrier.h>

#if BITS_PER_LONG == 32
# define _BITOPS_LONG_SHIFT 5
#elif BITS_PER_LONG == 64
# define _BITOPS_LONG_SHIFT 6
#else
# error "Unexpected BITS_PER_LONG"
#endif

#define BIT_64(n)                        (U64_C(1) << (n))

/*
 * These have to be done with inline assembly: that way the bit-setting
 * is guaranteed to be atomic. All bit operations return 0 if the bit
 * was cleared before the operation and != 0 if it was not.
 *
 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
 */

#define RLONG_ADDR(x)                         "m" (*(volatile long *) (x))
#define WBYTE_ADDR(x)                        "+m" (*(volatile char *) (x))

#define ADDR                                RLONG_ADDR(addr)

/*
 * We do the locked ops that don't return the old value as
 * a mask operation on a byte.
 */
#define IS_IMMEDIATE(nr)                (__builtin_constant_p(nr))
#define CONST_MASK_ADDR(nr, addr)        WBYTE_ADDR((void *)(addr) + ((nr)>>3))
#define CONST_MASK(nr)                        (1 << ((nr) & 7))

/**
 * set_bit - Atomically set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * This function is atomic and may not be reordered.  See __set_bit()
 * if you do not require the atomic guarantees.
 *
 * Note: there are no guarantees that this function will not be reordered
 * on non x86 architectures, so if you are writing portable code,
 * make sure not to rely on its reordering guarantees.
 *
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static __always_inline void
set_bit(long nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "orb %1,%0" /*covered*/
                        : CONST_MASK_ADDR(nr, addr) /*covered*/
                        : "iq" ((u8)CONST_MASK(nr))
                        : "memory");
        } else {
                asm volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0" /*covered*/
                        : : RLONG_ADDR(addr), "Ir" (nr) : "memory");
        }
}

/**
 * __set_bit - Set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * Unlike set_bit(), this function is non-atomic and may be reordered.
 * If it's called on the same region of memory simultaneously, the effect
 * may be that only one operation succeeds.
 */
static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
{
        asm volatile(__ASM_SIZE(bts) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); /*covered*/
}

/**
 * clear_bit - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * clear_bit() is atomic and may not be reordered.  However, it does
 * not contain a memory barrier, so if it is used for locking purposes,
 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
 * in order to ensure changes are visible on other processors.
 */
static __always_inline void
clear_bit(long nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "andb %1,%0" /*covered*/
                        : CONST_MASK_ADDR(nr, addr) /*covered*/
                        : "iq" ((u8)~CONST_MASK(nr)));
        } else {
                asm volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0" /*covered*/
                        : : RLONG_ADDR(addr), "Ir" (nr) : "memory");
        }
}

/*
 * clear_bit_unlock - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * clear_bit() is atomic and implies release semantics before the memory
 * operation. It can be used for an unlock.
 */
static __always_inline void clear_bit_unlock(long nr, volatile unsigned long *addr)
{
        barrier(); /*covered*/
        clear_bit(nr, addr);
}

static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
{
        asm volatile(__ASM_SIZE(btr) " %1,%0" : : ADDR, "Ir" (nr) : "memory"); /*covered*/
}

static __always_inline bool clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
{
        bool negative;
        asm volatile(LOCK_PREFIX "andb %2,%1"
                CC_SET(s)
                : CC_OUT(s) (negative), WBYTE_ADDR(addr)
                : "ir" ((char) ~(1 << nr)) : "memory");
        return negative;
}

// Let everybody know we have it
#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte

/*
 * __clear_bit_unlock - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * __clear_bit() is non-atomic and implies release semantics before the memory
 * operation. It can be used for an unlock if no other CPUs can concurrently
 * modify other bits in the word.
 */
static __always_inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
{
        __clear_bit(nr, addr); /*covered*/
}

/**
 * __change_bit - Toggle a bit in memory
 * @nr: the bit to change
 * @addr: the address to start counting from
 *
 * Unlike change_bit(), this function is non-atomic and may be reordered.
 * If it's called on the same region of memory simultaneously, the effect
 * may be that only one operation succeeds.
 */
static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
{
        asm volatile(__ASM_SIZE(btc) " %1,%0" : : ADDR, "Ir" (nr) : "memory");
}

/**
 * change_bit - Toggle a bit in memory
 * @nr: Bit to change
 * @addr: Address to start counting from
 *
 * change_bit() is atomic and may not be reordered.
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static __always_inline void change_bit(long nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "xorb %1,%0"
                        : CONST_MASK_ADDR(nr, addr)
                        : "iq" ((u8)CONST_MASK(nr)));
        } else {
                asm volatile(LOCK_PREFIX __ASM_SIZE(btc) " %1,%0"
                        : : RLONG_ADDR(addr), "Ir" (nr) : "memory");
        }
}

/**
 * test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts), *addr, c, "Ir", nr); /*covered*/
}

/**
 * test_and_set_bit_lock - Set a bit and return its old value for lock
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This is the same as test_and_set_bit on x86.
 */
static __always_inline bool
test_and_set_bit_lock(long nr, volatile unsigned long *addr)
{
        return test_and_set_bit(nr, addr); /*covered*/
}

/**
 * __test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is non-atomic and can be reordered.
 * If two examples of this operation race, one can appear to succeed
 * but actually fail.  You must protect multiple accesses with a lock.
 */
static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
{
        bool oldbit;

        asm(__ASM_SIZE(bts) " %2,%1"
            CC_SET(c)
            : CC_OUT(c) (oldbit)
            : ADDR, "Ir" (nr) : "memory");
        return oldbit;
}

/**
 * test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr), *addr, c, "Ir", nr); /*covered*/
}

/**
 * __test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is non-atomic and can be reordered.
 * If two examples of this operation race, one can appear to succeed
 * but actually fail.  You must protect multiple accesses with a lock.
 *
 * Note: the operation is performed atomically with respect to
 * the local CPU, but not other CPUs. Portable code should not
 * rely on this behaviour.
 * KVM relies on this behaviour on x86 for modifying memory that is also
 * accessed from a hypervisor on the same CPU if running in a VM: don't change
 * this without also updating arch/x86/kernel/kvm.c
 */
static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
{
        bool oldbit;

        asm volatile(__ASM_SIZE(btr) " %2,%1"
                     CC_SET(c)
                     : CC_OUT(c) (oldbit)
                     : ADDR, "Ir" (nr) : "memory");
        return oldbit;
}

/* WARNING: non atomic and it can be reordered! */
static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
{
        bool oldbit;

        asm volatile(__ASM_SIZE(btc) " %2,%1"
                     CC_SET(c)
                     : CC_OUT(c) (oldbit)
                     : ADDR, "Ir" (nr) : "memory");

        return oldbit;
}

/**
 * test_and_change_bit - Change a bit and return its old value
 * @nr: Bit to change
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
{
        return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc), *addr, c, "Ir", nr);
}

static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
{
        return ((1UL << (nr & (BITS_PER_LONG-1))) &
                (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; /*covered*/
}

static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)
{
        bool oldbit;

        asm volatile(__ASM_SIZE(bt) " %2,%1" /*covered*/
                     CC_SET(c)
                     : CC_OUT(c) (oldbit)
                     : "m" (*(unsigned long *)addr), "Ir" (nr) : "memory");

        return oldbit; /*covered*/
}

#if 0 /* Fool kernel-doc since it doesn't do macros yet */
/**
 * test_bit - Determine whether a bit is set
 * @nr: bit number to test
 * @addr: Address to start counting from
 */
static bool test_bit(int nr, const volatile unsigned long *addr);
#endif

#define test_bit(nr, addr)                        \
        (__builtin_constant_p((nr))                \
         ? constant_test_bit((nr), (addr))        \
         : variable_test_bit((nr), (addr)))

/**
 * __ffs - find first set bit in word
 * @word: The word to search
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
static __always_inline unsigned long __ffs(unsigned long word)
{
        asm("rep; bsf %1,%0" /*covered*/
                : "=r" (word)
                : "rm" (word));
        return word;
}

/**
 * ffz - find first zero bit in word
 * @word: The word to search
 *
 * Undefined if no zero exists, so code should check against ~0UL first.
 */
static __always_inline unsigned long ffz(unsigned long word)
{
        asm("rep; bsf %1,%0"
                : "=r" (word)
                : "r" (~word)); /*covered*/
        return word;
}

/*
 * __fls: find last set bit in word
 * @word: The word to search
 *
 * Undefined if no set bit exists, so code should check against 0 first.
 */
static __always_inline unsigned long __fls(unsigned long word)
{
        asm("bsr %1,%0"
            : "=r" (word)
            : "rm" (word));
        return word;
}

#undef ADDR

#ifdef __KERNEL__
/**
 * ffs - find first set bit in word
 * @x: the word to search
 *
 * This is defined the same way as the libc and compiler builtin ffs
 * routines, therefore differs in spirit from the other bitops.
 *
 * ffs(value) returns 0 if value is 0 or the position of the first
 * set bit if value is nonzero. The first (least significant) bit
 * is at position 1.
 */
static __always_inline int ffs(int x)
{
        int r;

#ifdef CONFIG_X86_64
        /*
         * AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the
         * dest reg is undefined if x==0, but their CPU architect says its
         * value is written to set it to the same as before, except that the
         * top 32 bits will be cleared.
         *
         * We cannot do this on 32 bits because at the very least some
         * 486 CPUs did not behave this way.
         */
        asm("bsfl %1,%0"
            : "=r" (r)
            : "rm" (x), "0" (-1));
#elif defined(CONFIG_X86_CMOV)
        asm("bsfl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "r" (-1));
#else
        asm("bsfl %1,%0\n\t"
            "jnz 1f\n\t"
            "movl $-1,%0\n"
            "1:" : "=r" (r) : "rm" (x));
#endif
        return r + 1;
}

/**
 * fls - find last set bit in word
 * @x: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffs, but returns the position of the most significant set bit.
 *
 * fls(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 32.
 */
static __always_inline int fls(unsigned int x)
{
        int r;

#ifdef CONFIG_X86_64
        /*
         * AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the
         * dest reg is undefined if x==0, but their CPU architect says its
         * value is written to set it to the same as before, except that the
         * top 32 bits will be cleared.
         *
         * We cannot do this on 32 bits because at the very least some
         * 486 CPUs did not behave this way.
         */
        asm("bsrl %1,%0" /*covered*/
            : "=r" (r)
            : "rm" (x), "0" (-1));
#elif defined(CONFIG_X86_CMOV)
        asm("bsrl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "rm" (-1));
#else
        asm("bsrl %1,%0\n\t"
            "jnz 1f\n\t"
            "movl $-1,%0\n"
            "1:" : "=r" (r) : "rm" (x));
#endif
        return r + 1;
}

/**
 * fls64 - find last set bit in a 64-bit word
 * @x: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffsll, but returns the position of the most significant set bit.
 *
 * fls64(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 64.
 */
#ifdef CONFIG_X86_64
static __always_inline int fls64(__u64 x)
{
        int bitpos = -1;
        /*
         * AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the
         * dest reg is undefined if x==0, but their CPU architect says its
         * value is written to set it to the same as before.
         */
        asm("bsrq %1,%q0" /*covered*/
            : "+r" (bitpos)
            : "rm" (x));
        return bitpos + 1;
}
#else
#include <asm-generic/bitops/fls64.h>
#endif

#include <asm-generic/bitops/find.h>

#include <asm-generic/bitops/sched.h>

#include <asm/arch_hweight.h>

#include <asm-generic/bitops/const_hweight.h>

#include <asm-generic/bitops/le.h>

#include <asm-generic/bitops/ext2-atomic-setbit.h>

#endif /* __KERNEL__ */
#endif /* _ASM_X86_BITOPS_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CHECKSUM_64_H
#define _ASM_X86_CHECKSUM_64_H

/*
 * Checksums for x86-64
 * Copyright 2002 by Andi Kleen, SuSE Labs
 * with some code from asm-x86/checksum.h
 */

#include <linux/compiler.h>
#include <linux/uaccess.h>
#include <asm/byteorder.h>

/**
 * csum_fold - Fold and invert a 32bit checksum.
 * sum: 32bit unfolded sum
 *
 * Fold a 32bit running checksum to 16bit and invert it. This is usually
 * the last step before putting a checksum into a packet.
 * Make sure not to mix with 64bit checksums.
 */
static inline __sum16 csum_fold(__wsum sum)
{
        asm("  addl %1,%0\n"
            "  adcl $0xffff,%0"
            : "=r" (sum)
            : "r" ((__force u32)sum << 16), /*covered*/
              "0" ((__force u32)sum & 0xffff0000));
        return (__force __sum16)(~(__force u32)sum >> 16); /*covered*/
}

/*
 *        This is a version of ip_compute_csum() optimized for IP headers,
 *        which always checksum on 4 octet boundaries.
 *
 *        By Jorge Cwik <jorge@laser.satlink.net>, adapted for linux by
 *        Arnt Gulbrandsen.
 */

/**
 * ip_fast_csum - Compute the IPv4 header checksum efficiently.
 * iph: ipv4 header
 * ihl: length of header / 4
 */
static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
{
        unsigned int sum;

        asm("  movl (%1), %0\n" /*covered*/
            "  subl $4, %2\n"
            "  jbe 2f\n"
            "  addl 4(%1), %0\n"
            "  adcl 8(%1), %0\n"
            "  adcl 12(%1), %0\n"
            "1: adcl 16(%1), %0\n"
            "  lea 4(%1), %1\n"
            "  decl %2\n"
            "  jne        1b\n"
            "  adcl $0, %0\n"
            "  movl %0, %2\n"
            "  shrl $16, %0\n"
            "  addw %w2, %w0\n"
            "  adcl $0, %0\n"
            "  notl %0\n"
            "2:"
        /* Since the input registers which are loaded with iph and ihl
           are modified, we must also specify them as outputs, or gcc
           will assume they contain their original values. */
            : "=r" (sum), "=r" (iph), "=r" (ihl)
            : "1" (iph), "2" (ihl)
            : "memory");
        return (__force __sum16)sum;
}

/**
 * csum_tcpup_nofold - Compute an IPv4 pseudo header checksum.
 * @saddr: source address
 * @daddr: destination address
 * @len: length of packet
 * @proto: ip protocol of packet
 * @sum: initial sum to be added in (32bit unfolded)
 *
 * Returns the pseudo header checksum the input data. Result is
 * 32bit unfolded.
 */
static inline __wsum
csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len,
                   __u8 proto, __wsum sum)
{
        asm("  addl %1, %0\n"
            "  adcl %2, %0\n"
            "  adcl %3, %0\n"
            "  adcl $0, %0\n"
            : "=r" (sum)
            : "g" (daddr), "g" (saddr),
              "g" ((len + proto)<<8), "0" (sum));
        return sum; /*covered*/
}


/**
 * csum_tcpup_magic - Compute an IPv4 pseudo header checksum.
 * @saddr: source address
 * @daddr: destination address
 * @len: length of packet
 * @proto: ip protocol of packet
 * @sum: initial sum to be added in (32bit unfolded)
 *
 * Returns the 16bit pseudo header checksum the input data already
 * complemented and ready to be filled in.
 */
static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
                                        __u32 len, __u8 proto,
                                        __wsum sum)
{
        return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); /*covered*/
}

/**
 * csum_partial - Compute an internet checksum.
 * @buff: buffer to be checksummed
 * @len: length of buffer.
 * @sum: initial sum to be added in (32bit unfolded)
 *
 * Returns the 32bit unfolded internet checksum of the buffer.
 * Before filling it in it needs to be csum_fold()'ed.
 * buff should be aligned to a 64bit boundary if possible.
 */
extern __wsum csum_partial(const void *buff, int len, __wsum sum);

#define  _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1
#define HAVE_CSUM_COPY_USER 1


/* Do not call this directly. Use the wrappers below */
extern __visible __wsum csum_partial_copy_generic(const void *src, const void *dst,
                                        int len, __wsum sum,
                                        int *src_err_ptr, int *dst_err_ptr);


extern __wsum csum_partial_copy_from_user(const void __user *src, void *dst,
                                          int len, __wsum isum, int *errp);
extern __wsum csum_partial_copy_to_user(const void *src, void __user *dst,
                                        int len, __wsum isum, int *errp);
extern __wsum csum_partial_copy_nocheck(const void *src, void *dst,
                                        int len, __wsum sum);

/* Old names. To be removed. */
#define csum_and_copy_to_user csum_partial_copy_to_user
#define csum_and_copy_from_user csum_partial_copy_from_user

/**
 * ip_compute_csum - Compute an 16bit IP checksum.
 * @buff: buffer address.
 * @len: length of buffer.
 *
 * Returns the 16bit folded/inverted checksum of the passed buffer.
 * Ready to fill in.
 */
extern __sum16 ip_compute_csum(const void *buff, int len);

/**
 * csum_ipv6_magic - Compute checksum of an IPv6 pseudo header.
 * @saddr: source address
 * @daddr: destination address
 * @len: length of packet
 * @proto: protocol of packet
 * @sum: initial sum (32bit unfolded) to be added in
 *
 * Computes an IPv6 pseudo header checksum. This sum is added the checksum
 * into UDP/TCP packets and contains some link layer information.
 * Returns the unfolded 32bit checksum.
 */

struct in6_addr;

#define _HAVE_ARCH_IPV6_CSUM 1
extern __sum16
csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr,
                __u32 len, __u8 proto, __wsum sum);

static inline unsigned add32_with_carry(unsigned a, unsigned b)
{
        asm("addl %2,%0\n\t" /*covered*/
            "adcl $0,%0"
            : "=r" (a)
            : "0" (a), "rm" (b));
        return a;
}

#define HAVE_ARCH_CSUM_ADD
static inline __wsum csum_add(__wsum csum, __wsum addend)
{
        return (__force __wsum)add32_with_carry((__force unsigned)csum, /*covered*/
                                                (__force unsigned)addend);
}

#endif /* _ASM_X86_CHECKSUM_64_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_COMPAT_H
#define _ASM_X86_COMPAT_H

/*
 * Architecture specific compatibility types
 */
#include <linux/types.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <asm/processor.h>
#include <asm/user32.h>
#include <asm/unistd.h>

#include <asm-generic/compat.h>

#define COMPAT_USER_HZ                100
#define COMPAT_UTS_MACHINE        "i686\0\0"

typedef u16                __compat_uid_t;
typedef u16                __compat_gid_t;
typedef u32                __compat_uid32_t;
typedef u32                __compat_gid32_t;
typedef u16                compat_mode_t;
typedef u16                compat_dev_t;
typedef u16                compat_nlink_t;
typedef u16                compat_ipc_pid_t;
typedef u32                compat_caddr_t;
typedef __kernel_fsid_t        compat_fsid_t;
typedef s64 __attribute__((aligned(4))) compat_s64;
typedef u64 __attribute__((aligned(4))) compat_u64;

struct compat_stat {
        compat_dev_t        st_dev;
        u16                __pad1;
        compat_ino_t        st_ino;
        compat_mode_t        st_mode;
        compat_nlink_t        st_nlink;
        __compat_uid_t        st_uid;
        __compat_gid_t        st_gid;
        compat_dev_t        st_rdev;
        u16                __pad2;
        u32                st_size;
        u32                st_blksize;
        u32                st_blocks;
        u32                st_atime;
        u32                st_atime_nsec;
        u32                st_mtime;
        u32                st_mtime_nsec;
        u32                st_ctime;
        u32                st_ctime_nsec;
        u32                __unused4;
        u32                __unused5;
};

struct compat_flock {
        short                l_type;
        short                l_whence;
        compat_off_t        l_start;
        compat_off_t        l_len;
        compat_pid_t        l_pid;
};

#define F_GETLK64        12        /*  using 'struct flock64' */
#define F_SETLK64        13
#define F_SETLKW64        14

/*
 * IA32 uses 4 byte alignment for 64 bit quantities,
 * so we need to pack this structure.
 */
struct compat_flock64 {
        short                l_type;
        short                l_whence;
        compat_loff_t        l_start;
        compat_loff_t        l_len;
        compat_pid_t        l_pid;
} __attribute__((packed));

struct compat_statfs {
        int                f_type;
        int                f_bsize;
        int                f_blocks;
        int                f_bfree;
        int                f_bavail;
        int                f_files;
        int                f_ffree;
        compat_fsid_t        f_fsid;
        int                f_namelen;        /* SunOS ignores this field. */
        int                f_frsize;
        int                f_flags;
        int                f_spare[4];
};

#define COMPAT_RLIM_INFINITY                0xffffffff

typedef u32                compat_old_sigset_t;        /* at least 32 bits */

#define _COMPAT_NSIG                64
#define _COMPAT_NSIG_BPW        32

typedef u32               compat_sigset_word;

#define COMPAT_OFF_T_MAX        0x7fffffff

struct compat_ipc64_perm {
        compat_key_t key;
        __compat_uid32_t uid;
        __compat_gid32_t gid;
        __compat_uid32_t cuid;
        __compat_gid32_t cgid;
        unsigned short mode;
        unsigned short __pad1;
        unsigned short seq;
        unsigned short __pad2;
        compat_ulong_t unused1;
        compat_ulong_t unused2;
};

struct compat_semid64_ds {
        struct compat_ipc64_perm sem_perm;
        compat_ulong_t sem_otime;
        compat_ulong_t sem_otime_high;
        compat_ulong_t sem_ctime;
        compat_ulong_t sem_ctime_high;
        compat_ulong_t sem_nsems;
        compat_ulong_t __unused3;
        compat_ulong_t __unused4;
};

struct compat_msqid64_ds {
        struct compat_ipc64_perm msg_perm;
        compat_ulong_t msg_stime;
        compat_ulong_t msg_stime_high;
        compat_ulong_t msg_rtime;
        compat_ulong_t msg_rtime_high;
        compat_ulong_t msg_ctime;
        compat_ulong_t msg_ctime_high;
        compat_ulong_t msg_cbytes;
        compat_ulong_t msg_qnum;
        compat_ulong_t msg_qbytes;
        compat_pid_t   msg_lspid;
        compat_pid_t   msg_lrpid;
        compat_ulong_t __unused4;
        compat_ulong_t __unused5;
};

struct compat_shmid64_ds {
        struct compat_ipc64_perm shm_perm;
        compat_size_t  shm_segsz;
        compat_ulong_t shm_atime;
        compat_ulong_t shm_atime_high;
        compat_ulong_t shm_dtime;
        compat_ulong_t shm_dtime_high;
        compat_ulong_t shm_ctime;
        compat_ulong_t shm_ctime_high;
        compat_pid_t   shm_cpid;
        compat_pid_t   shm_lpid;
        compat_ulong_t shm_nattch;
        compat_ulong_t __unused4;
        compat_ulong_t __unused5;
};

/*
 * The type of struct elf_prstatus.pr_reg in compatible core dumps.
 */
typedef struct user_regs_struct compat_elf_gregset_t;

/* Full regset -- prstatus on x32, otherwise on ia32 */
#define PRSTATUS_SIZE(S, R) (R != sizeof(S.pr_reg) ? 144 : 296)
#define SET_PR_FPVALID(S, V, R) \
  do { *(int *) (((void *) &((S)->pr_reg)) + R) = (V); } \
  while (0)

#ifdef CONFIG_X86_X32_ABI
#define COMPAT_USE_64BIT_TIME \
        (!!(task_pt_regs(current)->orig_ax & __X32_SYSCALL_BIT))
#endif

/*
 * A pointer passed in from user mode. This should not
 * be used for syscall parameters, just declare them
 * as pointers because the syscall entry code will have
 * appropriately converted them already.
 */

static inline void __user *compat_ptr(compat_uptr_t uptr)
{
        return (void __user *)(unsigned long)uptr;
}

static inline compat_uptr_t ptr_to_compat(void __user *uptr)
{
        return (u32)(unsigned long)uptr;
}

static inline void __user *arch_compat_alloc_user_space(long len)
{
        compat_uptr_t sp;

        if (test_thread_flag(TIF_IA32)) {
                sp = task_pt_regs(current)->sp;
        } else {
                /* -128 for the x32 ABI redzone */
                sp = task_pt_regs(current)->sp - 128;
        }

        return (void __user *)round_down(sp - len, 16);
}

static inline bool in_x32_syscall(void)
{
#ifdef CONFIG_X86_X32_ABI
        if (task_pt_regs(current)->orig_ax & __X32_SYSCALL_BIT) /*covered*/
                return true;
#endif
        return false;
}

static inline bool in_32bit_syscall(void)
{
        return in_ia32_syscall() || in_x32_syscall(); /*covered*/
}

#ifdef CONFIG_COMPAT
static inline bool in_compat_syscall(void)
{
        return in_32bit_syscall(); /*covered*/
}
#define in_compat_syscall in_compat_syscall        /* override the generic impl */
#endif

struct compat_siginfo;
int __copy_siginfo_to_user32(struct compat_siginfo __user *to,
                const kernel_siginfo_t *from, bool x32_ABI);

#endif /* _ASM_X86_COMPAT_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CPUFEATURE_H
#define _ASM_X86_CPUFEATURE_H

#include <asm/processor.h>

#if defined(__KERNEL__) && !defined(__ASSEMBLY__)

#include <asm/asm.h>
#include <linux/bitops.h>

enum cpuid_leafs
{
        CPUID_1_EDX                = 0,
        CPUID_8000_0001_EDX,
        CPUID_8086_0001_EDX,
        CPUID_LNX_1,
        CPUID_1_ECX,
        CPUID_C000_0001_EDX,
        CPUID_8000_0001_ECX,
        CPUID_LNX_2,
        CPUID_LNX_3,
        CPUID_7_0_EBX,
        CPUID_D_1_EAX,
        CPUID_F_0_EDX,
        CPUID_F_1_EDX,
        CPUID_8000_0008_EBX,
        CPUID_6_EAX,
        CPUID_8000_000A_EDX,
        CPUID_7_ECX,
        CPUID_8000_0007_EBX,
        CPUID_7_EDX,
};

#ifdef CONFIG_X86_FEATURE_NAMES
extern const char * const x86_cap_flags[NCAPINTS*32];
extern const char * const x86_power_flags[32];
#define X86_CAP_FMT "%s"
#define x86_cap_flag(flag) x86_cap_flags[flag]
#else
#define X86_CAP_FMT "%d:%d"
#define x86_cap_flag(flag) ((flag) >> 5), ((flag) & 31)
#endif

/*
 * In order to save room, we index into this array by doing
 * X86_BUG_<name> - NCAPINTS*32.
 */
extern const char * const x86_bug_flags[NBUGINTS*32];

#define test_cpu_cap(c, bit)                                                \
         test_bit(bit, (unsigned long *)((c)->x86_capability))

/*
 * There are 32 bits/features in each mask word.  The high bits
 * (selected with (bit>>5) give us the word number and the low 5
 * bits give us the bit/feature number inside the word.
 * (1UL<<((bit)&31) gives us a mask for the feature_bit so we can
 * see if it is set in the mask word.
 */
#define CHECK_BIT_IN_MASK_WORD(maskname, word, bit)        \
        (((bit)>>5)==(word) && (1UL<<((bit)&31) & maskname##word ))

#define REQUIRED_MASK_BIT_SET(feature_bit)                \
         ( CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  0, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  1, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  2, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  3, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  4, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  5, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  6, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  7, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  8, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  9, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 10, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 11, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 12, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 13, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 14, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) ||        \
           REQUIRED_MASK_CHECK                                          ||        \
           BUILD_BUG_ON_ZERO(NCAPINTS != 19))

#define DISABLED_MASK_BIT_SET(feature_bit)                                \
         ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  1, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  2, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  3, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  4, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  5, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  6, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  7, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  8, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  9, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 10, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 11, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 12, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 13, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 14, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) ||        \
           DISABLED_MASK_CHECK                                          ||        \
           BUILD_BUG_ON_ZERO(NCAPINTS != 19))

#define cpu_has(c, bit)                                                        \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :        \
         test_cpu_cap(c, bit))

#define this_cpu_has(bit)                                                \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :        \
         x86_this_cpu_test_bit(bit,                                        \
                (unsigned long __percpu *)&cpu_info.x86_capability))

/*
 * This macro is for detection of features which need kernel
 * infrastructure to be used.  It may *not* directly test the CPU
 * itself.  Use the cpu_has() family if you want true runtime
 * testing of CPU features, like in hypervisor code where you are
 * supporting a possible guest feature where host support for it
 * is not relevant.
 */
#define cpu_feature_enabled(bit)        \
        (__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))

#define boot_cpu_has(bit)        cpu_has(&boot_cpu_data, bit)

#define set_cpu_cap(c, bit)        set_bit(bit, (unsigned long *)((c)->x86_capability))

extern void setup_clear_cpu_cap(unsigned int bit);
extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);

#define setup_force_cpu_cap(bit) do { \
        set_cpu_cap(&boot_cpu_data, bit);        \
        set_bit(bit, (unsigned long *)cpu_caps_set);        \
} while (0)

#define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)

#if defined(__clang__) && !defined(CONFIG_CC_HAS_ASM_GOTO)

/*
 * Workaround for the sake of BPF compilation which utilizes kernel
 * headers, but clang does not support ASM GOTO and fails the build.
 */
#ifndef __BPF_TRACING__
#warning "Compiler lacks ASM_GOTO support. Add -D __BPF_TRACING__ to your compiler arguments"
#endif

#define static_cpu_has(bit)            boot_cpu_has(bit)

#else

/*
 * Static testing of CPU features.  Used the same as boot_cpu_has().
 * These will statically patch the target code for additional
 * performance.
 */
static __always_inline __pure bool _static_cpu_has(u16 bit)
{
        asm_volatile_goto("1: jmp 6f\n" /*covered*/
                 "2:\n"
                 ".skip -(((5f-4f) - (2b-1b)) > 0) * "
                         "((5f-4f) - (2b-1b)),0x90\n"
                 "3:\n"
                 ".section .altinstructions,\"a\"\n"
                 " .long 1b - .\n"                /* src offset */
                 " .long 4f - .\n"                /* repl offset */
                 " .word %P[always]\n"                /* always replace */
                 " .byte 3b - 1b\n"                /* src len */
                 " .byte 5f - 4f\n"                /* repl len */
                 " .byte 3b - 2b\n"                /* pad len */
                 ".previous\n"
                 ".section .altinstr_replacement,\"ax\"\n"
                 "4: jmp %l[t_no]\n"
                 "5:\n"
                 ".previous\n"
                 ".section .altinstructions,\"a\"\n"
                 " .long 1b - .\n"                /* src offset */
                 " .long 0\n"                        /* no replacement */
                 " .word %P[feature]\n"                /* feature bit */
                 " .byte 3b - 1b\n"                /* src len */
                 " .byte 0\n"                        /* repl len */
                 " .byte 0\n"                        /* pad len */
                 ".previous\n"
                 ".section .altinstr_aux,\"ax\"\n"
                 "6:\n"
                 " testb %[bitnum],%[cap_byte]\n"
                 " jnz %l[t_yes]\n"
                 " jmp %l[t_no]\n"
                 ".previous\n"
                 : : [feature]  "i" (bit),
                     [always]   "i" (X86_FEATURE_ALWAYS),
                     [bitnum]   "i" (1 << (bit & 7)),
                     [cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3])
                 : : t_yes, t_no);
t_yes:
        return true;
t_no:
        return false;
}

#define static_cpu_has(bit)                                        \
(                                                                \
        __builtin_constant_p(boot_cpu_has(bit)) ?                \
                boot_cpu_has(bit) :                                \
                _static_cpu_has(bit)                                \
)
#endif

#define cpu_has_bug(c, bit)                cpu_has(c, (bit))
#define set_cpu_bug(c, bit)                set_cpu_cap(c, (bit))
#define clear_cpu_bug(c, bit)                clear_cpu_cap(c, (bit))

#define static_cpu_has_bug(bit)                static_cpu_has((bit))
#define boot_cpu_has_bug(bit)                cpu_has_bug(&boot_cpu_data, (bit))
#define boot_cpu_set_bug(bit)                set_cpu_cap(&boot_cpu_data, (bit))

#define MAX_CPU_FEATURES                (NCAPINTS * 32)
#define cpu_have_feature                boot_cpu_has

#define CPU_FEATURE_TYPEFMT                "x86,ven%04Xfam%04Xmod%04X"
#define CPU_FEATURE_TYPEVAL                boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
                                        boot_cpu_data.x86_model

#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
#endif /* _ASM_X86_CPUFEATURE_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ASM_X86_CAMELLIA_H
#define ASM_X86_CAMELLIA_H

#include <crypto/b128ops.h>
#include <linux/crypto.h>
#include <linux/kernel.h>

#define CAMELLIA_MIN_KEY_SIZE        16
#define CAMELLIA_MAX_KEY_SIZE        32
#define CAMELLIA_BLOCK_SIZE        16
#define CAMELLIA_TABLE_BYTE_LEN        272
#define CAMELLIA_PARALLEL_BLOCKS 2

struct crypto_skcipher;

struct camellia_ctx {
        u64 key_table[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
        u32 key_length;
};

struct camellia_xts_ctx {
        struct camellia_ctx tweak_ctx;
        struct camellia_ctx crypt_ctx;
};

extern int __camellia_setkey(struct camellia_ctx *cctx,
                             const unsigned char *key,
                             unsigned int key_len, u32 *flags);

extern int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
                               unsigned int keylen);

/* regular block cipher functions */
asmlinkage void __camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, bool xor);
asmlinkage void camellia_dec_blk(struct camellia_ctx *ctx, u8 *dst,
                                 const u8 *src);

/* 2-way parallel cipher functions */
asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                        const u8 *src, bool xor);
asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                      const u8 *src);

/* 16-way parallel cipher functions (avx/aes-ni) */
asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);

asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);

asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);
asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);

static inline void camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
                                    const u8 *src)
{
        __camellia_enc_blk(ctx, dst, src, false); /*covered*/
}

static inline void camellia_enc_blk_xor(struct camellia_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __camellia_enc_blk(ctx, dst, src, true);
}

static inline void camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        __camellia_enc_blk_2way(ctx, dst, src, false); /*covered*/
}

static inline void camellia_enc_blk_xor_2way(struct camellia_ctx *ctx, u8 *dst,
                                             const u8 *src)
{
        __camellia_enc_blk_2way(ctx, dst, src, true);
}

/* glue helpers */
extern void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src);
extern void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src,
                               le128 *iv);
extern void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
                                    le128 *iv);

extern void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv);
extern void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv);

#endif /* ASM_X86_CAMELLIA_H */

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Shared glue code for 128bit block ciphers
 */

#ifndef _CRYPTO_GLUE_HELPER_H
#define _CRYPTO_GLUE_HELPER_H

#include <crypto/internal/skcipher.h>
#include <linux/kernel.h>
#include <asm/fpu/api.h>
#include <crypto/b128ops.h>

typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src);
typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src);
typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src,
                                       le128 *iv);
typedef void (*common_glue_xts_func_t)(void *ctx, u128 *dst, const u128 *src,
                                       le128 *iv);

#define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn))
#define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn))
#define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn))
#define GLUE_XTS_FUNC_CAST(fn) ((common_glue_xts_func_t)(fn))

struct common_glue_func_entry {
        unsigned int num_blocks; /* number of blocks that @fn will process */
        union {
                common_glue_func_t ecb;
                common_glue_cbc_func_t cbc;
                common_glue_ctr_func_t ctr;
                common_glue_xts_func_t xts;
        } fn_u;
};

struct common_glue_ctx {
        unsigned int num_funcs;
        int fpu_blocks_limit; /* -1 means fpu not needed at all */

        /*
         * First funcs entry must have largest num_blocks and last funcs entry
         * must have num_blocks == 1!
         */
        struct common_glue_func_entry funcs[];
};

static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit, /*covered*/
                                  struct skcipher_walk *walk,
                                  bool fpu_enabled, unsigned int nbytes)
{
        if (likely(fpu_blocks_limit < 0))
                return false;

        if (fpu_enabled) /*covered*/
                return true;

        /*
         * Vector-registers are only used when chunk to be processed is large
         * enough, so do not enable FPU until it is necessary.
         */
        if (nbytes < bsize * (unsigned int)fpu_blocks_limit) /*covered*/
                return false;

        /* prevent sleeping if FPU is in use */
        skcipher_walk_atomise(walk); /*covered*/

        kernel_fpu_begin();
        return true;
}

static inline void glue_fpu_end(bool fpu_enabled)
{
        if (fpu_enabled) /*covered*/
                kernel_fpu_end(); /*covered*/
}

static inline void le128_to_be128(be128 *dst, const le128 *src)
{
        dst->a = cpu_to_be64(le64_to_cpu(src->a)); /*covered*/
        dst->b = cpu_to_be64(le64_to_cpu(src->b));
}

static inline void be128_to_le128(le128 *dst, const be128 *src)
{
        dst->a = cpu_to_le64(be64_to_cpu(src->a));
        dst->b = cpu_to_le64(be64_to_cpu(src->b));
}

static inline void le128_inc(le128 *i)
{
        u64 a = le64_to_cpu(i->a);
        u64 b = le64_to_cpu(i->b);

        b++;
        if (!b)
                a++;

        i->a = cpu_to_le64(a); /*covered*/
        i->b = cpu_to_le64(b);
}

extern int glue_ecb_req_128bit(const struct common_glue_ctx *gctx,
                               struct skcipher_request *req);

extern int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn,
                                       struct skcipher_request *req);

extern int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx,
                                       struct skcipher_request *req);

extern int glue_ctr_req_128bit(const struct common_glue_ctx *gctx,
                               struct skcipher_request *req);

extern int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
                               struct skcipher_request *req,
                               common_glue_func_t tweak_fn, void *tweak_ctx,
                               void *crypt_ctx);

extern void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src,
                                      le128 *iv, common_glue_func_t fn);

#endif /* _CRYPTO_GLUE_HELPER_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ASM_X86_SERPENT_SSE2_H
#define ASM_X86_SERPENT_SSE2_H

#include <linux/crypto.h>
#include <crypto/serpent.h>

#ifdef CONFIG_X86_32

#define SERPENT_PARALLEL_BLOCKS 4

asmlinkage void __serpent_enc_blk_4way(struct serpent_ctx *ctx, u8 *dst,
                                       const u8 *src, bool xor);
asmlinkage void serpent_dec_blk_4way(struct serpent_ctx *ctx, u8 *dst,
                                     const u8 *src);

static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __serpent_enc_blk_4way(ctx, dst, src, false);
}

static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst,
                                            const u8 *src)
{
        __serpent_enc_blk_4way(ctx, dst, src, true);
}

static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        serpent_dec_blk_4way(ctx, dst, src);
}

#else

#define SERPENT_PARALLEL_BLOCKS 8

asmlinkage void __serpent_enc_blk_8way(struct serpent_ctx *ctx, u8 *dst,
                                       const u8 *src, bool xor);
asmlinkage void serpent_dec_blk_8way(struct serpent_ctx *ctx, u8 *dst,
                                     const u8 *src);

static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                   const u8 *src)
{
        __serpent_enc_blk_8way(ctx, dst, src, false);
}

static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst,
                                       const u8 *src)
{
        __serpent_enc_blk_8way(ctx, dst, src, true);
}

static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                   const u8 *src)
{
        serpent_dec_blk_8way(ctx, dst, src); /*covered*/
}

#endif

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CURRENT_H
#define _ASM_X86_CURRENT_H

#include <linux/compiler.h>
#include <asm/percpu.h>

#ifndef __ASSEMBLY__
struct task_struct;

DECLARE_PER_CPU(struct task_struct *, current_task);

static __always_inline struct task_struct *get_current(void)
{
        return this_cpu_read_stable(current_task); /*covered*/
}

#define current get_current()

#endif /* __ASSEMBLY__ */

#endif /* _ASM_X86_CURRENT_H */

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 1994 Linus Torvalds
 *
 * Pentium III FXSR, SSE support
 * General FPU state handling cleanups
 *        Gareth Hughes <gareth@valinux.com>, May 2000
 * x86-64 work by Andi Kleen 2002
 */

#ifndef _ASM_X86_FPU_INTERNAL_H
#define _ASM_X86_FPU_INTERNAL_H

#include <linux/compat.h>
#include <linux/sched.h>
#include <linux/slab.h>

#include <asm/user.h>
#include <asm/fpu/api.h>
#include <asm/fpu/xstate.h>
#include <asm/cpufeature.h>
#include <asm/trace/fpu.h>

/*
 * High level FPU state handling functions:
 */
extern void fpu__initialize(struct fpu *fpu);
extern void fpu__prepare_read(struct fpu *fpu);
extern void fpu__prepare_write(struct fpu *fpu);
extern void fpu__save(struct fpu *fpu);
extern void fpu__restore(struct fpu *fpu);
extern int  fpu__restore_sig(void __user *buf, int ia32_frame);
extern void fpu__drop(struct fpu *fpu);
extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
extern void fpu__clear(struct fpu *fpu);
extern int  fpu__exception_code(struct fpu *fpu, int trap_nr);
extern int  dump_fpu(struct pt_regs *ptregs, struct user_i387_struct *fpstate);

/*
 * Boot time FPU initialization functions:
 */
extern void fpu__init_cpu(void);
extern void fpu__init_system_xstate(void);
extern void fpu__init_cpu_xstate(void);
extern void fpu__init_system(struct cpuinfo_x86 *c);
extern void fpu__init_check_bugs(void);
extern void fpu__resume_cpu(void);
extern u64 fpu__get_supported_xfeatures_mask(void);

/*
 * Debugging facility:
 */
#ifdef CONFIG_X86_DEBUG_FPU
# define WARN_ON_FPU(x) WARN_ON_ONCE(x)
#else
# define WARN_ON_FPU(x) ({ (void)(x); 0; })
#endif

/*
 * FPU related CPU feature flag helper routines:
 */
static __always_inline __pure bool use_xsaveopt(void)
{
        return static_cpu_has(X86_FEATURE_XSAVEOPT);
}

static __always_inline __pure bool use_xsave(void)
{
        return static_cpu_has(X86_FEATURE_XSAVE); /*covered*/
}

static __always_inline __pure bool use_fxsr(void)
{
        return static_cpu_has(X86_FEATURE_FXSR);
}

/*
 * fpstate handling functions:
 */

extern union fpregs_state init_fpstate;

extern void fpstate_init(union fpregs_state *state);
#ifdef CONFIG_MATH_EMULATION
extern void fpstate_init_soft(struct swregs_state *soft);
#else
static inline void fpstate_init_soft(struct swregs_state *soft) {}
#endif

static inline void fpstate_init_xstate(struct xregs_state *xsave)
{
        /*
         * XRSTORS requires these bits set in xcomp_bv, or it will
         * trigger #GP:
         */
        xsave->header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT | xfeatures_mask;
}

static inline void fpstate_init_fxstate(struct fxregs_state *fx)
{
        fx->cwd = 0x37f; /*covered*/
        fx->mxcsr = MXCSR_DEFAULT;
}
extern void fpstate_sanitize_xstate(struct fpu *fpu);

#define user_insn(insn, output, input...)                                \
({                                                                        \
        int err;                                                        \
                                                                        \
        might_fault();                                                        \
                                                                        \
        asm volatile(ASM_STAC "\n"                                        \
                     "1:" #insn "\n\t"                                        \
                     "2: " ASM_CLAC "\n"                                \
                     ".section .fixup,\"ax\"\n"                                \
                     "3:  movl $-1,%[err]\n"                                \
                     "    jmp  2b\n"                                        \
                     ".previous\n"                                        \
                     _ASM_EXTABLE(1b, 3b)                                \
                     : [err] "=r" (err), output                                \
                     : "0"(0), input);                                        \
        err;                                                                \
})

#define kernel_insn(insn, output, input...)                                \
        asm volatile("1:" #insn "\n\t"                                        \
                     "2:\n"                                                \
                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_fprestore)        \
                     : output : input)

static inline int copy_fregs_to_user(struct fregs_state __user *fx)
{
        return user_insn(fnsave %[fx]; fwait,  [fx] "=m" (*fx), "m" (*fx));
}

static inline int copy_fxregs_to_user(struct fxregs_state __user *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
        else
                return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));

}

static inline void copy_kernel_to_fxregs(struct fxregs_state *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
        else
                kernel_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline int copy_user_to_fxregs(struct fxregs_state __user *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
        else
                return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline void copy_kernel_to_fregs(struct fregs_state *fx)
{
        kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline int copy_user_to_fregs(struct fregs_state __user *fx)
{
        return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline void copy_fxregs_to_kernel(struct fpu *fpu)
{
        if (IS_ENABLED(CONFIG_X86_32))
                asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state.fxsave));
        else
                asm volatile("fxsaveq %[fx]" : [fx] "=m" (fpu->state.fxsave));
}

/* These macros all use (%edi)/(%rdi) as the single memory argument. */
#define XSAVE                ".byte " REX_PREFIX "0x0f,0xae,0x27"
#define XSAVEOPT        ".byte " REX_PREFIX "0x0f,0xae,0x37"
#define XSAVES                ".byte " REX_PREFIX "0x0f,0xc7,0x2f"
#define XRSTOR                ".byte " REX_PREFIX "0x0f,0xae,0x2f"
#define XRSTORS                ".byte " REX_PREFIX "0x0f,0xc7,0x1f"

#define XSTATE_OP(op, st, lmask, hmask, err)                                \
        asm volatile("1:" op "\n\t"                                        \
                     "xor %[err], %[err]\n"                                \
                     "2:\n\t"                                                \
                     ".pushsection .fixup,\"ax\"\n\t"                        \
                     "3: movl $-2,%[err]\n\t"                                \
                     "jmp 2b\n\t"                                        \
                     ".popsection\n\t"                                        \
                     _ASM_EXTABLE(1b, 3b)                                \
                     : [err] "=r" (err)                                        \
                     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)        \
                     : "memory")

/*
 * If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact
 * format and supervisor states in addition to modified optimization in
 * XSAVEOPT.
 *
 * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT
 * supports modified optimization which is not supported by XSAVE.
 *
 * We use XSAVE as a fallback.
 *
 * The 661 label is defined in the ALTERNATIVE* macros as the address of the
 * original instruction which gets replaced. We need to use it here as the
 * address of the instruction where we might get an exception at.
 */
#define XSTATE_XSAVE(st, lmask, hmask, err)                                \
        asm volatile(ALTERNATIVE_2(XSAVE,                                \
                                   XSAVEOPT, X86_FEATURE_XSAVEOPT,        \
                                   XSAVES,   X86_FEATURE_XSAVES)        \
                     "\n"                                                \
                     "xor %[err], %[err]\n"                                \
                     "3:\n"                                                \
                     ".pushsection .fixup,\"ax\"\n"                        \
                     "4: movl $-2, %[err]\n"                                \
                     "jmp 3b\n"                                                \
                     ".popsection\n"                                        \
                     _ASM_EXTABLE(661b, 4b)                                \
                     : [err] "=r" (err)                                        \
                     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)        \
                     : "memory")

/*
 * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact
 * XSAVE area format.
 */
#define XSTATE_XRESTORE(st, lmask, hmask)                                \
        asm volatile(ALTERNATIVE(XRSTOR,                                \
                                 XRSTORS, X86_FEATURE_XSAVES)                \
                     "\n"                                                \
                     "3:\n"                                                \
                     _ASM_EXTABLE_HANDLE(661b, 3b, ex_handler_fprestore)\
                     :                                                        \
                     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)        \
                     : "memory")

/*
 * This function is called only during boot time when x86 caps are not set
 * up and alternative can not be used yet.
 */
static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate)
{
        u64 mask = -1;
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        WARN_ON(system_state != SYSTEM_BOOTING);

        if (static_cpu_has(X86_FEATURE_XSAVES))
                XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
        else
                XSTATE_OP(XSAVE, xstate, lmask, hmask, err);

        /* We should never fault when copying to a kernel buffer: */
        WARN_ON_FPU(err);
}

/*
 * This function is called only during boot time when x86 caps are not set
 * up and alternative can not be used yet.
 */
static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate)
{
        u64 mask = -1;
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        WARN_ON(system_state != SYSTEM_BOOTING);

        if (static_cpu_has(X86_FEATURE_XSAVES))
                XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
        else
                XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);

        /*
         * We should never fault when copying from a kernel buffer, and the FPU
         * state we set at boot time should be valid.
         */
        WARN_ON_FPU(err);
}

/*
 * Save processor xstate to xsave area.
 */
static inline void copy_xregs_to_kernel(struct xregs_state *xstate)
{
        u64 mask = -1;
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        WARN_ON_FPU(!alternatives_patched);

        XSTATE_XSAVE(xstate, lmask, hmask, err); /*covered*/

        /* We should never fault when copying to a kernel buffer: */
        WARN_ON_FPU(err);
}

/*
 * Restore processor xstate from xsave area.
 */
static inline void copy_kernel_to_xregs(struct xregs_state *xstate, u64 mask)
{
        u32 lmask = mask;
        u32 hmask = mask >> 32;

        XSTATE_XRESTORE(xstate, lmask, hmask); /*covered*/
}

/*
 * Save xstate to user space xsave area.
 *
 * We don't use modified optimization because xrstor/xrstors might track
 * a different application.
 *
 * We don't use compacted format xsave area for
 * backward compatibility for old applications which don't understand
 * compacted format of xsave area.
 */
static inline int copy_xregs_to_user(struct xregs_state __user *buf)
{
        int err;

        /*
         * Clear the xsave header first, so that reserved fields are
         * initialized to zero.
         */
        err = __clear_user(&buf->header, sizeof(buf->header));
        if (unlikely(err))
                return -EFAULT;

        stac(); /*covered*/
        XSTATE_OP(XSAVE, buf, -1, -1, err);
        clac();

        return err;
}

/*
 * Restore xstate from user space xsave area.
 */
static inline int copy_user_to_xregs(struct xregs_state __user *buf, u64 mask)
{
        struct xregs_state *xstate = ((__force struct xregs_state *)buf);
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        stac();
        XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
        clac();

        return err;
}

/*
 * These must be called with preempt disabled. Returns
 * 'true' if the FPU state is still intact and we can
 * keep registers active.
 *
 * The legacy FNSAVE instruction cleared all FPU state
 * unconditionally, so registers are essentially destroyed.
 * Modern FPU state can be kept in registers, if there are
 * no pending FP exceptions.
 */
static inline int copy_fpregs_to_fpstate(struct fpu *fpu)
{
        if (likely(use_xsave())) { /*covered*/
                copy_xregs_to_kernel(&fpu->state.xsave); /*covered*/

                /*
                 * AVX512 state is tracked here because its use is
                 * known to slow the max clock speed of the core.
                 */
                if (fpu->state.xsave.header.xfeatures & XFEATURE_MASK_AVX512) /*covered*/
                        fpu->avx512_timestamp = jiffies;
                return 1;
        }

        if (likely(use_fxsr())) {
                copy_fxregs_to_kernel(fpu);
                return 1;
        }

        /*
         * Legacy FPU register saving, FNSAVE always clears FPU registers,
         * so we have to mark them inactive:
         */
        asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->state.fsave));

        return 0;
}

static inline void __copy_kernel_to_fpregs(union fpregs_state *fpstate, u64 mask)
{
        if (use_xsave()) { /*covered*/
                copy_kernel_to_xregs(&fpstate->xsave, mask); /*covered*/
        } else {
                if (use_fxsr())
                        copy_kernel_to_fxregs(&fpstate->fxsave);
                else
                        copy_kernel_to_fregs(&fpstate->fsave);
        }
}

static inline void copy_kernel_to_fpregs(union fpregs_state *fpstate)
{
        /*
         * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is
         * pending. Clear the x87 state here by setting it to fixed values.
         * "m" is a random variable that should be in L1.
         */
        if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) {
                asm volatile(
                        "fnclex\n\t"
                        "emms\n\t"
                        "fildl %P[addr]"        /* set F?P to defined value */
                        : : [addr] "m" (fpstate));
        }

        __copy_kernel_to_fpregs(fpstate, -1); /*covered*/
}

extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fp, int size);

/*
 * FPU context switch related helper methods:
 */

DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);

/*
 * The in-register FPU state for an FPU context on a CPU is assumed to be
 * valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx
 * matches the FPU.
 *
 * If the FPU register state is valid, the kernel can skip restoring the
 * FPU state from memory.
 *
 * Any code that clobbers the FPU registers or updates the in-memory
 * FPU state for a task MUST let the rest of the kernel know that the
 * FPU registers are no longer valid for this task.
 *
 * Either one of these invalidation functions is enough. Invalidate
 * a resource you control: CPU if using the CPU for something else
 * (with preemption disabled), FPU for the current task, or a task that
 * is prevented from running by the current task.
 */
static inline void __cpu_invalidate_fpregs_state(void)
{
        __this_cpu_write(fpu_fpregs_owner_ctx, NULL);
}

static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu)
{
        fpu->last_cpu = -1;
}

static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu)
{
        return fpu == this_cpu_read_stable(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
}

/*
 * These generally need preemption protection to work,
 * do try to avoid using these on their own:
 */
static inline void fpregs_deactivate(struct fpu *fpu)
{
        this_cpu_write(fpu_fpregs_owner_ctx, NULL);
        trace_x86_fpu_regs_deactivated(fpu); /*covered*/
}

static inline void fpregs_activate(struct fpu *fpu)
{
        this_cpu_write(fpu_fpregs_owner_ctx, fpu);
        trace_x86_fpu_regs_activated(fpu); /*covered*/
}

/*
 * FPU state switching for scheduling.
 *
 * This is a two-stage process:
 *
 *  - switch_fpu_prepare() saves the old state.
 *    This is done within the context of the old process.
 *
 *  - switch_fpu_finish() restores the new state as
 *    necessary.
 */
static inline void
switch_fpu_prepare(struct fpu *old_fpu, int cpu)
{
        if (static_cpu_has(X86_FEATURE_FPU) && old_fpu->initialized) {
                if (!copy_fpregs_to_fpstate(old_fpu))
                        old_fpu->last_cpu = -1;
                else
                        old_fpu->last_cpu = cpu;

                /* But leave fpu_fpregs_owner_ctx! */
                trace_x86_fpu_regs_deactivated(old_fpu);
        } else
                old_fpu->last_cpu = -1;
}

/*
 * Misc helper functions:
 */

/*
 * Set up the userspace FPU context for the new task, if the task
 * has used the FPU.
 */
static inline void switch_fpu_finish(struct fpu *new_fpu, int cpu)
{
        bool preload = static_cpu_has(X86_FEATURE_FPU) &&
                       new_fpu->initialized;

        if (preload) {
                if (!fpregs_state_valid(new_fpu, cpu))
                        copy_kernel_to_fpregs(&new_fpu->state);
                fpregs_activate(new_fpu);
        }
}

/*
 * Needs to be preemption-safe.
 *
 * NOTE! user_fpu_begin() must be used only immediately before restoring
 * the save state. It does not do any saving/restoring on its own. In
 * lazy FPU mode, it is just an optimization to avoid a #NM exception,
 * the task can lose the FPU right after preempt_enable().
 */
static inline void user_fpu_begin(void)
{
        struct fpu *fpu = &current->thread.fpu;

        preempt_disable();
        fpregs_activate(fpu); /*covered*/
        preempt_enable(); /*covered*/
}

/*
 * MXCSR and XCR definitions:
 */

extern unsigned int mxcsr_feature_mask;

#define XCR_XFEATURE_ENABLED_MASK        0x00000000

static inline u64 xgetbv(u32 index)
{
        u32 eax, edx;

        asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
                     : "=a" (eax), "=d" (edx)
                     : "c" (index));
        return eax + ((u64)edx << 32);
}

static inline void xsetbv(u32 index, u64 value)
{
        u32 eax = value;
        u32 edx = value >> 32;

        asm volatile(".byte 0x0f,0x01,0xd1" /* xsetbv */
                     : : "a" (eax), "d" (edx), "c" (index));
}

#endif /* _ASM_X86_FPU_INTERNAL_H */

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Per-cpu current frame pointer - the location of the last exception frame on
 * the stack, stored in the per-cpu area.
 *
 * Jeremy Fitzhardinge <jeremy@goop.org>
 */
#ifndef _ASM_X86_IRQ_REGS_H
#define _ASM_X86_IRQ_REGS_H

#include <asm/percpu.h>

#define ARCH_HAS_OWN_IRQ_REGS

DECLARE_PER_CPU(struct pt_regs *, irq_regs);

static inline struct pt_regs *get_irq_regs(void)
{
        return this_cpu_read(irq_regs); /*covered*/
}

static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
{
        struct pt_regs *old_regs;

        old_regs = get_irq_regs();
        this_cpu_write(irq_regs, new_regs);

        return old_regs;
}

#endif /* _ASM_X86_IRQ_REGS_32_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _X86_IRQFLAGS_H_
#define _X86_IRQFLAGS_H_

#include <asm/processor-flags.h>

#ifndef __ASSEMBLY__

/* Provide __cpuidle; we can't safely include <linux/cpu.h> */
#define __cpuidle __attribute__((__section__(".cpuidle.text")))

/*
 * Interrupt control:
 */

/* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */
extern inline unsigned long native_save_fl(void);
extern inline unsigned long native_save_fl(void)
{
        unsigned long flags;

        /*
         * "=rm" is safe here, because "pop" adjusts the stack before
         * it evaluates its effective address -- this is part of the
         * documented behavior of the "pop" instruction.
         */
        asm volatile("# __raw_save_flags\n\t"
                     "pushf ; pop %0"
                     : "=rm" (flags)
                     : /* no input */
                     : "memory");

        return flags;
}

extern inline void native_restore_fl(unsigned long flags);
extern inline void native_restore_fl(unsigned long flags)
{
        asm volatile("push %0 ; popf"
                     : /* no output */
                     :"g" (flags)
                     :"memory", "cc");
}

static inline void native_irq_disable(void)
{
        asm volatile("cli": : :"memory");
}

static inline void native_irq_enable(void)
{
        asm volatile("sti": : :"memory");
}

static inline __cpuidle void native_safe_halt(void)
{
        asm volatile("sti; hlt": : :"memory");
}

static inline __cpuidle void native_halt(void)
{
        asm volatile("hlt": : :"memory");
}

#endif

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else
#ifndef __ASSEMBLY__
#include <linux/types.h>

static inline notrace unsigned long arch_local_save_flags(void)
{
        return native_save_fl();
}

static inline notrace void arch_local_irq_restore(unsigned long flags)
{
        native_restore_fl(flags);
}

static inline notrace void arch_local_irq_disable(void)
{
        native_irq_disable();
}

static inline notrace void arch_local_irq_enable(void)
{
        native_irq_enable();
}

/*
 * Used in the idle loop; sti takes one instruction cycle
 * to complete:
 */
static inline __cpuidle void arch_safe_halt(void)
{
        native_safe_halt();
}

/*
 * Used when interrupts are already enabled or to
 * shutdown the processor:
 */
static inline __cpuidle void halt(void)
{
        native_halt();
}

/*
 * For spinlocks, etc:
 */
static inline notrace unsigned long arch_local_irq_save(void)
{
        unsigned long flags = arch_local_save_flags();
        arch_local_irq_disable();
        return flags;
}
#else

#define ENABLE_INTERRUPTS(x)        sti
#define DISABLE_INTERRUPTS(x)        cli

#ifdef CONFIG_X86_64
#ifdef CONFIG_DEBUG_ENTRY
#define SAVE_FLAGS(x)                pushfq; popq %rax
#endif

#define SWAPGS        swapgs
/*
 * Currently paravirt can't handle swapgs nicely when we
 * don't have a stack we can rely on (such as a user space
 * stack).  So we either find a way around these or just fault
 * and emulate if a guest tries to call swapgs directly.
 *
 * Either way, this is a good way to document that we don't
 * have a reliable stack. x86_64 only.
 */
#define SWAPGS_UNSAFE_STACK        swapgs

#define INTERRUPT_RETURN        jmp native_iret
#define USERGS_SYSRET64                                \
        swapgs;                                        \
        sysretq;
#define USERGS_SYSRET32                                \
        swapgs;                                        \
        sysretl

#else
#define INTERRUPT_RETURN                iret
#endif

#endif /* __ASSEMBLY__ */
#endif /* CONFIG_PARAVIRT_XXL */

#ifndef __ASSEMBLY__
static inline int arch_irqs_disabled_flags(unsigned long flags)
{
        return !(flags & X86_EFLAGS_IF); /*covered*/
}

static inline int arch_irqs_disabled(void)
{
        unsigned long flags = arch_local_save_flags();

        return arch_irqs_disabled_flags(flags);
}
#endif /* !__ASSEMBLY__ */

#ifdef __ASSEMBLY__
#ifdef CONFIG_TRACE_IRQFLAGS
#  define TRACE_IRQS_ON                call trace_hardirqs_on_thunk;
#  define TRACE_IRQS_OFF        call trace_hardirqs_off_thunk;
#else
#  define TRACE_IRQS_ON
#  define TRACE_IRQS_OFF
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
#  ifdef CONFIG_X86_64
#    define LOCKDEP_SYS_EXIT                call lockdep_sys_exit_thunk
#    define LOCKDEP_SYS_EXIT_IRQ \
        TRACE_IRQS_ON; \
        sti; \
        call lockdep_sys_exit_thunk; \
        cli; \
        TRACE_IRQS_OFF;
#  else
#    define LOCKDEP_SYS_EXIT \
        pushl %eax;                                \
        pushl %ecx;                                \
        pushl %edx;                                \
        call lockdep_sys_exit;                        \
        popl %edx;                                \
        popl %ecx;                                \
        popl %eax;
#    define LOCKDEP_SYS_EXIT_IRQ
#  endif
#else
#  define LOCKDEP_SYS_EXIT
#  define LOCKDEP_SYS_EXIT_IRQ
#endif
#endif /* __ASSEMBLY__ */

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_JUMP_LABEL_H
#define _ASM_X86_JUMP_LABEL_H

#define JUMP_LABEL_NOP_SIZE 5

#ifdef CONFIG_X86_64
# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
#else
# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
#endif

#include <asm/asm.h>
#include <asm/nops.h>

#ifndef __ASSEMBLY__

#include <linux/stringify.h>
#include <linux/types.h>

static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{
        asm_volatile_goto("1:" /*covered*/
                ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
                ".pushsection __jump_table,  \"aw\" \n\t"
                _ASM_ALIGN "\n\t"
                ".long 1b - ., %l[l_yes] - . \n\t"
                _ASM_PTR "%c0 + %c1 - .\n\t"
                ".popsection \n\t"
                : :  "i" (key), "i" (branch) : : l_yes);

        return false;
l_yes:
        return true;
}

static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
        asm_volatile_goto("1:" /*covered*/
                ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
                "2:\n\t"
                ".pushsection __jump_table,  \"aw\" \n\t"
                _ASM_ALIGN "\n\t"
                ".long 1b - ., %l[l_yes] - . \n\t"
                _ASM_PTR "%c0 + %c1 - .\n\t"
                ".popsection \n\t"
                : :  "i" (key), "i" (branch) : : l_yes);

        return false;
l_yes:
        return true;
}

#else        /* __ASSEMBLY__ */

.macro STATIC_JUMP_IF_TRUE target, key, def
.Lstatic_jump_\@:
        .if \def
        /* Equivalent to "jmp.d32 \target" */
        .byte                0xe9
        .long                \target - .Lstatic_jump_after_\@
.Lstatic_jump_after_\@:
        .else
        .byte                STATIC_KEY_INIT_NOP
        .endif
        .pushsection __jump_table, "aw"
        _ASM_ALIGN
        .long                .Lstatic_jump_\@ - ., \target - .
        _ASM_PTR        \key - .
        .popsection
.endm

.macro STATIC_JUMP_IF_FALSE target, key, def
.Lstatic_jump_\@:
        .if \def
        .byte                STATIC_KEY_INIT_NOP
        .else
        /* Equivalent to "jmp.d32 \target" */
        .byte                0xe9
        .long                \target - .Lstatic_jump_after_\@
.Lstatic_jump_after_\@:
        .endif
        .pushsection __jump_table, "aw"
        _ASM_ALIGN
        .long                .Lstatic_jump_\@ - ., \target - .
        _ASM_PTR        \key + 1 - .
        .popsection
.endm

#endif        /* __ASSEMBLY__ */

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_MMU_CONTEXT_H
#define _ASM_X86_MMU_CONTEXT_H

#include <asm/desc.h>
#include <linux/atomic.h>
#include <linux/mm_types.h>
#include <linux/pkeys.h>

#include <trace/events/tlb.h>

#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm/paravirt.h>
#include <asm/mpx.h>

extern atomic64_t last_mm_ctx_id;

#ifndef CONFIG_PARAVIRT_XXL
static inline void paravirt_activate_mm(struct mm_struct *prev,
                                        struct mm_struct *next)
{
}
#endif        /* !CONFIG_PARAVIRT_XXL */

#ifdef CONFIG_PERF_EVENTS

DECLARE_STATIC_KEY_FALSE(rdpmc_always_available_key);

static inline void load_mm_cr4(struct mm_struct *mm)
{
        if (static_branch_unlikely(&rdpmc_always_available_key) ||
            atomic_read(&mm->context.perf_rdpmc_allowed))
                cr4_set_bits(X86_CR4_PCE);
        else
                cr4_clear_bits(X86_CR4_PCE);
}
#else
static inline void load_mm_cr4(struct mm_struct *mm) {}
#endif

#ifdef CONFIG_MODIFY_LDT_SYSCALL
/*
 * ldt_structs can be allocated, used, and freed, but they are never
 * modified while live.
 */
struct ldt_struct {
        /*
         * Xen requires page-aligned LDTs with special permissions.  This is
         * needed to prevent us from installing evil descriptors such as
         * call gates.  On native, we could merge the ldt_struct and LDT
         * allocations, but it's not worth trying to optimize.
         */
        struct desc_struct        *entries;
        unsigned int                nr_entries;

        /*
         * If PTI is in use, then the entries array is not mapped while we're
         * in user mode.  The whole array will be aliased at the addressed
         * given by ldt_slot_va(slot).  We use two slots so that we can allocate
         * and map, and enable a new LDT without invalidating the mapping
         * of an older, still-in-use LDT.
         *
         * slot will be -1 if this LDT doesn't have an alias mapping.
         */
        int                        slot;
};

/* This is a multiple of PAGE_SIZE. */
#define LDT_SLOT_STRIDE (LDT_ENTRIES * LDT_ENTRY_SIZE)

static inline void *ldt_slot_va(int slot)
{
        return (void *)(LDT_BASE_ADDR + LDT_SLOT_STRIDE * slot);
}

/*
 * Used for LDT copy/destruction.
 */
static inline void init_new_context_ldt(struct mm_struct *mm)
{
        mm->context.ldt = NULL;
        init_rwsem(&mm->context.ldt_usr_sem);
}
int ldt_dup_context(struct mm_struct *oldmm, struct mm_struct *mm);
void destroy_context_ldt(struct mm_struct *mm);
void ldt_arch_exit_mmap(struct mm_struct *mm);
#else        /* CONFIG_MODIFY_LDT_SYSCALL */
static inline void init_new_context_ldt(struct mm_struct *mm) { }
static inline int ldt_dup_context(struct mm_struct *oldmm,
                                  struct mm_struct *mm)
{
        return 0;
}
static inline void destroy_context_ldt(struct mm_struct *mm) { }
static inline void ldt_arch_exit_mmap(struct mm_struct *mm) { }
#endif

static inline void load_mm_ldt(struct mm_struct *mm)
{
#ifdef CONFIG_MODIFY_LDT_SYSCALL
        struct ldt_struct *ldt;

        /* READ_ONCE synchronizes with smp_store_release */
        ldt = READ_ONCE(mm->context.ldt);

        /*
         * Any change to mm->context.ldt is followed by an IPI to all
         * CPUs with the mm active.  The LDT will not be freed until
         * after the IPI is handled by all such CPUs.  This means that,
         * if the ldt_struct changes before we return, the values we see
         * will be safe, and the new values will be loaded before we run
         * any user code.
         *
         * NB: don't try to convert this to use RCU without extreme care.
         * We would still need IRQs off, because we don't want to change
         * the local LDT after an IPI loaded a newer value than the one
         * that we can see.
         */

        if (unlikely(ldt)) {
                if (static_cpu_has(X86_FEATURE_PTI)) {
                        if (WARN_ON_ONCE((unsigned long)ldt->slot > 1)) {
                                /*
                                 * Whoops -- either the new LDT isn't mapped
                                 * (if slot == -1) or is mapped into a bogus
                                 * slot (if slot > 1).
                                 */
                                clear_LDT();
                                return;
                        }

                        /*
                         * If page table isolation is enabled, ldt->entries
                         * will not be mapped in the userspace pagetables.
                         * Tell the CPU to access the LDT through the alias
                         * at ldt_slot_va(ldt->slot).
                         */
                        set_ldt(ldt_slot_va(ldt->slot), ldt->nr_entries);
                } else {
                        set_ldt(ldt->entries, ldt->nr_entries);
                }
        } else {
                clear_LDT();
        }
#else
        clear_LDT();
#endif
}

static inline void switch_ldt(struct mm_struct *prev, struct mm_struct *next)
{
#ifdef CONFIG_MODIFY_LDT_SYSCALL
        /*
         * Load the LDT if either the old or new mm had an LDT.
         *
         * An mm will never go from having an LDT to not having an LDT.  Two
         * mms never share an LDT, so we don't gain anything by checking to
         * see whether the LDT changed.  There's also no guarantee that
         * prev->context.ldt actually matches LDTR, but, if LDTR is non-NULL,
         * then prev->context.ldt will also be non-NULL.
         *
         * If we really cared, we could optimize the case where prev == next
         * and we're exiting lazy mode.  Most of the time, if this happens,
         * we don't actually need to reload LDTR, but modify_ldt() is mostly
         * used by legacy code and emulators where we don't need this level of
         * performance.
         *
         * This uses | instead of || because it generates better code.
         */
        if (unlikely((unsigned long)prev->context.ldt |
                     (unsigned long)next->context.ldt))
                load_mm_ldt(next);
#endif

        DEBUG_LOCKS_WARN_ON(preemptible());
}

void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);

/*
 * Init a new mm.  Used on mm copies, like at fork()
 * and on mm's that are brand-new, like at execve().
 */
static inline int init_new_context(struct task_struct *tsk,
                                   struct mm_struct *mm)
{
        mutex_init(&mm->context.lock);

        mm->context.ctx_id = atomic64_inc_return(&last_mm_ctx_id);
        atomic64_set(&mm->context.tlb_gen, 0);

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
        if (cpu_feature_enabled(X86_FEATURE_OSPKE)) {
                /* pkey 0 is the default and allocated implicitly */
                mm->context.pkey_allocation_map = 0x1;
                /* -1 means unallocated or invalid */
                mm->context.execute_only_pkey = -1;
        }
#endif
        init_new_context_ldt(mm);
        return 0;
}
static inline void destroy_context(struct mm_struct *mm)
{
        destroy_context_ldt(mm);
}

extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                      struct task_struct *tsk);

extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
                               struct task_struct *tsk);
#define switch_mm_irqs_off switch_mm_irqs_off

#define activate_mm(prev, next)                        \
do {                                                \
        paravirt_activate_mm((prev), (next));        \
        switch_mm((prev), (next), NULL);        \
} while (0);

#ifdef CONFIG_X86_32
#define deactivate_mm(tsk, mm)                        \
do {                                                \
        lazy_load_gs(0);                        \
} while (0)
#else
#define deactivate_mm(tsk, mm)                        \
do {                                                \
        load_gs_index(0);                        \
        loadsegment(fs, 0);                        \
} while (0)
#endif

static inline void arch_dup_pkeys(struct mm_struct *oldmm,
                                  struct mm_struct *mm)
{
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
        if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
                return;

        /* Duplicate the oldmm pkey state in mm: */
        mm->context.pkey_allocation_map = oldmm->context.pkey_allocation_map;
        mm->context.execute_only_pkey   = oldmm->context.execute_only_pkey;
#endif
}

static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
{
        arch_dup_pkeys(oldmm, mm);
        paravirt_arch_dup_mmap(oldmm, mm);
        return ldt_dup_context(oldmm, mm);
}

static inline void arch_exit_mmap(struct mm_struct *mm)
{
        paravirt_arch_exit_mmap(mm);
        ldt_arch_exit_mmap(mm);
}

#ifdef CONFIG_X86_64
static inline bool is_64bit_mm(struct mm_struct *mm)
{
        return        !IS_ENABLED(CONFIG_IA32_EMULATION) ||
                !(mm->context.ia32_compat == TIF_IA32);
}
#else
static inline bool is_64bit_mm(struct mm_struct *mm)
{
        return false;
}
#endif

static inline void arch_bprm_mm_init(struct mm_struct *mm,
                struct vm_area_struct *vma)
{
        mpx_mm_init(mm);
}

static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
                              unsigned long start, unsigned long end)
{
        /*
         * mpx_notify_unmap() goes and reads a rarely-hot
         * cacheline in the mm_struct.  That can be expensive
         * enough to be seen in profiles.
         *
         * The mpx_notify_unmap() call and its contents have been
         * observed to affect munmap() performance on hardware
         * where MPX is not present.
         *
         * The unlikely() optimizes for the fast case: no MPX
         * in the CPU, or no MPX use in the process.  Even if
         * we get this wrong (in the unlikely event that MPX
         * is widely enabled on some system) the overhead of
         * MPX itself (reading bounds tables) is expected to
         * overwhelm the overhead of getting this unlikely()
         * consistently wrong.
         */
        if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX)))
                mpx_notify_unmap(mm, vma, start, end);
}

/*
 * We only want to enforce protection keys on the current process
 * because we effectively have no access to PKRU for other
 * processes or any way to tell *which * PKRU in a threaded
 * process we could use.
 *
 * So do not enforce things if the VMA is not from the current
 * mm, or if we are in a kernel thread.
 */
static inline bool vma_is_foreign(struct vm_area_struct *vma)
{
        if (!current->mm) /*covered*/
                return true;
        /*
         * Should PKRU be enforced on the access to this VMA?  If
         * the VMA is from another process, then PKRU has no
         * relevance and should not be enforced.
         */
        if (current->mm != vma->vm_mm) /*covered*/
                return true;

        return false;
}

static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, /*covered*/
                bool write, bool execute, bool foreign)
{
        /* pkeys never affect instruction fetches */
        if (execute) /*covered*/
                return true;
        /* allow access if the VMA is not one from this process */
        if (foreign || vma_is_foreign(vma)) /*covered*/
                return true;
        return __pkru_allows_pkey(vma_pkey(vma), write); /*covered*/
}

/*
 * This can be used from process context to figure out what the value of
 * CR3 is without needing to do a (slow) __read_cr3().
 *
 * It's intended to be used for code like KVM that sneakily changes CR3
 * and needs to restore it.  It needs to be used very carefully.
 */
static inline unsigned long __get_current_cr3_fast(void)
{
        unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
                this_cpu_read(cpu_tlbstate.loaded_mm_asid));

        /* For now, be very restrictive about when this can be called. */
        VM_WARN_ON(in_nmi() || preemptible());

        VM_BUG_ON(cr3 != __read_cr3());
        return cr3;
}

#endif /* _ASM_X86_MMU_CONTEXT_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_MSR_H
#define _ASM_X86_MSR_H

#include "msr-index.h"

#ifndef __ASSEMBLY__

#include <asm/asm.h>
#include <asm/errno.h>
#include <asm/cpumask.h>
#include <uapi/asm/msr.h>

struct msr {
        union {
                struct {
                        u32 l;
                        u32 h;
                };
                u64 q;
        };
};

struct msr_info {
        u32 msr_no;
        struct msr reg;
        struct msr *msrs;
        int err;
};

struct msr_regs_info {
        u32 *regs;
        int err;
};

struct saved_msr {
        bool valid;
        struct msr_info info;
};

struct saved_msrs {
        unsigned int num;
        struct saved_msr *array;
};

/*
 * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A"
 * constraint has different meanings. For i386, "A" means exactly
 * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead,
 * it means rax *or* rdx.
 */
#ifdef CONFIG_X86_64
/* Using 64-bit values saves one instruction clearing the high half of low */
#define DECLARE_ARGS(val, low, high)        unsigned long low, high
#define EAX_EDX_VAL(val, low, high)        ((low) | (high) << 32)
#define EAX_EDX_RET(val, low, high)        "=a" (low), "=d" (high)
#else
#define DECLARE_ARGS(val, low, high)        unsigned long long val
#define EAX_EDX_VAL(val, low, high)        (val)
#define EAX_EDX_RET(val, low, high)        "=A" (val)
#endif

#ifdef CONFIG_TRACEPOINTS
/*
 * Be very careful with includes. This header is prone to include loops.
 */
#include <asm/atomic.h>
#include <linux/tracepoint-defs.h>

extern struct tracepoint __tracepoint_read_msr;
extern struct tracepoint __tracepoint_write_msr;
extern struct tracepoint __tracepoint_rdpmc;
#define msr_tracepoint_active(t) static_key_false(&(t).key)
extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
#else
#define msr_tracepoint_active(t) false
static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
#endif

/*
 * __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR
 * accessors and should not have any tracing or other functionality piggybacking
 * on them - those are *purely* for accessing MSRs and nothing more. So don't even
 * think of extending them - you will be slapped with a stinking trout or a frozen
 * shark will reach you, wherever you are! You've been warned.
 */
static inline unsigned long long notrace __rdmsr(unsigned int msr)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("1: rdmsr\n"
                     "2:\n"
                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_unsafe)
                     : EAX_EDX_RET(val, low, high) : "c" (msr));

        return EAX_EDX_VAL(val, low, high);
}

static inline void notrace __wrmsr(unsigned int msr, u32 low, u32 high)
{
        asm volatile("1: wrmsr\n"
                     "2:\n"
                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_unsafe)
                     : : "c" (msr), "a"(low), "d" (high) : "memory");
}

#define native_rdmsr(msr, val1, val2)                        \
do {                                                        \
        u64 __val = __rdmsr((msr));                        \
        (void)((val1) = (u32)__val);                        \
        (void)((val2) = (u32)(__val >> 32));                \
} while (0)

#define native_wrmsr(msr, low, high)                        \
        __wrmsr(msr, low, high)

#define native_wrmsrl(msr, val)                                \
        __wrmsr((msr), (u32)((u64)(val)),                \
                       (u32)((u64)(val) >> 32))

static inline unsigned long long native_read_msr(unsigned int msr)
{
        unsigned long long val;

        val = __rdmsr(msr);

        if (msr_tracepoint_active(__tracepoint_read_msr))
                do_trace_read_msr(msr, val, 0);

        return val;
}

static inline unsigned long long native_read_msr_safe(unsigned int msr,
                                                      int *err)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("2: rdmsr ; xor %[err],%[err]\n"
                     "1:\n\t"
                     ".section .fixup,\"ax\"\n\t"
                     "3: mov %[fault],%[err]\n\t"
                     "xorl %%eax, %%eax\n\t"
                     "xorl %%edx, %%edx\n\t"
                     "jmp 1b\n\t"
                     ".previous\n\t"
                     _ASM_EXTABLE(2b, 3b)
                     : [err] "=r" (*err), EAX_EDX_RET(val, low, high)
                     : "c" (msr), [fault] "i" (-EIO));
        if (msr_tracepoint_active(__tracepoint_read_msr))
                do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err);
        return EAX_EDX_VAL(val, low, high);
}

/* Can be uninlined because referenced by paravirt */
static inline void notrace
native_write_msr(unsigned int msr, u32 low, u32 high)
{
        __wrmsr(msr, low, high);

        if (msr_tracepoint_active(__tracepoint_write_msr))
                do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
}

/* Can be uninlined because referenced by paravirt */
static inline int notrace
native_write_msr_safe(unsigned int msr, u32 low, u32 high)
{
        int err;

        asm volatile("2: wrmsr ; xor %[err],%[err]\n"
                     "1:\n\t"
                     ".section .fixup,\"ax\"\n\t"
                     "3:  mov %[fault],%[err] ; jmp 1b\n\t"
                     ".previous\n\t"
                     _ASM_EXTABLE(2b, 3b)
                     : [err] "=a" (err)
                     : "c" (msr), "0" (low), "d" (high),
                       [fault] "i" (-EIO)
                     : "memory");
        if (msr_tracepoint_active(__tracepoint_write_msr))
                do_trace_write_msr(msr, ((u64)high << 32 | low), err);
        return err;
}

extern int rdmsr_safe_regs(u32 regs[8]);
extern int wrmsr_safe_regs(u32 regs[8]);

/**
 * rdtsc() - returns the current TSC without ordering constraints
 *
 * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
 * only ordering constraint it supplies is the ordering implied by
 * "asm volatile": it will put the RDTSC in the place you expect.  The
 * CPU can and will speculatively execute that RDTSC, though, so the
 * results can be non-monotonic if compared on different CPUs.
 */
static __always_inline unsigned long long rdtsc(void)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); /*covered*/

        return EAX_EDX_VAL(val, low, high);
}

/**
 * rdtsc_ordered() - read the current TSC in program order
 *
 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
 * It is ordered like a load to a global in-memory counter.  It should
 * be impossible to observe non-monotonic rdtsc_unordered() behavior
 * across multiple CPUs as long as the TSC is synced.
 */
static __always_inline unsigned long long rdtsc_ordered(void)
{
        DECLARE_ARGS(val, low, high);

        /*
         * The RDTSC instruction is not ordered relative to memory
         * access.  The Intel SDM and the AMD APM are both vague on this
         * point, but empirically an RDTSC instruction can be
         * speculatively executed before prior loads.  An RDTSC
         * immediately after an appropriate barrier appears to be
         * ordered as a normal load, that is, it provides the same
         * ordering guarantees as reading from a global memory location
         * that some other imaginary CPU is updating continuously with a
         * time stamp.
         *
         * Thus, use the preferred barrier on the respective CPU, aiming for
         * RDTSCP as the default.
         */
        asm volatile(ALTERNATIVE_3("rdtsc",
                                   "mfence; rdtsc", X86_FEATURE_MFENCE_RDTSC,
                                   "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
                                   "rdtscp", X86_FEATURE_RDTSCP)
                        : EAX_EDX_RET(val, low, high)
                        /* RDTSCP clobbers ECX with MSR_TSC_AUX. */
                        :: "ecx");

        return EAX_EDX_VAL(val, low, high);
}

static inline unsigned long long native_read_pmc(int counter)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
        if (msr_tracepoint_active(__tracepoint_rdpmc))
                do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0);
        return EAX_EDX_VAL(val, low, high);
}

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else
#include <linux/errno.h>
/*
 * Access to machine-specific registers (available on 586 and better only)
 * Note: the rd* operations modify the parameters directly (without using
 * pointer indirection), this allows gcc to optimize better
 */

#define rdmsr(msr, low, high)                                        \
do {                                                                \
        u64 __val = native_read_msr((msr));                        \
        (void)((low) = (u32)__val);                                \
        (void)((high) = (u32)(__val >> 32));                        \
} while (0)

static inline void wrmsr(unsigned int msr, u32 low, u32 high)
{
        native_write_msr(msr, low, high);
}

#define rdmsrl(msr, val)                        \
        ((val) = native_read_msr((msr)))

static inline void wrmsrl(unsigned int msr, u64 val)
{
        native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
}

/* wrmsr with exception handling */
static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
{
        return native_write_msr_safe(msr, low, high);
}

/* rdmsr with exception handling */
#define rdmsr_safe(msr, low, high)                                \
({                                                                \
        int __err;                                                \
        u64 __val = native_read_msr_safe((msr), &__err);        \
        (*low) = (u32)__val;                                        \
        (*high) = (u32)(__val >> 32);                                \
        __err;                                                        \
})

static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p)
{
        int err;

        *p = native_read_msr_safe(msr, &err);
        return err;
}

#define rdpmc(counter, low, high)                        \
do {                                                        \
        u64 _l = native_read_pmc((counter));                \
        (low)  = (u32)_l;                                \
        (high) = (u32)(_l >> 32);                        \
} while (0)

#define rdpmcl(counter, val) ((val) = native_read_pmc(counter))

#endif        /* !CONFIG_PARAVIRT_XXL */

/*
 * 64-bit version of wrmsr_safe():
 */
static inline int wrmsrl_safe(u32 msr, u64 val)
{
        return wrmsr_safe(msr, (u32)val,  (u32)(val >> 32));
}

#define write_tsc(low, high) wrmsr(MSR_IA32_TSC, (low), (high))

#define write_rdtscp_aux(val) wrmsr(MSR_TSC_AUX, (val), 0)

struct msr *msrs_alloc(void);
void msrs_free(struct msr *msrs);
int msr_set_bit(u32 msr, u8 bit);
int msr_clear_bit(u32 msr, u8 bit);

#ifdef CONFIG_SMP
int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
#else  /*  CONFIG_SMP  */
static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
{
        rdmsr(msr_no, *l, *h);
        return 0;
}
static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
{
        wrmsr(msr_no, l, h);
        return 0;
}
static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
{
        rdmsrl(msr_no, *q);
        return 0;
}
static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
{
        wrmsrl(msr_no, q);
        return 0;
}
static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no,
                                struct msr *msrs)
{
        rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h));
}
static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no,
                                struct msr *msrs)
{
        wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h);
}
static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no,
                                    u32 *l, u32 *h)
{
        return rdmsr_safe(msr_no, l, h);
}
static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
{
        return wrmsr_safe(msr_no, l, h);
}
static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
{
        return rdmsrl_safe(msr_no, q);
}
static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
{
        return wrmsrl_safe(msr_no, q);
}
static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
{
        return rdmsr_safe_regs(regs);
}
static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
{
        return wrmsr_safe_regs(regs);
}
#endif  /* CONFIG_SMP */
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_MSR_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PARAVIRT_H
#define _ASM_X86_PARAVIRT_H
/* Various instructions on x86 need to be replaced for
 * para-virtualization: those hooks are defined here. */

#ifdef CONFIG_PARAVIRT
#include <asm/pgtable_types.h>
#include <asm/asm.h>
#include <asm/nospec-branch.h>

#include <asm/paravirt_types.h>

#ifndef __ASSEMBLY__
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/cpumask.h>
#include <asm/frame.h>

static inline unsigned long long paravirt_sched_clock(void)
{
        return PVOP_CALL0(unsigned long long, time.sched_clock);
}

struct static_key;
extern struct static_key paravirt_steal_enabled;
extern struct static_key paravirt_steal_rq_enabled;

__visible void __native_queued_spin_unlock(struct qspinlock *lock);
bool pv_is_native_spin_unlock(void);
__visible bool __native_vcpu_is_preempted(long cpu);
bool pv_is_native_vcpu_is_preempted(void);

static inline u64 paravirt_steal_clock(int cpu)
{
        return PVOP_CALL1(u64, time.steal_clock, cpu);
}

/* The paravirtualized I/O functions */
static inline void slow_down_io(void)
{
        pv_ops.cpu.io_delay();
#ifdef REALLY_SLOW_IO
        pv_ops.cpu.io_delay();
        pv_ops.cpu.io_delay();
        pv_ops.cpu.io_delay();
#endif
}

static inline void __flush_tlb(void)
{
        PVOP_VCALL0(mmu.flush_tlb_user);
}

static inline void __flush_tlb_global(void)
{
        PVOP_VCALL0(mmu.flush_tlb_kernel);
}

static inline void __flush_tlb_one_user(unsigned long addr)
{
        PVOP_VCALL1(mmu.flush_tlb_one_user, addr); /*covered*/
}

static inline void flush_tlb_others(const struct cpumask *cpumask,
                                    const struct flush_tlb_info *info)
{
        PVOP_VCALL2(mmu.flush_tlb_others, cpumask, info);
}

static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
{
        PVOP_VCALL2(mmu.tlb_remove_table, tlb, table); /*covered*/
}

static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
{
        PVOP_VCALL1(mmu.exit_mmap, mm);
}

#ifdef CONFIG_PARAVIRT_XXL
static inline void load_sp0(unsigned long sp0)
{
        PVOP_VCALL1(cpu.load_sp0, sp0);
}

/* The paravirtualized CPUID instruction. */
static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
                           unsigned int *ecx, unsigned int *edx)
{
        PVOP_VCALL4(cpu.cpuid, eax, ebx, ecx, edx);
}

/*
 * These special macros can be used to get or set a debugging register
 */
static inline unsigned long paravirt_get_debugreg(int reg)
{
        return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg);
}
#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
static inline void set_debugreg(unsigned long val, int reg)
{
        PVOP_VCALL2(cpu.set_debugreg, reg, val);
}

static inline unsigned long read_cr0(void)
{
        return PVOP_CALL0(unsigned long, cpu.read_cr0);
}

static inline void write_cr0(unsigned long x)
{
        PVOP_VCALL1(cpu.write_cr0, x);
}

static inline unsigned long read_cr2(void)
{
        return PVOP_CALL0(unsigned long, mmu.read_cr2); /*covered*/
}

static inline void write_cr2(unsigned long x)
{
        PVOP_VCALL1(mmu.write_cr2, x);
}

static inline unsigned long __read_cr3(void)
{
        return PVOP_CALL0(unsigned long, mmu.read_cr3);
}

static inline void write_cr3(unsigned long x)
{
        PVOP_VCALL1(mmu.write_cr3, x);
}

static inline void __write_cr4(unsigned long x)
{
        PVOP_VCALL1(cpu.write_cr4, x);
}

#ifdef CONFIG_X86_64
static inline unsigned long read_cr8(void)
{
        return PVOP_CALL0(unsigned long, cpu.read_cr8);
}

static inline void write_cr8(unsigned long x)
{
        PVOP_VCALL1(cpu.write_cr8, x);
}
#endif

static inline void arch_safe_halt(void)
{
        PVOP_VCALL0(irq.safe_halt);
}

static inline void halt(void)
{
        PVOP_VCALL0(irq.halt);
}

static inline void wbinvd(void)
{
        PVOP_VCALL0(cpu.wbinvd);
}

#define get_kernel_rpl()  (pv_info.kernel_rpl)

static inline u64 paravirt_read_msr(unsigned msr)
{
        return PVOP_CALL1(u64, cpu.read_msr, msr);
}

static inline void paravirt_write_msr(unsigned msr,
                                      unsigned low, unsigned high)
{
        PVOP_VCALL3(cpu.write_msr, msr, low, high);
}

static inline u64 paravirt_read_msr_safe(unsigned msr, int *err)
{
        return PVOP_CALL2(u64, cpu.read_msr_safe, msr, err);
}

static inline int paravirt_write_msr_safe(unsigned msr,
                                          unsigned low, unsigned high)
{
        return PVOP_CALL3(int, cpu.write_msr_safe, msr, low, high);
}

#define rdmsr(msr, val1, val2)                        \
do {                                                \
        u64 _l = paravirt_read_msr(msr);        \
        val1 = (u32)_l;                                \
        val2 = _l >> 32;                        \
} while (0)

#define wrmsr(msr, val1, val2)                        \
do {                                                \
        paravirt_write_msr(msr, val1, val2);        \
} while (0)

#define rdmsrl(msr, val)                        \
do {                                                \
        val = paravirt_read_msr(msr);                \
} while (0)

static inline void wrmsrl(unsigned msr, u64 val)
{
        wrmsr(msr, (u32)val, (u32)(val>>32));
}

#define wrmsr_safe(msr, a, b)        paravirt_write_msr_safe(msr, a, b)

/* rdmsr with exception handling */
#define rdmsr_safe(msr, a, b)                                \
({                                                        \
        int _err;                                        \
        u64 _l = paravirt_read_msr_safe(msr, &_err);        \
        (*a) = (u32)_l;                                        \
        (*b) = _l >> 32;                                \
        _err;                                                \
})

static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
{
        int err;

        *p = paravirt_read_msr_safe(msr, &err);
        return err;
}

static inline unsigned long long paravirt_read_pmc(int counter)
{
        return PVOP_CALL1(u64, cpu.read_pmc, counter);
}

#define rdpmc(counter, low, high)                \
do {                                                \
        u64 _l = paravirt_read_pmc(counter);        \
        low = (u32)_l;                                \
        high = _l >> 32;                        \
} while (0)

#define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter))

static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
{
        PVOP_VCALL2(cpu.alloc_ldt, ldt, entries);
}

static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
{
        PVOP_VCALL2(cpu.free_ldt, ldt, entries);
}

static inline void load_TR_desc(void)
{
        PVOP_VCALL0(cpu.load_tr_desc);
}
static inline void load_gdt(const struct desc_ptr *dtr)
{
        PVOP_VCALL1(cpu.load_gdt, dtr);
}
static inline void load_idt(const struct desc_ptr *dtr)
{
        PVOP_VCALL1(cpu.load_idt, dtr);
}
static inline void set_ldt(const void *addr, unsigned entries)
{
        PVOP_VCALL2(cpu.set_ldt, addr, entries);
}
static inline unsigned long paravirt_store_tr(void)
{
        return PVOP_CALL0(unsigned long, cpu.store_tr);
}

#define store_tr(tr)        ((tr) = paravirt_store_tr())
static inline void load_TLS(struct thread_struct *t, unsigned cpu)
{
        PVOP_VCALL2(cpu.load_tls, t, cpu);
}

#ifdef CONFIG_X86_64
static inline void load_gs_index(unsigned int gs)
{
        PVOP_VCALL1(cpu.load_gs_index, gs);
}
#endif

static inline void write_ldt_entry(struct desc_struct *dt, int entry,
                                   const void *desc)
{
        PVOP_VCALL3(cpu.write_ldt_entry, dt, entry, desc);
}

static inline void write_gdt_entry(struct desc_struct *dt, int entry,
                                   void *desc, int type)
{
        PVOP_VCALL4(cpu.write_gdt_entry, dt, entry, desc, type);
}

static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
{
        PVOP_VCALL3(cpu.write_idt_entry, dt, entry, g);
}
static inline void set_iopl_mask(unsigned mask)
{
        PVOP_VCALL1(cpu.set_iopl_mask, mask);
}

static inline void paravirt_activate_mm(struct mm_struct *prev,
                                        struct mm_struct *next)
{
        PVOP_VCALL2(mmu.activate_mm, prev, next);
}

static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
                                          struct mm_struct *mm)
{
        PVOP_VCALL2(mmu.dup_mmap, oldmm, mm);
}

static inline int paravirt_pgd_alloc(struct mm_struct *mm)
{
        return PVOP_CALL1(int, mmu.pgd_alloc, mm);
}

static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
        PVOP_VCALL2(mmu.pgd_free, mm, pgd); /*covered*/
}

static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(mmu.alloc_pte, mm, pfn); /*covered*/
}
static inline void paravirt_release_pte(unsigned long pfn)
{
        PVOP_VCALL1(mmu.release_pte, pfn); /*covered*/
}

static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(mmu.alloc_pmd, mm, pfn);
}

static inline void paravirt_release_pmd(unsigned long pfn)
{
        PVOP_VCALL1(mmu.release_pmd, pfn);
}

static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(mmu.alloc_pud, mm, pfn);
}
static inline void paravirt_release_pud(unsigned long pfn)
{
        PVOP_VCALL1(mmu.release_pud, pfn);
}

static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(mmu.alloc_p4d, mm, pfn);
}

static inline void paravirt_release_p4d(unsigned long pfn)
{
        PVOP_VCALL1(mmu.release_p4d, pfn);
}

static inline pte_t __pte(pteval_t val)
{
        pteval_t ret;

        if (sizeof(pteval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pteval_t, mmu.make_pte, val, (u64)val >> 32);
        else
                ret = PVOP_CALLEE1(pteval_t, mmu.make_pte, val); /*covered*/

        return (pte_t) { .pte = ret };
}

static inline pteval_t pte_val(pte_t pte)
{
        pteval_t ret;

        if (sizeof(pteval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pteval_t, mmu.pte_val,
                                   pte.pte, (u64)pte.pte >> 32);
        else
                ret = PVOP_CALLEE1(pteval_t, mmu.pte_val, pte.pte); /*covered*/

        return ret;
}

static inline pgd_t __pgd(pgdval_t val)
{
        pgdval_t ret;

        if (sizeof(pgdval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pgdval_t, mmu.make_pgd, val, (u64)val >> 32);
        else
                ret = PVOP_CALLEE1(pgdval_t, mmu.make_pgd, val);

        return (pgd_t) { ret };
}

static inline pgdval_t pgd_val(pgd_t pgd)
{
        pgdval_t ret;

        if (sizeof(pgdval_t) > sizeof(long))
                ret =  PVOP_CALLEE2(pgdval_t, mmu.pgd_val,
                                    pgd.pgd, (u64)pgd.pgd >> 32);
        else
                ret =  PVOP_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd); /*covered*/

        return ret;
}

#define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
static inline pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,
                                           pte_t *ptep)
{
        pteval_t ret;

        ret = PVOP_CALL3(pteval_t, mmu.ptep_modify_prot_start, vma, addr, ptep);

        return (pte_t) { .pte = ret };
}

static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
                                           pte_t *ptep, pte_t old_pte, pte_t pte)
{

        if (sizeof(pteval_t) > sizeof(long))
                /* 5 arg words */
                pv_ops.mmu.ptep_modify_prot_commit(vma, addr, ptep, pte);
        else
                PVOP_VCALL4(mmu.ptep_modify_prot_commit,
                            vma, addr, ptep, pte.pte);
}

static inline void set_pte(pte_t *ptep, pte_t pte)
{
        if (sizeof(pteval_t) > sizeof(long))
                PVOP_VCALL3(mmu.set_pte, ptep, pte.pte, (u64)pte.pte >> 32);
        else
                PVOP_VCALL2(mmu.set_pte, ptep, pte.pte); /*covered*/
}

static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
                              pte_t *ptep, pte_t pte)
{
        if (sizeof(pteval_t) > sizeof(long))
                /* 5 arg words */
                pv_ops.mmu.set_pte_at(mm, addr, ptep, pte);
        else
                PVOP_VCALL4(mmu.set_pte_at, mm, addr, ptep, pte.pte); /*covered*/
}

static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
{
        pmdval_t val = native_pmd_val(pmd); /*covered*/

        if (sizeof(pmdval_t) > sizeof(long))
                PVOP_VCALL3(mmu.set_pmd, pmdp, val, (u64)val >> 32);
        else
                PVOP_VCALL2(mmu.set_pmd, pmdp, val); /*covered*/
}

#if CONFIG_PGTABLE_LEVELS >= 3
static inline pmd_t __pmd(pmdval_t val)
{
        pmdval_t ret;

        if (sizeof(pmdval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pmdval_t, mmu.make_pmd, val, (u64)val >> 32);
        else
                ret = PVOP_CALLEE1(pmdval_t, mmu.make_pmd, val); /*covered*/

        return (pmd_t) { ret };
}

static inline pmdval_t pmd_val(pmd_t pmd)
{
        pmdval_t ret;

        if (sizeof(pmdval_t) > sizeof(long))
                ret =  PVOP_CALLEE2(pmdval_t, mmu.pmd_val,
                                    pmd.pmd, (u64)pmd.pmd >> 32);
        else
                ret =  PVOP_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd); /*covered*/

        return ret;
}

static inline void set_pud(pud_t *pudp, pud_t pud)
{
        pudval_t val = native_pud_val(pud);

        if (sizeof(pudval_t) > sizeof(long))
                PVOP_VCALL3(mmu.set_pud, pudp, val, (u64)val >> 32);
        else
                PVOP_VCALL2(mmu.set_pud, pudp, val);
}
#if CONFIG_PGTABLE_LEVELS >= 4
static inline pud_t __pud(pudval_t val)
{
        pudval_t ret;

        ret = PVOP_CALLEE1(pudval_t, mmu.make_pud, val);

        return (pud_t) { ret };
}

static inline pudval_t pud_val(pud_t pud)
{
        return PVOP_CALLEE1(pudval_t, mmu.pud_val, pud.pud); /*covered*/
}

static inline void pud_clear(pud_t *pudp)
{
        set_pud(pudp, __pud(0));
}

static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
{
        p4dval_t val = native_p4d_val(p4d);

        PVOP_VCALL2(mmu.set_p4d, p4dp, val);
}

#if CONFIG_PGTABLE_LEVELS >= 5

static inline p4d_t __p4d(p4dval_t val)
{
        p4dval_t ret = PVOP_CALLEE1(p4dval_t, mmu.make_p4d, val);

        return (p4d_t) { ret };
}

static inline p4dval_t p4d_val(p4d_t p4d)
{
        return PVOP_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d);
}

static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd)
{
        PVOP_VCALL2(mmu.set_pgd, pgdp, native_pgd_val(pgd));
}

#define set_pgd(pgdp, pgdval) do {                                        \
        if (pgtable_l5_enabled())                                                \
                __set_pgd(pgdp, pgdval);                                \
        else                                                                \
                set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd });        \
} while (0)

#define pgd_clear(pgdp) do {                                                \
        if (pgtable_l5_enabled())                                                \
                set_pgd(pgdp, __pgd(0));                                \
} while (0)

#endif  /* CONFIG_PGTABLE_LEVELS == 5 */

static inline void p4d_clear(p4d_t *p4dp)
{
        set_p4d(p4dp, __p4d(0));
}

#endif        /* CONFIG_PGTABLE_LEVELS == 4 */

#endif        /* CONFIG_PGTABLE_LEVELS >= 3 */

#ifdef CONFIG_X86_PAE
/* Special-case pte-setting operations for PAE, which can't update a
   64-bit pte atomically */
static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
{
        PVOP_VCALL3(mmu.set_pte_atomic, ptep, pte.pte, pte.pte >> 32);
}

static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
                             pte_t *ptep)
{
        PVOP_VCALL3(mmu.pte_clear, mm, addr, ptep);
}

static inline void pmd_clear(pmd_t *pmdp)
{
        PVOP_VCALL1(mmu.pmd_clear, pmdp);
}
#else  /* !CONFIG_X86_PAE */
static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
{
        set_pte(ptep, pte); /*covered*/
}

static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
                             pte_t *ptep)
{
        set_pte_at(mm, addr, ptep, __pte(0));
}

static inline void pmd_clear(pmd_t *pmdp)
{
        set_pmd(pmdp, __pmd(0)); /*covered*/
}
#endif        /* CONFIG_X86_PAE */

#define  __HAVE_ARCH_START_CONTEXT_SWITCH
static inline void arch_start_context_switch(struct task_struct *prev)
{
        PVOP_VCALL1(cpu.start_context_switch, prev);
}

static inline void arch_end_context_switch(struct task_struct *next)
{
        PVOP_VCALL1(cpu.end_context_switch, next);
}

#define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
static inline void arch_enter_lazy_mmu_mode(void)
{
        PVOP_VCALL0(mmu.lazy_mode.enter); /*covered*/
}

static inline void arch_leave_lazy_mmu_mode(void)
{
        PVOP_VCALL0(mmu.lazy_mode.leave); /*covered*/
}

static inline void arch_flush_lazy_mmu_mode(void)
{
        PVOP_VCALL0(mmu.lazy_mode.flush);
}

static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
                                phys_addr_t phys, pgprot_t flags)
{
        pv_ops.mmu.set_fixmap(idx, phys, flags);
}
#endif

#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)

static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
                                                        u32 val)
{
        PVOP_VCALL2(lock.queued_spin_lock_slowpath, lock, val);
}

static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
{
        PVOP_VCALLEE1(lock.queued_spin_unlock, lock);
}

static __always_inline void pv_wait(u8 *ptr, u8 val)
{
        PVOP_VCALL2(lock.wait, ptr, val);
}

static __always_inline void pv_kick(int cpu)
{
        PVOP_VCALL1(lock.kick, cpu);
}

static __always_inline bool pv_vcpu_is_preempted(long cpu)
{
        return PVOP_CALLEE1(bool, lock.vcpu_is_preempted, cpu);
}

void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock);
bool __raw_callee_save___native_vcpu_is_preempted(long cpu);

#endif /* SMP && PARAVIRT_SPINLOCKS */

#ifdef CONFIG_X86_32
#define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
#define PV_RESTORE_REGS "popl %edx; popl %ecx;"

/* save and restore all caller-save registers, except return value */
#define PV_SAVE_ALL_CALLER_REGS                "pushl %ecx;"
#define PV_RESTORE_ALL_CALLER_REGS        "popl  %ecx;"

#define PV_FLAGS_ARG "0"
#define PV_EXTRA_CLOBBERS
#define PV_VEXTRA_CLOBBERS
#else
/* save and restore all caller-save registers, except return value */
#define PV_SAVE_ALL_CALLER_REGS                                                \
        "push %rcx;"                                                        \
        "push %rdx;"                                                        \
        "push %rsi;"                                                        \
        "push %rdi;"                                                        \
        "push %r8;"                                                        \
        "push %r9;"                                                        \
        "push %r10;"                                                        \
        "push %r11;"
#define PV_RESTORE_ALL_CALLER_REGS                                        \
        "pop %r11;"                                                        \
        "pop %r10;"                                                        \
        "pop %r9;"                                                        \
        "pop %r8;"                                                        \
        "pop %rdi;"                                                        \
        "pop %rsi;"                                                        \
        "pop %rdx;"                                                        \
        "pop %rcx;"

/* We save some registers, but all of them, that's too much. We clobber all
 * caller saved registers but the argument parameter */
#define PV_SAVE_REGS "pushq %%rdi;"
#define PV_RESTORE_REGS "popq %%rdi;"
#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
#define PV_FLAGS_ARG "D"
#endif

/*
 * Generate a thunk around a function which saves all caller-save
 * registers except for the return value.  This allows C functions to
 * be called from assembler code where fewer than normal registers are
 * available.  It may also help code generation around calls from C
 * code if the common case doesn't use many registers.
 *
 * When a callee is wrapped in a thunk, the caller can assume that all
 * arg regs and all scratch registers are preserved across the
 * call. The return value in rax/eax will not be saved, even for void
 * functions.
 */
#define PV_THUNK_NAME(func) "__raw_callee_save_" #func
#define PV_CALLEE_SAVE_REGS_THUNK(func)                                        \
        extern typeof(func) __raw_callee_save_##func;                        \
                                                                        \
        asm(".pushsection .text;"                                        \
            ".globl " PV_THUNK_NAME(func) ";"                                \
            ".type " PV_THUNK_NAME(func) ", @function;"                        \
            PV_THUNK_NAME(func) ":"                                        \
            FRAME_BEGIN                                                        \
            PV_SAVE_ALL_CALLER_REGS                                        \
            "call " #func ";"                                                \
            PV_RESTORE_ALL_CALLER_REGS                                        \
            FRAME_END                                                        \
            "ret;"                                                        \
            ".popsection")

/* Get a reference to a callee-save function */
#define PV_CALLEE_SAVE(func)                                                \
        ((struct paravirt_callee_save) { __raw_callee_save_##func })

/* Promise that "func" already uses the right calling convention */
#define __PV_IS_CALLEE_SAVE(func)                        \
        ((struct paravirt_callee_save) { func })

#ifdef CONFIG_PARAVIRT_XXL
static inline notrace unsigned long arch_local_save_flags(void)
{
        return PVOP_CALLEE0(unsigned long, irq.save_fl); /*covered*/
}

static inline notrace void arch_local_irq_restore(unsigned long f) /*covered*/
{
        PVOP_VCALLEE1(irq.restore_fl, f); /*covered*/
} /*covered*/

static inline notrace void arch_local_irq_disable(void)
{
        PVOP_VCALLEE0(irq.irq_disable); /*covered*/
}

static inline notrace void arch_local_irq_enable(void)
{
        PVOP_VCALLEE0(irq.irq_enable); /*covered*/
}

static inline notrace unsigned long arch_local_irq_save(void)
{
        unsigned long f;

        f = arch_local_save_flags(); /*covered*/
        arch_local_irq_disable(); /*covered*/
        return f;
}
#endif


/* Make sure as little as possible of this mess escapes. */
#undef PARAVIRT_CALL
#undef __PVOP_CALL
#undef __PVOP_VCALL
#undef PVOP_VCALL0
#undef PVOP_CALL0
#undef PVOP_VCALL1
#undef PVOP_CALL1
#undef PVOP_VCALL2
#undef PVOP_CALL2
#undef PVOP_VCALL3
#undef PVOP_CALL3
#undef PVOP_VCALL4
#undef PVOP_CALL4

extern void default_banner(void);

#else  /* __ASSEMBLY__ */

#define _PVSITE(ptype, ops, word, algn)                \
771:;                                                \
        ops;                                        \
772:;                                                \
        .pushsection .parainstructions,"a";        \
         .align        algn;                                \
         word 771b;                                \
         .byte ptype;                                \
         .byte 772b-771b;                        \
        .popsection


#define COND_PUSH(set, mask, reg)                        \
        .if ((~(set)) & mask); push %reg; .endif
#define COND_POP(set, mask, reg)                        \
        .if ((~(set)) & mask); pop %reg; .endif

#ifdef CONFIG_X86_64

#define PV_SAVE_REGS(set)                        \
        COND_PUSH(set, CLBR_RAX, rax);                \
        COND_PUSH(set, CLBR_RCX, rcx);                \
        COND_PUSH(set, CLBR_RDX, rdx);                \
        COND_PUSH(set, CLBR_RSI, rsi);                \
        COND_PUSH(set, CLBR_RDI, rdi);                \
        COND_PUSH(set, CLBR_R8, r8);                \
        COND_PUSH(set, CLBR_R9, r9);                \
        COND_PUSH(set, CLBR_R10, r10);                \
        COND_PUSH(set, CLBR_R11, r11)
#define PV_RESTORE_REGS(set)                        \
        COND_POP(set, CLBR_R11, r11);                \
        COND_POP(set, CLBR_R10, r10);                \
        COND_POP(set, CLBR_R9, r9);                \
        COND_POP(set, CLBR_R8, r8);                \
        COND_POP(set, CLBR_RDI, rdi);                \
        COND_POP(set, CLBR_RSI, rsi);                \
        COND_POP(set, CLBR_RDX, rdx);                \
        COND_POP(set, CLBR_RCX, rcx);                \
        COND_POP(set, CLBR_RAX, rax)

#define PARA_PATCH(off)                ((off) / 8)
#define PARA_SITE(ptype, ops)        _PVSITE(ptype, ops, .quad, 8)
#define PARA_INDIRECT(addr)        *addr(%rip)
#else
#define PV_SAVE_REGS(set)                        \
        COND_PUSH(set, CLBR_EAX, eax);                \
        COND_PUSH(set, CLBR_EDI, edi);                \
        COND_PUSH(set, CLBR_ECX, ecx);                \
        COND_PUSH(set, CLBR_EDX, edx)
#define PV_RESTORE_REGS(set)                        \
        COND_POP(set, CLBR_EDX, edx);                \
        COND_POP(set, CLBR_ECX, ecx);                \
        COND_POP(set, CLBR_EDI, edi);                \
        COND_POP(set, CLBR_EAX, eax)

#define PARA_PATCH(off)                ((off) / 4)
#define PARA_SITE(ptype, ops)        _PVSITE(ptype, ops, .long, 4)
#define PARA_INDIRECT(addr)        *%cs:addr
#endif

#ifdef CONFIG_PARAVIRT_XXL
#define INTERRUPT_RETURN                                                \
        PARA_SITE(PARA_PATCH(PV_CPU_iret),                                \
                  ANNOTATE_RETPOLINE_SAFE;                                \
                  jmp PARA_INDIRECT(pv_ops+PV_CPU_iret);)

#define DISABLE_INTERRUPTS(clobbers)                                        \
        PARA_SITE(PARA_PATCH(PV_IRQ_irq_disable),                        \
                  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);                \
                  ANNOTATE_RETPOLINE_SAFE;                                \
                  call PARA_INDIRECT(pv_ops+PV_IRQ_irq_disable);        \
                  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)

#define ENABLE_INTERRUPTS(clobbers)                                        \
        PARA_SITE(PARA_PATCH(PV_IRQ_irq_enable),                        \
                  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);                \
                  ANNOTATE_RETPOLINE_SAFE;                                \
                  call PARA_INDIRECT(pv_ops+PV_IRQ_irq_enable);                \
                  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
#endif

#ifdef CONFIG_X86_64
#ifdef CONFIG_PARAVIRT_XXL
/*
 * If swapgs is used while the userspace stack is still current,
 * there's no way to call a pvop.  The PV replacement *must* be
 * inlined, or the swapgs instruction must be trapped and emulated.
 */
#define SWAPGS_UNSAFE_STACK                                                \
        PARA_SITE(PARA_PATCH(PV_CPU_swapgs), swapgs)

/*
 * Note: swapgs is very special, and in practise is either going to be
 * implemented with a single "swapgs" instruction or something very
 * special.  Either way, we don't need to save any registers for
 * it.
 */
#define SWAPGS                                                                \
        PARA_SITE(PARA_PATCH(PV_CPU_swapgs),                                \
                  ANNOTATE_RETPOLINE_SAFE;                                \
                  call PARA_INDIRECT(pv_ops+PV_CPU_swapgs);                \
                 )
#endif

#define GET_CR2_INTO_RAX                                \
        ANNOTATE_RETPOLINE_SAFE;                                \
        call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2);

#ifdef CONFIG_PARAVIRT_XXL
#define USERGS_SYSRET64                                                        \
        PARA_SITE(PARA_PATCH(PV_CPU_usergs_sysret64),                        \
                  ANNOTATE_RETPOLINE_SAFE;                                \
                  jmp PARA_INDIRECT(pv_ops+PV_CPU_usergs_sysret64);)

#ifdef CONFIG_DEBUG_ENTRY
#define SAVE_FLAGS(clobbers)                                        \
        PARA_SITE(PARA_PATCH(PV_IRQ_save_fl),                            \
                  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);        \
                  ANNOTATE_RETPOLINE_SAFE;                            \
                  call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);            \
                  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
#endif
#endif

#endif        /* CONFIG_X86_32 */

#endif /* __ASSEMBLY__ */
#else  /* CONFIG_PARAVIRT */
# define default_banner x86_init_noop
#endif /* !CONFIG_PARAVIRT */

#ifndef __ASSEMBLY__
#ifndef CONFIG_PARAVIRT_XXL
static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
                                          struct mm_struct *mm)
{
}
#endif

#ifndef CONFIG_PARAVIRT
static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
{
}
#endif
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_PARAVIRT_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGALLOC_H
#define _ASM_X86_PGALLOC_H

#include <linux/threads.h>
#include <linux/mm.h>                /* for struct page */
#include <linux/pagemap.h>

static inline int  __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else
#define paravirt_pgd_alloc(mm)        __paravirt_pgd_alloc(mm)
static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {}
static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
                                            unsigned long start, unsigned long count) {}
static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_release_pte(unsigned long pfn) {}
static inline void paravirt_release_pmd(unsigned long pfn) {}
static inline void paravirt_release_pud(unsigned long pfn) {}
static inline void paravirt_release_p4d(unsigned long pfn) {}
#endif

/*
 * Flags to use when allocating a user page table page.
 */
extern gfp_t __userpte_alloc_gfp;

#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
 * Instead of one PGD, we acquire two PGDs.  Being order-1, it is
 * both 8k in size and 8k-aligned.  That lets us just flip bit 12
 * in a pointer to swap between the two 4k halves.
 */
#define PGD_ALLOCATION_ORDER 1
#else
#define PGD_ALLOCATION_ORDER 0
#endif

/*
 * Allocate and free page tables.
 */
extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);

extern pte_t *pte_alloc_one_kernel(struct mm_struct *);
extern pgtable_t pte_alloc_one(struct mm_struct *);

/* Should really implement gc for free page table pages. This could be
   done with a reference count in struct page. */

static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
        BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
        free_page((unsigned long)pte);
}

static inline void pte_free(struct mm_struct *mm, struct page *pte)
{
        pgtable_page_dtor(pte); /*covered*/
        __free_page(pte);
}

extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);

static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte,
                                  unsigned long address)
{
        ___pte_free_tlb(tlb, pte);
}

static inline void pmd_populate_kernel(struct mm_struct *mm,
                                       pmd_t *pmd, pte_t *pte)
{
        paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); /*covered*/
        set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); /*covered*/
}

static inline void pmd_populate_kernel_safe(struct mm_struct *mm,
                                       pmd_t *pmd, pte_t *pte)
{
        paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
        set_pmd_safe(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
}

static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
                                struct page *pte)
{
        unsigned long pfn = page_to_pfn(pte); /*covered*/

        paravirt_alloc_pte(mm, pfn); /*covered*/
        set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); /*covered*/
}

#define pmd_pgtable(pmd) pmd_page(pmd)

#if CONFIG_PGTABLE_LEVELS > 2
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
        struct page *page;
        gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;

        if (mm == &init_mm)
                gfp &= ~__GFP_ACCOUNT;
        page = alloc_pages(gfp, 0);
        if (!page)
                return NULL;
        if (!pgtable_pmd_page_ctor(page)) {
                __free_pages(page, 0);
                return NULL;
        }
        return (pmd_t *)page_address(page);
}

static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
        BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
        pgtable_pmd_page_dtor(virt_to_page(pmd));
        free_page((unsigned long)pmd);
}

extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);

static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
                                  unsigned long address)
{
        ___pmd_free_tlb(tlb, pmd);
}

#ifdef CONFIG_X86_PAE
extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
#else        /* !CONFIG_X86_PAE */
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
        paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
        set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
}

static inline void pud_populate_safe(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
        paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
        set_pud_safe(pud, __pud(_PAGE_TABLE | __pa(pmd)));
}
#endif        /* CONFIG_X86_PAE */

#if CONFIG_PGTABLE_LEVELS > 3
static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
{
        paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
        set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
}

static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
{
        paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
        set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
}

static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
        gfp_t gfp = GFP_KERNEL_ACCOUNT;

        if (mm == &init_mm)
                gfp &= ~__GFP_ACCOUNT;
        return (pud_t *)get_zeroed_page(gfp);
}

static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
        BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
        free_page((unsigned long)pud);
}

extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);

static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
                                  unsigned long address)
{
        ___pud_free_tlb(tlb, pud);
}

#if CONFIG_PGTABLE_LEVELS > 4
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
{
        if (!pgtable_l5_enabled())
                return;
        paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
        set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
}

static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
{
        if (!pgtable_l5_enabled())
                return;
        paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
        set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
}

static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
{
        gfp_t gfp = GFP_KERNEL_ACCOUNT;

        if (mm == &init_mm)
                gfp &= ~__GFP_ACCOUNT;
        return (p4d_t *)get_zeroed_page(gfp);
}

static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
{
        if (!pgtable_l5_enabled())
                return;

        BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
        free_page((unsigned long)p4d);
}

extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);

static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
                                  unsigned long address)
{
        if (pgtable_l5_enabled())
                ___p4d_free_tlb(tlb, p4d);
}

#endif        /* CONFIG_PGTABLE_LEVELS > 4 */
#endif        /* CONFIG_PGTABLE_LEVELS > 3 */
#endif        /* CONFIG_PGTABLE_LEVELS > 2 */

#endif /* _ASM_X86_PGALLOC_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_PGTABLE_INVERT_H
#define _ASM_PGTABLE_INVERT_H 1

#ifndef __ASSEMBLY__

/*
 * A clear pte value is special, and doesn't get inverted.
 *
 * Note that even users that only pass a pgprot_t (rather
 * than a full pte) won't trigger the special zero case,
 * because even PAGE_NONE has _PAGE_PROTNONE | _PAGE_ACCESSED
 * set. So the all zero case really is limited to just the
 * cleared page table entry case.
 */
static inline bool __pte_needs_invert(u64 val)
{
        return val && !(val & _PAGE_PRESENT); /*covered*/
}

/* Get a mask to xor with the page table entry to get the correct pfn. */
static inline u64 protnone_mask(u64 val)
{
        return __pte_needs_invert(val) ?  ~0ull : 0; /*covered*/
}

static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask)
{
        /*
         * When a PTE transitions from NONE to !NONE or vice-versa
         * invert the PFN part to stop speculation.
         * pte_pfn undoes this when needed.
         */
        if (__pte_needs_invert(oldval) != __pte_needs_invert(val))
                val = (val & ~mask) | (~val & mask);
        return val;
}

#endif /* __ASSEMBLY__ */

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGTABLE_H
#define _ASM_X86_PGTABLE_H

#include <linux/mem_encrypt.h>
#include <asm/page.h>
#include <asm/pgtable_types.h>

/*
 * Macro to mark a page protection value as UC-
 */
#define pgprot_noncached(prot)                                                \
        ((boot_cpu_data.x86 > 3)                                        \
         ? (__pgprot(pgprot_val(prot) |                                        \
                     cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))        \
         : (prot))

/*
 * Macros to add or remove encryption attribute
 */
#define pgprot_encrypted(prot)        __pgprot(__sme_set(pgprot_val(prot)))
#define pgprot_decrypted(prot)        __pgprot(__sme_clr(pgprot_val(prot)))

#ifndef __ASSEMBLY__
#include <asm/x86_init.h>

extern pgd_t early_top_pgt[PTRS_PER_PGD];
int __init __early_make_pgtable(unsigned long address, pmdval_t pmd);

void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user);
void ptdump_walk_pgd_level_checkwx(void);
void ptdump_walk_user_pgd_level_checkwx(void);

#ifdef CONFIG_DEBUG_WX
#define debug_checkwx()                ptdump_walk_pgd_level_checkwx()
#define debug_checkwx_user()        ptdump_walk_user_pgd_level_checkwx()
#else
#define debug_checkwx()                do { } while (0)
#define debug_checkwx_user()        do { } while (0)
#endif

/*
 * ZERO_PAGE is a global shared page that is always zero: used
 * for zero-mapped memory areas etc..
 */
extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
        __visible;
#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))

extern spinlock_t pgd_lock;
extern struct list_head pgd_list;

extern struct mm_struct *pgd_page_get_mm(struct page *page);

extern pmdval_t early_pmd_flags;

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else  /* !CONFIG_PARAVIRT_XXL */
#define set_pte(ptep, pte)                native_set_pte(ptep, pte)
#define set_pte_at(mm, addr, ptep, pte)        native_set_pte_at(mm, addr, ptep, pte)

#define set_pte_atomic(ptep, pte)                                        \
        native_set_pte_atomic(ptep, pte)

#define set_pmd(pmdp, pmd)                native_set_pmd(pmdp, pmd)

#ifndef __PAGETABLE_P4D_FOLDED
#define set_pgd(pgdp, pgd)                native_set_pgd(pgdp, pgd)
#define pgd_clear(pgd)                        (pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
#endif

#ifndef set_p4d
# define set_p4d(p4dp, p4d)                native_set_p4d(p4dp, p4d)
#endif

#ifndef __PAGETABLE_PUD_FOLDED
#define p4d_clear(p4d)                        native_p4d_clear(p4d)
#endif

#ifndef set_pud
# define set_pud(pudp, pud)                native_set_pud(pudp, pud)
#endif

#ifndef __PAGETABLE_PUD_FOLDED
#define pud_clear(pud)                        native_pud_clear(pud)
#endif

#define pte_clear(mm, addr, ptep)        native_pte_clear(mm, addr, ptep)
#define pmd_clear(pmd)                        native_pmd_clear(pmd)

#define pgd_val(x)        native_pgd_val(x)
#define __pgd(x)        native_make_pgd(x)

#ifndef __PAGETABLE_P4D_FOLDED
#define p4d_val(x)        native_p4d_val(x)
#define __p4d(x)        native_make_p4d(x)
#endif

#ifndef __PAGETABLE_PUD_FOLDED
#define pud_val(x)        native_pud_val(x)
#define __pud(x)        native_make_pud(x)
#endif

#ifndef __PAGETABLE_PMD_FOLDED
#define pmd_val(x)        native_pmd_val(x)
#define __pmd(x)        native_make_pmd(x)
#endif

#define pte_val(x)        native_pte_val(x)
#define __pte(x)        native_make_pte(x)

#define arch_end_context_switch(prev)        do {} while(0)
#endif        /* CONFIG_PARAVIRT_XXL */

/*
 * The following only work if pte_present() is true.
 * Undefined behaviour if not..
 */
static inline int pte_dirty(pte_t pte)
{
        return pte_flags(pte) & _PAGE_DIRTY; /*covered*/
}


static inline u32 read_pkru(void)
{
        if (boot_cpu_has(X86_FEATURE_OSPKE)) /*covered*/
                return __read_pkru();
        return 0;
}

static inline void write_pkru(u32 pkru)
{
        if (boot_cpu_has(X86_FEATURE_OSPKE))
                __write_pkru(pkru);
}

static inline int pte_young(pte_t pte)
{
        return pte_flags(pte) & _PAGE_ACCESSED; /*covered*/
}

static inline int pmd_dirty(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_DIRTY; /*covered*/
}

static inline int pmd_young(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_ACCESSED;
}

static inline int pud_dirty(pud_t pud)
{
        return pud_flags(pud) & _PAGE_DIRTY;
}

static inline int pud_young(pud_t pud)
{
        return pud_flags(pud) & _PAGE_ACCESSED;
}

static inline int pte_write(pte_t pte)
{
        return pte_flags(pte) & _PAGE_RW; /*covered*/
}

static inline int pte_huge(pte_t pte)
{
        return pte_flags(pte) & _PAGE_PSE;
}

static inline int pte_global(pte_t pte)
{
        return pte_flags(pte) & _PAGE_GLOBAL;
}

static inline int pte_exec(pte_t pte)
{
        return !(pte_flags(pte) & _PAGE_NX);
}

static inline int pte_special(pte_t pte)
{
        return pte_flags(pte) & _PAGE_SPECIAL; /*covered*/
}

/* Entries that were set to PROT_NONE are inverted */

static inline u64 protnone_mask(u64 val);

static inline unsigned long pte_pfn(pte_t pte)
{
        phys_addr_t pfn = pte_val(pte); /*covered*/
        pfn ^= protnone_mask(pfn); /*covered*/
        return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT; /*covered*/
}

static inline unsigned long pmd_pfn(pmd_t pmd)
{
        phys_addr_t pfn = pmd_val(pmd); /*covered*/
        pfn ^= protnone_mask(pfn); /*covered*/
        return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT; /*covered*/
}

static inline unsigned long pud_pfn(pud_t pud)
{
        phys_addr_t pfn = pud_val(pud); /*covered*/
        pfn ^= protnone_mask(pfn); /*covered*/
        return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT; /*covered*/
}

static inline unsigned long p4d_pfn(p4d_t p4d)
{
        return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
}

static inline unsigned long pgd_pfn(pgd_t pgd)
{
        return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
}

static inline int p4d_large(p4d_t p4d)
{
        /* No 512 GiB pages yet */
        return 0;
}

#define pte_page(pte)        pfn_to_page(pte_pfn(pte))

static inline int pmd_large(pmd_t pte)
{
        return pmd_flags(pte) & _PAGE_PSE; /*covered*/
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int pmd_trans_huge(pmd_t pmd)
{
        return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE; /*covered*/
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static inline int pud_trans_huge(pud_t pud)
{
        return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
}
#endif

#define has_transparent_hugepage has_transparent_hugepage
static inline int has_transparent_hugepage(void)
{
        return boot_cpu_has(X86_FEATURE_PSE);
}

#ifdef __HAVE_ARCH_PTE_DEVMAP
static inline int pmd_devmap(pmd_t pmd)
{
        return !!(pmd_val(pmd) & _PAGE_DEVMAP); /*covered*/
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static inline int pud_devmap(pud_t pud)
{
        return !!(pud_val(pud) & _PAGE_DEVMAP); /*covered*/
}
#else
static inline int pud_devmap(pud_t pud)
{
        return 0;
}
#endif

static inline int pgd_devmap(pgd_t pgd)
{
        return 0;
}
#endif
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
{
        pteval_t v = native_pte_val(pte);

        return native_make_pte(v | set); /*covered*/
}

static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
{
        pteval_t v = native_pte_val(pte);

        return native_make_pte(v & ~clear);
}

static inline pte_t pte_mkclean(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_DIRTY);
}

static inline pte_t pte_mkold(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_ACCESSED);
}

static inline pte_t pte_wrprotect(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_RW);
}

static inline pte_t pte_mkexec(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_NX);
}

static inline pte_t pte_mkdirty(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
}

static inline pte_t pte_mkyoung(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_ACCESSED); /*covered*/
}

static inline pte_t pte_mkwrite(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_RW); /*covered*/
}

static inline pte_t pte_mkhuge(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_PSE);
}

static inline pte_t pte_clrhuge(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_PSE);
}

static inline pte_t pte_mkglobal(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_GLOBAL);
}

static inline pte_t pte_clrglobal(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_GLOBAL);
}

static inline pte_t pte_mkspecial(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SPECIAL); /*covered*/
}

static inline pte_t pte_mkdevmap(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
}

static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
{
        pmdval_t v = native_pmd_val(pmd);

        return native_make_pmd(v | set); /*covered*/
}

static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
{
        pmdval_t v = native_pmd_val(pmd);

        return native_make_pmd(v & ~clear);
}

static inline pmd_t pmd_mkold(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_ACCESSED);
}

static inline pmd_t pmd_mkclean(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_DIRTY);
}

static inline pmd_t pmd_wrprotect(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_RW);
}

static inline pmd_t pmd_mkdirty(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); /*covered*/
}

static inline pmd_t pmd_mkdevmap(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_DEVMAP);
}

static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_PSE);
}

static inline pmd_t pmd_mkyoung(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_ACCESSED); /*covered*/
}

static inline pmd_t pmd_mkwrite(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_RW); /*covered*/
}

static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
{
        pudval_t v = native_pud_val(pud);

        return native_make_pud(v | set);
}

static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
{
        pudval_t v = native_pud_val(pud);

        return native_make_pud(v & ~clear);
}

static inline pud_t pud_mkold(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_ACCESSED);
}

static inline pud_t pud_mkclean(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_DIRTY);
}

static inline pud_t pud_wrprotect(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_RW);
}

static inline pud_t pud_mkdirty(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
}

static inline pud_t pud_mkdevmap(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_DEVMAP);
}

static inline pud_t pud_mkhuge(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_PSE);
}

static inline pud_t pud_mkyoung(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_ACCESSED);
}

static inline pud_t pud_mkwrite(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_RW);
}

#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
static inline int pte_soft_dirty(pte_t pte)
{
        return pte_flags(pte) & _PAGE_SOFT_DIRTY;
}

static inline int pmd_soft_dirty(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
}

static inline int pud_soft_dirty(pud_t pud)
{
        return pud_flags(pud) & _PAGE_SOFT_DIRTY;
}

static inline pte_t pte_mksoft_dirty(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SOFT_DIRTY); /*covered*/
}

static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
}

static inline pud_t pud_mksoft_dirty(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
}

static inline pte_t pte_clear_soft_dirty(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
}

static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
}

static inline pud_t pud_clear_soft_dirty(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
}

#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */

/*
 * Mask out unsupported bits in a present pgprot.  Non-present pgprots
 * can use those bits for other purposes, so leave them be.
 */
static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
{
        pgprotval_t protval = pgprot_val(pgprot);

        if (protval & _PAGE_PRESENT) /*covered*/
                protval &= __supported_pte_mask; /*covered*/

        return protval;
}

static inline pgprotval_t check_pgprot(pgprot_t pgprot)
{
        pgprotval_t massaged_val = massage_pgprot(pgprot); /*covered*/

        /* mmdebug.h can not be included here because of dependencies */
#ifdef CONFIG_DEBUG_VM
        WARN_ONCE(pgprot_val(pgprot) != massaged_val, /*covered*/
                  "attempted to set unsupported pgprot: %016llx "
                  "bits: %016llx supported: %016llx\n",
                  (u64)pgprot_val(pgprot),
                  (u64)pgprot_val(pgprot) ^ massaged_val,
                  (u64)__supported_pte_mask);
#endif

        return massaged_val;
}

static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
{
        phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT; /*covered*/
        pfn ^= protnone_mask(pgprot_val(pgprot)); /*covered*/
        pfn &= PTE_PFN_MASK; /*covered*/
        return __pte(pfn | check_pgprot(pgprot)); /*covered*/
}

static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
{
        phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
        pfn ^= protnone_mask(pgprot_val(pgprot)); /*covered*/
        pfn &= PHYSICAL_PMD_PAGE_MASK; /*covered*/
        return __pmd(pfn | check_pgprot(pgprot)); /*covered*/
}

static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
{
        phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
        pfn ^= protnone_mask(pgprot_val(pgprot));
        pfn &= PHYSICAL_PUD_PAGE_MASK;
        return __pud(pfn | check_pgprot(pgprot));
}

static inline pmd_t pmd_mknotpresent(pmd_t pmd)
{
        return pfn_pmd(pmd_pfn(pmd), /*covered*/
                      __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE))); /*covered*/
}

static inline pud_t pud_mknotpresent(pud_t pud)
{
        return pfn_pud(pud_pfn(pud),
              __pgprot(pud_flags(pud) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
}

static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);

static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
        pteval_t val = pte_val(pte), oldval = val;

        /*
         * Chop off the NX bit (if present), and add the NX portion of
         * the newprot (if present):
         */
        val &= _PAGE_CHG_MASK;
        val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
        val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
        return __pte(val);
}

static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
{
        pmdval_t val = pmd_val(pmd), oldval = val;

        val &= _HPAGE_CHG_MASK;
        val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
        val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
        return __pmd(val);
}

/* mprotect needs to preserve PAT bits when updating vm_page_prot */
#define pgprot_modify pgprot_modify
static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
{
        pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK; /*covered*/
        pgprotval_t addbits = pgprot_val(newprot);
        return __pgprot(preservebits | addbits);
}

#define pte_pgprot(x) __pgprot(pte_flags(x))
#define pmd_pgprot(x) __pgprot(pmd_flags(x))
#define pud_pgprot(x) __pgprot(pud_flags(x))
#define p4d_pgprot(x) __pgprot(p4d_flags(x))

#define canon_pgprot(p) __pgprot(massage_pgprot(p))

static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
{
        return canon_pgprot(prot); /*covered*/
}

static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
                                         enum page_cache_mode pcm,
                                         enum page_cache_mode new_pcm)
{
        /*
         * PAT type is always WB for untracked ranges, so no need to check.
         */
        if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
                return 1;

        /*
         * Certain new memtypes are not allowed with certain
         * requested memtype:
         * - request is uncached, return cannot be write-back
         * - request is write-combine, return cannot be write-back
         * - request is write-through, return cannot be write-back
         * - request is write-through, return cannot be write-combine
         */
        if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
             new_pcm == _PAGE_CACHE_MODE_WB) ||
            (pcm == _PAGE_CACHE_MODE_WC &&
             new_pcm == _PAGE_CACHE_MODE_WB) ||
            (pcm == _PAGE_CACHE_MODE_WT &&
             new_pcm == _PAGE_CACHE_MODE_WB) ||
            (pcm == _PAGE_CACHE_MODE_WT &&
             new_pcm == _PAGE_CACHE_MODE_WC)) {
                return 0;
        }

        return 1;
}

pmd_t *populate_extra_pmd(unsigned long vaddr);
pte_t *populate_extra_pte(unsigned long vaddr);

#ifdef CONFIG_PAGE_TABLE_ISOLATION
pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);

/*
 * Take a PGD location (pgdp) and a pgd value that needs to be set there.
 * Populates the user and returns the resulting PGD that must be set in
 * the kernel copy of the page tables.
 */
static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
{
        if (!static_cpu_has(X86_FEATURE_PTI))
                return pgd;
        return __pti_set_user_pgtbl(pgdp, pgd);
}
#else   /* CONFIG_PAGE_TABLE_ISOLATION */
static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
{
        return pgd;
}
#endif  /* CONFIG_PAGE_TABLE_ISOLATION */

#endif        /* __ASSEMBLY__ */


#ifdef CONFIG_X86_32
# include <asm/pgtable_32.h>
#else
# include <asm/pgtable_64.h>
#endif

#ifndef __ASSEMBLY__
#include <linux/mm_types.h>
#include <linux/mmdebug.h>
#include <linux/log2.h>
#include <asm/fixmap.h>

static inline int pte_none(pte_t pte)
{
        return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK)); /*covered*/
}

#define __HAVE_ARCH_PTE_SAME
static inline int pte_same(pte_t a, pte_t b)
{
        return a.pte == b.pte;
}

static inline int pte_present(pte_t a)
{
        return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE); /*covered*/
}

#ifdef __HAVE_ARCH_PTE_DEVMAP
static inline int pte_devmap(pte_t a)
{
        return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; /*covered*/
}
#endif

#define pte_accessible pte_accessible
static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
{
        if (pte_flags(a) & _PAGE_PRESENT)
                return true;

        if ((pte_flags(a) & _PAGE_PROTNONE) &&
                        mm_tlb_flush_pending(mm))
                return true;

        return false;
}

static inline int pmd_present(pmd_t pmd)
{
        /*
         * Checking for _PAGE_PSE is needed too because
         * split_huge_page will temporarily clear the present bit (but
         * the _PAGE_PSE flag will remain set at all times while the
         * _PAGE_PRESENT bit is clear).
         */
        return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE); /*covered*/
}

#ifdef CONFIG_NUMA_BALANCING
/*
 * These work without NUMA balancing but the kernel does not care. See the
 * comment in include/asm-generic/pgtable.h
 */
static inline int pte_protnone(pte_t pte)
{
        return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT)) /*covered*/
                == _PAGE_PROTNONE;
}

static inline int pmd_protnone(pmd_t pmd)
{
        return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT)) /*covered*/
                == _PAGE_PROTNONE;
}
#endif /* CONFIG_NUMA_BALANCING */

static inline int pmd_none(pmd_t pmd)
{
        /* Only check low word on 32-bit platforms, since it might be
           out of sync with upper half. */
        unsigned long val = native_pmd_val(pmd);
        return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0; /*covered*/
}

static inline unsigned long pmd_page_vaddr(pmd_t pmd)
{
        return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd)); /*covered*/
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define pmd_page(pmd)        pfn_to_page(pmd_pfn(pmd))

/*
 * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
 *
 * this macro returns the index of the entry in the pmd page which would
 * control the given virtual address
 */
static inline unsigned long pmd_index(unsigned long address)
{
        return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); /*covered*/
}

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 *
 * (Currently stuck as a macro because of indirect forward reference
 * to linux/mm.h:page_to_nid())
 */
#define mk_pte(page, pgprot)   pfn_pte(page_to_pfn(page), (pgprot))

/*
 * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
 *
 * this function returns the index of the entry in the pte page which would
 * control the given virtual address
 */
static inline unsigned long pte_index(unsigned long address)
{
        return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); /*covered*/
}

static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
{
        return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); /*covered*/
}

static inline int pmd_bad(pmd_t pmd)
{
        return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; /*covered*/
}

static inline unsigned long pages_to_mb(unsigned long npg)
{
        return npg >> (20 - PAGE_SHIFT);
}

#if CONFIG_PGTABLE_LEVELS > 2
static inline int pud_none(pud_t pud)
{
        return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0; /*covered*/
}

static inline int pud_present(pud_t pud)
{
        return pud_flags(pud) & _PAGE_PRESENT; /*covered*/
}

static inline unsigned long pud_page_vaddr(pud_t pud)
{
        return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud)); /*covered*/
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define pud_page(pud)        pfn_to_page(pud_pfn(pud))

/* Find an entry in the second-level page table.. */
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
{
        return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); /*covered*/
}

static inline int pud_large(pud_t pud)
{
        return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) == /*covered*/
                (_PAGE_PSE | _PAGE_PRESENT);
}

static inline int pud_bad(pud_t pud)
{
        return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0; /*covered*/
}
#else
static inline int pud_large(pud_t pud)
{
        return 0;
}
#endif        /* CONFIG_PGTABLE_LEVELS > 2 */

static inline unsigned long pud_index(unsigned long address)
{
        return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); /*covered*/
}

#if CONFIG_PGTABLE_LEVELS > 3
static inline int p4d_none(p4d_t p4d)
{
        return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0; /*covered*/
}

static inline int p4d_present(p4d_t p4d)
{
        return p4d_flags(p4d) & _PAGE_PRESENT;
}

static inline unsigned long p4d_page_vaddr(p4d_t p4d)
{
        return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d)); /*covered*/
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define p4d_page(p4d)        pfn_to_page(p4d_pfn(p4d))

/* Find an entry in the third-level page table.. */
static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
{
        return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); /*covered*/
}

static inline int p4d_bad(p4d_t p4d)
{
        unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;

        if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
                ignore_flags |= _PAGE_NX;

        return (p4d_flags(p4d) & ~ignore_flags) != 0; /*covered*/
}
#endif  /* CONFIG_PGTABLE_LEVELS > 3 */

static inline unsigned long p4d_index(unsigned long address)
{
        return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
}

#if CONFIG_PGTABLE_LEVELS > 4
static inline int pgd_present(pgd_t pgd)
{
        if (!pgtable_l5_enabled())
                return 1;
        return pgd_flags(pgd) & _PAGE_PRESENT;
}

static inline unsigned long pgd_page_vaddr(pgd_t pgd)
{
        return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define pgd_page(pgd)        pfn_to_page(pgd_pfn(pgd))

/* to find an entry in a page-table-directory. */
static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
{
        if (!pgtable_l5_enabled())
                return (p4d_t *)pgd;
        return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
}

static inline int pgd_bad(pgd_t pgd)
{
        unsigned long ignore_flags = _PAGE_USER;

        if (!pgtable_l5_enabled())
                return 0;

        if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
                ignore_flags |= _PAGE_NX;

        return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
}

static inline int pgd_none(pgd_t pgd)
{
        if (!pgtable_l5_enabled())
                return 0;
        /*
         * There is no need to do a workaround for the KNL stray
         * A/D bit erratum here.  PGDs only point to page tables
         * except on 32-bit non-PAE which is not supported on
         * KNL.
         */
        return !native_pgd_val(pgd);
}
#endif        /* CONFIG_PGTABLE_LEVELS > 4 */

#endif        /* __ASSEMBLY__ */

/*
 * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
 *
 * this macro returns the index of the entry in the pgd page which would
 * control the given virtual address
 */
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))

/*
 * pgd_offset() returns a (pgd_t *)
 * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
 */
#define pgd_offset_pgd(pgd, address) (pgd + pgd_index((address)))
/*
 * a shortcut to get a pgd_t in a given mm
 */
#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
/*
 * a shortcut which implies the use of the kernel's pgd, instead
 * of a process's
 */
#define pgd_offset_k(address) pgd_offset(&init_mm, (address))


#define KERNEL_PGD_BOUNDARY        pgd_index(PAGE_OFFSET)
#define KERNEL_PGD_PTRS                (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)

#ifndef __ASSEMBLY__

extern int direct_gbpages;
void init_mem_mapping(void);
void early_alloc_pgt_buf(void);
extern void memblock_find_dma_reserve(void);

#ifdef CONFIG_X86_64
/* Realmode trampoline initialization. */
extern pgd_t trampoline_pgd_entry;
static inline void __meminit init_trampoline_default(void)
{
        /* Default trampoline pgd value */
        trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
}
# ifdef CONFIG_RANDOMIZE_MEMORY
void __meminit init_trampoline(void);
# else
#  define init_trampoline init_trampoline_default
# endif
#else
static inline void init_trampoline(void) { }
#endif

/* local pte updates need not use xchg for locking */
static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
{
        pte_t res = *ptep;

        /* Pure native function needs no input for mm, addr */
        native_pte_clear(NULL, 0, ptep);
        return res;
}

static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
{
        pmd_t res = *pmdp;

        native_pmd_clear(pmdp);
        return res;
}

static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
{
        pud_t res = *pudp;

        native_pud_clear(pudp);
        return res;
}

static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
                                     pte_t *ptep , pte_t pte)
{
        native_set_pte(ptep, pte);
}

static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
                              pmd_t *pmdp, pmd_t pmd)
{
        set_pmd(pmdp, pmd); /*covered*/
}

static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
                              pud_t *pudp, pud_t pud)
{
        native_set_pud(pudp, pud);
}

/*
 * We only update the dirty/accessed state if we set
 * the dirty bit by hand in the kernel, since the hardware
 * will do the accessed bit for us, and we don't want to
 * race with other CPU's that might be updating the dirty
 * bit at the same time.
 */
struct vm_area_struct;

#define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
extern int ptep_set_access_flags(struct vm_area_struct *vma,
                                 unsigned long address, pte_t *ptep,
                                 pte_t entry, int dirty);

#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
                                     unsigned long addr, pte_t *ptep);

#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
extern int ptep_clear_flush_young(struct vm_area_struct *vma,
                                  unsigned long address, pte_t *ptep);

#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
                                       pte_t *ptep)
{
        pte_t pte = native_ptep_get_and_clear(ptep); /*covered*/
        return pte;
}

#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
                                            unsigned long addr, pte_t *ptep,
                                            int full)
{
        pte_t pte;
        if (full) {
                /*
                 * Full address destruction in progress; paravirt does not
                 * care about updates and native needs no locking
                 */
                pte = native_local_ptep_get_and_clear(ptep);
        } else {
                pte = ptep_get_and_clear(mm, addr, ptep); /*covered*/
        }
        return pte;
}

#define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(struct mm_struct *mm,
                                      unsigned long addr, pte_t *ptep)
{
        clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
}

#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)

#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))

#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
extern int pmdp_set_access_flags(struct vm_area_struct *vma,
                                 unsigned long address, pmd_t *pmdp,
                                 pmd_t entry, int dirty);
extern int pudp_set_access_flags(struct vm_area_struct *vma,
                                 unsigned long address, pud_t *pudp,
                                 pud_t entry, int dirty);

#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
                                     unsigned long addr, pmd_t *pmdp);
extern int pudp_test_and_clear_young(struct vm_area_struct *vma,
                                     unsigned long addr, pud_t *pudp);

#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
                                  unsigned long address, pmd_t *pmdp);


#define pmd_write pmd_write
static inline int pmd_write(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_RW; /*covered*/
}

#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
                                       pmd_t *pmdp)
{
        return native_pmdp_get_and_clear(pmdp); /*covered*/
}

#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
                                        unsigned long addr, pud_t *pudp)
{
        return native_pudp_get_and_clear(pudp);
}

#define __HAVE_ARCH_PMDP_SET_WRPROTECT
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
                                      unsigned long addr, pmd_t *pmdp)
{
        clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
}

#define pud_write pud_write
static inline int pud_write(pud_t pud)
{
        return pud_flags(pud) & _PAGE_RW;
}

#ifndef pmdp_establish
#define pmdp_establish pmdp_establish
static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
                unsigned long address, pmd_t *pmdp, pmd_t pmd)
{
        if (IS_ENABLED(CONFIG_SMP)) {
                return xchg(pmdp, pmd);
        } else {
                pmd_t old = *pmdp;
                WRITE_ONCE(*pmdp, pmd);
                return old;
        }
}
#endif
/*
 * Page table pages are page-aligned.  The lower half of the top
 * level is used for userspace and the top half for the kernel.
 *
 * Returns true for parts of the PGD that map userspace and
 * false for the parts that map the kernel.
 */
static inline bool pgdp_maps_userspace(void *__ptr)
{
        unsigned long ptr = (unsigned long)__ptr;

        return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
}

static inline int pgd_large(pgd_t pgd) { return 0; }

#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
 * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
 * (8k-aligned and 8k in size).  The kernel one is at the beginning 4k and
 * the user one is in the last 4k.  To switch between them, you
 * just need to flip the 12th bit in their addresses.
 */
#define PTI_PGTABLE_SWITCH_BIT        PAGE_SHIFT

/*
 * This generates better code than the inline assembly in
 * __set_bit().
 */
static inline void *ptr_set_bit(void *ptr, int bit)
{
        unsigned long __ptr = (unsigned long)ptr;

        __ptr |= BIT(bit);
        return (void *)__ptr;
}
static inline void *ptr_clear_bit(void *ptr, int bit)
{
        unsigned long __ptr = (unsigned long)ptr;

        __ptr &= ~BIT(bit);
        return (void *)__ptr;
}

static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
{
        return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
{
        return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
{
        return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
{
        return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}
#endif /* CONFIG_PAGE_TABLE_ISOLATION */

/*
 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
 *
 *  dst - pointer to pgd range anwhere on a pgd page
 *  src - ""
 *  count - the number of pgds to copy.
 *
 * dst and src can be on the same page, but the range must not overlap,
 * and must not cross a page boundary.
 */
static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
{
        memcpy(dst, src, count * sizeof(pgd_t));
#ifdef CONFIG_PAGE_TABLE_ISOLATION
        if (!static_cpu_has(X86_FEATURE_PTI))
                return;
        /* Clone the user space pgd as well */
        memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
               count * sizeof(pgd_t));
#endif
}

#define PTE_SHIFT ilog2(PTRS_PER_PTE)
static inline int page_level_shift(enum pg_level level)
{
        return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
}
static inline unsigned long page_level_size(enum pg_level level)
{
        return 1UL << page_level_shift(level);
}
static inline unsigned long page_level_mask(enum pg_level level)
{
        return ~(page_level_size(level) - 1);
}

/*
 * The x86 doesn't have any external MMU info: the kernel page
 * tables contain all the necessary information.
 */
static inline void update_mmu_cache(struct vm_area_struct *vma,
                unsigned long addr, pte_t *ptep)
{
}
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
                unsigned long addr, pmd_t *pmd)
{
}
static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
                unsigned long addr, pud_t *pud)
{
}

#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
}

static inline int pte_swp_soft_dirty(pte_t pte)
{
        return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
}

static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
}

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
}

static inline int pmd_swp_soft_dirty(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY; /*covered*/
}

static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
}
#endif
#endif

#define PKRU_AD_BIT 0x1
#define PKRU_WD_BIT 0x2
#define PKRU_BITS_PER_PKEY 2

static inline bool __pkru_allows_read(u32 pkru, u16 pkey)
{
        int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
        return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits));
}

static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
{
        int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
        /*
         * Access-disable disables writes too so we need to check
         * both bits here.
         */
        return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits));
}

static inline u16 pte_flags_pkey(unsigned long pte_flags)
{
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
        /* ifdef to avoid doing 59-bit shift on 32-bit values */
        return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
#else
        return 0;
#endif
}

static inline bool __pkru_allows_pkey(u16 pkey, bool write)
{
        u32 pkru = read_pkru(); /*covered*/

        if (!__pkru_allows_read(pkru, pkey))
                return false;
        if (write && !__pkru_allows_write(pkru, pkey)) /*covered*/
                return false;

        return true;
}

/*
 * 'pteval' can come from a PTE, PMD or PUD.  We only check
 * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
 * same value on all 3 types.
 */
static inline bool __pte_access_permitted(unsigned long pteval, bool write) /*covered*/
{
        unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;

        if (write)
                need_pte_bits |= _PAGE_RW;

        if ((pteval & need_pte_bits) != need_pte_bits)
                return 0;

        return __pkru_allows_pkey(pte_flags_pkey(pteval), write); /*covered*/
}

#define pte_access_permitted pte_access_permitted
static inline bool pte_access_permitted(pte_t pte, bool write)
{
        return __pte_access_permitted(pte_val(pte), write); /*covered*/
}

#define pmd_access_permitted pmd_access_permitted
static inline bool pmd_access_permitted(pmd_t pmd, bool write)
{
        return __pte_access_permitted(pmd_val(pmd), write); /*covered*/
}

#define pud_access_permitted pud_access_permitted
static inline bool pud_access_permitted(pud_t pud, bool write)
{
        return __pte_access_permitted(pud_val(pud), write);
}

#define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1
extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot);

static inline bool arch_has_pfn_modify_check(void)
{
        return boot_cpu_has_bug(X86_BUG_L1TF);
}

#include <asm-generic/pgtable.h>
#endif        /* __ASSEMBLY__ */

#endif /* _ASM_X86_PGTABLE_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGTABLE_64_H
#define _ASM_X86_PGTABLE_64_H

#include <linux/const.h>
#include <asm/pgtable_64_types.h>

#ifndef __ASSEMBLY__

/*
 * This file contains the functions and defines necessary to modify and use
 * the x86-64 page table tree.
 */
#include <asm/processor.h>
#include <linux/bitops.h>
#include <linux/threads.h>
#include <asm/fixmap.h>

extern p4d_t level4_kernel_pgt[512];
extern p4d_t level4_ident_pgt[512];
extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
extern pmd_t level2_kernel_pgt[512];
extern pmd_t level2_fixmap_pgt[512];
extern pmd_t level2_ident_pgt[512];
extern pte_t level1_fixmap_pgt[512 * FIXMAP_PMD_NUM];
extern pgd_t init_top_pgt[];

#define swapper_pg_dir init_top_pgt

extern void paging_init(void);
static inline void sync_initial_page_table(void) { }

#define pte_ERROR(e)                                        \
        pr_err("%s:%d: bad pte %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pte_val(e))
#define pmd_ERROR(e)                                        \
        pr_err("%s:%d: bad pmd %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pmd_val(e))
#define pud_ERROR(e)                                        \
        pr_err("%s:%d: bad pud %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pud_val(e))

#if CONFIG_PGTABLE_LEVELS >= 5
#define p4d_ERROR(e)                                        \
        pr_err("%s:%d: bad p4d %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), p4d_val(e))
#endif

#define pgd_ERROR(e)                                        \
        pr_err("%s:%d: bad pgd %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pgd_val(e))

struct mm_struct;

void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte);
void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);

static inline void native_set_pte(pte_t *ptep, pte_t pte)
{
        WRITE_ONCE(*ptep, pte);
}

static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
                                    pte_t *ptep)
{
        native_set_pte(ptep, native_make_pte(0));
}

static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
        native_set_pte(ptep, pte);
}

static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
        WRITE_ONCE(*pmdp, pmd);
}

static inline void native_pmd_clear(pmd_t *pmd)
{
        native_set_pmd(pmd, native_make_pmd(0));
}

static inline pte_t native_ptep_get_and_clear(pte_t *xp)
{
#ifdef CONFIG_SMP
        return native_make_pte(xchg(&xp->pte, 0)); /*covered*/
#else
        /* native_local_ptep_get_and_clear,
           but duplicated because of cyclic dependency */
        pte_t ret = *xp;
        native_pte_clear(NULL, 0, xp);
        return ret;
#endif
}

static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
{
#ifdef CONFIG_SMP
        return native_make_pmd(xchg(&xp->pmd, 0)); /*covered*/
#else
        /* native_local_pmdp_get_and_clear,
           but duplicated because of cyclic dependency */
        pmd_t ret = *xp;
        native_pmd_clear(xp);
        return ret;
#endif
}

static inline void native_set_pud(pud_t *pudp, pud_t pud)
{
        WRITE_ONCE(*pudp, pud);
}

static inline void native_pud_clear(pud_t *pud)
{
        native_set_pud(pud, native_make_pud(0));
}

static inline pud_t native_pudp_get_and_clear(pud_t *xp)
{
#ifdef CONFIG_SMP
        return native_make_pud(xchg(&xp->pud, 0));
#else
        /* native_local_pudp_get_and_clear,
         * but duplicated because of cyclic dependency
         */
        pud_t ret = *xp;

        native_pud_clear(xp);
        return ret;
#endif
}

static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
{
        pgd_t pgd;

        if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) {
                WRITE_ONCE(*p4dp, p4d);
                return;
        }

        pgd = native_make_pgd(native_p4d_val(p4d));
        pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd);
        WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd)));
}

static inline void native_p4d_clear(p4d_t *p4d)
{
        native_set_p4d(p4d, native_make_p4d(0));
}

static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
{
        WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd));
}

static inline void native_pgd_clear(pgd_t *pgd)
{
        native_set_pgd(pgd, native_make_pgd(0));
}

extern void sync_global_pgds(unsigned long start, unsigned long end);

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */

/*
 * Level 4 access.
 */
#define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE)

/* PUD - Level3 access */

/* PMD  - Level 2 access */

/* PTE - Level 1 access. */

/* x86-64 always has all page tables mapped. */
#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
#define pte_unmap(pte) ((void)(pte))/* NOP */

/*
 * Encode and de-code a swap entry
 *
 * |     ...            | 11| 10|  9|8|7|6|5| 4| 3|2| 1|0| <- bit number
 * |     ...            |SW3|SW2|SW1|G|L|D|A|CD|WT|U| W|P| <- bit names
 * | TYPE (59-63) | ~OFFSET (9-58)  |0|0|X|X| X| X|X|SD|0| <- swp entry
 *
 * G (8) is aliased and used as a PROT_NONE indicator for
 * !present ptes.  We need to start storing swap entries above
 * there.  We also need to avoid using A and D because of an
 * erratum where they can be incorrectly set by hardware on
 * non-present PTEs.
 *
 * SD (1) in swp entry is used to store soft dirty bit, which helps us
 * remember soft dirty over page migration
 *
 * Bit 7 in swp entry should be 0 because pmd_present checks not only P,
 * but also L and G.
 *
 * The offset is inverted by a binary not operation to make the high
 * physical bits set.
 */
#define SWP_TYPE_BITS                5

#define SWP_OFFSET_FIRST_BIT        (_PAGE_BIT_PROTNONE + 1)

/* We always extract/encode the offset by shifting it all the way up, and then down again */
#define SWP_OFFSET_SHIFT        (SWP_OFFSET_FIRST_BIT+SWP_TYPE_BITS)

#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)

/* Extract the high bits for type */
#define __swp_type(x) ((x).val >> (64 - SWP_TYPE_BITS))

/* Shift up (to get rid of type), then down to get value */
#define __swp_offset(x) (~(x).val << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT)

/*
 * Shift the offset up "too far" by TYPE bits, then down again
 * The offset is inverted by a binary not operation to make the high
 * physical bits set.
 */
#define __swp_entry(type, offset) ((swp_entry_t) { \
        (~(unsigned long)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \
        | ((unsigned long)(type) << (64-SWP_TYPE_BITS)) })

#define __pte_to_swp_entry(pte)                ((swp_entry_t) { pte_val((pte)) })
#define __pmd_to_swp_entry(pmd)                ((swp_entry_t) { pmd_val((pmd)) })
#define __swp_entry_to_pte(x)                ((pte_t) { .pte = (x).val })
#define __swp_entry_to_pmd(x)                ((pmd_t) { .pmd = (x).val })

extern int kern_addr_valid(unsigned long addr);
extern void cleanup_highmap(void);

#define HAVE_ARCH_UNMAPPED_AREA
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN

#define pgtable_cache_init()   do { } while (0)
#define check_pgt_cache()      do { } while (0)

#define PAGE_AGP    PAGE_KERNEL_NOCACHE
#define HAVE_PAGE_AGP 1

/* fs/proc/kcore.c */
#define        kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK)
#define        kc_offset_to_vaddr(o) ((o) | ~__VIRTUAL_MASK)

#define __HAVE_ARCH_PTE_SAME

#define vmemmap ((struct page *)VMEMMAP_START)

extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);

#define gup_fast_permitted gup_fast_permitted
static inline bool gup_fast_permitted(unsigned long start, int nr_pages)
{
        unsigned long len, end;

        len = (unsigned long)nr_pages << PAGE_SHIFT;
        end = start + len;
        if (end < start) /*covered*/
                return false;
        if (end >> __VIRTUAL_MASK_SHIFT) /*covered*/
                return false;
        return true;
}

#include <asm/pgtable-invert.h>

#endif /* !__ASSEMBLY__ */
#endif /* _ASM_X86_PGTABLE_64_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGTABLE_DEFS_H
#define _ASM_X86_PGTABLE_DEFS_H

#include <linux/const.h>
#include <linux/mem_encrypt.h>

#include <asm/page_types.h>

#define FIRST_USER_ADDRESS        0UL

#define _PAGE_BIT_PRESENT        0        /* is present */
#define _PAGE_BIT_RW                1        /* writeable */
#define _PAGE_BIT_USER                2        /* userspace addressable */
#define _PAGE_BIT_PWT                3        /* page write through */
#define _PAGE_BIT_PCD                4        /* page cache disabled */
#define _PAGE_BIT_ACCESSED        5        /* was accessed (raised by CPU) */
#define _PAGE_BIT_DIRTY                6        /* was written to (raised by CPU) */
#define _PAGE_BIT_PSE                7        /* 4 MB (or 2MB) page */
#define _PAGE_BIT_PAT                7        /* on 4KB pages */
#define _PAGE_BIT_GLOBAL        8        /* Global TLB entry PPro+ */
#define _PAGE_BIT_SOFTW1        9        /* available for programmer */
#define _PAGE_BIT_SOFTW2        10        /* " */
#define _PAGE_BIT_SOFTW3        11        /* " */
#define _PAGE_BIT_PAT_LARGE        12        /* On 2MB or 1GB pages */
#define _PAGE_BIT_SOFTW4        58        /* available for programmer */
#define _PAGE_BIT_PKEY_BIT0        59        /* Protection Keys, bit 1/4 */
#define _PAGE_BIT_PKEY_BIT1        60        /* Protection Keys, bit 2/4 */
#define _PAGE_BIT_PKEY_BIT2        61        /* Protection Keys, bit 3/4 */
#define _PAGE_BIT_PKEY_BIT3        62        /* Protection Keys, bit 4/4 */
#define _PAGE_BIT_NX                63        /* No execute: only valid after cpuid check */

#define _PAGE_BIT_SPECIAL        _PAGE_BIT_SOFTW1
#define _PAGE_BIT_CPA_TEST        _PAGE_BIT_SOFTW1
#define _PAGE_BIT_SOFT_DIRTY        _PAGE_BIT_SOFTW3 /* software dirty tracking */
#define _PAGE_BIT_DEVMAP        _PAGE_BIT_SOFTW4

/* If _PAGE_BIT_PRESENT is clear, we use these: */
/* - if the user mapped it with PROT_NONE; pte_present gives true */
#define _PAGE_BIT_PROTNONE        _PAGE_BIT_GLOBAL

#define _PAGE_PRESENT        (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT)
#define _PAGE_RW        (_AT(pteval_t, 1) << _PAGE_BIT_RW)
#define _PAGE_USER        (_AT(pteval_t, 1) << _PAGE_BIT_USER)
#define _PAGE_PWT        (_AT(pteval_t, 1) << _PAGE_BIT_PWT)
#define _PAGE_PCD        (_AT(pteval_t, 1) << _PAGE_BIT_PCD)
#define _PAGE_ACCESSED        (_AT(pteval_t, 1) << _PAGE_BIT_ACCESSED)
#define _PAGE_DIRTY        (_AT(pteval_t, 1) << _PAGE_BIT_DIRTY)
#define _PAGE_PSE        (_AT(pteval_t, 1) << _PAGE_BIT_PSE)
#define _PAGE_GLOBAL        (_AT(pteval_t, 1) << _PAGE_BIT_GLOBAL)
#define _PAGE_SOFTW1        (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW1)
#define _PAGE_SOFTW2        (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW2)
#define _PAGE_SOFTW3        (_AT(pteval_t, 1) << _PAGE_BIT_SOFTW3)
#define _PAGE_PAT        (_AT(pteval_t, 1) << _PAGE_BIT_PAT)
#define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE)
#define _PAGE_SPECIAL        (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL)
#define _PAGE_CPA_TEST        (_AT(pteval_t, 1) << _PAGE_BIT_CPA_TEST)
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
#define _PAGE_PKEY_BIT0        (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT0)
#define _PAGE_PKEY_BIT1        (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT1)
#define _PAGE_PKEY_BIT2        (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT2)
#define _PAGE_PKEY_BIT3        (_AT(pteval_t, 1) << _PAGE_BIT_PKEY_BIT3)
#else
#define _PAGE_PKEY_BIT0        (_AT(pteval_t, 0))
#define _PAGE_PKEY_BIT1        (_AT(pteval_t, 0))
#define _PAGE_PKEY_BIT2        (_AT(pteval_t, 0))
#define _PAGE_PKEY_BIT3        (_AT(pteval_t, 0))
#endif

#define _PAGE_PKEY_MASK (_PAGE_PKEY_BIT0 | \
                         _PAGE_PKEY_BIT1 | \
                         _PAGE_PKEY_BIT2 | \
                         _PAGE_PKEY_BIT3)

#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
#define _PAGE_KNL_ERRATUM_MASK (_PAGE_DIRTY | _PAGE_ACCESSED)
#else
#define _PAGE_KNL_ERRATUM_MASK 0
#endif

#ifdef CONFIG_MEM_SOFT_DIRTY
#define _PAGE_SOFT_DIRTY        (_AT(pteval_t, 1) << _PAGE_BIT_SOFT_DIRTY)
#else
#define _PAGE_SOFT_DIRTY        (_AT(pteval_t, 0))
#endif

/*
 * Tracking soft dirty bit when a page goes to a swap is tricky.
 * We need a bit which can be stored in pte _and_ not conflict
 * with swap entry format. On x86 bits 1-4 are *not* involved
 * into swap entry computation, but bit 7 is used for thp migration,
 * so we borrow bit 1 for soft dirty tracking.
 *
 * Please note that this bit must be treated as swap dirty page
 * mark if and only if the PTE/PMD has present bit clear!
 */
#ifdef CONFIG_MEM_SOFT_DIRTY
#define _PAGE_SWP_SOFT_DIRTY        _PAGE_RW
#else
#define _PAGE_SWP_SOFT_DIRTY        (_AT(pteval_t, 0))
#endif

#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
#define _PAGE_NX        (_AT(pteval_t, 1) << _PAGE_BIT_NX)
#define _PAGE_DEVMAP        (_AT(u64, 1) << _PAGE_BIT_DEVMAP)
#define __HAVE_ARCH_PTE_DEVMAP
#else
#define _PAGE_NX        (_AT(pteval_t, 0))
#define _PAGE_DEVMAP        (_AT(pteval_t, 0))
#endif

#define _PAGE_PROTNONE        (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)

#define _PAGE_TABLE_NOENC        (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER |\
                                 _PAGE_ACCESSED | _PAGE_DIRTY)
#define _KERNPG_TABLE_NOENC        (_PAGE_PRESENT | _PAGE_RW |                \
                                 _PAGE_ACCESSED | _PAGE_DIRTY)

/*
 * Set of bits not changed in pte_modify.  The pte's
 * protection key is treated like _PAGE_RW, for
 * instance, and is *not* included in this mask since
 * pte_modify() does modify it.
 */
#define _PAGE_CHG_MASK        (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT |                \
                         _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY |        \
                         _PAGE_SOFT_DIRTY | _PAGE_DEVMAP)
#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE)

/*
 * The cache modes defined here are used to translate between pure SW usage
 * and the HW defined cache mode bits and/or PAT entries.
 *
 * The resulting bits for PWT, PCD and PAT should be chosen in a way
 * to have the WB mode at index 0 (all bits clear). This is the default
 * right now and likely would break too much if changed.
 */
#ifndef __ASSEMBLY__
enum page_cache_mode {
        _PAGE_CACHE_MODE_WB = 0,
        _PAGE_CACHE_MODE_WC = 1,
        _PAGE_CACHE_MODE_UC_MINUS = 2,
        _PAGE_CACHE_MODE_UC = 3,
        _PAGE_CACHE_MODE_WT = 4,
        _PAGE_CACHE_MODE_WP = 5,
        _PAGE_CACHE_MODE_NUM = 8
};
#endif

#define _PAGE_CACHE_MASK        (_PAGE_PAT | _PAGE_PCD | _PAGE_PWT)
#define _PAGE_NOCACHE                (cachemode2protval(_PAGE_CACHE_MODE_UC))
#define _PAGE_CACHE_WP                (cachemode2protval(_PAGE_CACHE_MODE_WP))

#define PAGE_NONE        __pgprot(_PAGE_PROTNONE | _PAGE_ACCESSED)
#define PAGE_SHARED        __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \
                                 _PAGE_ACCESSED | _PAGE_NX)

#define PAGE_SHARED_EXEC        __pgprot(_PAGE_PRESENT | _PAGE_RW |        \
                                         _PAGE_USER | _PAGE_ACCESSED)
#define PAGE_COPY_NOEXEC        __pgprot(_PAGE_PRESENT | _PAGE_USER |        \
                                         _PAGE_ACCESSED | _PAGE_NX)
#define PAGE_COPY_EXEC                __pgprot(_PAGE_PRESENT | _PAGE_USER |        \
                                         _PAGE_ACCESSED)
#define PAGE_COPY                PAGE_COPY_NOEXEC
#define PAGE_READONLY                __pgprot(_PAGE_PRESENT | _PAGE_USER |        \
                                         _PAGE_ACCESSED | _PAGE_NX)
#define PAGE_READONLY_EXEC        __pgprot(_PAGE_PRESENT | _PAGE_USER |        \
                                         _PAGE_ACCESSED)

#define __PAGE_KERNEL_EXEC                                                \
        (_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_GLOBAL)
#define __PAGE_KERNEL                (__PAGE_KERNEL_EXEC | _PAGE_NX)

#define __PAGE_KERNEL_RO                (__PAGE_KERNEL & ~_PAGE_RW)
#define __PAGE_KERNEL_RX                (__PAGE_KERNEL_EXEC & ~_PAGE_RW)
#define __PAGE_KERNEL_NOCACHE                (__PAGE_KERNEL | _PAGE_NOCACHE)
#define __PAGE_KERNEL_VVAR                (__PAGE_KERNEL_RO | _PAGE_USER)
#define __PAGE_KERNEL_LARGE                (__PAGE_KERNEL | _PAGE_PSE)
#define __PAGE_KERNEL_LARGE_EXEC        (__PAGE_KERNEL_EXEC | _PAGE_PSE)
#define __PAGE_KERNEL_WP                (__PAGE_KERNEL | _PAGE_CACHE_WP)

#define __PAGE_KERNEL_IO                (__PAGE_KERNEL)
#define __PAGE_KERNEL_IO_NOCACHE        (__PAGE_KERNEL_NOCACHE)

#ifndef __ASSEMBLY__

#define _PAGE_ENC        (_AT(pteval_t, sme_me_mask))

#define _KERNPG_TABLE        (_PAGE_PRESENT | _PAGE_RW | _PAGE_ACCESSED |        \
                         _PAGE_DIRTY | _PAGE_ENC)
#define _PAGE_TABLE        (_KERNPG_TABLE | _PAGE_USER)

#define __PAGE_KERNEL_ENC        (__PAGE_KERNEL | _PAGE_ENC)
#define __PAGE_KERNEL_ENC_WP        (__PAGE_KERNEL_WP | _PAGE_ENC)

#define __PAGE_KERNEL_NOENC        (__PAGE_KERNEL)
#define __PAGE_KERNEL_NOENC_WP        (__PAGE_KERNEL_WP)

#define default_pgprot(x)        __pgprot((x) & __default_kernel_pte_mask)

#define PAGE_KERNEL                default_pgprot(__PAGE_KERNEL | _PAGE_ENC)
#define PAGE_KERNEL_NOENC        default_pgprot(__PAGE_KERNEL)
#define PAGE_KERNEL_RO                default_pgprot(__PAGE_KERNEL_RO | _PAGE_ENC)
#define PAGE_KERNEL_EXEC        default_pgprot(__PAGE_KERNEL_EXEC | _PAGE_ENC)
#define PAGE_KERNEL_EXEC_NOENC        default_pgprot(__PAGE_KERNEL_EXEC)
#define PAGE_KERNEL_RX                default_pgprot(__PAGE_KERNEL_RX | _PAGE_ENC)
#define PAGE_KERNEL_NOCACHE        default_pgprot(__PAGE_KERNEL_NOCACHE | _PAGE_ENC)
#define PAGE_KERNEL_LARGE        default_pgprot(__PAGE_KERNEL_LARGE | _PAGE_ENC)
#define PAGE_KERNEL_LARGE_EXEC        default_pgprot(__PAGE_KERNEL_LARGE_EXEC | _PAGE_ENC)
#define PAGE_KERNEL_VVAR        default_pgprot(__PAGE_KERNEL_VVAR | _PAGE_ENC)

#define PAGE_KERNEL_IO                default_pgprot(__PAGE_KERNEL_IO)
#define PAGE_KERNEL_IO_NOCACHE        default_pgprot(__PAGE_KERNEL_IO_NOCACHE)

#endif        /* __ASSEMBLY__ */

/*         xwr */
#define __P000        PAGE_NONE
#define __P001        PAGE_READONLY
#define __P010        PAGE_COPY
#define __P011        PAGE_COPY
#define __P100        PAGE_READONLY_EXEC
#define __P101        PAGE_READONLY_EXEC
#define __P110        PAGE_COPY_EXEC
#define __P111        PAGE_COPY_EXEC

#define __S000        PAGE_NONE
#define __S001        PAGE_READONLY
#define __S010        PAGE_SHARED
#define __S011        PAGE_SHARED
#define __S100        PAGE_READONLY_EXEC
#define __S101        PAGE_READONLY_EXEC
#define __S110        PAGE_SHARED_EXEC
#define __S111        PAGE_SHARED_EXEC

/*
 * early identity mapping  pte attrib macros.
 */
#ifdef CONFIG_X86_64
#define __PAGE_KERNEL_IDENT_LARGE_EXEC        __PAGE_KERNEL_LARGE_EXEC
#else
#define PTE_IDENT_ATTR         0x003                /* PRESENT+RW */
#define PDE_IDENT_ATTR         0x063                /* PRESENT+RW+DIRTY+ACCESSED */
#define PGD_IDENT_ATTR         0x001                /* PRESENT (no other attributes) */
#endif

#ifdef CONFIG_X86_32
# include <asm/pgtable_32_types.h>
#else
# include <asm/pgtable_64_types.h>
#endif

#ifndef __ASSEMBLY__

#include <linux/types.h>

/* Extracts the PFN from a (pte|pmd|pud|pgd)val_t of a 4KB page */
#define PTE_PFN_MASK                ((pteval_t)PHYSICAL_PAGE_MASK)

/*
 *  Extracts the flags from a (pte|pmd|pud|pgd)val_t
 *  This includes the protection key value.
 */
#define PTE_FLAGS_MASK                (~PTE_PFN_MASK)

typedef struct pgprot { pgprotval_t pgprot; } pgprot_t;

typedef struct { pgdval_t pgd; } pgd_t;

#ifdef CONFIG_X86_PAE

/*
 * PHYSICAL_PAGE_MASK might be non-constant when SME is compiled in, so we can't
 * use it here.
 */

#define PGD_PAE_PAGE_MASK        ((signed long)PAGE_MASK)
#define PGD_PAE_PHYS_MASK        (((1ULL << __PHYSICAL_MASK_SHIFT)-1) & PGD_PAE_PAGE_MASK)

/*
 * PAE allows Base Address, P, PWT, PCD and AVL bits to be set in PGD entries.
 * All other bits are Reserved MBZ
 */
#define PGD_ALLOWED_BITS        (PGD_PAE_PHYS_MASK | _PAGE_PRESENT | \
                                 _PAGE_PWT | _PAGE_PCD | \
                                 _PAGE_SOFTW1 | _PAGE_SOFTW2 | _PAGE_SOFTW3)

#else
/* No need to mask any bits for !PAE */
#define PGD_ALLOWED_BITS        (~0ULL)
#endif

static inline pgd_t native_make_pgd(pgdval_t val)
{
        return (pgd_t) { val & PGD_ALLOWED_BITS };
}

static inline pgdval_t native_pgd_val(pgd_t pgd)
{
        return pgd.pgd & PGD_ALLOWED_BITS;
}

static inline pgdval_t pgd_flags(pgd_t pgd)
{
        return native_pgd_val(pgd) & PTE_FLAGS_MASK;
}

#if CONFIG_PGTABLE_LEVELS > 4
typedef struct { p4dval_t p4d; } p4d_t;

static inline p4d_t native_make_p4d(pudval_t val)
{
        return (p4d_t) { val };
}

static inline p4dval_t native_p4d_val(p4d_t p4d)
{
        return p4d.p4d;
}
#else
#include <asm-generic/pgtable-nop4d.h>

static inline p4d_t native_make_p4d(pudval_t val)
{
        return (p4d_t) { .pgd = native_make_pgd((pgdval_t)val) };
}

static inline p4dval_t native_p4d_val(p4d_t p4d)
{
        return native_pgd_val(p4d.pgd);
}
#endif

#if CONFIG_PGTABLE_LEVELS > 3
typedef struct { pudval_t pud; } pud_t;

static inline pud_t native_make_pud(pmdval_t val)
{
        return (pud_t) { val };
}

static inline pudval_t native_pud_val(pud_t pud)
{
        return pud.pud;
}
#else
#include <asm-generic/pgtable-nopud.h>

static inline pud_t native_make_pud(pudval_t val)
{
        return (pud_t) { .p4d.pgd = native_make_pgd(val) };
}

static inline pudval_t native_pud_val(pud_t pud)
{
        return native_pgd_val(pud.p4d.pgd);
}
#endif

#if CONFIG_PGTABLE_LEVELS > 2
typedef struct { pmdval_t pmd; } pmd_t;

static inline pmd_t native_make_pmd(pmdval_t val)
{
        return (pmd_t) { val };
}

static inline pmdval_t native_pmd_val(pmd_t pmd)
{
        return pmd.pmd;
}
#else
#include <asm-generic/pgtable-nopmd.h>

static inline pmd_t native_make_pmd(pmdval_t val)
{
        return (pmd_t) { .pud.p4d.pgd = native_make_pgd(val) };
}

static inline pmdval_t native_pmd_val(pmd_t pmd)
{
        return native_pgd_val(pmd.pud.p4d.pgd);
}
#endif

static inline p4dval_t p4d_pfn_mask(p4d_t p4d)
{
        /* No 512 GiB huge pages yet */
        return PTE_PFN_MASK;
}

static inline p4dval_t p4d_flags_mask(p4d_t p4d)
{
        return ~p4d_pfn_mask(p4d);
}

static inline p4dval_t p4d_flags(p4d_t p4d)
{
        return native_p4d_val(p4d) & p4d_flags_mask(p4d);
}

static inline pudval_t pud_pfn_mask(pud_t pud)
{
        if (native_pud_val(pud) & _PAGE_PSE) /*covered*/
                return PHYSICAL_PUD_PAGE_MASK;
        else
                return PTE_PFN_MASK;
}

static inline pudval_t pud_flags_mask(pud_t pud)
{
        return ~pud_pfn_mask(pud); /*covered*/
}

static inline pudval_t pud_flags(pud_t pud)
{
        return native_pud_val(pud) & pud_flags_mask(pud); /*covered*/
}

static inline pmdval_t pmd_pfn_mask(pmd_t pmd)
{
        if (native_pmd_val(pmd) & _PAGE_PSE) /*covered*/
                return PHYSICAL_PMD_PAGE_MASK;
        else
                return PTE_PFN_MASK;
}

static inline pmdval_t pmd_flags_mask(pmd_t pmd)
{
        return ~pmd_pfn_mask(pmd); /*covered*/
}

static inline pmdval_t pmd_flags(pmd_t pmd)
{
        return native_pmd_val(pmd) & pmd_flags_mask(pmd); /*covered*/
}

static inline pte_t native_make_pte(pteval_t val)
{
        return (pte_t) { .pte = val };
}

static inline pteval_t native_pte_val(pte_t pte)
{
        return pte.pte;
}

static inline pteval_t pte_flags(pte_t pte)
{
        return native_pte_val(pte) & PTE_FLAGS_MASK; /*covered*/
}

#define pgprot_val(x)        ((x).pgprot)
#define __pgprot(x)        ((pgprot_t) { (x) } )

extern uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM];
extern uint8_t __pte2cachemode_tbl[8];

#define __pte2cm_idx(cb)                                \
        ((((cb) >> (_PAGE_BIT_PAT - 2)) & 4) |                \
         (((cb) >> (_PAGE_BIT_PCD - 1)) & 2) |                \
         (((cb) >> _PAGE_BIT_PWT) & 1))
#define __cm_idx2pte(i)                                        \
        ((((i) & 4) << (_PAGE_BIT_PAT - 2)) |                \
         (((i) & 2) << (_PAGE_BIT_PCD - 1)) |                \
         (((i) & 1) << _PAGE_BIT_PWT))

static inline unsigned long cachemode2protval(enum page_cache_mode pcm)
{
        if (likely(pcm == 0))
                return 0;
        return __cachemode2pte_tbl[pcm];
}
static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm)
{
        return __pgprot(cachemode2protval(pcm));
}
static inline enum page_cache_mode pgprot2cachemode(pgprot_t pgprot)
{
        unsigned long masked;

        masked = pgprot_val(pgprot) & _PAGE_CACHE_MASK; /*covered*/
        if (likely(masked == 0))
                return 0;
        return __pte2cachemode_tbl[__pte2cm_idx(masked)];
}
static inline pgprot_t pgprot_4k_2_large(pgprot_t pgprot)
{
        pgprotval_t val = pgprot_val(pgprot);
        pgprot_t new;

        pgprot_val(new) = (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) |
                ((val & _PAGE_PAT) << (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
        return new;
}
static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot)
{
        pgprotval_t val = pgprot_val(pgprot);
        pgprot_t new;

        pgprot_val(new) = (val & ~(_PAGE_PAT | _PAGE_PAT_LARGE)) | /*covered*/
                          ((val & _PAGE_PAT_LARGE) >>
                           (_PAGE_BIT_PAT_LARGE - _PAGE_BIT_PAT));
        return new;
}


typedef struct page *pgtable_t;

extern pteval_t __supported_pte_mask;
extern pteval_t __default_kernel_pte_mask;
extern void set_nx(void);
extern int nx_enabled;

#define pgprot_writecombine        pgprot_writecombine
extern pgprot_t pgprot_writecombine(pgprot_t prot);

#define pgprot_writethrough        pgprot_writethrough
extern pgprot_t pgprot_writethrough(pgprot_t prot);

/* Indicate that x86 has its own track and untrack pfn vma functions */
#define __HAVE_PFNMAP_TRACKING

#define __HAVE_PHYS_MEM_ACCESS_PROT
struct file;
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
                              unsigned long size, pgprot_t vma_prot);

/* Install a pte for a particular vaddr in kernel space. */
void set_pte_vaddr(unsigned long vaddr, pte_t pte);

#ifdef CONFIG_X86_32
extern void native_pagetable_init(void);
#else
#define native_pagetable_init        paging_init
#endif

struct seq_file;
extern void arch_report_meminfo(struct seq_file *m);

enum pg_level {
        PG_LEVEL_NONE,
        PG_LEVEL_4K,
        PG_LEVEL_2M,
        PG_LEVEL_1G,
        PG_LEVEL_512G,
        PG_LEVEL_NUM
};

#ifdef CONFIG_PROC_FS
extern void update_page_count(int level, unsigned long pages);
#else
static inline void update_page_count(int level, unsigned long pages) { }
#endif

/*
 * Helper function that returns the kernel pagetable entry controlling
 * the virtual address 'address'. NULL means no pagetable entry present.
 * NOTE: the return type is pte_t but if the pmd is PSE then we return it
 * as a pte too.
 */
extern pte_t *lookup_address(unsigned long address, unsigned int *level);
extern pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
                                    unsigned int *level);
extern pmd_t *lookup_pmd_address(unsigned long address);
extern phys_addr_t slow_virt_to_phys(void *__address);
extern int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn,
                                          unsigned long address,
                                          unsigned numpages,
                                          unsigned long page_flags);
extern int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address,
                                            unsigned long numpages);
#endif        /* !__ASSEMBLY__ */

#endif /* _ASM_X86_PGTABLE_DEFS_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PKEYS_H
#define _ASM_X86_PKEYS_H

#define ARCH_DEFAULT_PKEY        0

#define arch_max_pkey() (boot_cpu_has(X86_FEATURE_OSPKE) ? 16 : 1)

extern int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
                unsigned long init_val);

static inline bool arch_pkeys_enabled(void)
{
        return boot_cpu_has(X86_FEATURE_OSPKE);
}

/*
 * Try to dedicate one of the protection keys to be used as an
 * execute-only protection key.
 */
extern int __execute_only_pkey(struct mm_struct *mm);
static inline int execute_only_pkey(struct mm_struct *mm)
{
        if (!boot_cpu_has(X86_FEATURE_OSPKE)) /*covered*/
                return ARCH_DEFAULT_PKEY;

        return __execute_only_pkey(mm);
}

extern int __arch_override_mprotect_pkey(struct vm_area_struct *vma,
                int prot, int pkey);
static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma,
                int prot, int pkey)
{
        if (!boot_cpu_has(X86_FEATURE_OSPKE))
                return 0;

        return __arch_override_mprotect_pkey(vma, prot, pkey);
}

extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
                unsigned long init_val);

#define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | VM_PKEY_BIT3)

#define mm_pkey_allocation_map(mm)        (mm->context.pkey_allocation_map)
#define mm_set_pkey_allocated(mm, pkey) do {                \
        mm_pkey_allocation_map(mm) |= (1U << pkey);        \
} while (0)
#define mm_set_pkey_free(mm, pkey) do {                        \
        mm_pkey_allocation_map(mm) &= ~(1U << pkey);        \
} while (0)

static inline
bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey)
{
        /*
         * "Allocated" pkeys are those that have been returned
         * from pkey_alloc() or pkey 0 which is allocated
         * implicitly when the mm is created.
         */
        if (pkey < 0)
                return false;
        if (pkey >= arch_max_pkey())
                return false;
        /*
         * The exec-only pkey is set in the allocation map, but
         * is not available to any of the user interfaces like
         * mprotect_pkey().
         */
        if (pkey == mm->context.execute_only_pkey)
                return false;

        return mm_pkey_allocation_map(mm) & (1U << pkey);
}

/*
 * Returns a positive, 4-bit key on success, or -1 on failure.
 */
static inline
int mm_pkey_alloc(struct mm_struct *mm)
{
        /*
         * Note: this is the one and only place we make sure
         * that the pkey is valid as far as the hardware is
         * concerned.  The rest of the kernel trusts that
         * only good, valid pkeys come out of here.
         */
        u16 all_pkeys_mask = ((1U << arch_max_pkey()) - 1);
        int ret;

        /*
         * Are we out of pkeys?  We must handle this specially
         * because ffz() behavior is undefined if there are no
         * zeros.
         */
        if (mm_pkey_allocation_map(mm) == all_pkeys_mask)
                return -1;

        ret = ffz(mm_pkey_allocation_map(mm));

        mm_set_pkey_allocated(mm, ret);

        return ret;
}

static inline
int mm_pkey_free(struct mm_struct *mm, int pkey)
{
        if (!mm_pkey_is_allocated(mm, pkey))
                return -EINVAL;

        mm_set_pkey_free(mm, pkey);

        return 0;
}

extern int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
                unsigned long init_val);
extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
                unsigned long init_val);
extern void copy_init_pkru_to_fpregs(void);

static inline int vma_pkey(struct vm_area_struct *vma)
{
        unsigned long vma_pkey_mask = VM_PKEY_BIT0 | VM_PKEY_BIT1 |
                                      VM_PKEY_BIT2 | VM_PKEY_BIT3;

        return (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT;
}

#endif /*_ASM_X86_PKEYS_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PREEMPT_H
#define __ASM_PREEMPT_H

#include <asm/rmwcc.h>
#include <asm/percpu.h>
#include <linux/thread_info.h>

DECLARE_PER_CPU(int, __preempt_count);

/* We use the MSB mostly because its available */
#define PREEMPT_NEED_RESCHED        0x80000000

/*
 * We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
 * that a decrement hitting 0 means we can and should reschedule.
 */
#define PREEMPT_ENABLED        (0 + PREEMPT_NEED_RESCHED)

/*
 * We mask the PREEMPT_NEED_RESCHED bit so as not to confuse all current users
 * that think a non-zero value indicates we cannot preempt.
 */
static __always_inline int preempt_count(void)
{
        return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED; /*covered*/
}

static __always_inline void preempt_count_set(int pc)
{
        int old, new;

        do {
                old = raw_cpu_read_4(__preempt_count);
                new = (old & PREEMPT_NEED_RESCHED) |
                        (pc & ~PREEMPT_NEED_RESCHED);
        } while (raw_cpu_cmpxchg_4(__preempt_count, old, new) != old);
}

/*
 * must be macros to avoid header recursion hell
 */
#define init_task_preempt_count(p) do { } while (0)

#define init_idle_preempt_count(p, cpu) do { \
        per_cpu(__preempt_count, (cpu)) = PREEMPT_ENABLED; \
} while (0)

/*
 * We fold the NEED_RESCHED bit into the preempt count such that
 * preempt_enable() can decrement and test for needing to reschedule with a
 * single instruction.
 *
 * We invert the actual bit, so that when the decrement hits 0 we know we both
 * need to resched (the bit is cleared) and can resched (no preempt count).
 */

static __always_inline void set_preempt_need_resched(void)
{
        raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
}

static __always_inline void clear_preempt_need_resched(void)
{
        raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
}

static __always_inline bool test_preempt_need_resched(void)
{
        return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
}

/*
 * The various preempt_count add/sub methods
 */

static __always_inline void __preempt_count_add(int val)
{
        raw_cpu_add_4(__preempt_count, val); /*covered*/
}

static __always_inline void __preempt_count_sub(int val)
{
        raw_cpu_add_4(__preempt_count, -val);
}

/*
 * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
 * a decrement which hits zero means we have no preempt_count and should
 * reschedule.
 */
static __always_inline bool __preempt_count_dec_and_test(void)
{
        return GEN_UNARY_RMWcc("decl", __preempt_count, e, __percpu_arg([var]));
}

/*
 * Returns true when we need to resched and can (barring IRQ state).
 */
static __always_inline bool should_resched(int preempt_offset)
{
        return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
}

#ifdef CONFIG_PREEMPT
  extern asmlinkage void ___preempt_schedule(void);
# define __preempt_schedule() \
        asm volatile ("call ___preempt_schedule" : ASM_CALL_CONSTRAINT)

  extern asmlinkage void preempt_schedule(void);
  extern asmlinkage void ___preempt_schedule_notrace(void);
# define __preempt_schedule_notrace() \
        asm volatile ("call ___preempt_schedule_notrace" : ASM_CALL_CONSTRAINT)

  extern asmlinkage void preempt_schedule_notrace(void);
#endif

#endif /* __ASM_PREEMPT_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PROCESSOR_H
#define _ASM_X86_PROCESSOR_H

#include <asm/processor-flags.h>

/* Forward declaration, a strange C thing */
struct task_struct;
struct mm_struct;
struct vm86;

#include <asm/math_emu.h>
#include <asm/segment.h>
#include <asm/types.h>
#include <uapi/asm/sigcontext.h>
#include <asm/current.h>
#include <asm/cpufeatures.h>
#include <asm/page.h>
#include <asm/pgtable_types.h>
#include <asm/percpu.h>
#include <asm/msr.h>
#include <asm/desc_defs.h>
#include <asm/nops.h>
#include <asm/special_insns.h>
#include <asm/fpu/types.h>
#include <asm/unwind_hints.h>

#include <linux/personality.h>
#include <linux/cache.h>
#include <linux/threads.h>
#include <linux/math64.h>
#include <linux/err.h>
#include <linux/irqflags.h>
#include <linux/mem_encrypt.h>

/*
 * We handle most unaligned accesses in hardware.  On the other hand
 * unaligned DMA can be quite expensive on some Nehalem processors.
 *
 * Based on this we disable the IP header alignment in network drivers.
 */
#define NET_IP_ALIGN        0

#define HBP_NUM 4

/*
 * These alignment constraints are for performance in the vSMP case,
 * but in the task_struct case we must also meet hardware imposed
 * alignment requirements of the FPU state:
 */
#ifdef CONFIG_X86_VSMP
# define ARCH_MIN_TASKALIGN                (1 << INTERNODE_CACHE_SHIFT)
# define ARCH_MIN_MMSTRUCT_ALIGN        (1 << INTERNODE_CACHE_SHIFT)
#else
# define ARCH_MIN_TASKALIGN                __alignof__(union fpregs_state)
# define ARCH_MIN_MMSTRUCT_ALIGN        0
#endif

enum tlb_infos {
        ENTRIES,
        NR_INFO
};

extern u16 __read_mostly tlb_lli_4k[NR_INFO];
extern u16 __read_mostly tlb_lli_2m[NR_INFO];
extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_1g[NR_INFO];

/*
 *  CPU type and hardware bug flags. Kept separately for each CPU.
 *  Members of this structure are referenced in head_32.S, so think twice
 *  before touching them. [mj]
 */

struct cpuinfo_x86 {
        __u8                        x86;                /* CPU family */
        __u8                        x86_vendor;        /* CPU vendor */
        __u8                        x86_model;
        __u8                        x86_stepping;
#ifdef CONFIG_X86_64
        /* Number of 4K pages in DTLB/ITLB combined(in pages): */
        int                        x86_tlbsize;
#endif
        __u8                        x86_virt_bits;
        __u8                        x86_phys_bits;
        /* CPUID returned core id bits: */
        __u8                        x86_coreid_bits;
        __u8                        cu_id;
        /* Max extended CPUID function supported: */
        __u32                        extended_cpuid_level;
        /* Maximum supported CPUID level, -1=no CPUID: */
        int                        cpuid_level;
        __u32                        x86_capability[NCAPINTS + NBUGINTS];
        char                        x86_vendor_id[16];
        char                        x86_model_id[64];
        /* in KB - valid for CPUS which support this call: */
        unsigned int                x86_cache_size;
        int                        x86_cache_alignment;        /* In bytes */
        /* Cache QoS architectural values: */
        int                        x86_cache_max_rmid;        /* max index */
        int                        x86_cache_occ_scale;        /* scale to bytes */
        int                        x86_power;
        unsigned long                loops_per_jiffy;
        /* cpuid returned max cores value: */
        u16                         x86_max_cores;
        u16                        apicid;
        u16                        initial_apicid;
        u16                        x86_clflush_size;
        /* number of cores as seen by the OS: */
        u16                        booted_cores;
        /* Physical processor id: */
        u16                        phys_proc_id;
        /* Logical processor id: */
        u16                        logical_proc_id;
        /* Core id: */
        u16                        cpu_core_id;
        /* Index into per_cpu list: */
        u16                        cpu_index;
        u32                        microcode;
        /* Address space bits used by the cache internally */
        u8                        x86_cache_bits;
        unsigned                initialized : 1;
} __randomize_layout;

struct cpuid_regs {
        u32 eax, ebx, ecx, edx;
};

enum cpuid_regs_idx {
        CPUID_EAX = 0,
        CPUID_EBX,
        CPUID_ECX,
        CPUID_EDX,
};

#define X86_VENDOR_INTEL        0
#define X86_VENDOR_CYRIX        1
#define X86_VENDOR_AMD                2
#define X86_VENDOR_UMC                3
#define X86_VENDOR_CENTAUR        5
#define X86_VENDOR_TRANSMETA        7
#define X86_VENDOR_NSC                8
#define X86_VENDOR_HYGON        9
#define X86_VENDOR_NUM                10

#define X86_VENDOR_UNKNOWN        0xff

/*
 * capabilities of CPUs
 */
extern struct cpuinfo_x86        boot_cpu_data;
extern struct cpuinfo_x86        new_cpu_data;

extern struct x86_hw_tss        doublefault_tss;
extern __u32                        cpu_caps_cleared[NCAPINTS + NBUGINTS];
extern __u32                        cpu_caps_set[NCAPINTS + NBUGINTS];

#ifdef CONFIG_SMP
DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
#define cpu_data(cpu)                per_cpu(cpu_info, cpu)
#else
#define cpu_info                boot_cpu_data
#define cpu_data(cpu)                boot_cpu_data
#endif

extern const struct seq_operations cpuinfo_op;

#define cache_line_size()        (boot_cpu_data.x86_cache_alignment)

extern void cpu_detect(struct cpuinfo_x86 *c);

static inline unsigned long long l1tf_pfn_limit(void)
{
        return BIT_ULL(boot_cpu_data.x86_cache_bits - 1 - PAGE_SHIFT);
}

extern void early_cpu_init(void);
extern void identify_boot_cpu(void);
extern void identify_secondary_cpu(struct cpuinfo_x86 *);
extern void print_cpu_info(struct cpuinfo_x86 *);
void print_cpu_msr(struct cpuinfo_x86 *);

#ifdef CONFIG_X86_32
extern int have_cpuid_p(void);
#else
static inline int have_cpuid_p(void)
{
        return 1;
}
#endif
static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
                                unsigned int *ecx, unsigned int *edx)
{
        /* ecx is often an input as well as an output. */
        asm volatile("cpuid"
            : "=a" (*eax),
              "=b" (*ebx),
              "=c" (*ecx),
              "=d" (*edx)
            : "0" (*eax), "2" (*ecx)
            : "memory");
}

#define native_cpuid_reg(reg)                                        \
static inline unsigned int native_cpuid_##reg(unsigned int op)        \
{                                                                \
        unsigned int eax = op, ebx, ecx = 0, edx;                \
                                                                \
        native_cpuid(&eax, &ebx, &ecx, &edx);                        \
                                                                \
        return reg;                                                \
}

/*
 * Native CPUID functions returning a single datum.
 */
native_cpuid_reg(eax)
native_cpuid_reg(ebx)
native_cpuid_reg(ecx)
native_cpuid_reg(edx)

/*
 * Friendlier CR3 helpers.
 */
static inline unsigned long read_cr3_pa(void)
{
        return __read_cr3() & CR3_ADDR_MASK;
}

static inline unsigned long native_read_cr3_pa(void)
{
        return __native_read_cr3() & CR3_ADDR_MASK;
}

static inline void load_cr3(pgd_t *pgdir)
{
        write_cr3(__sme_pa(pgdir));
}

/*
 * Note that while the legacy 'TSS' name comes from 'Task State Segment',
 * on modern x86 CPUs the TSS also holds information important to 64-bit mode,
 * unrelated to the task-switch mechanism:
 */
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
        unsigned short                back_link, __blh;
        unsigned long                sp0;
        unsigned short                ss0, __ss0h;
        unsigned long                sp1;

        /*
         * We don't use ring 1, so ss1 is a convenient scratch space in
         * the same cacheline as sp0.  We use ss1 to cache the value in
         * MSR_IA32_SYSENTER_CS.  When we context switch
         * MSR_IA32_SYSENTER_CS, we first check if the new value being
         * written matches ss1, and, if it's not, then we wrmsr the new
         * value and update ss1.
         *
         * The only reason we context switch MSR_IA32_SYSENTER_CS is
         * that we set it to zero in vm86 tasks to avoid corrupting the
         * stack if we were to go through the sysenter path from vm86
         * mode.
         */
        unsigned short                ss1;        /* MSR_IA32_SYSENTER_CS */

        unsigned short                __ss1h;
        unsigned long                sp2;
        unsigned short                ss2, __ss2h;
        unsigned long                __cr3;
        unsigned long                ip;
        unsigned long                flags;
        unsigned long                ax;
        unsigned long                cx;
        unsigned long                dx;
        unsigned long                bx;
        unsigned long                sp;
        unsigned long                bp;
        unsigned long                si;
        unsigned long                di;
        unsigned short                es, __esh;
        unsigned short                cs, __csh;
        unsigned short                ss, __ssh;
        unsigned short                ds, __dsh;
        unsigned short                fs, __fsh;
        unsigned short                gs, __gsh;
        unsigned short                ldt, __ldth;
        unsigned short                trace;
        unsigned short                io_bitmap_base;

} __attribute__((packed));
#else
struct x86_hw_tss {
        u32                        reserved1;
        u64                        sp0;

        /*
         * We store cpu_current_top_of_stack in sp1 so it's always accessible.
         * Linux does not use ring 1, so sp1 is not otherwise needed.
         */
        u64                        sp1;

        /*
         * Since Linux does not use ring 2, the 'sp2' slot is unused by
         * hardware.  entry_SYSCALL_64 uses it as scratch space to stash
         * the user RSP value.
         */
        u64                        sp2;

        u64                        reserved2;
        u64                        ist[7];
        u32                        reserved3;
        u32                        reserved4;
        u16                        reserved5;
        u16                        io_bitmap_base;

} __attribute__((packed));
#endif

/*
 * IO-bitmap sizes:
 */
#define IO_BITMAP_BITS                        65536
#define IO_BITMAP_BYTES                        (IO_BITMAP_BITS/8)
#define IO_BITMAP_LONGS                        (IO_BITMAP_BYTES/sizeof(long))
#define IO_BITMAP_OFFSET                (offsetof(struct tss_struct, io_bitmap) - offsetof(struct tss_struct, x86_tss))
#define INVALID_IO_BITMAP_OFFSET        0x8000

struct entry_stack {
        unsigned long                words[64];
};

struct entry_stack_page {
        struct entry_stack stack;
} __aligned(PAGE_SIZE);

struct tss_struct {
        /*
         * The fixed hardware portion.  This must not cross a page boundary
         * at risk of violating the SDM's advice and potentially triggering
         * errata.
         */
        struct x86_hw_tss        x86_tss;

        /*
         * The extra 1 is there because the CPU will access an
         * additional byte beyond the end of the IO permission
         * bitmap. The extra byte must be all 1 bits, and must
         * be within the limit.
         */
        unsigned long                io_bitmap[IO_BITMAP_LONGS + 1];
} __aligned(PAGE_SIZE);

DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss_rw);

/*
 * sizeof(unsigned long) coming from an extra "long" at the end
 * of the iobitmap.
 *
 * -1? seg base+limit should be pointing to the address of the
 * last valid byte
 */
#define __KERNEL_TSS_LIMIT        \
        (IO_BITMAP_OFFSET + IO_BITMAP_BYTES + sizeof(unsigned long) - 1)

#ifdef CONFIG_X86_32
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
#else
/* The RO copy can't be accessed with this_cpu_xyz(), so use the RW copy. */
#define cpu_current_top_of_stack cpu_tss_rw.x86_tss.sp1
#endif

/*
 * Save the original ist values for checking stack pointers during debugging
 */
struct orig_ist {
        unsigned long                ist[7];
};

#ifdef CONFIG_X86_64
DECLARE_PER_CPU(struct orig_ist, orig_ist);

union irq_stack_union {
        char irq_stack[IRQ_STACK_SIZE];
        /*
         * GCC hardcodes the stack canary as %gs:40.  Since the
         * irq_stack is the object at %gs:0, we reserve the bottom
         * 48 bytes of the irq stack for the canary.
         */
        struct {
                char gs_base[40];
                unsigned long stack_canary;
        };
};

DECLARE_PER_CPU_FIRST(union irq_stack_union, irq_stack_union) __visible;
DECLARE_INIT_PER_CPU(irq_stack_union);

static inline unsigned long cpu_kernelmode_gs_base(int cpu)
{
        return (unsigned long)per_cpu(irq_stack_union.gs_base, cpu);
}

DECLARE_PER_CPU(char *, irq_stack_ptr);
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);

#if IS_ENABLED(CONFIG_KVM)
/* Save actual FS/GS selectors and bases to current->thread */
void save_fsgs_for_kvm(void);
#endif
#else        /* X86_64 */
#ifdef CONFIG_STACKPROTECTOR
/*
 * Make sure stack canary segment base is cached-aligned:
 *   "For Intel Atom processors, avoid non zero segment base address
 *    that is not aligned to cache line boundary at all cost."
 * (Optim Ref Manual Assembly/Compiler Coding Rule 15.)
 */
struct stack_canary {
        char __pad[20];                /* canary at %gs:20 */
        unsigned long canary;
};
DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
#endif
/*
 * per-CPU IRQ handling stacks
 */
struct irq_stack {
        u32                     stack[THREAD_SIZE/sizeof(u32)];
} __aligned(THREAD_SIZE);

DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
#endif        /* X86_64 */

extern unsigned int fpu_kernel_xstate_size;
extern unsigned int fpu_user_xstate_size;

struct perf_event;

typedef struct {
        unsigned long                seg;
} mm_segment_t;

struct thread_struct {
        /* Cached TLS descriptors: */
        struct desc_struct        tls_array[GDT_ENTRY_TLS_ENTRIES];
#ifdef CONFIG_X86_32
        unsigned long                sp0;
#endif
        unsigned long                sp;
#ifdef CONFIG_X86_32
        unsigned long                sysenter_cs;
#else
        unsigned short                es;
        unsigned short                ds;
        unsigned short                fsindex;
        unsigned short                gsindex;
#endif

#ifdef CONFIG_X86_64
        unsigned long                fsbase;
        unsigned long                gsbase;
#else
        /*
         * XXX: this could presumably be unsigned short.  Alternatively,
         * 32-bit kernels could be taught to use fsindex instead.
         */
        unsigned long fs;
        unsigned long gs;
#endif

        /* Save middle states of ptrace breakpoints */
        struct perf_event        *ptrace_bps[HBP_NUM];
        /* Debug status used for traps, single steps, etc... */
        unsigned long           debugreg6;
        /* Keep track of the exact dr7 value set by the user */
        unsigned long           ptrace_dr7;
        /* Fault info: */
        unsigned long                cr2;
        unsigned long                trap_nr;
        unsigned long                error_code;
#ifdef CONFIG_VM86
        /* Virtual 86 mode info */
        struct vm86                *vm86;
#endif
        /* IO permissions: */
        unsigned long                *io_bitmap_ptr;
        unsigned long                iopl;
        /* Max allowed port in the bitmap, in bytes: */
        unsigned                io_bitmap_max;

        mm_segment_t                addr_limit;

        unsigned int                sig_on_uaccess_err:1;
        unsigned int                uaccess_err:1;        /* uaccess failed */

        /* Floating point and extended processor state */
        struct fpu                fpu;
        /*
         * WARNING: 'fpu' is dynamically-sized.  It *MUST* be at
         * the end.
         */
};

/* Whitelist the FPU state from the task_struct for hardened usercopy. */
static inline void arch_thread_struct_whitelist(unsigned long *offset,
                                                unsigned long *size)
{
        *offset = offsetof(struct thread_struct, fpu.state);
        *size = fpu_kernel_xstate_size;
}

/*
 * Thread-synchronous status.
 *
 * This is different from the flags in that nobody else
 * ever touches our thread-synchronous status, so we don't
 * have to worry about atomic accesses.
 */
#define TS_COMPAT                0x0002        /* 32bit syscall active (64BIT)*/

/*
 * Set IOPL bits in EFLAGS from given mask
 */
static inline void native_set_iopl_mask(unsigned mask)
{
#ifdef CONFIG_X86_32
        unsigned int reg;

        asm volatile ("pushfl;"
                      "popl %0;"
                      "andl %1, %0;"
                      "orl %2, %0;"
                      "pushl %0;"
                      "popfl"
                      : "=&r" (reg)
                      : "i" (~X86_EFLAGS_IOPL), "r" (mask));
#endif
}

static inline void
native_load_sp0(unsigned long sp0)
{
        this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0);
}

static inline void native_swapgs(void)
{
#ifdef CONFIG_X86_64
        asm volatile("swapgs" ::: "memory");
#endif
}

static inline unsigned long current_top_of_stack(void)
{
        /*
         *  We can't read directly from tss.sp0: sp0 on x86_32 is special in
         *  and around vm86 mode and sp0 on x86_64 is special because of the
         *  entry trampoline.
         */
        return this_cpu_read_stable(cpu_current_top_of_stack);
}

static inline bool on_thread_stack(void)
{
        return (unsigned long)(current_top_of_stack() -
                               current_stack_pointer) < THREAD_SIZE;
}

#ifdef CONFIG_PARAVIRT_XXL
#include <asm/paravirt.h>
#else
#define __cpuid                        native_cpuid

static inline void load_sp0(unsigned long sp0)
{
        native_load_sp0(sp0);
}

#define set_iopl_mask native_set_iopl_mask
#endif /* CONFIG_PARAVIRT_XXL */

/* Free all resources held by a thread. */
extern void release_thread(struct task_struct *);

unsigned long get_wchan(struct task_struct *p);

/*
 * Generic CPUID function
 * clear %ecx since some cpus (Cyrix MII) do not set or clear %ecx
 * resulting in stale register contents being returned.
 */
static inline void cpuid(unsigned int op,
                         unsigned int *eax, unsigned int *ebx,
                         unsigned int *ecx, unsigned int *edx)
{
        *eax = op;
        *ecx = 0;
        __cpuid(eax, ebx, ecx, edx);
}

/* Some CPUID calls want 'count' to be placed in ecx */
static inline void cpuid_count(unsigned int op, int count,
                               unsigned int *eax, unsigned int *ebx,
                               unsigned int *ecx, unsigned int *edx)
{
        *eax = op;
        *ecx = count;
        __cpuid(eax, ebx, ecx, edx);
}

/*
 * CPUID functions returning a single datum
 */
static inline unsigned int cpuid_eax(unsigned int op)
{
        unsigned int eax, ebx, ecx, edx;

        cpuid(op, &eax, &ebx, &ecx, &edx);

        return eax;
}

static inline unsigned int cpuid_ebx(unsigned int op)
{
        unsigned int eax, ebx, ecx, edx;

        cpuid(op, &eax, &ebx, &ecx, &edx);

        return ebx;
}

static inline unsigned int cpuid_ecx(unsigned int op)
{
        unsigned int eax, ebx, ecx, edx;

        cpuid(op, &eax, &ebx, &ecx, &edx);

        return ecx;
}

static inline unsigned int cpuid_edx(unsigned int op)
{
        unsigned int eax, ebx, ecx, edx;

        cpuid(op, &eax, &ebx, &ecx, &edx);

        return edx;
}

/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
static __always_inline void rep_nop(void)
{
        asm volatile("rep; nop" ::: "memory"); /*covered*/
}

static __always_inline void cpu_relax(void)
{
        rep_nop(); /*covered*/
}

/*
 * This function forces the icache and prefetched instruction stream to
 * catch up with reality in two very specific cases:
 *
 *  a) Text was modified using one virtual address and is about to be executed
 *     from the same physical page at a different virtual address.
 *
 *  b) Text was modified on a different CPU, may subsequently be
 *     executed on this CPU, and you want to make sure the new version
 *     gets executed.  This generally means you're calling this in a IPI.
 *
 * If you're calling this for a different reason, you're probably doing
 * it wrong.
 */
static inline void sync_core(void)
{
        /*
         * There are quite a few ways to do this.  IRET-to-self is nice
         * because it works on every CPU, at any CPL (so it's compatible
         * with paravirtualization), and it never exits to a hypervisor.
         * The only down sides are that it's a bit slow (it seems to be
         * a bit more than 2x slower than the fastest options) and that
         * it unmasks NMIs.  The "push %cs" is needed because, in
         * paravirtual environments, __KERNEL_CS may not be a valid CS
         * value when we do IRET directly.
         *
         * In case NMI unmasking or performance ever becomes a problem,
         * the next best option appears to be MOV-to-CR2 and an
         * unconditional jump.  That sequence also works on all CPUs,
         * but it will fault at CPL3 (i.e. Xen PV).
         *
         * CPUID is the conventional way, but it's nasty: it doesn't
         * exist on some 486-like CPUs, and it usually exits to a
         * hypervisor.
         *
         * Like all of Linux's memory ordering operations, this is a
         * compiler barrier as well.
         */
#ifdef CONFIG_X86_32
        asm volatile (
                "pushfl\n\t"
                "pushl %%cs\n\t"
                "pushl $1f\n\t"
                "iret\n\t"
                "1:"
                : ASM_CALL_CONSTRAINT : : "memory");
#else
        unsigned int tmp;

        asm volatile (
                UNWIND_HINT_SAVE
                "mov %%ss, %0\n\t"
                "pushq %q0\n\t"
                "pushq %%rsp\n\t"
                "addq $8, (%%rsp)\n\t"
                "pushfq\n\t"
                "mov %%cs, %0\n\t"
                "pushq %q0\n\t"
                "pushq $1f\n\t"
                "iretq\n\t"
                UNWIND_HINT_RESTORE
                "1:"
                : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
#endif
}

extern void select_idle_routine(const struct cpuinfo_x86 *c);
extern void amd_e400_c1e_apic_setup(void);

extern unsigned long                boot_option_idle_override;

enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT,
                         IDLE_POLL};

extern void enable_sep_cpu(void);
extern int sysenter_setup(void);


/* Defined in head.S */
extern struct desc_ptr                early_gdt_descr;

extern void switch_to_new_gdt(int);
extern void load_direct_gdt(int);
extern void load_fixmap_gdt(int);
extern void load_percpu_segment(int);
extern void cpu_init(void);

static inline unsigned long get_debugctlmsr(void)
{
        unsigned long debugctlmsr = 0;

#ifndef CONFIG_X86_DEBUGCTLMSR
        if (boot_cpu_data.x86 < 6)
                return 0;
#endif
        rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);

        return debugctlmsr;
}

static inline void update_debugctlmsr(unsigned long debugctlmsr)
{
#ifndef CONFIG_X86_DEBUGCTLMSR
        if (boot_cpu_data.x86 < 6)
                return;
#endif
        wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
}

extern void set_task_blockstep(struct task_struct *task, bool on);

/* Boot loader type from the setup header: */
extern int                        bootloader_type;
extern int                        bootloader_version;

extern char                        ignore_fpu_irq;

#define HAVE_ARCH_PICK_MMAP_LAYOUT 1
#define ARCH_HAS_PREFETCHW
#define ARCH_HAS_SPINLOCK_PREFETCH

#ifdef CONFIG_X86_32
# define BASE_PREFETCH                ""
# define ARCH_HAS_PREFETCH
#else
# define BASE_PREFETCH                "prefetcht0 %P1"
#endif

/*
 * Prefetch instructions for Pentium III (+) and AMD Athlon (+)
 *
 * It's not worth to care about 3dnow prefetches for the K6
 * because they are microcoded there and very slow.
 */
static inline void prefetch(const void *x)
{
        alternative_input(BASE_PREFETCH, "prefetchnta %P1",
                          X86_FEATURE_XMM,
                          "m" (*(const char *)x));
}

/*
 * 3dnow prefetch to get an exclusive cache line.
 * Useful for spinlocks to avoid one state transition in the
 * cache coherency protocol:
 */
static inline void prefetchw(const void *x)
{
        alternative_input(BASE_PREFETCH, "prefetchw %P1", /*covered*/
                          X86_FEATURE_3DNOWPREFETCH,
                          "m" (*(const char *)x));
}

static inline void spin_lock_prefetch(const void *x)
{
        prefetchw(x);
}

#define TOP_OF_INIT_STACK ((unsigned long)&init_stack + sizeof(init_stack) - \
                           TOP_OF_KERNEL_STACK_PADDING)

#define task_top_of_stack(task) ((unsigned long)(task_pt_regs(task) + 1))

#define task_pt_regs(task) \
({                                                                        \
        unsigned long __ptr = (unsigned long)task_stack_page(task);        \
        __ptr += THREAD_SIZE - TOP_OF_KERNEL_STACK_PADDING;                \
        ((struct pt_regs *)__ptr) - 1;                                        \
})

#ifdef CONFIG_X86_32
/*
 * User space process size: 3GB (default).
 */
#define IA32_PAGE_OFFSET        PAGE_OFFSET
#define TASK_SIZE                PAGE_OFFSET
#define TASK_SIZE_LOW                TASK_SIZE
#define TASK_SIZE_MAX                TASK_SIZE
#define DEFAULT_MAP_WINDOW        TASK_SIZE
#define STACK_TOP                TASK_SIZE
#define STACK_TOP_MAX                STACK_TOP

#define INIT_THREAD  {                                                          \
        .sp0                        = TOP_OF_INIT_STACK,                          \
        .sysenter_cs                = __KERNEL_CS,                                  \
        .io_bitmap_ptr                = NULL,                                          \
        .addr_limit                = KERNEL_DS,                                  \
}

#define KSTK_ESP(task)                (task_pt_regs(task)->sp)

#else
/*
 * User space process size.  This is the first address outside the user range.
 * There are a few constraints that determine this:
 *
 * On Intel CPUs, if a SYSCALL instruction is at the highest canonical
 * address, then that syscall will enter the kernel with a
 * non-canonical return address, and SYSRET will explode dangerously.
 * We avoid this particular problem by preventing anything executable
 * from being mapped at the maximum canonical address.
 *
 * On AMD CPUs in the Ryzen family, there's a nasty bug in which the
 * CPUs malfunction if they execute code from the highest canonical page.
 * They'll speculate right off the end of the canonical space, and
 * bad things happen.  This is worked around in the same way as the
 * Intel problem.
 *
 * With page table isolation enabled, we map the LDT in ... [stay tuned]
 */
#define TASK_SIZE_MAX        ((1UL << __VIRTUAL_MASK_SHIFT) - PAGE_SIZE)

#define DEFAULT_MAP_WINDOW        ((1UL << 47) - PAGE_SIZE)

/* This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
#define IA32_PAGE_OFFSET        ((current->personality & ADDR_LIMIT_3GB) ? \
                                        0xc0000000 : 0xFFFFe000)

#define TASK_SIZE_LOW                (test_thread_flag(TIF_ADDR32) ? \
                                        IA32_PAGE_OFFSET : DEFAULT_MAP_WINDOW)
#define TASK_SIZE                (test_thread_flag(TIF_ADDR32) ? \
                                        IA32_PAGE_OFFSET : TASK_SIZE_MAX)
#define TASK_SIZE_OF(child)        ((test_tsk_thread_flag(child, TIF_ADDR32)) ? \
                                        IA32_PAGE_OFFSET : TASK_SIZE_MAX)

#define STACK_TOP                TASK_SIZE_LOW
#define STACK_TOP_MAX                TASK_SIZE_MAX

#define INIT_THREAD  {                                                \
        .addr_limit                = KERNEL_DS,                        \
}

extern unsigned long KSTK_ESP(struct task_struct *task);

#endif /* CONFIG_X86_64 */

extern void start_thread(struct pt_regs *regs, unsigned long new_ip,
                                               unsigned long new_sp);

/*
 * This decides where the kernel will search for a free chunk of vm
 * space during mmap's.
 */
#define __TASK_UNMAPPED_BASE(task_size)        (PAGE_ALIGN(task_size / 3))
#define TASK_UNMAPPED_BASE                __TASK_UNMAPPED_BASE(TASK_SIZE_LOW)

#define KSTK_EIP(task)                (task_pt_regs(task)->ip)

/* Get/set a process' ability to use the timestamp counter instruction */
#define GET_TSC_CTL(adr)        get_tsc_mode((adr))
#define SET_TSC_CTL(val)        set_tsc_mode((val))

extern int get_tsc_mode(unsigned long adr);
extern int set_tsc_mode(unsigned int val);

DECLARE_PER_CPU(u64, msr_misc_features_shadow);

/* Register/unregister a process' MPX related resource */
#define MPX_ENABLE_MANAGEMENT()        mpx_enable_management()
#define MPX_DISABLE_MANAGEMENT()        mpx_disable_management()

#ifdef CONFIG_X86_INTEL_MPX
extern int mpx_enable_management(void);
extern int mpx_disable_management(void);
#else
static inline int mpx_enable_management(void)
{
        return -EINVAL;
}
static inline int mpx_disable_management(void)
{
        return -EINVAL;
}
#endif /* CONFIG_X86_INTEL_MPX */

#ifdef CONFIG_CPU_SUP_AMD
extern u16 amd_get_nb_id(int cpu);
extern u32 amd_get_nodes_per_socket(void);
#else
static inline u16 amd_get_nb_id(int cpu)                { return 0; }
static inline u32 amd_get_nodes_per_socket(void)        { return 0; }
#endif

static inline uint32_t hypervisor_cpuid_base(const char *sig, uint32_t leaves)
{
        uint32_t base, eax, signature[3];

        for (base = 0x40000000; base < 0x40010000; base += 0x100) {
                cpuid(base, &eax, &signature[0], &signature[1], &signature[2]);

                if (!memcmp(sig, signature, 12) &&
                    (leaves == 0 || ((eax - base) >= leaves)))
                        return base;
        }

        return 0;
}

extern unsigned long arch_align_stack(unsigned long sp);
void free_init_pages(const char *what, unsigned long begin, unsigned long end);
extern void free_kernel_image_pages(void *begin, void *end);

void default_idle(void);
#ifdef        CONFIG_XEN
bool xen_set_default_idle(void);
#else
#define xen_set_default_idle 0
#endif

void stop_this_cpu(void *dummy);
void df_debug(struct pt_regs *regs, long error_code);
void microcode_check(void);

enum l1tf_mitigations {
        L1TF_MITIGATION_OFF,
        L1TF_MITIGATION_FLUSH_NOWARN,
        L1TF_MITIGATION_FLUSH,
        L1TF_MITIGATION_FLUSH_NOSMT,
        L1TF_MITIGATION_FULL,
        L1TF_MITIGATION_FULL_FORCE
};

extern enum l1tf_mitigations l1tf_mitigation;

#endif /* _ASM_X86_PROCESSOR_H */

/* SPDX-License-Identifier: GPL-2.0 */

#include <asm/fpu/api.h>

/*
 * may_use_simd - whether it is allowable at this time to issue SIMD
 *                instructions or access the SIMD register file
 */
static __must_check inline bool may_use_simd(void)
{
        return irq_fpu_usable(); /*covered*/
}

/*
 * Supervisor Mode Access Prevention support
 *
 * Copyright (C) 2012 Intel Corporation
 * Author: H. Peter Anvin <hpa@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#ifndef _ASM_X86_SMAP_H
#define _ASM_X86_SMAP_H

#include <linux/stringify.h>
#include <asm/nops.h>
#include <asm/cpufeatures.h>

/* "Raw" instruction opcodes */
#define __ASM_CLAC        .byte 0x0f,0x01,0xca
#define __ASM_STAC        .byte 0x0f,0x01,0xcb

#ifdef __ASSEMBLY__

#include <asm/alternative-asm.h>

#ifdef CONFIG_X86_SMAP

#define ASM_CLAC \
        ALTERNATIVE "", __stringify(__ASM_CLAC), X86_FEATURE_SMAP

#define ASM_STAC \
        ALTERNATIVE "", __stringify(__ASM_STAC), X86_FEATURE_SMAP

#else /* CONFIG_X86_SMAP */

#define ASM_CLAC
#define ASM_STAC

#endif /* CONFIG_X86_SMAP */

#else /* __ASSEMBLY__ */

#include <asm/alternative.h>

#ifdef CONFIG_X86_SMAP

static __always_inline void clac(void)
{
        /* Note: a barrier is implicit in alternative() */
        alternative("", __stringify(__ASM_CLAC), X86_FEATURE_SMAP); /*covered*/
}

static __always_inline void stac(void)
{
        /* Note: a barrier is implicit in alternative() */
        alternative("", __stringify(__ASM_STAC), X86_FEATURE_SMAP); /*covered*/
}

/* These macros can be used in asm() statements */
#define ASM_CLAC \
        ALTERNATIVE("", __stringify(__ASM_CLAC), X86_FEATURE_SMAP)
#define ASM_STAC \
        ALTERNATIVE("", __stringify(__ASM_STAC), X86_FEATURE_SMAP)

#else /* CONFIG_X86_SMAP */

static inline void clac(void) { }
static inline void stac(void) { }

#define ASM_CLAC
#define ASM_STAC

#endif /* CONFIG_X86_SMAP */

#endif /* __ASSEMBLY__ */

#endif /* _ASM_X86_SMAP_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_SMP_H
#define _ASM_X86_SMP_H
#ifndef __ASSEMBLY__
#include <linux/cpumask.h>
#include <asm/percpu.h>

/*
 * We need the APIC definitions automatically as part of 'smp.h'
 */
#ifdef CONFIG_X86_LOCAL_APIC
# include <asm/mpspec.h>
# include <asm/apic.h>
# ifdef CONFIG_X86_IO_APIC
#  include <asm/io_apic.h>
# endif
#endif
#include <asm/thread_info.h>
#include <asm/cpumask.h>

extern int smp_num_siblings;
extern unsigned int num_processors;

DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
/* cpus sharing the last level cache: */
DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);
DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);

static inline struct cpumask *cpu_llc_shared_mask(int cpu)
{
        return per_cpu(cpu_llc_shared_map, cpu);
}

DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);
DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid);
DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid);
#endif

struct task_struct;

struct smp_ops {
        void (*smp_prepare_boot_cpu)(void);
        void (*smp_prepare_cpus)(unsigned max_cpus);
        void (*smp_cpus_done)(unsigned max_cpus);

        void (*stop_other_cpus)(int wait);
        void (*crash_stop_other_cpus)(void);
        void (*smp_send_reschedule)(int cpu);

        int (*cpu_up)(unsigned cpu, struct task_struct *tidle);
        int (*cpu_disable)(void);
        void (*cpu_die)(unsigned int cpu);
        void (*play_dead)(void);

        void (*send_call_func_ipi)(const struct cpumask *mask);
        void (*send_call_func_single_ipi)(int cpu);
};

/* Globals due to paravirt */
extern void set_cpu_sibling_map(int cpu);

#ifdef CONFIG_SMP
extern struct smp_ops smp_ops;

static inline void smp_send_stop(void)
{
        smp_ops.stop_other_cpus(0);
}

static inline void stop_other_cpus(void)
{
        smp_ops.stop_other_cpus(1);
}

static inline void smp_prepare_boot_cpu(void)
{
        smp_ops.smp_prepare_boot_cpu();
}

static inline void smp_prepare_cpus(unsigned int max_cpus)
{
        smp_ops.smp_prepare_cpus(max_cpus);
}

static inline void smp_cpus_done(unsigned int max_cpus)
{
        smp_ops.smp_cpus_done(max_cpus);
}

static inline int __cpu_up(unsigned int cpu, struct task_struct *tidle)
{
        return smp_ops.cpu_up(cpu, tidle);
}

static inline int __cpu_disable(void)
{
        return smp_ops.cpu_disable();
}

static inline void __cpu_die(unsigned int cpu)
{
        smp_ops.cpu_die(cpu);
}

static inline void play_dead(void)
{
        smp_ops.play_dead();
}

static inline void smp_send_reschedule(int cpu)
{
        smp_ops.smp_send_reschedule(cpu);
}

static inline void arch_send_call_function_single_ipi(int cpu)
{
        smp_ops.send_call_func_single_ipi(cpu); /*covered*/
}

static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
{
        smp_ops.send_call_func_ipi(mask);
}

void cpu_disable_common(void);
void native_smp_prepare_boot_cpu(void);
void native_smp_prepare_cpus(unsigned int max_cpus);
void calculate_max_logical_packages(void);
void native_smp_cpus_done(unsigned int max_cpus);
void common_cpu_up(unsigned int cpunum, struct task_struct *tidle);
int native_cpu_up(unsigned int cpunum, struct task_struct *tidle);
int native_cpu_disable(void);
int common_cpu_die(unsigned int cpu);
void native_cpu_die(unsigned int cpu);
void hlt_play_dead(void);
void native_play_dead(void);
void play_dead_common(void);
void wbinvd_on_cpu(int cpu);
int wbinvd_on_all_cpus(void);

void native_send_call_func_ipi(const struct cpumask *mask);
void native_send_call_func_single_ipi(int cpu);
void x86_idle_thread_init(unsigned int cpu, struct task_struct *idle);

void smp_store_boot_cpu_info(void);
void smp_store_cpu_info(int id);

asmlinkage __visible void smp_reboot_interrupt(void);
__visible void smp_reschedule_interrupt(struct pt_regs *regs);
__visible void smp_call_function_interrupt(struct pt_regs *regs);
__visible void smp_call_function_single_interrupt(struct pt_regs *r);

#define cpu_physical_id(cpu)        per_cpu(x86_cpu_to_apicid, cpu)
#define cpu_acpi_id(cpu)        per_cpu(x86_cpu_to_acpiid, cpu)

/*
 * This function is needed by all SMP systems. It must _always_ be valid
 * from the initial startup. We map APIC_BASE very early in page_setup(),
 * so this is correct in the x86 case.
 */
#define raw_smp_processor_id() (this_cpu_read(cpu_number))

#ifdef CONFIG_X86_32
extern int safe_smp_processor_id(void);
#else
# define safe_smp_processor_id()        smp_processor_id()
#endif

#else /* !CONFIG_SMP */
#define wbinvd_on_cpu(cpu)     wbinvd()
static inline int wbinvd_on_all_cpus(void)
{
        wbinvd();
        return 0;
}
#endif /* CONFIG_SMP */

extern unsigned disabled_cpus;

#ifdef CONFIG_X86_LOCAL_APIC
extern int hard_smp_processor_id(void);

#else /* CONFIG_X86_LOCAL_APIC */
#define hard_smp_processor_id()        0
#endif /* CONFIG_X86_LOCAL_APIC */

#ifdef CONFIG_DEBUG_NMI_SELFTEST
extern void nmi_selftest(void);
#else
#define nmi_selftest() do { } while (0)
#endif

#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_SMP_H */

/* SPDX-License-Identifier: GPL-2.0 */
/* thread_info.h: low-level thread information
 *
 * Copyright (C) 2002  David Howells (dhowells@redhat.com)
 * - Incorporating suggestions made by Linus Torvalds and Dave Miller
 */

#ifndef _ASM_X86_THREAD_INFO_H
#define _ASM_X86_THREAD_INFO_H

#include <linux/compiler.h>
#include <asm/page.h>
#include <asm/percpu.h>
#include <asm/types.h>

/*
 * TOP_OF_KERNEL_STACK_PADDING is a number of unused bytes that we
 * reserve at the top of the kernel stack.  We do it because of a nasty
 * 32-bit corner case.  On x86_32, the hardware stack frame is
 * variable-length.  Except for vm86 mode, struct pt_regs assumes a
 * maximum-length frame.  If we enter from CPL 0, the top 8 bytes of
 * pt_regs don't actually exist.  Ordinarily this doesn't matter, but it
 * does in at least one case:
 *
 * If we take an NMI early enough in SYSENTER, then we can end up with
 * pt_regs that extends above sp0.  On the way out, in the espfix code,
 * we can read the saved SS value, but that value will be above sp0.
 * Without this offset, that can result in a page fault.  (We are
 * careful that, in this case, the value we read doesn't matter.)
 *
 * In vm86 mode, the hardware frame is much longer still, so add 16
 * bytes to make room for the real-mode segments.
 *
 * x86_64 has a fixed-length stack frame.
 */
#ifdef CONFIG_X86_32
# ifdef CONFIG_VM86
#  define TOP_OF_KERNEL_STACK_PADDING 16
# else
#  define TOP_OF_KERNEL_STACK_PADDING 8
# endif
#else
# define TOP_OF_KERNEL_STACK_PADDING 0
#endif

/*
 * low level task data that entry.S needs immediate access to
 * - this struct should fit entirely inside of one cache line
 * - this struct shares the supervisor stack pages
 */
#ifndef __ASSEMBLY__
struct task_struct;
#include <asm/cpufeature.h>
#include <linux/atomic.h>

struct thread_info {
        unsigned long                flags;                /* low level flags */
        u32                        status;                /* thread synchronous flags */
};

#define INIT_THREAD_INFO(tsk)                        \
{                                                \
        .flags                = 0,                        \
}

#else /* !__ASSEMBLY__ */

#include <asm/asm-offsets.h>

#endif

/*
 * thread information flags
 * - these are process state flags that various assembly files
 *   may need to access
 */
#define TIF_SYSCALL_TRACE        0        /* syscall trace active */
#define TIF_NOTIFY_RESUME        1        /* callback before returning to user */
#define TIF_SIGPENDING                2        /* signal pending */
#define TIF_NEED_RESCHED        3        /* rescheduling necessary */
#define TIF_SINGLESTEP                4        /* reenable singlestep on user return*/
#define TIF_SSBD                5        /* Speculative store bypass disable */
#define TIF_SYSCALL_EMU                6        /* syscall emulation active */
#define TIF_SYSCALL_AUDIT        7        /* syscall auditing active */
#define TIF_SECCOMP                8        /* secure computing */
#define TIF_SPEC_IB                9        /* Indirect branch speculation mitigation */
#define TIF_SPEC_FORCE_UPDATE        10        /* Force speculation MSR update in context switch */
#define TIF_USER_RETURN_NOTIFY        11        /* notify kernel of userspace return */
#define TIF_UPROBE                12        /* breakpointed or singlestepping */
#define TIF_PATCH_PENDING        13        /* pending live patching update */
#define TIF_NOCPUID                15        /* CPUID is not accessible in userland */
#define TIF_NOTSC                16        /* TSC is not accessible in userland */
#define TIF_IA32                17        /* IA32 compatibility process */
#define TIF_NOHZ                19        /* in adaptive nohz mode */
#define TIF_MEMDIE                20        /* is terminating due to OOM killer */
#define TIF_POLLING_NRFLAG        21        /* idle is polling for TIF_NEED_RESCHED */
#define TIF_IO_BITMAP                22        /* uses I/O bitmap */
#define TIF_FORCED_TF                24        /* true if TF in eflags artificially */
#define TIF_BLOCKSTEP                25        /* set when we want DEBUGCTLMSR_BTF */
#define TIF_LAZY_MMU_UPDATES        27        /* task is updating the mmu lazily */
#define TIF_SYSCALL_TRACEPOINT        28        /* syscall tracepoint instrumentation */
#define TIF_ADDR32                29        /* 32-bit address space on 64 bits */
#define TIF_X32                        30        /* 32-bit native x86-64 binary */
#define TIF_FSCHECK                31        /* Check FS is USER_DS on return */

#define _TIF_SYSCALL_TRACE        (1 << TIF_SYSCALL_TRACE)
#define _TIF_NOTIFY_RESUME        (1 << TIF_NOTIFY_RESUME)
#define _TIF_SIGPENDING                (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED        (1 << TIF_NEED_RESCHED)
#define _TIF_SINGLESTEP                (1 << TIF_SINGLESTEP)
#define _TIF_SSBD                (1 << TIF_SSBD)
#define _TIF_SYSCALL_EMU        (1 << TIF_SYSCALL_EMU)
#define _TIF_SYSCALL_AUDIT        (1 << TIF_SYSCALL_AUDIT)
#define _TIF_SECCOMP                (1 << TIF_SECCOMP)
#define _TIF_SPEC_IB                (1 << TIF_SPEC_IB)
#define _TIF_SPEC_FORCE_UPDATE        (1 << TIF_SPEC_FORCE_UPDATE)
#define _TIF_USER_RETURN_NOTIFY        (1 << TIF_USER_RETURN_NOTIFY)
#define _TIF_UPROBE                (1 << TIF_UPROBE)
#define _TIF_PATCH_PENDING        (1 << TIF_PATCH_PENDING)
#define _TIF_NOCPUID                (1 << TIF_NOCPUID)
#define _TIF_NOTSC                (1 << TIF_NOTSC)
#define _TIF_IA32                (1 << TIF_IA32)
#define _TIF_NOHZ                (1 << TIF_NOHZ)
#define _TIF_POLLING_NRFLAG        (1 << TIF_POLLING_NRFLAG)
#define _TIF_IO_BITMAP                (1 << TIF_IO_BITMAP)
#define _TIF_FORCED_TF                (1 << TIF_FORCED_TF)
#define _TIF_BLOCKSTEP                (1 << TIF_BLOCKSTEP)
#define _TIF_LAZY_MMU_UPDATES        (1 << TIF_LAZY_MMU_UPDATES)
#define _TIF_SYSCALL_TRACEPOINT        (1 << TIF_SYSCALL_TRACEPOINT)
#define _TIF_ADDR32                (1 << TIF_ADDR32)
#define _TIF_X32                (1 << TIF_X32)
#define _TIF_FSCHECK                (1 << TIF_FSCHECK)

/*
 * work to do in syscall_trace_enter().  Also includes TIF_NOHZ for
 * enter_from_user_mode()
 */
#define _TIF_WORK_SYSCALL_ENTRY        \
        (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_AUDIT |        \
         _TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT |        \
         _TIF_NOHZ)

/* flags to check in __switch_to() */
#define _TIF_WORK_CTXSW_BASE                                                \
        (_TIF_IO_BITMAP|_TIF_NOCPUID|_TIF_NOTSC|_TIF_BLOCKSTEP|                \
         _TIF_SSBD | _TIF_SPEC_FORCE_UPDATE)

/*
 * Avoid calls to __switch_to_xtra() on UP as STIBP is not evaluated.
 */
#ifdef CONFIG_SMP
# define _TIF_WORK_CTXSW        (_TIF_WORK_CTXSW_BASE | _TIF_SPEC_IB)
#else
# define _TIF_WORK_CTXSW        (_TIF_WORK_CTXSW_BASE)
#endif

#define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
#define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)

#define STACK_WARN                (THREAD_SIZE/8)

/*
 * macros/functions for gaining access to the thread information structure
 *
 * preempt_count needs to be 1 initially, until the scheduler is functional.
 */
#ifndef __ASSEMBLY__

/*
 * Walks up the stack frames to make sure that the specified object is
 * entirely contained by a single stack frame.
 *
 * Returns:
 *        GOOD_FRAME        if within a frame
 *        BAD_STACK        if placed across a frame boundary (or outside stack)
 *        NOT_STACK        unable to determine (no frame pointers, etc)
 */
static inline int arch_within_stack_frames(const void * const stack,
                                           const void * const stackend,
                                           const void *obj, unsigned long len)
{
#if defined(CONFIG_FRAME_POINTER)
        const void *frame = NULL;
        const void *oldframe;

        oldframe = __builtin_frame_address(1); /*covered*/
        if (oldframe)
                frame = __builtin_frame_address(2); /*covered*/
        /*
         * low ----------------------------------------------> high
         * [saved bp][saved ip][args][local vars][saved bp][saved ip]
         *                     ^----------------^
         *               allow copies only within here
         */
        while (stack <= frame && frame < stackend) { /*covered*/
                /*
                 * If obj + len extends past the last frame, this
                 * check won't pass and the next frame will be 0,
                 * causing us to bail out and correctly report
                 * the copy as invalid.
                 */
                if (obj + len <= frame) /*covered*/
                        return obj >= oldframe + 2 * sizeof(void *) ? /*covered*/
                                GOOD_FRAME : BAD_STACK;
                oldframe = frame;
                frame = *(const void * const *)frame; /*covered*/
        }
        return BAD_STACK;
#else
        return NOT_STACK;
#endif
}

#else /* !__ASSEMBLY__ */

#ifdef CONFIG_X86_64
# define cpu_current_top_of_stack (cpu_tss_rw + TSS_sp1)
#endif

#endif

#ifdef CONFIG_COMPAT
#define TS_I386_REGS_POKED        0x0004        /* regs poked by 32-bit ptracer */
#endif
#ifndef __ASSEMBLY__

#ifdef CONFIG_X86_32
#define in_ia32_syscall() true
#else
#define in_ia32_syscall() (IS_ENABLED(CONFIG_IA32_EMULATION) && \
                           current_thread_info()->status & TS_COMPAT)
#endif

/*
 * Force syscall return via IRET by making it look as if there was
 * some work pending. IRET is our most capable (but slowest) syscall
 * return path, which is able to restore modified SS, CS and certain
 * EFLAGS values that other (fast) syscall return instructions
 * are not able to restore properly.
 */
#define force_iret() set_thread_flag(TIF_NOTIFY_RESUME)

extern void arch_task_cache_init(void);
extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
extern void arch_release_task_struct(struct task_struct *tsk);
extern void arch_setup_new_exec(void);
#define arch_setup_new_exec arch_setup_new_exec
#endif        /* !__ASSEMBLY__ */

#endif /* _ASM_X86_THREAD_INFO_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_TLB_H
#define _ASM_X86_TLB_H

#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)

static inline void tlb_flush(struct mmu_gather *tlb);

#include <asm-generic/tlb.h>

static inline void tlb_flush(struct mmu_gather *tlb)
{
        unsigned long start = 0UL, end = TLB_FLUSH_ALL;
        unsigned int stride_shift = tlb_get_unmap_shift(tlb); /*covered*/

        if (!tlb->fullmm && !tlb->need_flush_all) { /*covered*/
                start = tlb->start; /*covered*/
                end = tlb->end;
        }

        flush_tlb_mm_range(tlb->mm, start, end, stride_shift, tlb->freed_tables); /*covered*/
}

/*
 * While x86 architecture in general requires an IPI to perform TLB
 * shootdown, enablement code for several hypervisors overrides
 * .flush_tlb_others hook in pv_mmu_ops and implements it by issuing
 * a hypercall. To keep software pagetable walkers safe in this case we
 * switch to RCU based table free (HAVE_RCU_TABLE_FREE). See the comment
 * below 'ifdef CONFIG_HAVE_RCU_TABLE_FREE' in include/asm-generic/tlb.h
 * for more details.
 */
static inline void __tlb_remove_table(void *table)
{
        free_page_and_swap_cache(table);
}

#endif /* _ASM_X86_TLB_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_TLBFLUSH_H
#define _ASM_X86_TLBFLUSH_H

#include <linux/mm.h>
#include <linux/sched.h>

#include <asm/processor.h>
#include <asm/cpufeature.h>
#include <asm/special_insns.h>
#include <asm/smp.h>
#include <asm/invpcid.h>
#include <asm/pti.h>
#include <asm/processor-flags.h>

/*
 * The x86 feature is called PCID (Process Context IDentifier). It is similar
 * to what is traditionally called ASID on the RISC processors.
 *
 * We don't use the traditional ASID implementation, where each process/mm gets
 * its own ASID and flush/restart when we run out of ASID space.
 *
 * Instead we have a small per-cpu array of ASIDs and cache the last few mm's
 * that came by on this CPU, allowing cheaper switch_mm between processes on
 * this CPU.
 *
 * We end up with different spaces for different things. To avoid confusion we
 * use different names for each of them:
 *
 * ASID  - [0, TLB_NR_DYN_ASIDS-1]
 *         the canonical identifier for an mm
 *
 * kPCID - [1, TLB_NR_DYN_ASIDS]
 *         the value we write into the PCID part of CR3; corresponds to the
 *         ASID+1, because PCID 0 is special.
 *
 * uPCID - [2048 + 1, 2048 + TLB_NR_DYN_ASIDS]
 *         for KPTI each mm has two address spaces and thus needs two
 *         PCID values, but we can still do with a single ASID denomination
 *         for each mm. Corresponds to kPCID + 2048.
 *
 */

/* There are 12 bits of space for ASIDS in CR3 */
#define CR3_HW_ASID_BITS                12

/*
 * When enabled, PAGE_TABLE_ISOLATION consumes a single bit for
 * user/kernel switches
 */
#ifdef CONFIG_PAGE_TABLE_ISOLATION
# define PTI_CONSUMED_PCID_BITS        1
#else
# define PTI_CONSUMED_PCID_BITS        0
#endif

#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS)

/*
 * ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid.  -1 below to account
 * for them being zero-based.  Another -1 is because PCID 0 is reserved for
 * use by non-PCID-aware users.
 */
#define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_PCID_BITS) - 2)

/*
 * 6 because 6 should be plenty and struct tlb_state will fit in two cache
 * lines.
 */
#define TLB_NR_DYN_ASIDS        6

/*
 * Given @asid, compute kPCID
 */
static inline u16 kern_pcid(u16 asid)
{
        VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);

#ifdef CONFIG_PAGE_TABLE_ISOLATION
        /*
         * Make sure that the dynamic ASID space does not confict with the
         * bit we are using to switch between user and kernel ASIDs.
         */
        BUILD_BUG_ON(TLB_NR_DYN_ASIDS >= (1 << X86_CR3_PTI_PCID_USER_BIT));

        /*
         * The ASID being passed in here should have respected the
         * MAX_ASID_AVAILABLE and thus never have the switch bit set.
         */
        VM_WARN_ON_ONCE(asid & (1 << X86_CR3_PTI_PCID_USER_BIT));
#endif
        /*
         * The dynamically-assigned ASIDs that get passed in are small
         * (<TLB_NR_DYN_ASIDS).  They never have the high switch bit set,
         * so do not bother to clear it.
         *
         * If PCID is on, ASID-aware code paths put the ASID+1 into the
         * PCID bits.  This serves two purposes.  It prevents a nasty
         * situation in which PCID-unaware code saves CR3, loads some other
         * value (with PCID == 0), and then restores CR3, thus corrupting
         * the TLB for ASID 0 if the saved ASID was nonzero.  It also means
         * that any bugs involving loading a PCID-enabled CR3 with
         * CR4.PCIDE off will trigger deterministically.
         */
        return asid + 1;
}

/*
 * Given @asid, compute uPCID
 */
static inline u16 user_pcid(u16 asid)
{
        u16 ret = kern_pcid(asid);
#ifdef CONFIG_PAGE_TABLE_ISOLATION
        ret |= 1 << X86_CR3_PTI_PCID_USER_BIT;
#endif
        return ret;
}

struct pgd_t;
static inline unsigned long build_cr3(pgd_t *pgd, u16 asid)
{
        if (static_cpu_has(X86_FEATURE_PCID)) {
                return __sme_pa(pgd) | kern_pcid(asid);
        } else {
                VM_WARN_ON_ONCE(asid != 0);
                return __sme_pa(pgd);
        }
}

static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
{
        VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
        /*
         * Use boot_cpu_has() instead of this_cpu_has() as this function
         * might be called during early boot. This should work even after
         * boot because all CPU's the have same capabilities:
         */
        VM_WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_PCID));
        return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH;
}

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#define __flush_tlb() __native_flush_tlb()
#define __flush_tlb_global() __native_flush_tlb_global()
#define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr)
#endif

struct tlb_context {
        u64 ctx_id;
        u64 tlb_gen;
};

struct tlb_state {
        /*
         * cpu_tlbstate.loaded_mm should match CR3 whenever interrupts
         * are on.  This means that it may not match current->active_mm,
         * which will contain the previous user mm when we're in lazy TLB
         * mode even if we've already switched back to swapper_pg_dir.
         *
         * During switch_mm_irqs_off(), loaded_mm will be set to
         * LOADED_MM_SWITCHING during the brief interrupts-off window
         * when CR3 and loaded_mm would otherwise be inconsistent.  This
         * is for nmi_uaccess_okay()'s benefit.
         */
        struct mm_struct *loaded_mm;

#define LOADED_MM_SWITCHING ((struct mm_struct *)1)

        /* Last user mm for optimizing IBPB */
        union {
                struct mm_struct        *last_user_mm;
                unsigned long                last_user_mm_ibpb;
        };

        u16 loaded_mm_asid;
        u16 next_asid;

        /*
         * We can be in one of several states:
         *
         *  - Actively using an mm.  Our CPU's bit will be set in
         *    mm_cpumask(loaded_mm) and is_lazy == false;
         *
         *  - Not using a real mm.  loaded_mm == &init_mm.  Our CPU's bit
         *    will not be set in mm_cpumask(&init_mm) and is_lazy == false.
         *
         *  - Lazily using a real mm.  loaded_mm != &init_mm, our bit
         *    is set in mm_cpumask(loaded_mm), but is_lazy == true.
         *    We're heuristically guessing that the CR3 load we
         *    skipped more than makes up for the overhead added by
         *    lazy mode.
         */
        bool is_lazy;

        /*
         * If set we changed the page tables in such a way that we
         * needed an invalidation of all contexts (aka. PCIDs / ASIDs).
         * This tells us to go invalidate all the non-loaded ctxs[]
         * on the next context switch.
         *
         * The current ctx was kept up-to-date as it ran and does not
         * need to be invalidated.
         */
        bool invalidate_other;

        /*
         * Mask that contains TLB_NR_DYN_ASIDS+1 bits to indicate
         * the corresponding user PCID needs a flush next time we
         * switch to it; see SWITCH_TO_USER_CR3.
         */
        unsigned short user_pcid_flush_mask;

        /*
         * Access to this CR4 shadow and to H/W CR4 is protected by
         * disabling interrupts when modifying either one.
         */
        unsigned long cr4;

        /*
         * This is a list of all contexts that might exist in the TLB.
         * There is one per ASID that we use, and the ASID (what the
         * CPU calls PCID) is the index into ctxts.
         *
         * For each context, ctx_id indicates which mm the TLB's user
         * entries came from.  As an invariant, the TLB will never
         * contain entries that are out-of-date as when that mm reached
         * the tlb_gen in the list.
         *
         * To be clear, this means that it's legal for the TLB code to
         * flush the TLB without updating tlb_gen.  This can happen
         * (for now, at least) due to paravirt remote flushes.
         *
         * NB: context 0 is a bit special, since it's also used by
         * various bits of init code.  This is fine -- code that
         * isn't aware of PCID will end up harmlessly flushing
         * context 0.
         */
        struct tlb_context ctxs[TLB_NR_DYN_ASIDS];
};
DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);

/*
 * Blindly accessing user memory from NMI context can be dangerous
 * if we're in the middle of switching the current user task or
 * switching the loaded mm.  It can also be dangerous if we
 * interrupted some kernel code that was temporarily using a
 * different mm.
 */
static inline bool nmi_uaccess_okay(void)
{
        struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
        struct mm_struct *current_mm = current->mm;

        VM_WARN_ON_ONCE(!loaded_mm);

        /*
         * The condition we want to check is
         * current_mm->pgd == __va(read_cr3_pa()).  This may be slow, though,
         * if we're running in a VM with shadow paging, and nmi_uaccess_okay()
         * is supposed to be reasonably fast.
         *
         * Instead, we check the almost equivalent but somewhat conservative
         * condition below, and we rely on the fact that switch_mm_irqs_off()
         * sets loaded_mm to LOADED_MM_SWITCHING before writing to CR3.
         */
        if (loaded_mm != current_mm)
                return false;

        VM_WARN_ON_ONCE(current_mm->pgd != __va(read_cr3_pa()));

        return true;
}

/* Initialize cr4 shadow for this CPU. */
static inline void cr4_init_shadow(void)
{
        this_cpu_write(cpu_tlbstate.cr4, __read_cr4());
}

static inline void __cr4_set(unsigned long cr4)
{
        lockdep_assert_irqs_disabled();
        this_cpu_write(cpu_tlbstate.cr4, cr4);
        __write_cr4(cr4);
}

/* Set in this cpu's CR4. */
static inline void cr4_set_bits(unsigned long mask)
{
        unsigned long cr4, flags;

        local_irq_save(flags);
        cr4 = this_cpu_read(cpu_tlbstate.cr4);
        if ((cr4 | mask) != cr4)
                __cr4_set(cr4 | mask);
        local_irq_restore(flags);
}

/* Clear in this cpu's CR4. */
static inline void cr4_clear_bits(unsigned long mask)
{
        unsigned long cr4, flags;

        local_irq_save(flags);
        cr4 = this_cpu_read(cpu_tlbstate.cr4);
        if ((cr4 & ~mask) != cr4)
                __cr4_set(cr4 & ~mask);
        local_irq_restore(flags);
}

static inline void cr4_toggle_bits_irqsoff(unsigned long mask)
{
        unsigned long cr4;

        cr4 = this_cpu_read(cpu_tlbstate.cr4);
        __cr4_set(cr4 ^ mask);
}

/* Read the CR4 shadow. */
static inline unsigned long cr4_read_shadow(void)
{
        return this_cpu_read(cpu_tlbstate.cr4);
}

/*
 * Mark all other ASIDs as invalid, preserves the current.
 */
static inline void invalidate_other_asid(void)
{
        this_cpu_write(cpu_tlbstate.invalidate_other, true);
}

/*
 * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
 * enable and PPro Global page enable), so that any CPU's that boot
 * up after us can get the correct flags.  This should only be used
 * during boot on the boot cpu.
 */
extern unsigned long mmu_cr4_features;
extern u32 *trampoline_cr4_features;

static inline void cr4_set_bits_and_update_boot(unsigned long mask)
{
        mmu_cr4_features |= mask;
        if (trampoline_cr4_features)
                *trampoline_cr4_features = mmu_cr4_features;
        cr4_set_bits(mask);
}

extern void initialize_tlbstate_and_flush(void);

/*
 * Given an ASID, flush the corresponding user ASID.  We can delay this
 * until the next time we switch to it.
 *
 * See SWITCH_TO_USER_CR3.
 */
static inline void invalidate_user_asid(u16 asid)
{
        /* There is no user ASID if address space separation is off */
        if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
                return;

        /*
         * We only have a single ASID if PCID is off and the CR3
         * write will have flushed it.
         */
        if (!cpu_feature_enabled(X86_FEATURE_PCID))
                return;

        if (!static_cpu_has(X86_FEATURE_PTI))
                return;

        __set_bit(kern_pcid(asid),
                  (unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));
}

/*
 * flush the entire current user mapping
 */
static inline void __native_flush_tlb(void)
{
        /*
         * Preemption or interrupts must be disabled to protect the access
         * to the per CPU variable and to prevent being preempted between
         * read_cr3() and write_cr3().
         */
        WARN_ON_ONCE(preemptible());

        invalidate_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));

        /* If current->mm == NULL then the read_cr3() "borrows" an mm */
        native_write_cr3(__native_read_cr3());
}

/*
 * flush everything
 */
static inline void __native_flush_tlb_global(void)
{
        unsigned long cr4, flags;

        if (static_cpu_has(X86_FEATURE_INVPCID)) {
                /*
                 * Using INVPCID is considerably faster than a pair of writes
                 * to CR4 sandwiched inside an IRQ flag save/restore.
                 *
                 * Note, this works with CR4.PCIDE=0 or 1.
                 */
                invpcid_flush_all();
                return;
        }

        /*
         * Read-modify-write to CR4 - protect it from preemption and
         * from interrupts. (Use the raw variant because this code can
         * be called from deep inside debugging code.)
         */
        raw_local_irq_save(flags);

        cr4 = this_cpu_read(cpu_tlbstate.cr4);
        /* toggle PGE */
        native_write_cr4(cr4 ^ X86_CR4_PGE);
        /* write old PGE again and flush TLBs */
        native_write_cr4(cr4);

        raw_local_irq_restore(flags);
}

/*
 * flush one page in the user mapping
 */
static inline void __native_flush_tlb_one_user(unsigned long addr)
{
        u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);

        asm volatile("invlpg (%0)" ::"r" (addr) : "memory");

        if (!static_cpu_has(X86_FEATURE_PTI))
                return;

        /*
         * Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.
         * Just use invalidate_user_asid() in case we are called early.
         */
        if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))
                invalidate_user_asid(loaded_mm_asid);
        else
                invpcid_flush_one(user_pcid(loaded_mm_asid), addr);
}

/*
 * flush everything
 */
static inline void __flush_tlb_all(void)
{
        /*
         * This is to catch users with enabled preemption and the PGE feature
         * and don't trigger the warning in __native_flush_tlb().
         */
        VM_WARN_ON_ONCE(preemptible());

        if (boot_cpu_has(X86_FEATURE_PGE)) {
                __flush_tlb_global();
        } else {
                /*
                 * !PGE -> !PCID (setup_pcid()), thus every flush is total.
                 */
                __flush_tlb();
        }
}

/*
 * flush one page in the kernel mapping
 */
static inline void __flush_tlb_one_kernel(unsigned long addr)
{
        count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);

        /*
         * If PTI is off, then __flush_tlb_one_user() is just INVLPG or its
         * paravirt equivalent.  Even with PCID, this is sufficient: we only
         * use PCID if we also use global PTEs for the kernel mapping, and
         * INVLPG flushes global translations across all address spaces.
         *
         * If PTI is on, then the kernel is mapped with non-global PTEs, and
         * __flush_tlb_one_user() will flush the given address for the current
         * kernel address space and for its usermode counterpart, but it does
         * not flush it for other address spaces.
         */
        __flush_tlb_one_user(addr); /*covered*/

        if (!static_cpu_has(X86_FEATURE_PTI))
                return;

        /*
         * See above.  We need to propagate the flush to all other address
         * spaces.  In principle, we only need to propagate it to kernelmode
         * address spaces, but the extra bookkeeping we would need is not
         * worth it.
         */
        invalidate_other_asid();
}

#define TLB_FLUSH_ALL        -1UL

/*
 * TLB flushing:
 *
 *  - flush_tlb_all() flushes all processes TLBs
 *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
 *  - flush_tlb_page(vma, vmaddr) flushes one page
 *  - flush_tlb_range(vma, start, end) flushes a range of pages
 *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
 *  - flush_tlb_others(cpumask, info) flushes TLBs on other cpus
 *
 * ..but the i386 has somewhat limited tlb flushing capabilities,
 * and page-granular flushes are available only on i486 and up.
 */
struct flush_tlb_info {
        /*
         * We support several kinds of flushes.
         *
         * - Fully flush a single mm.  .mm will be set, .end will be
         *   TLB_FLUSH_ALL, and .new_tlb_gen will be the tlb_gen to
         *   which the IPI sender is trying to catch us up.
         *
         * - Partially flush a single mm.  .mm will be set, .start and
         *   .end will indicate the range, and .new_tlb_gen will be set
         *   such that the changes between generation .new_tlb_gen-1 and
         *   .new_tlb_gen are entirely contained in the indicated range.
         *
         * - Fully flush all mms whose tlb_gens have been updated.  .mm
         *   will be NULL, .end will be TLB_FLUSH_ALL, and .new_tlb_gen
         *   will be zero.
         */
        struct mm_struct        *mm;
        unsigned long                start;
        unsigned long                end;
        u64                        new_tlb_gen;
        unsigned int                stride_shift;
        bool                        freed_tables;
};

#define local_flush_tlb() __flush_tlb()

#define flush_tlb_mm(mm)                                                \
                flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL, true)

#define flush_tlb_range(vma, start, end)                                \
        flush_tlb_mm_range((vma)->vm_mm, start, end,                        \
                           ((vma)->vm_flags & VM_HUGETLB)                \
                                ? huge_page_shift(hstate_vma(vma))        \
                                : PAGE_SHIFT, false)

extern void flush_tlb_all(void);
extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
                                unsigned long end, unsigned int stride_shift,
                                bool freed_tables);
extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);

static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a)
{
        flush_tlb_mm_range(vma->vm_mm, a, a + PAGE_SIZE, PAGE_SHIFT, false);
}

void native_flush_tlb_others(const struct cpumask *cpumask,
                             const struct flush_tlb_info *info);

static inline u64 inc_mm_tlb_gen(struct mm_struct *mm)
{
        /*
         * Bump the generation count.  This also serves as a full barrier
         * that synchronizes with switch_mm(): callers are required to order
         * their read of mm_cpumask after their writes to the paging
         * structures.
         */
        return atomic64_inc_return(&mm->context.tlb_gen);
}

static inline void arch_tlbbatch_add_mm(struct arch_tlbflush_unmap_batch *batch,
                                        struct mm_struct *mm)
{
        inc_mm_tlb_gen(mm);
        cpumask_or(&batch->cpumask, &batch->cpumask, mm_cpumask(mm));
}

extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);

#ifndef CONFIG_PARAVIRT
#define flush_tlb_others(mask, info)        \
        native_flush_tlb_others(mask, info)

#define paravirt_tlb_remove_table(tlb, page) \
        tlb_remove_page(tlb, (void *)(page))
#endif

#endif /* _ASM_X86_TLBFLUSH_H */

/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM x86_fpu

#if !defined(_TRACE_FPU_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_FPU_H

#include <linux/tracepoint.h>

DECLARE_EVENT_CLASS(x86_fpu,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu),

        TP_STRUCT__entry(
                __field(struct fpu *, fpu)
                __field(bool, initialized)
                __field(u64, xfeatures)
                __field(u64, xcomp_bv)
                ),

        TP_fast_assign(
                __entry->fpu                = fpu;
                __entry->initialized        = fpu->initialized;
                if (boot_cpu_has(X86_FEATURE_OSXSAVE)) {
                        __entry->xfeatures = fpu->state.xsave.header.xfeatures;
                        __entry->xcomp_bv  = fpu->state.xsave.header.xcomp_bv;
                }
        ),
        TP_printk("x86/fpu: %p initialized: %d xfeatures: %llx xcomp_bv: %llx",
                        __entry->fpu,
                        __entry->initialized,
                        __entry->xfeatures,
                        __entry->xcomp_bv
        )
);

DEFINE_EVENT(x86_fpu, x86_fpu_before_save,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_after_save,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_before_restore,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_after_restore,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_regs_activated, /*covered*/
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_regs_deactivated, /*covered*/
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_activate_state, /*covered*/
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_init_state, /*covered*/
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_dropped, /*covered*/
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_copy_src,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_copy_dst,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

DEFINE_EVENT(x86_fpu, x86_fpu_xstate_check_failed,
        TP_PROTO(struct fpu *fpu),
        TP_ARGS(fpu)
);

#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH asm/trace/
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE fpu
#endif /* _TRACE_FPU_H */

/* This part must be outside protection */
#include <trace/define_trace.h>

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * x86 TSC related functions
 */
#ifndef _ASM_X86_TSC_H
#define _ASM_X86_TSC_H

#include <asm/processor.h>

#define NS_SCALE        10 /* 2^10, carefully chosen */
#define US_SCALE        32 /* 2^32, arbitralrily chosen */

/*
 * Standard way to access the cycle counter.
 */
typedef unsigned long long cycles_t;

extern unsigned int cpu_khz;
extern unsigned int tsc_khz;

extern void disable_TSC(void);

static inline cycles_t get_cycles(void)
{
#ifndef CONFIG_X86_TSC
        if (!boot_cpu_has(X86_FEATURE_TSC))
                return 0;
#endif

        return rdtsc(); /*covered*/
}

extern struct system_counterval_t convert_art_to_tsc(u64 art);
extern struct system_counterval_t convert_art_ns_to_tsc(u64 art_ns);

extern void tsc_early_init(void);
extern void tsc_init(void);
extern unsigned long calibrate_delay_is_known(void);
extern void mark_tsc_unstable(char *reason);
extern int unsynchronized_tsc(void);
extern int check_tsc_unstable(void);
extern void mark_tsc_async_resets(char *reason);
extern unsigned long native_calibrate_cpu_early(void);
extern unsigned long native_calibrate_tsc(void);
extern unsigned long long native_sched_clock_from_tsc(u64 tsc);

extern int tsc_clocksource_reliable;
#ifdef CONFIG_X86_TSC
extern bool tsc_async_resets;
#else
# define tsc_async_resets        false
#endif

/*
 * Boot-time check whether the TSCs are synchronized across
 * all CPUs/cores:
 */
#ifdef CONFIG_X86_TSC
extern bool tsc_store_and_check_tsc_adjust(bool bootcpu);
extern void tsc_verify_tsc_adjust(bool resume);
extern void check_tsc_sync_source(int cpu);
extern void check_tsc_sync_target(void);
#else
static inline bool tsc_store_and_check_tsc_adjust(bool bootcpu) { return false; }
static inline void tsc_verify_tsc_adjust(bool resume) { }
static inline void check_tsc_sync_source(int cpu) { }
static inline void check_tsc_sync_target(void) { }
#endif

extern int notsc_setup(char *);
extern void tsc_save_sched_clock_state(void);
extern void tsc_restore_sched_clock_state(void);

unsigned long cpu_khz_from_msr(void);

#endif /* _ASM_X86_TSC_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_UACCESS_H
#define _ASM_X86_UACCESS_H
/*
 * User space memory access functions
 */
#include <linux/compiler.h>
#include <linux/kasan-checks.h>
#include <linux/string.h>
#include <asm/asm.h>
#include <asm/page.h>
#include <asm/smap.h>
#include <asm/extable.h>

/*
 * The fs value determines whether argument validity checking should be
 * performed or not.  If get_fs() == USER_DS, checking is performed, with
 * get_fs() == KERNEL_DS, checking is bypassed.
 *
 * For historical reasons, these macros are grossly misnamed.
 */

#define MAKE_MM_SEG(s)        ((mm_segment_t) { (s) })

#define KERNEL_DS        MAKE_MM_SEG(-1UL)
#define USER_DS         MAKE_MM_SEG(TASK_SIZE_MAX)

#define get_fs()        (current->thread.addr_limit)
static inline void set_fs(mm_segment_t fs)
{
        current->thread.addr_limit = fs; /*covered*/
        /* On user-mode return, check fs is correct */
        set_thread_flag(TIF_FSCHECK);
}

#define segment_eq(a, b)        ((a).seg == (b).seg)
#define user_addr_max() (current->thread.addr_limit.seg)

/*
 * Test whether a block of memory is a valid user space address.
 * Returns 0 if the range is valid, nonzero otherwise.
 */
static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
{
        /*
         * If we have used "sizeof()" for the size,
         * we know it won't overflow the limit (but
         * it might overflow the 'addr', so it's
         * important to subtract the size from the
         * limit, not add it to the address).
         */
        if (__builtin_constant_p(size))
                return unlikely(addr > limit - size);

        /* Arbitrary sizes? Be careful about overflow */
        addr += size;
        if (unlikely(addr < size))
                return true;
        return unlikely(addr > limit);
}

#define __range_not_ok(addr, size, limit)                                \
({                                                                        \
        __chk_user_ptr(addr);                                                \
        __chk_range_not_ok((unsigned long __force)(addr), size, limit); \
})

#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
# define WARN_ON_IN_IRQ()        WARN_ON_ONCE(!in_task())
#else
# define WARN_ON_IN_IRQ()
#endif

/**
 * access_ok - Checks if a user space pointer is valid
 * @addr: User space pointer to start of block to check
 * @size: Size of block to check
 *
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
 *
 * Checks if a pointer to a block of memory in user space is valid.
 *
 * Note that, depending on architecture, this function probably just
 * checks that the pointer is in the user space range - after calling
 * this function, memory access functions may still return -EFAULT.
 *
 * Return: true (nonzero) if the memory block may be valid, false (zero)
 * if it is definitely invalid.
 */
#define access_ok(addr, size)                                        \
({                                                                        \
        WARN_ON_IN_IRQ();                                                \
        likely(!__range_not_ok(addr, size, user_addr_max()));                \
})

/*
 * These are the main single-value transfer routines.  They automatically
 * use the right size if we just have the right pointer type.
 *
 * This gets kind of ugly. We want to return _two_ values in "get_user()"
 * and yet we don't want to do any pointers, because that is too much
 * of a performance impact. Thus we have a few rather ugly macros here,
 * and hide all the ugliness from the user.
 *
 * The "__xxx" versions of the user access functions are versions that
 * do not verify the address space, that must have been done previously
 * with a separate "access_ok()" call (this is used when we do multiple
 * accesses to the same area of user memory).
 */

extern int __get_user_1(void);
extern int __get_user_2(void);
extern int __get_user_4(void);
extern int __get_user_8(void);
extern int __get_user_bad(void);

#define __uaccess_begin() stac()
#define __uaccess_end()   clac()
#define __uaccess_begin_nospec()        \
({                                        \
        stac();                                \
        barrier_nospec();                \
})

/*
 * This is a type: either unsigned long, if the argument fits into
 * that type, or otherwise unsigned long long.
 */
#define __inttype(x) \
__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))

/**
 * get_user - Get a simple variable from user space.
 * @x:   Variable to store result.
 * @ptr: Source address, in user space.
 *
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
 *
 * This macro copies a single simple variable from user space to kernel
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and the result of
 * dereferencing @ptr must be assignable to @x without a cast.
 *
 * Return: zero on success, or -EFAULT on error.
 * On error, the variable @x is set to zero.
 */
/*
 * Careful: we have to cast the result to the type of the pointer
 * for sign reasons.
 *
 * The use of _ASM_DX as the register specifier is a bit of a
 * simplification, as gcc only cares about it as the starting point
 * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
 * (%ecx being the next register in gcc's x86 register sequence), and
 * %rdx on 64 bits.
 *
 * Clang/LLVM cares about the size of the register, but still wants
 * the base register for something that ends up being a pair.
 */
#define get_user(x, ptr)                                                \
({                                                                        \
        int __ret_gu;                                                        \
        register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);                \
        __chk_user_ptr(ptr);                                                \
        might_fault();                                                        \
        asm volatile("call __get_user_%P4"                                \
                     : "=a" (__ret_gu), "=r" (__val_gu),                \
                        ASM_CALL_CONSTRAINT                                \
                     : "0" (ptr), "i" (sizeof(*(ptr))));                \
        (x) = (__force __typeof__(*(ptr))) __val_gu;                        \
        __builtin_expect(__ret_gu, 0);                                        \
})

#define __put_user_x(size, x, ptr, __ret_pu)                        \
        asm volatile("call __put_user_" #size : "=a" (__ret_pu)        \
                     : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")



#ifdef CONFIG_X86_32
#define __put_user_goto_u64(x, addr, label)                        \
        asm_volatile_goto("\n"                                        \
                     "1:        movl %%eax,0(%1)\n"                \
                     "2:        movl %%edx,4(%1)\n"                \
                     _ASM_EXTABLE_UA(1b, %l2)                        \
                     _ASM_EXTABLE_UA(2b, %l2)                        \
                     : : "A" (x), "r" (addr)                        \
                     : : label)

#define __put_user_asm_ex_u64(x, addr)                                        \
        asm volatile("\n"                                                \
                     "1:        movl %%eax,0(%1)\n"                        \
                     "2:        movl %%edx,4(%1)\n"                        \
                     "3:"                                                \
                     _ASM_EXTABLE_EX(1b, 2b)                                \
                     _ASM_EXTABLE_EX(2b, 3b)                                \
                     : : "A" (x), "r" (addr))

#define __put_user_x8(x, ptr, __ret_pu)                                \
        asm volatile("call __put_user_8" : "=a" (__ret_pu)        \
                     : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
#else
#define __put_user_goto_u64(x, ptr, label) \
        __put_user_goto(x, ptr, "q", "", "er", label)
#define __put_user_asm_ex_u64(x, addr)        \
        __put_user_asm_ex(x, addr, "q", "", "er")
#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
#endif

extern void __put_user_bad(void);

/*
 * Strange magic calling convention: pointer in %ecx,
 * value in %eax(:%edx), return value in %eax. clobbers %rbx
 */
extern void __put_user_1(void);
extern void __put_user_2(void);
extern void __put_user_4(void);
extern void __put_user_8(void);

/**
 * put_user - Write a simple value into user space.
 * @x:   Value to copy to user space.
 * @ptr: Destination address, in user space.
 *
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
 *
 * This macro copies a single simple value from kernel space to user
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 * to the result of dereferencing @ptr.
 *
 * Return: zero on success, or -EFAULT on error.
 */
#define put_user(x, ptr)                                        \
({                                                                \
        int __ret_pu;                                                \
        __typeof__(*(ptr)) __pu_val;                                \
        __chk_user_ptr(ptr);                                        \
        might_fault();                                                \
        __pu_val = x;                                                \
        switch (sizeof(*(ptr))) {                                \
        case 1:                                                        \
                __put_user_x(1, __pu_val, ptr, __ret_pu);        \
                break;                                                \
        case 2:                                                        \
                __put_user_x(2, __pu_val, ptr, __ret_pu);        \
                break;                                                \
        case 4:                                                        \
                __put_user_x(4, __pu_val, ptr, __ret_pu);        \
                break;                                                \
        case 8:                                                        \
                __put_user_x8(__pu_val, ptr, __ret_pu);                \
                break;                                                \
        default:                                                \
                __put_user_x(X, __pu_val, ptr, __ret_pu);        \
                break;                                                \
        }                                                        \
        __builtin_expect(__ret_pu, 0);                                \
})

#define __put_user_size(x, ptr, size, label)                                \
do {                                                                        \
        __chk_user_ptr(ptr);                                                \
        switch (size) {                                                        \
        case 1:                                                                \
                __put_user_goto(x, ptr, "b", "b", "iq", label);        \
                break;                                                        \
        case 2:                                                                \
                __put_user_goto(x, ptr, "w", "w", "ir", label);                \
                break;                                                        \
        case 4:                                                                \
                __put_user_goto(x, ptr, "l", "k", "ir", label);                \
                break;                                                        \
        case 8:                                                                \
                __put_user_goto_u64(x, ptr, label);                        \
                break;                                                        \
        default:                                                        \
                __put_user_bad();                                        \
        }                                                                \
} while (0)

/*
 * This doesn't do __uaccess_begin/end - the exception handling
 * around it must do that.
 */
#define __put_user_size_ex(x, ptr, size)                                \
do {                                                                        \
        __chk_user_ptr(ptr);                                                \
        switch (size) {                                                        \
        case 1:                                                                \
                __put_user_asm_ex(x, ptr, "b", "b", "iq");                \
                break;                                                        \
        case 2:                                                                \
                __put_user_asm_ex(x, ptr, "w", "w", "ir");                \
                break;                                                        \
        case 4:                                                                \
                __put_user_asm_ex(x, ptr, "l", "k", "ir");                \
                break;                                                        \
        case 8:                                                                \
                __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr);        \
                break;                                                        \
        default:                                                        \
                __put_user_bad();                                        \
        }                                                                \
} while (0)

#ifdef CONFIG_X86_32
#define __get_user_asm_u64(x, ptr, retval, errret)                        \
({                                                                        \
        __typeof__(ptr) __ptr = (ptr);                                        \
        asm volatile("\n"                                        \
                     "1:        movl %2,%%eax\n"                        \
                     "2:        movl %3,%%edx\n"                        \
                     "3:\n"                                \
                     ".section .fixup,\"ax\"\n"                                \
                     "4:        mov %4,%0\n"                                \
                     "        xorl %%eax,%%eax\n"                                \
                     "        xorl %%edx,%%edx\n"                                \
                     "        jmp 3b\n"                                        \
                     ".previous\n"                                        \
                     _ASM_EXTABLE_UA(1b, 4b)                                \
                     _ASM_EXTABLE_UA(2b, 4b)                                \
                     : "=r" (retval), "=&A"(x)                                \
                     : "m" (__m(__ptr)), "m" __m(((u32 __user *)(__ptr)) + 1),        \
                       "i" (errret), "0" (retval));                        \
})

#define __get_user_asm_ex_u64(x, ptr)                        (x) = __get_user_bad()
#else
#define __get_user_asm_u64(x, ptr, retval, errret) \
         __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
#define __get_user_asm_ex_u64(x, ptr) \
         __get_user_asm_ex(x, ptr, "q", "", "=r")
#endif

#define __get_user_size(x, ptr, size, retval, errret)                        \
do {                                                                        \
        retval = 0;                                                        \
        __chk_user_ptr(ptr);                                                \
        switch (size) {                                                        \
        case 1:                                                                \
                __get_user_asm(x, ptr, retval, "b", "b", "=q", errret);        \
                break;                                                        \
        case 2:                                                                \
                __get_user_asm(x, ptr, retval, "w", "w", "=r", errret);        \
                break;                                                        \
        case 4:                                                                \
                __get_user_asm(x, ptr, retval, "l", "k", "=r", errret);        \
                break;                                                        \
        case 8:                                                                \
                __get_user_asm_u64(x, ptr, retval, errret);                \
                break;                                                        \
        default:                                                        \
                (x) = __get_user_bad();                                        \
        }                                                                \
} while (0)

#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)        \
        asm volatile("\n"                                                \
                     "1:        mov"itype" %2,%"rtype"1\n"                \
                     "2:\n"                                                \
                     ".section .fixup,\"ax\"\n"                                \
                     "3:        mov %3,%0\n"                                \
                     "        xor"itype" %"rtype"1,%"rtype"1\n"                \
                     "        jmp 2b\n"                                        \
                     ".previous\n"                                        \
                     _ASM_EXTABLE_UA(1b, 3b)                                \
                     : "=r" (err), ltype(x)                                \
                     : "m" (__m(addr)), "i" (errret), "0" (err))

#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret)        \
        asm volatile("\n"                                                \
                     "1:        mov"itype" %2,%"rtype"1\n"                \
                     "2:\n"                                                \
                     ".section .fixup,\"ax\"\n"                                \
                     "3:        mov %3,%0\n"                                \
                     "        jmp 2b\n"                                        \
                     ".previous\n"                                        \
                     _ASM_EXTABLE_UA(1b, 3b)                                \
                     : "=r" (err), ltype(x)                                \
                     : "m" (__m(addr)), "i" (errret), "0" (err))

/*
 * This doesn't do __uaccess_begin/end - the exception handling
 * around it must do that.
 */
#define __get_user_size_ex(x, ptr, size)                                \
do {                                                                        \
        __chk_user_ptr(ptr);                                                \
        switch (size) {                                                        \
        case 1:                                                                \
                __get_user_asm_ex(x, ptr, "b", "b", "=q");                \
                break;                                                        \
        case 2:                                                                \
                __get_user_asm_ex(x, ptr, "w", "w", "=r");                \
                break;                                                        \
        case 4:                                                                \
                __get_user_asm_ex(x, ptr, "l", "k", "=r");                \
                break;                                                        \
        case 8:                                                                \
                __get_user_asm_ex_u64(x, ptr);                                \
                break;                                                        \
        default:                                                        \
                (x) = __get_user_bad();                                        \
        }                                                                \
} while (0)

#define __get_user_asm_ex(x, addr, itype, rtype, ltype)                        \
        asm volatile("1:        mov"itype" %1,%"rtype"0\n"                \
                     "2:\n"                                                \
                     ".section .fixup,\"ax\"\n"                                \
                     "3:xor"itype" %"rtype"0,%"rtype"0\n"                \
                     "  jmp 2b\n"                                        \
                     ".previous\n"                                        \
                     _ASM_EXTABLE_EX(1b, 3b)                                \
                     : ltype(x) : "m" (__m(addr)))

#define __put_user_nocheck(x, ptr, size)                        \
({                                                                \
        __label__ __pu_label;                                        \
        int __pu_err = -EFAULT;                                        \
        __typeof__(*(ptr)) __pu_val;                                \
        __pu_val = x;                                                \
        __uaccess_begin();                                        \
        __put_user_size(__pu_val, (ptr), (size), __pu_label);        \
        __pu_err = 0;                                                \
__pu_label:                                                        \
        __uaccess_end();                                        \
        __builtin_expect(__pu_err, 0);                                \
})

#define __get_user_nocheck(x, ptr, size)                                \
({                                                                        \
        int __gu_err;                                                        \
        __inttype(*(ptr)) __gu_val;                                        \
        __uaccess_begin_nospec();                                        \
        __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);        \
        __uaccess_end();                                                \
        (x) = (__force __typeof__(*(ptr)))__gu_val;                        \
        __builtin_expect(__gu_err, 0);                                        \
})

/* FIXME: this hack is definitely wrong -AK */
struct __large_struct { unsigned long buf[100]; };
#define __m(x) (*(struct __large_struct __user *)(x))

/*
 * Tell gcc we read from memory instead of writing: this is because
 * we do not write to any memory gcc knows about, so there are no
 * aliasing issues.
 */
#define __put_user_goto(x, addr, itype, rtype, ltype, label)        \
        asm_volatile_goto("\n"                                                \
                "1:        mov"itype" %"rtype"0,%1\n"                        \
                _ASM_EXTABLE_UA(1b, %l2)                                        \
                : : ltype(x), "m" (__m(addr))                                \
                : : label)

#define __put_user_failed(x, addr, itype, rtype, ltype, errret)                \
        ({        __label__ __puflab;                                        \
                int __pufret = errret;                                        \
                __put_user_goto(x,addr,itype,rtype,ltype,__puflab);        \
                __pufret = 0;                                                \
        __puflab: __pufret; })

#define __put_user_asm(x, addr, retval, itype, rtype, ltype, errret)        do {        \
        retval = __put_user_failed(x, addr, itype, rtype, ltype, errret);        \
} while (0)

#define __put_user_asm_ex(x, addr, itype, rtype, ltype)                        \
        asm volatile("1:        mov"itype" %"rtype"0,%1\n"                \
                     "2:\n"                                                \
                     _ASM_EXTABLE_EX(1b, 2b)                                \
                     : : ltype(x), "m" (__m(addr)))

/*
 * uaccess_try and catch
 */
#define uaccess_try        do {                                                \
        current->thread.uaccess_err = 0;                                \
        __uaccess_begin();                                                \
        barrier();

#define uaccess_try_nospec do {                                                \
        current->thread.uaccess_err = 0;                                \
        __uaccess_begin_nospec();                                        \

#define uaccess_catch(err)                                                \
        __uaccess_end();                                                \
        (err) |= (current->thread.uaccess_err ? -EFAULT : 0);                \
} while (0)

/**
 * __get_user - Get a simple variable from user space, with less checking.
 * @x:   Variable to store result.
 * @ptr: Source address, in user space.
 *
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
 *
 * This macro copies a single simple variable from user space to kernel
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and the result of
 * dereferencing @ptr must be assignable to @x without a cast.
 *
 * Caller must check the pointer with access_ok() before calling this
 * function.
 *
 * Return: zero on success, or -EFAULT on error.
 * On error, the variable @x is set to zero.
 */

#define __get_user(x, ptr)                                                \
        __get_user_nocheck((x), (ptr), sizeof(*(ptr)))

/**
 * __put_user - Write a simple value into user space, with less checking.
 * @x:   Value to copy to user space.
 * @ptr: Destination address, in user space.
 *
 * Context: User context only. This function may sleep if pagefaults are
 *          enabled.
 *
 * This macro copies a single simple value from kernel space to user
 * space.  It supports simple types like char and int, but not larger
 * data types like structures or arrays.
 *
 * @ptr must have pointer-to-simple-variable type, and @x must be assignable
 * to the result of dereferencing @ptr.
 *
 * Caller must check the pointer with access_ok() before calling this
 * function.
 *
 * Return: zero on success, or -EFAULT on error.
 */

#define __put_user(x, ptr)                                                \
        __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))

/*
 * {get|put}_user_try and catch
 *
 * get_user_try {
 *        get_user_ex(...);
 * } get_user_catch(err)
 */
#define get_user_try                uaccess_try_nospec
#define get_user_catch(err)        uaccess_catch(err)

#define get_user_ex(x, ptr)        do {                                        \
        unsigned long __gue_val;                                        \
        __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr))));        \
        (x) = (__force __typeof__(*(ptr)))__gue_val;                        \
} while (0)

#define put_user_try                uaccess_try
#define put_user_catch(err)        uaccess_catch(err)

#define put_user_ex(x, ptr)                                                \
        __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))

extern unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
extern __must_check long
strncpy_from_user(char *dst, const char __user *src, long count);

extern __must_check long strnlen_user(const char __user *str, long n);

unsigned long __must_check clear_user(void __user *mem, unsigned long len);
unsigned long __must_check __clear_user(void __user *mem, unsigned long len);

extern void __cmpxchg_wrong_size(void)
        __compiletime_error("Bad argument size for cmpxchg");

#define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size)        \
({                                                                        \
        int __ret = 0;                                                        \
        __typeof__(ptr) __uval = (uval);                                \
        __typeof__(*(ptr)) __old = (old);                                \
        __typeof__(*(ptr)) __new = (new);                                \
        __uaccess_begin_nospec();                                        \
        switch (size) {                                                        \
        case 1:                                                                \
        {