/*
 * The AEGIS-128 Authenticated-Encryption Algorithm
 *   Glue for AES-NI + SSE2 implementation
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define AEGIS128_BLOCK_ALIGN 16
#define AEGIS128_BLOCK_SIZE 16
#define AEGIS128_NONCE_SIZE 16
#define AEGIS128_STATE_BLOCKS 5
#define AEGIS128_KEY_SIZE 16
#define AEGIS128_MIN_AUTH_SIZE 8
#define AEGIS128_MAX_AUTH_SIZE 16

asmlinkage void crypto_aegis128_aesni_init(void *state, void *key, void *iv);

asmlinkage void crypto_aegis128_aesni_ad(
                void *state, unsigned int length, const void *data);

asmlinkage void crypto_aegis128_aesni_enc(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_dec(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_enc_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_dec_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128_aesni_final(
                void *state, void *tag_xor, unsigned int cryptlen,
                unsigned int assoclen);

struct aegis_block {
        u8 bytes[AEGIS128_BLOCK_SIZE] __aligned(AEGIS128_BLOCK_ALIGN);
};

struct aegis_state {
        struct aegis_block blocks[AEGIS128_STATE_BLOCKS];
};

struct aegis_ctx {
        struct aegis_block key;
};

struct aegis_crypt_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, unsigned int length, const void *src,
                             void *dst);
        void (*crypt_tail)(void *state, unsigned int length, const void *src,
                           void *dst);
};

static void crypto_aegis128_aesni_process_ad(
                struct aegis_state *state, struct scatterlist *sg_src,
                unsigned int assoclen)
{
        struct scatter_walk walk;
        struct aegis_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) {
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= AEGIS128_BLOCK_SIZE) {
                        if (pos > 0) { /*covered*/
                                unsigned int fill = AEGIS128_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                crypto_aegis128_aesni_ad(state,
                                                         AEGIS128_BLOCK_SIZE,
                                                         buf.bytes);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        crypto_aegis128_aesni_ad(state, left, src); /*covered*/

                        src += left & ~(AEGIS128_BLOCK_SIZE - 1);
                        left &= AEGIS128_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/
                pos += left;
                assoclen -= size;

                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos);
                crypto_aegis128_aesni_ad(state, AEGIS128_BLOCK_SIZE, buf.bytes); /*covered*/
        }
}

static void crypto_aegis128_aesni_process_crypt(
                struct aegis_state *state, struct aead_request *req,
                const struct aegis_crypt_ops *ops)
{
        struct skcipher_walk walk; /*covered*/
        u8 *src, *dst;
        unsigned int chunksize, base;

        ops->skcipher_walk_init(&walk, req, false);

        while (walk.nbytes) { /*covered*/
                src = walk.src.virt.addr; /*covered*/
                dst = walk.dst.virt.addr;
                chunksize = walk.nbytes;

                ops->crypt_blocks(state, chunksize, src, dst);

                base = chunksize & ~(AEGIS128_BLOCK_SIZE - 1); /*covered*/
                src += base;
                dst += base;
                chunksize &= AEGIS128_BLOCK_SIZE - 1;

                if (chunksize > 0)
                        ops->crypt_tail(state, chunksize, src, dst);

                skcipher_walk_done(&walk, 0); /*covered*/
        }
}

static struct aegis_ctx *crypto_aegis128_aesni_ctx(struct crypto_aead *aead)
{
        u8 *ctx = crypto_aead_ctx(aead); /*covered*/
        ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
        return (void *)ctx;
}

static int crypto_aegis128_aesni_setkey(struct crypto_aead *aead, const u8 *key,
                                        unsigned int keylen)
{
        struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(aead); /*covered*/

        if (keylen != AEGIS128_KEY_SIZE) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key.bytes, key, AEGIS128_KEY_SIZE);

        return 0; /*covered*/
}

static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm,
                                                unsigned int authsize)
{
        if (authsize > AEGIS128_MAX_AUTH_SIZE) /*covered*/
                return -EINVAL;
        if (authsize < AEGIS128_MIN_AUTH_SIZE) /*covered*/
                return -EINVAL;
        return 0; /*covered*/
}

static void crypto_aegis128_aesni_crypt(struct aead_request *req,
                                        struct aegis_block *tag_xor,
                                        unsigned int cryptlen,
                                        const struct aegis_crypt_ops *ops)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
        struct aegis_state state;

        kernel_fpu_begin();

        crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
        crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen); /*covered*/
        crypto_aegis128_aesni_process_crypt(&state, req, ops);
        crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);

        kernel_fpu_end();
}

static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
{
        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = crypto_aegis128_aesni_enc,
                .crypt_tail = crypto_aegis128_aesni_enc_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}

static int crypto_aegis128_aesni_decrypt(struct aead_request *req)
{
        static const struct aegis_block zeros = {};

        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = crypto_aegis128_aesni_dec,
                .crypt_tail = crypto_aegis128_aesni_dec_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);

        return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
}

static int crypto_aegis128_aesni_init_tfm(struct crypto_aead *aead)
{
        return 0; /*covered*/
}

static void crypto_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
{
} /*covered*/

static int cryptd_aegis128_aesni_setkey(struct crypto_aead *aead,
                                        const u8 *key, unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}

static int cryptd_aegis128_aesni_setauthsize(struct crypto_aead *aead,
                                             unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int cryptd_aegis128_aesni_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int cryptd_aegis128_aesni_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req);
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() ||
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm);

        aead_request_set_tfm(req, aead);

        return crypto_aead_decrypt(req);
}

static int cryptd_aegis128_aesni_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__aegis128-aesni", CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void cryptd_aegis128_aesni_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}

static struct aead_alg crypto_aegis128_aesni_alg[] = {
        {
                .setkey = crypto_aegis128_aesni_setkey,
                .setauthsize = crypto_aegis128_aesni_setauthsize,
                .encrypt = crypto_aegis128_aesni_encrypt,
                .decrypt = crypto_aegis128_aesni_decrypt,
                .init = crypto_aegis128_aesni_init_tfm,
                .exit = crypto_aegis128_aesni_exit_tfm,

                .ivsize = AEGIS128_NONCE_SIZE,
                .maxauthsize = AEGIS128_MAX_AUTH_SIZE,
                .chunksize = AEGIS128_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct aegis_ctx) +
                                __alignof__(struct aegis_ctx),
                        .cra_alignmask = 0,

                        .cra_name = "__aegis128",
                        .cra_driver_name = "__aegis128-aesni",

                        .cra_module = THIS_MODULE,
                }
        }, {
                .setkey = cryptd_aegis128_aesni_setkey,
                .setauthsize = cryptd_aegis128_aesni_setauthsize,
                .encrypt = cryptd_aegis128_aesni_encrypt,
                .decrypt = cryptd_aegis128_aesni_decrypt,
                .init = cryptd_aegis128_aesni_init_tfm,
                .exit = cryptd_aegis128_aesni_exit_tfm,

                .ivsize = AEGIS128_NONCE_SIZE,
                .maxauthsize = AEGIS128_MAX_AUTH_SIZE,
                .chunksize = AEGIS128_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_ASYNC,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct cryptd_aead *),
                        .cra_alignmask = 0,

                        .cra_priority = 400,

                        .cra_name = "aegis128",
                        .cra_driver_name = "aegis128-aesni",

                        .cra_module = THIS_MODULE,
                }
        }
};

static int __init crypto_aegis128_aesni_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_aegis128_aesni_alg,
                                     ARRAY_SIZE(crypto_aegis128_aesni_alg));
}

static void __exit crypto_aegis128_aesni_module_exit(void)
{
        crypto_unregister_aeads(crypto_aegis128_aesni_alg,
                                ARRAY_SIZE(crypto_aegis128_aesni_alg));
}

module_init(crypto_aegis128_aesni_module_init);
module_exit(crypto_aegis128_aesni_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE2 implementation");
MODULE_ALIAS_CRYPTO("aegis128");
MODULE_ALIAS_CRYPTO("aegis128-aesni");

/*
 * The AEGIS-128L Authenticated-Encryption Algorithm
 *   Glue for AES-NI + SSE2 implementation
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define AEGIS128L_BLOCK_ALIGN 16
#define AEGIS128L_BLOCK_SIZE 32
#define AEGIS128L_NONCE_SIZE 16
#define AEGIS128L_STATE_BLOCKS 8
#define AEGIS128L_KEY_SIZE 16
#define AEGIS128L_MIN_AUTH_SIZE 8
#define AEGIS128L_MAX_AUTH_SIZE 16

asmlinkage void crypto_aegis128l_aesni_init(void *state, void *key, void *iv);

asmlinkage void crypto_aegis128l_aesni_ad(
                void *state, unsigned int length, const void *data);

asmlinkage void crypto_aegis128l_aesni_enc(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_dec(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_enc_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_dec_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis128l_aesni_final(
                void *state, void *tag_xor, unsigned int cryptlen,
                unsigned int assoclen);

struct aegis_block {
        u8 bytes[AEGIS128L_BLOCK_SIZE] __aligned(AEGIS128L_BLOCK_ALIGN);
};

struct aegis_state {
        struct aegis_block blocks[AEGIS128L_STATE_BLOCKS];
};

struct aegis_ctx {
        struct aegis_block key;
};

struct aegis_crypt_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, unsigned int length, const void *src,
                             void *dst);
        void (*crypt_tail)(void *state, unsigned int length, const void *src,
                           void *dst);
};

static void crypto_aegis128l_aesni_process_ad(
                struct aegis_state *state, struct scatterlist *sg_src,
                unsigned int assoclen)
{
        struct scatter_walk walk;
        struct aegis_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) {
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= AEGIS128L_BLOCK_SIZE) {
                        if (pos > 0) { /*covered*/
                                unsigned int fill = AEGIS128L_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                crypto_aegis128l_aesni_ad(state,
                                                          AEGIS128L_BLOCK_SIZE,
                                                          buf.bytes);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        crypto_aegis128l_aesni_ad(state, left, src); /*covered*/

                        src += left & ~(AEGIS128L_BLOCK_SIZE - 1);
                        left &= AEGIS128L_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/
                pos += left;
                assoclen -= size;

                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, AEGIS128L_BLOCK_SIZE - pos);
                crypto_aegis128l_aesni_ad(state, AEGIS128L_BLOCK_SIZE, buf.bytes); /*covered*/
        }
}

static void crypto_aegis128l_aesni_process_crypt(
                struct aegis_state *state, struct aead_request *req,
                const struct aegis_crypt_ops *ops)
{
        struct skcipher_walk walk; /*covered*/
        u8 *src, *dst;
        unsigned int chunksize, base;

        ops->skcipher_walk_init(&walk, req, false);

        while (walk.nbytes) { /*covered*/
                src = walk.src.virt.addr; /*covered*/
                dst = walk.dst.virt.addr;
                chunksize = walk.nbytes;

                ops->crypt_blocks(state, chunksize, src, dst);

                base = chunksize & ~(AEGIS128L_BLOCK_SIZE - 1); /*covered*/
                src += base;
                dst += base;
                chunksize &= AEGIS128L_BLOCK_SIZE - 1;

                if (chunksize > 0)
                        ops->crypt_tail(state, chunksize, src, dst);

                skcipher_walk_done(&walk, 0); /*covered*/
        }
}

static struct aegis_ctx *crypto_aegis128l_aesni_ctx(struct crypto_aead *aead)
{
        u8 *ctx = crypto_aead_ctx(aead); /*covered*/
        ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
        return (void *)ctx;
}

static int crypto_aegis128l_aesni_setkey(struct crypto_aead *aead,
                                         const u8 *key, unsigned int keylen)
{
        struct aegis_ctx *ctx = crypto_aegis128l_aesni_ctx(aead); /*covered*/

        if (keylen != AEGIS128L_KEY_SIZE) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key.bytes, key, AEGIS128L_KEY_SIZE);

        return 0; /*covered*/
}

static int crypto_aegis128l_aesni_setauthsize(struct crypto_aead *tfm,
                                              unsigned int authsize)
{
        if (authsize > AEGIS128L_MAX_AUTH_SIZE) /*covered*/
                return -EINVAL;
        if (authsize < AEGIS128L_MIN_AUTH_SIZE) /*covered*/
                return -EINVAL;
        return 0; /*covered*/
}

static void crypto_aegis128l_aesni_crypt(struct aead_request *req,
                                         struct aegis_block *tag_xor,
                                         unsigned int cryptlen,
                                         const struct aegis_crypt_ops *ops)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_ctx *ctx = crypto_aegis128l_aesni_ctx(tfm);
        struct aegis_state state;

        kernel_fpu_begin();

        crypto_aegis128l_aesni_init(&state, ctx->key.bytes, req->iv);
        crypto_aegis128l_aesni_process_ad(&state, req->src, req->assoclen); /*covered*/
        crypto_aegis128l_aesni_process_crypt(&state, req, ops);
        crypto_aegis128l_aesni_final(&state, tag_xor, req->assoclen, cryptlen);

        kernel_fpu_end();
}

static int crypto_aegis128l_aesni_encrypt(struct aead_request *req)
{
        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = crypto_aegis128l_aesni_enc,
                .crypt_tail = crypto_aegis128l_aesni_enc_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_aegis128l_aesni_crypt(req, &tag, cryptlen, &OPS);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}

static int crypto_aegis128l_aesni_decrypt(struct aead_request *req)
{
        static const struct aegis_block zeros = {};

        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = crypto_aegis128l_aesni_dec,
                .crypt_tail = crypto_aegis128l_aesni_dec_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_aegis128l_aesni_crypt(req, &tag, cryptlen, &OPS);

        return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0; /*covered*/
}

static int crypto_aegis128l_aesni_init_tfm(struct crypto_aead *aead)
{
        return 0; /*covered*/
}

static void crypto_aegis128l_aesni_exit_tfm(struct crypto_aead *aead)
{
}

static int cryptd_aegis128l_aesni_setkey(struct crypto_aead *aead,
                                         const u8 *key, unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}

static int cryptd_aegis128l_aesni_setauthsize(struct crypto_aead *aead,
                                              unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int cryptd_aegis128l_aesni_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int cryptd_aegis128l_aesni_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}

static int cryptd_aegis128l_aesni_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__aegis128l-aesni", CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void cryptd_aegis128l_aesni_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx);
}

static struct aead_alg crypto_aegis128l_aesni_alg[] = {
        {
                .setkey = crypto_aegis128l_aesni_setkey,
                .setauthsize = crypto_aegis128l_aesni_setauthsize,
                .encrypt = crypto_aegis128l_aesni_encrypt,
                .decrypt = crypto_aegis128l_aesni_decrypt,
                .init = crypto_aegis128l_aesni_init_tfm,
                .exit = crypto_aegis128l_aesni_exit_tfm,

                .ivsize = AEGIS128L_NONCE_SIZE,
                .maxauthsize = AEGIS128L_MAX_AUTH_SIZE,
                .chunksize = AEGIS128L_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct aegis_ctx) +
                                __alignof__(struct aegis_ctx),
                        .cra_alignmask = 0,

                        .cra_name = "__aegis128l",
                        .cra_driver_name = "__aegis128l-aesni",

                        .cra_module = THIS_MODULE,
                }
        }, {
                .setkey = cryptd_aegis128l_aesni_setkey,
                .setauthsize = cryptd_aegis128l_aesni_setauthsize,
                .encrypt = cryptd_aegis128l_aesni_encrypt,
                .decrypt = cryptd_aegis128l_aesni_decrypt,
                .init = cryptd_aegis128l_aesni_init_tfm,
                .exit = cryptd_aegis128l_aesni_exit_tfm,

                .ivsize = AEGIS128L_NONCE_SIZE,
                .maxauthsize = AEGIS128L_MAX_AUTH_SIZE,
                .chunksize = AEGIS128L_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_ASYNC,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct cryptd_aead *),
                        .cra_alignmask = 0,

                        .cra_priority = 400,

                        .cra_name = "aegis128l",
                        .cra_driver_name = "aegis128l-aesni",

                        .cra_module = THIS_MODULE,
                }
        }
};

static int __init crypto_aegis128l_aesni_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_aegis128l_aesni_alg,
                                     ARRAY_SIZE(crypto_aegis128l_aesni_alg));
}

static void __exit crypto_aegis128l_aesni_module_exit(void)
{
        crypto_unregister_aeads(crypto_aegis128l_aesni_alg,
                                ARRAY_SIZE(crypto_aegis128l_aesni_alg));
}

module_init(crypto_aegis128l_aesni_module_init);
module_exit(crypto_aegis128l_aesni_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("AEGIS-128L AEAD algorithm -- AESNI+SSE2 implementation");
MODULE_ALIAS_CRYPTO("aegis128l");
MODULE_ALIAS_CRYPTO("aegis128l-aesni");

/*
 * The AEGIS-256 Authenticated-Encryption Algorithm
 *   Glue for AES-NI + SSE2 implementation
 *
 * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define AEGIS256_BLOCK_ALIGN 16
#define AEGIS256_BLOCK_SIZE 16
#define AEGIS256_NONCE_SIZE 32
#define AEGIS256_STATE_BLOCKS 6
#define AEGIS256_KEY_SIZE 32
#define AEGIS256_MIN_AUTH_SIZE 8
#define AEGIS256_MAX_AUTH_SIZE 16

asmlinkage void crypto_aegis256_aesni_init(void *state, void *key, void *iv);

asmlinkage void crypto_aegis256_aesni_ad(
                void *state, unsigned int length, const void *data);

asmlinkage void crypto_aegis256_aesni_enc(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_dec(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_enc_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_dec_tail(
                void *state, unsigned int length, const void *src, void *dst);

asmlinkage void crypto_aegis256_aesni_final(
                void *state, void *tag_xor, unsigned int cryptlen,
                unsigned int assoclen);

struct aegis_block {
        u8 bytes[AEGIS256_BLOCK_SIZE] __aligned(AEGIS256_BLOCK_ALIGN);
};

struct aegis_state {
        struct aegis_block blocks[AEGIS256_STATE_BLOCKS];
};

struct aegis_ctx {
        struct aegis_block key[AEGIS256_KEY_SIZE / AEGIS256_BLOCK_SIZE];
};

struct aegis_crypt_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, unsigned int length, const void *src,
                             void *dst);
        void (*crypt_tail)(void *state, unsigned int length, const void *src,
                           void *dst);
};

static void crypto_aegis256_aesni_process_ad(
                struct aegis_state *state, struct scatterlist *sg_src,
                unsigned int assoclen)
{
        struct scatter_walk walk;
        struct aegis_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) {
                unsigned int size = scatterwalk_clamp(&walk, assoclen);
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk);
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= AEGIS256_BLOCK_SIZE) {
                        if (pos > 0) {
                                unsigned int fill = AEGIS256_BLOCK_SIZE - pos;
                                memcpy(buf.bytes + pos, src, fill);
                                crypto_aegis256_aesni_ad(state,
                                                         AEGIS256_BLOCK_SIZE,
                                                         buf.bytes);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        crypto_aegis256_aesni_ad(state, left, src);

                        src += left & ~(AEGIS256_BLOCK_SIZE - 1);
                        left &= AEGIS256_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left);
                pos += left;
                assoclen -= size;

                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size);
                scatterwalk_done(&walk, 0, assoclen);
        }

        if (pos > 0) {
                memset(buf.bytes + pos, 0, AEGIS256_BLOCK_SIZE - pos);
                crypto_aegis256_aesni_ad(state, AEGIS256_BLOCK_SIZE, buf.bytes); /*covered*/
        }
}

static void crypto_aegis256_aesni_process_crypt(
                struct aegis_state *state, struct aead_request *req,
                const struct aegis_crypt_ops *ops)
{
        struct skcipher_walk walk; /*covered*/
        u8 *src, *dst;
        unsigned int chunksize, base;

        ops->skcipher_walk_init(&walk, req, false);

        while (walk.nbytes) { /*covered*/
                src = walk.src.virt.addr; /*covered*/
                dst = walk.dst.virt.addr;
                chunksize = walk.nbytes;

                ops->crypt_blocks(state, chunksize, src, dst);

                base = chunksize & ~(AEGIS256_BLOCK_SIZE - 1); /*covered*/
                src += base;
                dst += base;
                chunksize &= AEGIS256_BLOCK_SIZE - 1;

                if (chunksize > 0)
                        ops->crypt_tail(state, chunksize, src, dst);

                skcipher_walk_done(&walk, 0); /*covered*/
        }
}

static struct aegis_ctx *crypto_aegis256_aesni_ctx(struct crypto_aead *aead)
{
        u8 *ctx = crypto_aead_ctx(aead); /*covered*/
        ctx = PTR_ALIGN(ctx, __alignof__(struct aegis_ctx));
        return (void *)ctx;
}

static int crypto_aegis256_aesni_setkey(struct crypto_aead *aead, const u8 *key,
                                        unsigned int keylen)
{
        struct aegis_ctx *ctx = crypto_aegis256_aesni_ctx(aead); /*covered*/

        if (keylen != AEGIS256_KEY_SIZE) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key, key, AEGIS256_KEY_SIZE);

        return 0; /*covered*/
}

static int crypto_aegis256_aesni_setauthsize(struct crypto_aead *tfm,
                                                unsigned int authsize)
{
        if (authsize > AEGIS256_MAX_AUTH_SIZE) /*covered*/
                return -EINVAL;
        if (authsize < AEGIS256_MIN_AUTH_SIZE) /*covered*/
                return -EINVAL;
        return 0; /*covered*/
}

static void crypto_aegis256_aesni_crypt(struct aead_request *req,
                                        struct aegis_block *tag_xor,
                                        unsigned int cryptlen,
                                        const struct aegis_crypt_ops *ops)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_ctx *ctx = crypto_aegis256_aesni_ctx(tfm);
        struct aegis_state state;

        kernel_fpu_begin();

        crypto_aegis256_aesni_init(&state, ctx->key, req->iv);
        crypto_aegis256_aesni_process_ad(&state, req->src, req->assoclen); /*covered*/
        crypto_aegis256_aesni_process_crypt(&state, req, ops);
        crypto_aegis256_aesni_final(&state, tag_xor, req->assoclen, cryptlen);

        kernel_fpu_end();
}

static int crypto_aegis256_aesni_encrypt(struct aead_request *req)
{
        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = crypto_aegis256_aesni_enc,
                .crypt_tail = crypto_aegis256_aesni_enc_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct aegis_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}

static int crypto_aegis256_aesni_decrypt(struct aead_request *req)
{
        static const struct aegis_block zeros = {};

        static const struct aegis_crypt_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = crypto_aegis256_aesni_dec,
                .crypt_tail = crypto_aegis256_aesni_dec_tail,
        };

        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aegis_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_aegis256_aesni_crypt(req, &tag, cryptlen, &OPS);

        return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
}

static int crypto_aegis256_aesni_init_tfm(struct crypto_aead *aead)
{
        return 0; /*covered*/
}

static void crypto_aegis256_aesni_exit_tfm(struct crypto_aead *aead)
{
} /*covered*/

static int cryptd_aegis256_aesni_setkey(struct crypto_aead *aead,
                                        const u8 *key, unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}

static int cryptd_aegis256_aesni_setauthsize(struct crypto_aead *aead,
                                             unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int cryptd_aegis256_aesni_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int cryptd_aegis256_aesni_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req);
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() ||
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm);

        aead_request_set_tfm(req, aead);

        return crypto_aead_decrypt(req);
}

static int cryptd_aegis256_aesni_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__aegis256-aesni", CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void cryptd_aegis256_aesni_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}

static struct aead_alg crypto_aegis256_aesni_alg[] = {
        {
                .setkey = crypto_aegis256_aesni_setkey,
                .setauthsize = crypto_aegis256_aesni_setauthsize,
                .encrypt = crypto_aegis256_aesni_encrypt,
                .decrypt = crypto_aegis256_aesni_decrypt,
                .init = crypto_aegis256_aesni_init_tfm,
                .exit = crypto_aegis256_aesni_exit_tfm,

                .ivsize = AEGIS256_NONCE_SIZE,
                .maxauthsize = AEGIS256_MAX_AUTH_SIZE,
                .chunksize = AEGIS256_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct aegis_ctx) +
                                __alignof__(struct aegis_ctx),
                        .cra_alignmask = 0,

                        .cra_name = "__aegis256",
                        .cra_driver_name = "__aegis256-aesni",

                        .cra_module = THIS_MODULE,
                }
        }, {
                .setkey = cryptd_aegis256_aesni_setkey,
                .setauthsize = cryptd_aegis256_aesni_setauthsize,
                .encrypt = cryptd_aegis256_aesni_encrypt,
                .decrypt = cryptd_aegis256_aesni_decrypt,
                .init = cryptd_aegis256_aesni_init_tfm,
                .exit = cryptd_aegis256_aesni_exit_tfm,

                .ivsize = AEGIS256_NONCE_SIZE,
                .maxauthsize = AEGIS256_MAX_AUTH_SIZE,
                .chunksize = AEGIS256_BLOCK_SIZE,

                .base = {
                        .cra_flags = CRYPTO_ALG_ASYNC,
                        .cra_blocksize = 1,
                        .cra_ctxsize = sizeof(struct cryptd_aead *),
                        .cra_alignmask = 0,

                        .cra_priority = 400,

                        .cra_name = "aegis256",
                        .cra_driver_name = "aegis256-aesni",

                        .cra_module = THIS_MODULE,
                }
        }
};

static int __init crypto_aegis256_aesni_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_aegis256_aesni_alg,
                                    ARRAY_SIZE(crypto_aegis256_aesni_alg));
}

static void __exit crypto_aegis256_aesni_module_exit(void)
{
        crypto_unregister_aeads(crypto_aegis256_aesni_alg,
                                ARRAY_SIZE(crypto_aegis256_aesni_alg));
}

module_init(crypto_aegis256_aesni_module_init);
module_exit(crypto_aegis256_aesni_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("AEGIS-256 AEAD algorithm -- AESNI+SSE2 implementation");
MODULE_ALIAS_CRYPTO("aegis256");
MODULE_ALIAS_CRYPTO("aegis256-aesni");

/*
 * Glue Code for the asm optimized version of the AES Cipher Algorithm
 *
 */

#include <linux/module.h>
#include <crypto/aes.h>
#include <asm/crypto/aes.h>

asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);

void crypto_aes_encrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
{
        aes_enc_blk(ctx, dst, src);
}
EXPORT_SYMBOL_GPL(crypto_aes_encrypt_x86);

void crypto_aes_decrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
{
        aes_dec_blk(ctx, dst, src);
}
EXPORT_SYMBOL_GPL(crypto_aes_decrypt_x86);

static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        aes_enc_blk(crypto_tfm_ctx(tfm), dst, src); /*covered*/
}

static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        aes_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static struct crypto_alg aes_alg = {
        .cra_name                = "aes",
        .cra_driver_name        = "aes-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = AES_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct crypto_aes_ctx),
        .cra_module                = THIS_MODULE,
        .cra_u        = {
                .cipher        = {
                        .cia_min_keysize        = AES_MIN_KEY_SIZE,
                        .cia_max_keysize        = AES_MAX_KEY_SIZE,
                        .cia_setkey                = crypto_aes_set_key,
                        .cia_encrypt                = aes_encrypt,
                        .cia_decrypt                = aes_decrypt
                }
        }
};

static int __init aes_init(void)
{
        return crypto_register_alg(&aes_alg);
}

static void __exit aes_fini(void)
{
        crypto_unregister_alg(&aes_alg);
}

module_init(aes_init);
module_exit(aes_fini);

MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, asm optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");
MODULE_ALIAS_CRYPTO("aes-asm");

/*
 * Support for Intel AES-NI instructions. This file contains glue
 * code, the real AES implementation is in intel-aes_asm.S.
 *
 * Copyright (C) 2008, Intel Corp.
 *    Author: Huang Ying <ying.huang@intel.com>
 *
 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
 * interface for 64-bit kernels.
 *    Authors: Adrian Hoban <adrian.hoban@intel.com>
 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
 *             Tadeusz Struk (tadeusz.struk@intel.com)
 *             Aidan O'Mahony (aidan.o.mahony@intel.com)
 *    Copyright (c) 2010, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <linux/hardirq.h>
#include <linux/types.h>
#include <linux/module.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/aes.h>
#include <crypto/cryptd.h>
#include <crypto/ctr.h>
#include <crypto/b128ops.h>
#include <crypto/gcm.h>
#include <crypto/xts.h>
#include <asm/cpu_device_id.h>
#include <asm/fpu/api.h>
#include <asm/crypto/aes.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
#ifdef CONFIG_X86_64
#include <asm/crypto/glue_helper.h>
#endif


#define AESNI_ALIGN        16
#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
#define AES_BLOCK_MASK        (~(AES_BLOCK_SIZE - 1))
#define RFC4106_HASH_SUBKEY_SIZE 16
#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)

/* This data is stored at the end of the crypto_tfm struct.
 * It's a type of per "session" data storage location.
 * This needs to be 16 byte aligned.
 */
struct aesni_rfc4106_gcm_ctx {
        u8 hash_subkey[16] AESNI_ALIGN_ATTR;
        struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
        u8 nonce[4];
};

struct generic_gcmaes_ctx {
        u8 hash_subkey[16] AESNI_ALIGN_ATTR;
        struct crypto_aes_ctx aes_key_expanded AESNI_ALIGN_ATTR;
};

struct aesni_xts_ctx {
        u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
        u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx)] AESNI_ALIGN_ATTR;
};

#define GCM_BLOCK_LEN 16

struct gcm_context_data {
        /* init, update and finalize context data */
        u8 aad_hash[GCM_BLOCK_LEN];
        u64 aad_length;
        u64 in_length;
        u8 partial_block_enc_key[GCM_BLOCK_LEN];
        u8 orig_IV[GCM_BLOCK_LEN];
        u8 current_counter[GCM_BLOCK_LEN];
        u64 partial_block_len;
        u64 unused;
        u8 hash_keys[GCM_BLOCK_LEN * 8];
};

asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
                             unsigned int key_len);
asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
                          const u8 *in);
asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
                          const u8 *in);
asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len);
asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len);
asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);
asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);

int crypto_fpu_init(void);
void crypto_fpu_exit(void);

#define AVX_GEN2_OPTSIZE 640
#define AVX_GEN4_OPTSIZE 4096

#ifdef CONFIG_X86_64

static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);
asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv);

asmlinkage void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, u8 *out,
                                 const u8 *in, bool enc, u8 *iv);

/* asmlinkage void aesni_gcm_enc()
 * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 * struct gcm_context_data.  May be uninitialized.
 * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 * const u8 *in, Plaintext input
 * unsigned long plaintext_len, Length of data in bytes for encryption.
 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 *         16-byte aligned pointer.
 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 * const u8 *aad, Additional Authentication Data (AAD)
 * unsigned long aad_len, Length of AAD in bytes.
 * u8 *auth_tag, Authenticated Tag output.
 * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 *          Valid values are 16 (most likely), 12 or 8.
 */
asmlinkage void aesni_gcm_enc(void *ctx,
                        struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

/* asmlinkage void aesni_gcm_dec()
 * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 * struct gcm_context_data.  May be uninitialized.
 * u8 *out, Plaintext output. Decrypt in-place is allowed.
 * const u8 *in, Ciphertext input
 * unsigned long ciphertext_len, Length of data in bytes for decryption.
 * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001.
 *         16-byte aligned pointer.
 * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 * const u8 *aad, Additional Authentication Data (AAD)
 * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 * to be 8 or 12 bytes
 * u8 *auth_tag, Authenticated Tag output.
 * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 * Valid values are 16 (most likely), 12 or 8.
 */
asmlinkage void aesni_gcm_dec(void *ctx,
                        struct gcm_context_data *gdata, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

/* Scatter / Gather routines, with args similar to above */
asmlinkage void aesni_gcm_init(void *ctx,
                               struct gcm_context_data *gdata,
                               u8 *iv,
                               u8 *hash_subkey, const u8 *aad,
                               unsigned long aad_len);
asmlinkage void aesni_gcm_enc_update(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in, unsigned long plaintext_len);
asmlinkage void aesni_gcm_dec_update(void *ctx,
                                     struct gcm_context_data *gdata, u8 *out,
                                     const u8 *in,
                                     unsigned long ciphertext_len);
asmlinkage void aesni_gcm_finalize(void *ctx,
                                   struct gcm_context_data *gdata,
                                   u8 *auth_tag, unsigned long auth_tag_len);

#ifdef CONFIG_AS_AVX
asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,
                void *keys, u8 *out, unsigned int num_bytes);
asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
                void *keys, u8 *out, unsigned int num_bytes);
asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
                void *keys, u8 *out, unsigned int num_bytes);
/*
 * asmlinkage void aesni_gcm_precomp_avx_gen2()
 * gcm_data *my_ctx_data, context data
 * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 */
asmlinkage void aesni_gcm_precomp_avx_gen2(void *my_ctx_data, u8 *hash_subkey);

asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

static void aesni_gcm_enc_avx(void *ctx,
                        struct gcm_context_data *data, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len)
{
        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
        if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){
                aesni_gcm_enc(ctx, data, out, in,
                        plaintext_len, iv, hash_subkey, aad,
                        aad_len, auth_tag, auth_tag_len);
        } else {
                aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
                aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
                                        aad_len, auth_tag, auth_tag_len);
        }
}

static void aesni_gcm_dec_avx(void *ctx,
                        struct gcm_context_data *data, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len)
{
        struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
        if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) {
                aesni_gcm_dec(ctx, data, out, in,
                        ciphertext_len, iv, hash_subkey, aad,
                        aad_len, auth_tag, auth_tag_len);
        } else {
                aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
                aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
                                        aad_len, auth_tag, auth_tag_len);
        }
}
#endif

#ifdef CONFIG_AS_AVX2
/*
 * asmlinkage void aesni_gcm_precomp_avx_gen4()
 * gcm_data *my_ctx_data, context data
 * u8 *hash_subkey,  the Hash sub key input. Data starts on a 16-byte boundary.
 */
asmlinkage void aesni_gcm_precomp_avx_gen4(void *my_ctx_data, u8 *hash_subkey);

asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len);

static void aesni_gcm_enc_avx2(void *ctx,
                        struct gcm_context_data *data, u8 *out,
                        const u8 *in, unsigned long plaintext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len)
{
       struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
        if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { /*covered*/
                aesni_gcm_enc(ctx, data, out, in,
                              plaintext_len, iv, hash_subkey, aad,
                              aad_len, auth_tag, auth_tag_len);
        } else if (plaintext_len < AVX_GEN4_OPTSIZE) { /*covered*/
                aesni_gcm_precomp_avx_gen2(ctx, hash_subkey); /*covered*/
                aesni_gcm_enc_avx_gen2(ctx, out, in, plaintext_len, iv, aad,
                                        aad_len, auth_tag, auth_tag_len);
        } else {
                aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); /*covered*/
                aesni_gcm_enc_avx_gen4(ctx, out, in, plaintext_len, iv, aad,
                                        aad_len, auth_tag, auth_tag_len);
        }
} /*covered*/

static void aesni_gcm_dec_avx2(void *ctx,
        struct gcm_context_data *data, u8 *out,
                        const u8 *in, unsigned long ciphertext_len, u8 *iv,
                        u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
                        u8 *auth_tag, unsigned long auth_tag_len)
{
       struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx;
        if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { /*covered*/
                aesni_gcm_dec(ctx, data, out, in, /*covered*/
                              ciphertext_len, iv, hash_subkey,
                              aad, aad_len, auth_tag, auth_tag_len);
        } else if (ciphertext_len < AVX_GEN4_OPTSIZE) { /*covered*/
                aesni_gcm_precomp_avx_gen2(ctx, hash_subkey);
                aesni_gcm_dec_avx_gen2(ctx, out, in, ciphertext_len, iv, aad,
                                        aad_len, auth_tag, auth_tag_len);
        } else {
                aesni_gcm_precomp_avx_gen4(ctx, hash_subkey); /*covered*/
                aesni_gcm_dec_avx_gen4(ctx, out, in, ciphertext_len, iv, aad,
                                        aad_len, auth_tag, auth_tag_len);
        }
} /*covered*/
#endif

static void (*aesni_gcm_enc_tfm)(void *ctx,
                                 struct gcm_context_data *data, u8 *out,
                                 const u8 *in, unsigned long plaintext_len,
                                 u8 *iv, u8 *hash_subkey, const u8 *aad,
                                 unsigned long aad_len, u8 *auth_tag,
                                 unsigned long auth_tag_len);

static void (*aesni_gcm_dec_tfm)(void *ctx,
                                 struct gcm_context_data *data, u8 *out,
                                 const u8 *in, unsigned long ciphertext_len,
                                 u8 *iv, u8 *hash_subkey, const u8 *aad,
                                 unsigned long aad_len, u8 *auth_tag,
                                 unsigned long auth_tag_len);

static inline struct
aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
{
        unsigned long align = AESNI_ALIGN;

        if (align <= crypto_tfm_ctx_alignment())
                align = 1;
        return PTR_ALIGN(crypto_aead_ctx(tfm), align); /*covered*/
}

static inline struct
generic_gcmaes_ctx *generic_gcmaes_ctx_get(struct crypto_aead *tfm)
{
        unsigned long align = AESNI_ALIGN;

        if (align <= crypto_tfm_ctx_alignment())
                align = 1;
        return PTR_ALIGN(crypto_aead_ctx(tfm), align); /*covered*/
}
#endif

static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
{
        unsigned long addr = (unsigned long)raw_ctx; /*covered*/
        unsigned long align = AESNI_ALIGN;

        if (align <= crypto_tfm_ctx_alignment())
                align = 1;
        return (struct crypto_aes_ctx *)ALIGN(addr, align);
}

static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
                              const u8 *in_key, unsigned int key_len)
{
        struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); /*covered*/
        u32 *flags = &tfm->crt_flags;
        int err;

        if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && /*covered*/
            key_len != AES_KEYSIZE_256) {
                *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; /*covered*/
                return -EINVAL;
        }

        if (!irq_fpu_usable()) /*covered*/
                err = crypto_aes_expand_key(ctx, in_key, key_len);
        else {
                kernel_fpu_begin(); /*covered*/
                err = aesni_set_key(ctx, in_key, key_len);
                kernel_fpu_end(); /*covered*/
        }

        return err;
}

static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
                       unsigned int key_len)
{
        return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); /*covered*/
}

static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); /*covered*/

        if (!irq_fpu_usable())
                crypto_aes_encrypt_x86(ctx, dst, src);
        else {
                kernel_fpu_begin(); /*covered*/
                aesni_enc(ctx, dst, src);
                kernel_fpu_end();
        }
} /*covered*/

static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); /*covered*/

        if (!irq_fpu_usable())
                crypto_aes_decrypt_x86(ctx, dst, src);
        else {
                kernel_fpu_begin(); /*covered*/
                aesni_dec(ctx, dst, src);
                kernel_fpu_end();
        }
} /*covered*/

static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));

        aesni_enc(ctx, dst, src);
}

static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));

        aesni_dec(ctx, dst, src);
}

static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
                                 unsigned int len)
{
        return aes_set_key_common(crypto_skcipher_tfm(tfm), /*covered*/
                                  crypto_skcipher_ctx(tfm), key, len);
}

static int ecb_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
                              nbytes & AES_BLOCK_MASK);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end();

        return err;
}

static int ecb_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
                              nbytes & AES_BLOCK_MASK);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end(); /*covered*/

        return err;
}

static int cbc_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
                              nbytes & AES_BLOCK_MASK, walk.iv); /*covered*/
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end(); /*covered*/

        return err;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes)) {
                aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
                              nbytes & AES_BLOCK_MASK, walk.iv);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        kernel_fpu_end();

        return err;
}

#ifdef CONFIG_X86_64
static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
                            struct skcipher_walk *walk)
{
        u8 *ctrblk = walk->iv; /*covered*/
        u8 keystream[AES_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        aesni_enc(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, AES_BLOCK_SIZE);
}

#ifdef CONFIG_AS_AVX
static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
                              const u8 *in, unsigned int len, u8 *iv)
{
        /*
         * based on key length, override with the by8 version
         * of ctr mode encryption/decryption for improved performance
         * aes_set_key_common() ensures that key length is one of
         * {128,192,256}
         */
        if (ctx->key_length == AES_KEYSIZE_128) /*covered*/
                aes_ctr_enc_128_avx_by8(in, iv, (void *)ctx, out, len); /*covered*/
        else if (ctx->key_length == AES_KEYSIZE_192) /*covered*/
                aes_ctr_enc_192_avx_by8(in, iv, (void *)ctx, out, len); /*covered*/
        else
                aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len); /*covered*/
} /*covered*/
#endif

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm));
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, true);

        kernel_fpu_begin();
        while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
                aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, /*covered*/
                                      nbytes & AES_BLOCK_MASK, walk.iv);
                nbytes &= AES_BLOCK_SIZE - 1;
                err = skcipher_walk_done(&walk, nbytes);
        }
        if (walk.nbytes) { /*covered*/
                ctr_crypt_final(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, 0);
        }
        kernel_fpu_end(); /*covered*/

        return err;
}

static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key,
                            unsigned int keylen)
{
        struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        keylen /= 2;

        /* first half of xts-key is for crypt */
        err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, /*covered*/
                                 key, keylen);
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, /*covered*/
                                  key + keylen, keylen);
}


static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
{
        aesni_enc(ctx, out, in); /*covered*/
}

static void aesni_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_enc)); /*covered*/
}

static void aesni_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, GLUE_FUNC_CAST(aesni_dec)); /*covered*/
}

static void aesni_xts_enc8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, true, (u8 *)iv); /*covered*/
}

static void aesni_xts_dec8(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        aesni_xts_crypt8(ctx, (u8 *)dst, (const u8 *)src, false, (u8 *)iv); /*covered*/
}

static const struct common_glue_ctx aesni_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = 1,

        .funcs = { {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc8) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_enc) }
        } }
};

static const struct common_glue_ctx aesni_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = 1,

        .funcs = { {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec8) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(aesni_xts_dec) }
        } }
};

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&aesni_enc_xts, req,
                                   XTS_TWEAK_CAST(aesni_xts_tweak),
                                   aes_ctx(ctx->raw_tweak_ctx),
                                   aes_ctx(ctx->raw_crypt_ctx));
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&aesni_dec_xts, req,
                                   XTS_TWEAK_CAST(aesni_xts_tweak),
                                   aes_ctx(ctx->raw_tweak_ctx),
                                   aes_ctx(ctx->raw_crypt_ctx));
}

static int rfc4106_init(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", /*covered*/
                                       CRYPTO_ALG_INTERNAL,
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}

static void rfc4106_exit(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}

static int
rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
{
        struct crypto_cipher *tfm;
        int ret;

        tfm = crypto_alloc_cipher("aes", 0, 0); /*covered*/
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        ret = crypto_cipher_setkey(tfm, key, key_len); /*covered*/
        if (ret)
                goto out_free_cipher;

        /* Clear the data in the hash sub key container to zero.*/
        /* We want to cipher all zeros to create the hash sub key. */
        memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE); /*covered*/

        crypto_cipher_encrypt_one(tfm, hash_subkey, hash_subkey);

out_free_cipher:
        crypto_free_cipher(tfm); /*covered*/
        return ret; /*covered*/
}

static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key,
                                  unsigned int key_len)
{
        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); /*covered*/

        if (key_len < 4) {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }
        /*Account for 4 byte nonce at the end.*/
        key_len -= 4; /*covered*/

        memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));

        return aes_set_key_common(crypto_aead_tfm(aead),
                                  &ctx->aes_key_expanded, key, key_len) ?: /*covered*/
               rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); /*covered*/
}

static int gcmaes_wrapper_set_key(struct crypto_aead *parent, const u8 *key,
                                  unsigned int key_len)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(parent);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, key_len);
}

static int common_rfc4106_set_authsize(struct crypto_aead *aead,
                                       unsigned int authsize)
{ /*covered*/
        switch (authsize) {
        case 8:
        case 12:
        case 16:
                break;
        default:
                return -EINVAL;
        }

        return 0; /*covered*/
}

/* This is the Integrity Check Value (aka the authentication tag length and can
 * be 8, 12 or 16 bytes long. */
static int gcmaes_wrapper_set_authsize(struct crypto_aead *parent,
                                       unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(parent);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}

static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
                                       unsigned int authsize)
{ /*covered*/
        switch (authsize) {
        case 4:
        case 8:
        case 12:
        case 13:
        case 14:
        case 15:
        case 16:
                break;
        default:
                return -EINVAL;
        }

        return 0; /*covered*/
}

static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req,
                              unsigned int assoclen, u8 *hash_subkey,
                              u8 *iv, void *aes_ctx)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        unsigned long auth_tag_len = crypto_aead_authsize(tfm);
        struct gcm_context_data data AESNI_ALIGN_ATTR;
        struct scatter_walk dst_sg_walk = {};
        unsigned long left = req->cryptlen;
        unsigned long len, srclen, dstlen;
        struct scatter_walk assoc_sg_walk;
        struct scatter_walk src_sg_walk;
        struct scatterlist src_start[2];
        struct scatterlist dst_start[2];
        struct scatterlist *src_sg;
        struct scatterlist *dst_sg;
        u8 *src, *dst, *assoc;
        u8 *assocmem = NULL;
        u8 authTag[16];

        if (!enc)
                left -= auth_tag_len; /*covered*/

        /* Linearize assoc, if not already linear */
        if (req->src->length >= assoclen && req->src->length && /*covered*/
                (!PageHighMem(sg_page(req->src)) || /*covered*/
                        req->src->offset + req->src->length < PAGE_SIZE)) {
                scatterwalk_start(&assoc_sg_walk, req->src); /*covered*/
                assoc = scatterwalk_map(&assoc_sg_walk); /*covered*/
        } else {
                /* assoc can be any length, so must be on heap */
                assocmem = kmalloc(assoclen, GFP_ATOMIC); /*covered*/
                if (unlikely(!assocmem))
                        return -ENOMEM;
                assoc = assocmem;

                scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); /*covered*/
        }

        src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); /*covered*/
        scatterwalk_start(&src_sg_walk, src_sg);
        if (req->src != req->dst) {
                dst_sg = scatterwalk_ffwd(dst_start, req->dst, req->assoclen); /*covered*/
                scatterwalk_start(&dst_sg_walk, dst_sg);
        }

        kernel_fpu_begin(); /*covered*/
        aesni_gcm_init(aes_ctx, &data, iv,
                hash_subkey, assoc, assoclen);
        if (req->src != req->dst) {
                while (left) { /*covered*/
                        src = scatterwalk_map(&src_sg_walk); /*covered*/
                        dst = scatterwalk_map(&dst_sg_walk); /*covered*/
                        srclen = scatterwalk_clamp(&src_sg_walk, left);
                        dstlen = scatterwalk_clamp(&dst_sg_walk, left);
                        len = min(srclen, dstlen);
                        if (len) { /*covered*/
                                if (enc)
                                        aesni_gcm_enc_update(aes_ctx, &data, /*covered*/
                                                             dst, src, len);
                                else
                                        aesni_gcm_dec_update(aes_ctx, &data,
                                                             dst, src, len);
                        }
                        left -= len; /*covered*/

                        scatterwalk_unmap(src);
                        scatterwalk_unmap(dst); /*covered*/
                        scatterwalk_advance(&src_sg_walk, len); /*covered*/
                        scatterwalk_advance(&dst_sg_walk, len);
                        scatterwalk_done(&src_sg_walk, 0, left); /*covered*/
                        scatterwalk_done(&dst_sg_walk, 1, left); /*covered*/
                }
        } else {
                while (left) { /*covered*/
                        dst = src = scatterwalk_map(&src_sg_walk); /*covered*/
                        len = scatterwalk_clamp(&src_sg_walk, left);
                        if (len) { /*covered*/
                                if (enc)
                                        aesni_gcm_enc_update(aes_ctx, &data, /*covered*/
                                                             src, src, len);
                                else
                                        aesni_gcm_dec_update(aes_ctx, &data, /*covered*/
                                                             src, src, len);
                        }
                        left -= len; /*covered*/
                        scatterwalk_unmap(src);
                        scatterwalk_advance(&src_sg_walk, len); /*covered*/
                        scatterwalk_done(&src_sg_walk, 1, left); /*covered*/
                }
        }
        aesni_gcm_finalize(aes_ctx, &data, authTag, auth_tag_len); /*covered*/
        kernel_fpu_end();

        if (!assocmem)
                scatterwalk_unmap(assoc); /*covered*/
        else
                kfree(assocmem); /*covered*/

        if (!enc) { /*covered*/
                u8 authTagMsg[16]; /*covered*/

                /* Copy out original authTag */
                scatterwalk_map_and_copy(authTagMsg, req->src,
                                         req->assoclen + req->cryptlen -
                                         auth_tag_len,
                                         auth_tag_len, 0);

                /* Compare generated tag with passed in tag. */
                return crypto_memneq(authTagMsg, authTag, auth_tag_len) ?
                        -EBADMSG : 0; /*covered*/
        }

        /* Copy in the authTag */
        scatterwalk_map_and_copy(authTag, req->dst,
                                 req->assoclen + req->cryptlen, /*covered*/
                                 auth_tag_len, 1);

        return 0; /*covered*/
}

static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen,
                          u8 *hash_subkey, u8 *iv, void *aes_ctx)
{
        u8 one_entry_in_sg = 0;
        u8 *src, *dst, *assoc;
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        unsigned long auth_tag_len = crypto_aead_authsize(tfm); /*covered*/
        struct scatter_walk src_sg_walk;
        struct scatter_walk dst_sg_walk = {};
        struct gcm_context_data data AESNI_ALIGN_ATTR;

        if (((struct crypto_aes_ctx *)aes_ctx)->key_length != AES_KEYSIZE_128 ||
                aesni_gcm_enc_tfm == aesni_gcm_enc || /*covered*/
                req->cryptlen < AVX_GEN2_OPTSIZE) { /*covered*/
                return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, /*covered*/
                                          aes_ctx);
        }
        if (sg_is_last(req->src) &&
            (!PageHighMem(sg_page(req->src)) || /*covered*/
            req->src->offset + req->src->length <= PAGE_SIZE) &&
            sg_is_last(req->dst) && /*covered*/
            (!PageHighMem(sg_page(req->dst)) || /*covered*/
            req->dst->offset + req->dst->length <= PAGE_SIZE)) {
                one_entry_in_sg = 1;
                scatterwalk_start(&src_sg_walk, req->src); /*covered*/
                assoc = scatterwalk_map(&src_sg_walk); /*covered*/
                src = assoc + req->assoclen;
                dst = src;
                if (unlikely(req->src != req->dst)) {
                        scatterwalk_start(&dst_sg_walk, req->dst);
                        dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
                }
        } else {
                /* Allocate memory for src, dst, assoc */
                assoc = kmalloc(req->cryptlen + auth_tag_len + req->assoclen, /*covered*/
                        GFP_ATOMIC);
                if (unlikely(!assoc))
                        return -ENOMEM;
                scatterwalk_map_and_copy(assoc, req->src, 0,
                                         req->assoclen + req->cryptlen, 0); /*covered*/
                src = assoc + req->assoclen;
                dst = src;
        }

        kernel_fpu_begin(); /*covered*/
        aesni_gcm_enc_tfm(aes_ctx, &data, dst, src, req->cryptlen, iv,
                          hash_subkey, assoc, assoclen,
                          dst + req->cryptlen, auth_tag_len);
        kernel_fpu_end();

        /* The authTag (aka the Integrity Check Value) needs to be written
         * back to the packet. */
        if (one_entry_in_sg) {
                if (unlikely(req->src != req->dst)) {
                        scatterwalk_unmap(dst - req->assoclen);
                        scatterwalk_advance(&dst_sg_walk, req->dst->length);
                        scatterwalk_done(&dst_sg_walk, 1, 0);
                }
                scatterwalk_unmap(assoc); /*covered*/
                scatterwalk_advance(&src_sg_walk, req->src->length); /*covered*/
                scatterwalk_done(&src_sg_walk, req->src == req->dst, 0); /*covered*/
        } else {
                scatterwalk_map_and_copy(dst, req->dst, req->assoclen,
                                         req->cryptlen + auth_tag_len, 1);
                kfree(assoc);
        }
        return 0;
}

static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen,
                          u8 *hash_subkey, u8 *iv, void *aes_ctx)
{
        u8 one_entry_in_sg = 0;
        u8 *src, *dst, *assoc;
        unsigned long tempCipherLen = 0;
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        unsigned long auth_tag_len = crypto_aead_authsize(tfm); /*covered*/
        u8 authTag[16];
        struct scatter_walk src_sg_walk;
        struct scatter_walk dst_sg_walk = {};
        struct gcm_context_data data AESNI_ALIGN_ATTR;
        int retval = 0;

        if (((struct crypto_aes_ctx *)aes_ctx)->key_length != AES_KEYSIZE_128 ||
                aesni_gcm_enc_tfm == aesni_gcm_enc || /*covered*/
                req->cryptlen < AVX_GEN2_OPTSIZE) { /*covered*/
                return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, /*covered*/
                                          aes_ctx);
        }
        tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);

        if (sg_is_last(req->src) &&
            (!PageHighMem(sg_page(req->src)) ||
            req->src->offset + req->src->length <= PAGE_SIZE) &&
            sg_is_last(req->dst) && req->dst->length &&
            (!PageHighMem(sg_page(req->dst)) ||
            req->dst->offset + req->dst->length <= PAGE_SIZE)) {
                one_entry_in_sg = 1;
                scatterwalk_start(&src_sg_walk, req->src);
                assoc = scatterwalk_map(&src_sg_walk);
                src = assoc + req->assoclen;
                dst = src;
                if (unlikely(req->src != req->dst)) {
                        scatterwalk_start(&dst_sg_walk, req->dst);
                        dst = scatterwalk_map(&dst_sg_walk) + req->assoclen;
                }
        } else {
                /* Allocate memory for src, dst, assoc */
                assoc = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC); /*covered*/
                if (!assoc)
                        return -ENOMEM;
                scatterwalk_map_and_copy(assoc, req->src, 0,
                                         req->assoclen + req->cryptlen, 0); /*covered*/
                src = assoc + req->assoclen;
                dst = src;
        }


        kernel_fpu_begin();
        aesni_gcm_dec_tfm(aes_ctx, &data, dst, src, tempCipherLen, iv,
                          hash_subkey, assoc, assoclen,
                          authTag, auth_tag_len);
        kernel_fpu_end();

        /* Compare generated tag with passed in tag. */
        retval = crypto_memneq(src + tempCipherLen, authTag, auth_tag_len) ?
                -EBADMSG : 0;

        if (one_entry_in_sg) {
                if (unlikely(req->src != req->dst)) {
                        scatterwalk_unmap(dst - req->assoclen);
                        scatterwalk_advance(&dst_sg_walk, req->dst->length);
                        scatterwalk_done(&dst_sg_walk, 1, 0);
                }
                scatterwalk_unmap(assoc);
                scatterwalk_advance(&src_sg_walk, req->src->length);
                scatterwalk_done(&src_sg_walk, req->src == req->dst, 0);
        } else {
                scatterwalk_map_and_copy(dst, req->dst, req->assoclen, /*covered*/
                                         tempCipherLen, 1);
                kfree(assoc);
        }
        return retval;

}

static int helper_rfc4106_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
        unsigned int i;
        __be32 counter = cpu_to_be32(1);

        /* Assuming we are supporting rfc4106 64-bit extended */
        /* sequence numbers We need to have the AAD length equal */
        /* to 16 or 20 bytes */
        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
                return -EINVAL;

        /* IV below built */
        for (i = 0; i < 4; i++)
                *(iv+i) = ctx->nonce[i];
        for (i = 0; i < 8; i++)
                *(iv+4+i) = req->iv[i];
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_encrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int helper_rfc4106_decrypt(struct aead_request *req)
{
        __be32 counter = cpu_to_be32(1);
        struct crypto_aead *tfm = crypto_aead_reqtfm(req);
        struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
        unsigned int i;

        if (unlikely(req->assoclen != 16 && req->assoclen != 20))
                return -EINVAL;

        /* Assuming we are supporting rfc4106 64-bit extended */
        /* sequence numbers We need to have the AAD length */
        /* equal to 16 or 20 bytes */

        /* IV below built */
        for (i = 0; i < 4; i++)
                *(iv+i) = ctx->nonce[i];
        for (i = 0; i < 8; i++)
                *(iv+4+i) = req->iv[i];
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_decrypt(req, req->assoclen - 8, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int gcmaes_wrapper_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
        struct cryptd_aead *cryptd_tfm = *ctx;

        tfm = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                tfm = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, tfm); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}

static int gcmaes_wrapper_decrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(tfm);
        struct cryptd_aead *cryptd_tfm = *ctx;

        tfm = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                tfm = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, tfm); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}
#endif

static struct crypto_alg aesni_algs[] = { {
        .cra_name                = "aes",
        .cra_driver_name        = "aes-aesni",
        .cra_priority                = 300,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = AES_BLOCK_SIZE,
        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
        .cra_module                = THIS_MODULE,
        .cra_u        = {
                .cipher        = {
                        .cia_min_keysize        = AES_MIN_KEY_SIZE,
                        .cia_max_keysize        = AES_MAX_KEY_SIZE,
                        .cia_setkey                = aes_set_key,
                        .cia_encrypt                = aes_encrypt,
                        .cia_decrypt                = aes_decrypt
                }
        }
}, {
        .cra_name                = "__aes",
        .cra_driver_name        = "__aes-aesni",
        .cra_priority                = 300,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL,
        .cra_blocksize                = AES_BLOCK_SIZE,
        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
        .cra_module                = THIS_MODULE,
        .cra_u        = {
                .cipher        = {
                        .cia_min_keysize        = AES_MIN_KEY_SIZE,
                        .cia_max_keysize        = AES_MAX_KEY_SIZE,
                        .cia_setkey                = aes_set_key,
                        .cia_encrypt                = __aes_encrypt,
                        .cia_decrypt                = __aes_decrypt
                }
        }
} };

static struct skcipher_alg aesni_skciphers[] = {
        {
                .base = {
                        .cra_name                = "__ecb(aes)",
                        .cra_driver_name        = "__ecb-aes-aesni",
                        .cra_priority                = 400,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = AES_BLOCK_SIZE,
                        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = AES_MIN_KEY_SIZE,
                .max_keysize        = AES_MAX_KEY_SIZE,
                .setkey                = aesni_skcipher_setkey,
                .encrypt        = ecb_encrypt,
                .decrypt        = ecb_decrypt,
        }, {
                .base = {
                        .cra_name                = "__cbc(aes)",
                        .cra_driver_name        = "__cbc-aes-aesni",
                        .cra_priority                = 400,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = AES_BLOCK_SIZE,
                        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = AES_MIN_KEY_SIZE,
                .max_keysize        = AES_MAX_KEY_SIZE,
                .ivsize                = AES_BLOCK_SIZE,
                .setkey                = aesni_skcipher_setkey,
                .encrypt        = cbc_encrypt,
                .decrypt        = cbc_decrypt,
#ifdef CONFIG_X86_64
        }, {
                .base = {
                        .cra_name                = "__ctr(aes)",
                        .cra_driver_name        = "__ctr-aes-aesni",
                        .cra_priority                = 400,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = 1,
                        .cra_ctxsize                = CRYPTO_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = AES_MIN_KEY_SIZE,
                .max_keysize        = AES_MAX_KEY_SIZE,
                .ivsize                = AES_BLOCK_SIZE,
                .chunksize        = AES_BLOCK_SIZE,
                .setkey                = aesni_skcipher_setkey,
                .encrypt        = ctr_crypt,
                .decrypt        = ctr_crypt,
        }, {
                .base = {
                        .cra_name                = "__xts(aes)",
                        .cra_driver_name        = "__xts-aes-aesni",
                        .cra_priority                = 401,
                        .cra_flags                = CRYPTO_ALG_INTERNAL,
                        .cra_blocksize                = AES_BLOCK_SIZE,
                        .cra_ctxsize                = XTS_AES_CTX_SIZE,
                        .cra_module                = THIS_MODULE,
                },
                .min_keysize        = 2 * AES_MIN_KEY_SIZE,
                .max_keysize        = 2 * AES_MAX_KEY_SIZE,
                .ivsize                = AES_BLOCK_SIZE,
                .setkey                = xts_aesni_setkey,
                .encrypt        = xts_encrypt,
                .decrypt        = xts_decrypt,
#endif
        }
};

static
struct simd_skcipher_alg *aesni_simd_skciphers[ARRAY_SIZE(aesni_skciphers)];

static struct {
        const char *algname;
        const char *drvname;
        const char *basename;
        struct simd_skcipher_alg *simd;
} aesni_simd_skciphers2[] = {
#if (defined(MODULE) && IS_ENABLED(CONFIG_CRYPTO_PCBC)) || \
    IS_BUILTIN(CONFIG_CRYPTO_PCBC)
        {
                .algname        = "pcbc(aes)",
                .drvname        = "pcbc-aes-aesni",
                .basename        = "fpu(pcbc(__aes-aesni))",
        },
#endif
};

#ifdef CONFIG_X86_64
static int generic_gcmaes_set_key(struct crypto_aead *aead, const u8 *key,
                                  unsigned int key_len)
{
        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(aead); /*covered*/

        return aes_set_key_common(crypto_aead_tfm(aead),
                                  &ctx->aes_key_expanded, key, key_len) ?: /*covered*/
               rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); /*covered*/
}

static int generic_gcmaes_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));
        __be32 counter = cpu_to_be32(1);

        memcpy(iv, req->iv, 12);
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_encrypt(req, req->assoclen, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int generic_gcmaes_decrypt(struct aead_request *req)
{
        __be32 counter = cpu_to_be32(1);
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm);
        void *aes_ctx = &(ctx->aes_key_expanded);
        u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN)));

        memcpy(iv, req->iv, 12);
        *((__be32 *)(iv+12)) = counter;

        return gcmaes_decrypt(req, req->assoclen, ctx->hash_subkey, iv,
                              aes_ctx);
}

static int generic_gcmaes_init(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_tfm = cryptd_alloc_aead("__driver-generic-gcm-aes-aesni", /*covered*/
                                       CRYPTO_ALG_INTERNAL,
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));

        return 0; /*covered*/
}

static void generic_gcmaes_exit(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx);
}

static struct aead_alg aesni_aead_algs[] = { {
        .setkey                        = common_rfc4106_set_key,
        .setauthsize                = common_rfc4106_set_authsize,
        .encrypt                = helper_rfc4106_encrypt,
        .decrypt                = helper_rfc4106_decrypt,
        .ivsize                        = GCM_RFC4106_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "__gcm-aes-aesni",
                .cra_driver_name        = "__driver-gcm-aes-aesni",
                .cra_flags                = CRYPTO_ALG_INTERNAL,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct aesni_rfc4106_gcm_ctx),
                .cra_alignmask                = AESNI_ALIGN - 1,
                .cra_module                = THIS_MODULE,
        },
}, {
        .init                        = rfc4106_init,
        .exit                        = rfc4106_exit,
        .setkey                        = gcmaes_wrapper_set_key,
        .setauthsize                = gcmaes_wrapper_set_authsize,
        .encrypt                = gcmaes_wrapper_encrypt,
        .decrypt                = gcmaes_wrapper_decrypt,
        .ivsize                        = GCM_RFC4106_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "rfc4106(gcm(aes))",
                .cra_driver_name        = "rfc4106-gcm-aesni",
                .cra_priority                = 400,
                .cra_flags                = CRYPTO_ALG_ASYNC,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct cryptd_aead *),
                .cra_module                = THIS_MODULE,
        },
}, {
        .setkey                        = generic_gcmaes_set_key,
        .setauthsize                = generic_gcmaes_set_authsize,
        .encrypt                = generic_gcmaes_encrypt,
        .decrypt                = generic_gcmaes_decrypt,
        .ivsize                        = GCM_AES_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "__generic-gcm-aes-aesni",
                .cra_driver_name        = "__driver-generic-gcm-aes-aesni",
                .cra_priority                = 0,
                .cra_flags                = CRYPTO_ALG_INTERNAL,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct generic_gcmaes_ctx),
                .cra_alignmask                = AESNI_ALIGN - 1,
                .cra_module                = THIS_MODULE,
        },
}, {
        .init                        = generic_gcmaes_init,
        .exit                        = generic_gcmaes_exit,
        .setkey                        = gcmaes_wrapper_set_key,
        .setauthsize                = gcmaes_wrapper_set_authsize,
        .encrypt                = gcmaes_wrapper_encrypt,
        .decrypt                = gcmaes_wrapper_decrypt,
        .ivsize                        = GCM_AES_IV_SIZE,
        .maxauthsize                = 16,
        .base = {
                .cra_name                = "gcm(aes)",
                .cra_driver_name        = "generic-gcm-aesni",
                .cra_priority                = 400,
                .cra_flags                = CRYPTO_ALG_ASYNC,
                .cra_blocksize                = 1,
                .cra_ctxsize                = sizeof(struct cryptd_aead *),
                .cra_module                = THIS_MODULE,
        },
} };
#else
static struct aead_alg aesni_aead_algs[0];
#endif


static const struct x86_cpu_id aesni_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_AES),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);

static void aesni_free_simds(void)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers) &&
                    aesni_simd_skciphers[i]; i++)
                simd_skcipher_free(aesni_simd_skciphers[i]);

        for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++)
                if (aesni_simd_skciphers2[i].simd)
                        simd_skcipher_free(aesni_simd_skciphers2[i].simd);
}

static int __init aesni_init(void)
{
        struct simd_skcipher_alg *simd;
        const char *basename;
        const char *algname;
        const char *drvname;
        int err;
        int i;

        if (!x86_match_cpu(aesni_cpu_id))
                return -ENODEV;
#ifdef CONFIG_X86_64
#ifdef CONFIG_AS_AVX2
        if (boot_cpu_has(X86_FEATURE_AVX2)) {
                pr_info("AVX2 version of gcm_enc/dec engaged.\n");
                aesni_gcm_enc_tfm = aesni_gcm_enc_avx2;
                aesni_gcm_dec_tfm = aesni_gcm_dec_avx2;
        } else
#endif
#ifdef CONFIG_AS_AVX
        if (boot_cpu_has(X86_FEATURE_AVX)) {
                pr_info("AVX version of gcm_enc/dec engaged.\n");
                aesni_gcm_enc_tfm = aesni_gcm_enc_avx;
                aesni_gcm_dec_tfm = aesni_gcm_dec_avx;
        } else
#endif
        {
                pr_info("SSE version of gcm_enc/dec engaged.\n");
                aesni_gcm_enc_tfm = aesni_gcm_enc;
                aesni_gcm_dec_tfm = aesni_gcm_dec;
        }
        aesni_ctr_enc_tfm = aesni_ctr_enc;
#ifdef CONFIG_AS_AVX
        if (boot_cpu_has(X86_FEATURE_AVX)) {
                /* optimize performance of ctr mode encryption transform */
                aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
                pr_info("AES CTR mode by8 optimization enabled\n");
        }
#endif
#endif

        err = crypto_fpu_init();
        if (err)
                return err;

        err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
        if (err)
                goto fpu_exit;

        err = crypto_register_skciphers(aesni_skciphers,
                                        ARRAY_SIZE(aesni_skciphers));
        if (err)
                goto unregister_algs;

        err = crypto_register_aeads(aesni_aead_algs,
                                    ARRAY_SIZE(aesni_aead_algs));
        if (err)
                goto unregister_skciphers;

        for (i = 0; i < ARRAY_SIZE(aesni_skciphers); i++) {
                algname = aesni_skciphers[i].base.cra_name + 2;
                drvname = aesni_skciphers[i].base.cra_driver_name + 2;
                basename = aesni_skciphers[i].base.cra_driver_name;
                simd = simd_skcipher_create_compat(algname, drvname, basename);
                err = PTR_ERR(simd);
                if (IS_ERR(simd))
                        goto unregister_simds;

                aesni_simd_skciphers[i] = simd;
        }

        for (i = 0; i < ARRAY_SIZE(aesni_simd_skciphers2); i++) {
                algname = aesni_simd_skciphers2[i].algname;
                drvname = aesni_simd_skciphers2[i].drvname;
                basename = aesni_simd_skciphers2[i].basename;
                simd = simd_skcipher_create_compat(algname, drvname, basename);
                err = PTR_ERR(simd);
                if (IS_ERR(simd))
                        continue;

                aesni_simd_skciphers2[i].simd = simd;
        }

        return 0;

unregister_simds:
        aesni_free_simds();
        crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
unregister_skciphers:
        crypto_unregister_skciphers(aesni_skciphers,
                                    ARRAY_SIZE(aesni_skciphers));
unregister_algs:
        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
fpu_exit:
        crypto_fpu_exit();
        return err;
}

static void __exit aesni_exit(void)
{
        aesni_free_simds();
        crypto_unregister_aeads(aesni_aead_algs, ARRAY_SIZE(aesni_aead_algs));
        crypto_unregister_skciphers(aesni_skciphers,
                                    ARRAY_SIZE(aesni_skciphers));
        crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));

        crypto_fpu_exit();
}

late_initcall(aesni_init);
module_exit(aesni_exit);

MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");

/*
 * Glue Code for assembler optimized version of Blowfish
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <crypto/algapi.h>
#include <crypto/blowfish.h>
#include <crypto/internal/skcipher.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

/* regular block cipher functions */
asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
                                   bool xor);
asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);

/* 4-way parallel cipher functions */
asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                        const u8 *src, bool xor);
asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                      const u8 *src);

static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
{
        __blowfish_enc_blk(ctx, dst, src, false); /*covered*/
}

static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __blowfish_enc_blk(ctx, dst, src, true);
}

static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        __blowfish_enc_blk_4way(ctx, dst, src, false); /*covered*/
}

static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
                                      const u8 *src)
{
        __blowfish_enc_blk_4way(ctx, dst, src, true);
}

static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
}

static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static int blowfish_setkey_skcipher(struct crypto_skcipher *tfm,
                                    const u8 *key, unsigned int keylen)
{
        return blowfish_setkey(&tfm->base, key, keylen); /*covered*/
}

static int ecb_crypt(struct skcipher_request *req,
                     void (*fn)(struct bf_ctx *, u8 *, const u8 *),
                     void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
{
        unsigned int bsize = BF_BLOCK_SIZE;
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                u8 *wsrc = walk.src.virt.addr; /*covered*/
                u8 *wdst = walk.dst.virt.addr;

                /* Process four block batch */
                if (nbytes >= bsize * 4) {
                        do {
                                fn_4way(ctx, wdst, wsrc); /*covered*/

                                wsrc += bsize * 4;
                                wdst += bsize * 4;
                                nbytes -= bsize * 4;
                        } while (nbytes >= bsize * 4);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }

                /* Handle leftovers */
                do {
                        fn(ctx, wdst, wsrc); /*covered*/

                        wsrc += bsize;
                        wdst += bsize;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

done: /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static int ecb_encrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, blowfish_enc_blk, blowfish_enc_blk_4way); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, blowfish_dec_blk, blowfish_dec_blk_4way); /*covered*/
}

static unsigned int __cbc_encrypt(struct bf_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = BF_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 *iv = (u64 *)walk->iv;

        do {
                *dst = *src ^ *iv; /*covered*/
                blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
                iv = dst;

                src += 1;
                dst += 1;
                nbytes -= bsize;
        } while (nbytes >= bsize);

        *(u64 *)walk->iv = *iv; /*covered*/
        return nbytes;
}

static int cbc_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                nbytes = __cbc_encrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static unsigned int __cbc_decrypt(struct bf_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = BF_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 ivs[4 - 1];
        u64 last_iv;

        /* Start of the last block. */
        src += nbytes / bsize - 1;
        dst += nbytes / bsize - 1;

        last_iv = *src;

        /* Process four block batch */
        if (nbytes >= bsize * 4) {
                do {
                        nbytes -= bsize * 4 - bsize;
                        src -= 4 - 1; /*covered*/
                        dst -= 4 - 1;

                        ivs[0] = src[0];
                        ivs[1] = src[1];
                        ivs[2] = src[2];

                        blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);

                        dst[1] ^= ivs[0];
                        dst[2] ^= ivs[1];
                        dst[3] ^= ivs[2];

                        nbytes -= bsize;
                        if (nbytes < bsize)
                                goto done;

                        *dst ^= *(src - 1); /*covered*/
                        src -= 1;
                        dst -= 1;
                } while (nbytes >= bsize * 4);
        }

        /* Handle leftovers */
        for (;;) {
                blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src); /*covered*/

                nbytes -= bsize;
                if (nbytes < bsize)
                        break;

                *dst ^= *(src - 1); /*covered*/
                src -= 1;
                dst -= 1;
        }

done:
        *dst ^= *(u64 *)walk->iv; /*covered*/
        *(u64 *)walk->iv = last_iv;

        return nbytes;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                nbytes = __cbc_decrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static void ctr_crypt_final(struct bf_ctx *ctx, struct skcipher_walk *walk)
{
        u8 *ctrblk = walk->iv; /*covered*/
        u8 keystream[BF_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        blowfish_enc_blk(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, BF_BLOCK_SIZE);
}

static unsigned int __ctr_crypt(struct bf_ctx *ctx, struct skcipher_walk *walk)
{
        unsigned int bsize = BF_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
        __be64 ctrblocks[4];

        /* Process four block batch */
        if (nbytes >= bsize * 4) {
                do {
                        if (dst != src) { /*covered*/
                                dst[0] = src[0]; /*covered*/
                                dst[1] = src[1];
                                dst[2] = src[2];
                                dst[3] = src[3];
                        }

                        /* create ctrblks for parallel encrypt */
                        ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/
                        ctrblocks[1] = cpu_to_be64(ctrblk++);
                        ctrblocks[2] = cpu_to_be64(ctrblk++);
                        ctrblocks[3] = cpu_to_be64(ctrblk++);

                        blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
                                                  (u8 *)ctrblocks);

                        src += 4;
                        dst += 4;
                } while ((nbytes -= bsize * 4) >= bsize * 4);

                if (nbytes < bsize) /*covered*/
                        goto done;
        }

        /* Handle leftovers */
        do {
                if (dst != src) /*covered*/
                        *dst = *src; /*covered*/

                ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/

                blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);

                src += 1;
                dst += 1;
        } while ((nbytes -= bsize) >= bsize);

done: /*covered*/
        *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
        return nbytes;
}

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct bf_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
                nbytes = __ctr_crypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        if (nbytes) { /*covered*/
                ctr_crypt_final(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, 0);
        }

        return err; /*covered*/
}

static struct crypto_alg bf_cipher_alg = {
        .cra_name                = "blowfish",
        .cra_driver_name        = "blowfish-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = BF_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct bf_ctx),
        .cra_alignmask                = 0,
        .cra_module                = THIS_MODULE,
        .cra_u = {
                .cipher = {
                        .cia_min_keysize        = BF_MIN_KEY_SIZE,
                        .cia_max_keysize        = BF_MAX_KEY_SIZE,
                        .cia_setkey                = blowfish_setkey,
                        .cia_encrypt                = blowfish_encrypt,
                        .cia_decrypt                = blowfish_decrypt,
                }
        }
};

static struct skcipher_alg bf_skcipher_algs[] = {
        {
                .base.cra_name                = "ecb(blowfish)",
                .base.cra_driver_name        = "ecb-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = BF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(blowfish)",
                .base.cra_driver_name        = "cbc-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = BF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .ivsize                        = BF_BLOCK_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(blowfish)",
                .base.cra_driver_name        = "ctr-blowfish-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct bf_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = BF_MIN_KEY_SIZE,
                .max_keysize                = BF_MAX_KEY_SIZE,
                .ivsize                        = BF_BLOCK_SIZE,
                .chunksize                = BF_BLOCK_SIZE,
                .setkey                        = blowfish_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        },
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, blowfish-x86_64 is slower than generic C
                 * implementation because use of 64bit rotates (which are really
                 * slow on P4). Therefore blacklist P4s.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "blowfish-x86_64: performance on this CPU "
                        "would be suboptimal: disabling "
                        "blowfish-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&bf_cipher_alg);
        if (err)
                return err;

        err = crypto_register_skciphers(bf_skcipher_algs,
                                        ARRAY_SIZE(bf_skcipher_algs));
        if (err)
                crypto_unregister_alg(&bf_cipher_alg);

        return err;
}

static void __exit fini(void)
{
        crypto_unregister_alg(&bf_cipher_alg);
        crypto_unregister_skciphers(bf_skcipher_algs,
                                    ARRAY_SIZE(bf_skcipher_algs));
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("blowfish");
MODULE_ALIAS_CRYPTO("blowfish-asm");

/*
 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 */

#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/xts.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/types.h>

#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
#define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32

/* 32-way AVX2/AES-NI parallel cipher functions */
asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);

asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);

asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);
asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);

static const struct common_glue_ctx camellia_enc = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
        } }
};

static const struct common_glue_ctx camellia_ctr = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
        } }
};

static const struct common_glue_ctx camellia_enc_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
        } }
};

static const struct common_glue_ctx camellia_dec = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_cbc = {
        .num_funcs = 4,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) }
        }, {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
        } }
};

static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
                           unsigned int keylen)
{
        return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen, /*covered*/
                                 &tfm->base.crt_flags);
}

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&camellia_ctr, req);
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&camellia_enc_xts, req,
                                   XTS_TWEAK_CAST(camellia_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&camellia_dec_xts, req,
                                   XTS_TWEAK_CAST(camellia_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg camellia_algs[] = {
        {
                .base.cra_name                = "__ecb(camellia)",
                .base.cra_driver_name        = "__ecb-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(camellia)",
                .base.cra_driver_name        = "__cbc-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(camellia)",
                .base.cra_driver_name        = "__ctr-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .chunksize                = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(camellia)",
                .base.cra_driver_name        = "__xts-camellia-aesni-avx2",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = 2 * CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = xts_camellia_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];

static int __init camellia_aesni_init(void)
{
        const char *feature_name;

        if (!boot_cpu_has(X86_FEATURE_AVX) ||
            !boot_cpu_has(X86_FEATURE_AVX2) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
                pr_info("AVX2 or AES-NI instructions are not detected.\n");
                return -ENODEV;
        }

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(camellia_algs,
                                              ARRAY_SIZE(camellia_algs),
                                              camellia_simd_algs);
}

static void __exit camellia_aesni_fini(void)
{
        simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
                                  camellia_simd_algs);
}

module_init(camellia_aesni_init);
module_exit(camellia_aesni_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-asm");

/*
 * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia
 *
 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 */

#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/xts.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/types.h>

#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16

/* 16-way parallel cipher functions (avx/aes-ni) */
asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way);

asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way);

asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way);

asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(camellia_ctr_16way);

asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(camellia_xts_enc_16way);

asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(camellia_xts_dec_16way);

void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(camellia_enc_blk));
}
EXPORT_SYMBOL_GPL(camellia_xts_enc);

void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(camellia_dec_blk));
}
EXPORT_SYMBOL_GPL(camellia_xts_dec);

static const struct common_glue_ctx camellia_enc = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
        } }
};

static const struct common_glue_ctx camellia_ctr = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
        } }
};

static const struct common_glue_ctx camellia_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
        } }
};

static const struct common_glue_ctx camellia_dec = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_cbc = {
        .num_funcs = 3,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
        }, {
                .num_blocks = 2,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
        } }
};

static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
                           unsigned int keylen)
{
        return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen,
                                 &tfm->base.crt_flags);
}

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&camellia_ctr, req);
}

int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
                        unsigned int keylen)
{
        struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        u32 *flags = &tfm->base.crt_flags; /*covered*/
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        /* first half of xts-key is for crypt */
        err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); /*covered*/
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, /*covered*/
                                flags);
}
EXPORT_SYMBOL_GPL(xts_camellia_setkey);

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&camellia_enc_xts, req,
                                   XTS_TWEAK_CAST(camellia_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&camellia_dec_xts, req,
                                   XTS_TWEAK_CAST(camellia_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg camellia_algs[] = {
        {
                .base.cra_name                = "__ecb(camellia)",
                .base.cra_driver_name        = "__ecb-camellia-aesni",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(camellia)",
                .base.cra_driver_name        = "__cbc-camellia-aesni",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(camellia)",
                .base.cra_driver_name        = "__ctr-camellia-aesni",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .chunksize                = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(camellia)",
                .base.cra_driver_name        = "__xts-camellia-aesni",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = 2 * CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = xts_camellia_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];

static int __init camellia_aesni_init(void)
{
        const char *feature_name;

        if (!boot_cpu_has(X86_FEATURE_AVX) ||
            !boot_cpu_has(X86_FEATURE_AES) ||
            !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
                pr_info("AVX or AES-NI instructions are not detected.\n");
                return -ENODEV;
        }

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(camellia_algs,
                                              ARRAY_SIZE(camellia_algs),
                                              camellia_simd_algs);
}

static void __exit camellia_aesni_fini(void)
{
        simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
                                  camellia_simd_algs);
}

module_init(camellia_aesni_init);
module_exit(camellia_aesni_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-asm");

/*
 * Glue Code for assembler optimized version of Camellia
 *
 * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * Camellia parts based on code by:
 *  Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation)
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <asm/unaligned.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>
#include <crypto/algapi.h>
#include <asm/crypto/camellia.h>
#include <asm/crypto/glue_helper.h>

/* regular block cipher functions */
asmlinkage void __camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, bool xor);
EXPORT_SYMBOL_GPL(__camellia_enc_blk);
asmlinkage void camellia_dec_blk(struct camellia_ctx *ctx, u8 *dst,
                                 const u8 *src);
EXPORT_SYMBOL_GPL(camellia_dec_blk);

/* 2-way parallel cipher functions */
asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                        const u8 *src, bool xor);
EXPORT_SYMBOL_GPL(__camellia_enc_blk_2way);
asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                      const u8 *src);
EXPORT_SYMBOL_GPL(camellia_dec_blk_2way);

static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        camellia_enc_blk(crypto_tfm_ctx(tfm), dst, src); /*covered*/
}

static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        camellia_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

/* camellia sboxes */
__visible const u64 camellia_sp10011110[256] = {
        0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL,
        0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL,
        0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL,
        0x8500008585858500ULL, 0x5700005757575700ULL, 0x3500003535353500ULL,
        0xea0000eaeaeaea00ULL, 0x0c00000c0c0c0c00ULL, 0xae0000aeaeaeae00ULL,
        0x4100004141414100ULL, 0x2300002323232300ULL, 0xef0000efefefef00ULL,
        0x6b00006b6b6b6b00ULL, 0x9300009393939300ULL, 0x4500004545454500ULL,
        0x1900001919191900ULL, 0xa50000a5a5a5a500ULL, 0x2100002121212100ULL,
        0xed0000edededed00ULL, 0x0e00000e0e0e0e00ULL, 0x4f00004f4f4f4f00ULL,
        0x4e00004e4e4e4e00ULL, 0x1d00001d1d1d1d00ULL, 0x6500006565656500ULL,
        0x9200009292929200ULL, 0xbd0000bdbdbdbd00ULL, 0x8600008686868600ULL,
        0xb80000b8b8b8b800ULL, 0xaf0000afafafaf00ULL, 0x8f00008f8f8f8f00ULL,
        0x7c00007c7c7c7c00ULL, 0xeb0000ebebebeb00ULL, 0x1f00001f1f1f1f00ULL,
        0xce0000cececece00ULL, 0x3e00003e3e3e3e00ULL, 0x3000003030303000ULL,
        0xdc0000dcdcdcdc00ULL, 0x5f00005f5f5f5f00ULL, 0x5e00005e5e5e5e00ULL,
        0xc50000c5c5c5c500ULL, 0x0b00000b0b0b0b00ULL, 0x1a00001a1a1a1a00ULL,
        0xa60000a6a6a6a600ULL, 0xe10000e1e1e1e100ULL, 0x3900003939393900ULL,
        0xca0000cacacaca00ULL, 0xd50000d5d5d5d500ULL, 0x4700004747474700ULL,
        0x5d00005d5d5d5d00ULL, 0x3d00003d3d3d3d00ULL, 0xd90000d9d9d9d900ULL,
        0x0100000101010100ULL, 0x5a00005a5a5a5a00ULL, 0xd60000d6d6d6d600ULL,
        0x5100005151515100ULL, 0x5600005656565600ULL, 0x6c00006c6c6c6c00ULL,
        0x4d00004d4d4d4d00ULL, 0x8b00008b8b8b8b00ULL, 0x0d00000d0d0d0d00ULL,
        0x9a00009a9a9a9a00ULL, 0x6600006666666600ULL, 0xfb0000fbfbfbfb00ULL,
        0xcc0000cccccccc00ULL, 0xb00000b0b0b0b000ULL, 0x2d00002d2d2d2d00ULL,
        0x7400007474747400ULL, 0x1200001212121200ULL, 0x2b00002b2b2b2b00ULL,
        0x2000002020202000ULL, 0xf00000f0f0f0f000ULL, 0xb10000b1b1b1b100ULL,
        0x8400008484848400ULL, 0x9900009999999900ULL, 0xdf0000dfdfdfdf00ULL,
        0x4c00004c4c4c4c00ULL, 0xcb0000cbcbcbcb00ULL, 0xc20000c2c2c2c200ULL,
        0x3400003434343400ULL, 0x7e00007e7e7e7e00ULL, 0x7600007676767600ULL,
        0x0500000505050500ULL, 0x6d00006d6d6d6d00ULL, 0xb70000b7b7b7b700ULL,
        0xa90000a9a9a9a900ULL, 0x3100003131313100ULL, 0xd10000d1d1d1d100ULL,
        0x1700001717171700ULL, 0x0400000404040400ULL, 0xd70000d7d7d7d700ULL,
        0x1400001414141400ULL, 0x5800005858585800ULL, 0x3a00003a3a3a3a00ULL,
        0x6100006161616100ULL, 0xde0000dededede00ULL, 0x1b00001b1b1b1b00ULL,
        0x1100001111111100ULL, 0x1c00001c1c1c1c00ULL, 0x3200003232323200ULL,
        0x0f00000f0f0f0f00ULL, 0x9c00009c9c9c9c00ULL, 0x1600001616161600ULL,
        0x5300005353535300ULL, 0x1800001818181800ULL, 0xf20000f2f2f2f200ULL,
        0x2200002222222200ULL, 0xfe0000fefefefe00ULL, 0x4400004444444400ULL,
        0xcf0000cfcfcfcf00ULL, 0xb20000b2b2b2b200ULL, 0xc30000c3c3c3c300ULL,
        0xb50000b5b5b5b500ULL, 0x7a00007a7a7a7a00ULL, 0x9100009191919100ULL,
        0x2400002424242400ULL, 0x0800000808080800ULL, 0xe80000e8e8e8e800ULL,
        0xa80000a8a8a8a800ULL, 0x6000006060606000ULL, 0xfc0000fcfcfcfc00ULL,
        0x6900006969696900ULL, 0x5000005050505000ULL, 0xaa0000aaaaaaaa00ULL,
        0xd00000d0d0d0d000ULL, 0xa00000a0a0a0a000ULL, 0x7d00007d7d7d7d00ULL,
        0xa10000a1a1a1a100ULL, 0x8900008989898900ULL, 0x6200006262626200ULL,
        0x9700009797979700ULL, 0x5400005454545400ULL, 0x5b00005b5b5b5b00ULL,
        0x1e00001e1e1e1e00ULL, 0x9500009595959500ULL, 0xe00000e0e0e0e000ULL,
        0xff0000ffffffff00ULL, 0x6400006464646400ULL, 0xd20000d2d2d2d200ULL,
        0x1000001010101000ULL, 0xc40000c4c4c4c400ULL, 0x0000000000000000ULL,
        0x4800004848484800ULL, 0xa30000a3a3a3a300ULL, 0xf70000f7f7f7f700ULL,
        0x7500007575757500ULL, 0xdb0000dbdbdbdb00ULL, 0x8a00008a8a8a8a00ULL,
        0x0300000303030300ULL, 0xe60000e6e6e6e600ULL, 0xda0000dadadada00ULL,
        0x0900000909090900ULL, 0x3f00003f3f3f3f00ULL, 0xdd0000dddddddd00ULL,
        0x9400009494949400ULL, 0x8700008787878700ULL, 0x5c00005c5c5c5c00ULL,
        0x8300008383838300ULL, 0x0200000202020200ULL, 0xcd0000cdcdcdcd00ULL,
        0x4a00004a4a4a4a00ULL, 0x9000009090909000ULL, 0x3300003333333300ULL,
        0x7300007373737300ULL, 0x6700006767676700ULL, 0xf60000f6f6f6f600ULL,
        0xf30000f3f3f3f300ULL, 0x9d00009d9d9d9d00ULL, 0x7f00007f7f7f7f00ULL,
        0xbf0000bfbfbfbf00ULL, 0xe20000e2e2e2e200ULL, 0x5200005252525200ULL,
        0x9b00009b9b9b9b00ULL, 0xd80000d8d8d8d800ULL, 0x2600002626262600ULL,
        0xc80000c8c8c8c800ULL, 0x3700003737373700ULL, 0xc60000c6c6c6c600ULL,
        0x3b00003b3b3b3b00ULL, 0x8100008181818100ULL, 0x9600009696969600ULL,
        0x6f00006f6f6f6f00ULL, 0x4b00004b4b4b4b00ULL, 0x1300001313131300ULL,
        0xbe0000bebebebe00ULL, 0x6300006363636300ULL, 0x2e00002e2e2e2e00ULL,
        0xe90000e9e9e9e900ULL, 0x7900007979797900ULL, 0xa70000a7a7a7a700ULL,
        0x8c00008c8c8c8c00ULL, 0x9f00009f9f9f9f00ULL, 0x6e00006e6e6e6e00ULL,
        0xbc0000bcbcbcbc00ULL, 0x8e00008e8e8e8e00ULL, 0x2900002929292900ULL,
        0xf50000f5f5f5f500ULL, 0xf90000f9f9f9f900ULL, 0xb60000b6b6b6b600ULL,
        0x2f00002f2f2f2f00ULL, 0xfd0000fdfdfdfd00ULL, 0xb40000b4b4b4b400ULL,
        0x5900005959595900ULL, 0x7800007878787800ULL, 0x9800009898989800ULL,
        0x0600000606060600ULL, 0x6a00006a6a6a6a00ULL, 0xe70000e7e7e7e700ULL,
        0x4600004646464600ULL, 0x7100007171717100ULL, 0xba0000babababa00ULL,
        0xd40000d4d4d4d400ULL, 0x2500002525252500ULL, 0xab0000abababab00ULL,
        0x4200004242424200ULL, 0x8800008888888800ULL, 0xa20000a2a2a2a200ULL,
        0x8d00008d8d8d8d00ULL, 0xfa0000fafafafa00ULL, 0x7200007272727200ULL,
        0x0700000707070700ULL, 0xb90000b9b9b9b900ULL, 0x5500005555555500ULL,
        0xf80000f8f8f8f800ULL, 0xee0000eeeeeeee00ULL, 0xac0000acacacac00ULL,
        0x0a00000a0a0a0a00ULL, 0x3600003636363600ULL, 0x4900004949494900ULL,
        0x2a00002a2a2a2a00ULL, 0x6800006868686800ULL, 0x3c00003c3c3c3c00ULL,
        0x3800003838383800ULL, 0xf10000f1f1f1f100ULL, 0xa40000a4a4a4a400ULL,
        0x4000004040404000ULL, 0x2800002828282800ULL, 0xd30000d3d3d3d300ULL,
        0x7b00007b7b7b7b00ULL, 0xbb0000bbbbbbbb00ULL, 0xc90000c9c9c9c900ULL,
        0x4300004343434300ULL, 0xc10000c1c1c1c100ULL, 0x1500001515151500ULL,
        0xe30000e3e3e3e300ULL, 0xad0000adadadad00ULL, 0xf40000f4f4f4f400ULL,
        0x7700007777777700ULL, 0xc70000c7c7c7c700ULL, 0x8000008080808000ULL,
        0x9e00009e9e9e9e00ULL,
};

__visible const u64 camellia_sp22000222[256] = {
        0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL,
        0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL,
        0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL,
        0x0b0b0000000b0b0bULL, 0xaeae000000aeaeaeULL, 0x6a6a0000006a6a6aULL,
        0xd5d5000000d5d5d5ULL, 0x1818000000181818ULL, 0x5d5d0000005d5d5dULL,
        0x8282000000828282ULL, 0x4646000000464646ULL, 0xdfdf000000dfdfdfULL,
        0xd6d6000000d6d6d6ULL, 0x2727000000272727ULL, 0x8a8a0000008a8a8aULL,
        0x3232000000323232ULL, 0x4b4b0000004b4b4bULL, 0x4242000000424242ULL,
        0xdbdb000000dbdbdbULL, 0x1c1c0000001c1c1cULL, 0x9e9e0000009e9e9eULL,
        0x9c9c0000009c9c9cULL, 0x3a3a0000003a3a3aULL, 0xcaca000000cacacaULL,
        0x2525000000252525ULL, 0x7b7b0000007b7b7bULL, 0x0d0d0000000d0d0dULL,
        0x7171000000717171ULL, 0x5f5f0000005f5f5fULL, 0x1f1f0000001f1f1fULL,
        0xf8f8000000f8f8f8ULL, 0xd7d7000000d7d7d7ULL, 0x3e3e0000003e3e3eULL,
        0x9d9d0000009d9d9dULL, 0x7c7c0000007c7c7cULL, 0x6060000000606060ULL,
        0xb9b9000000b9b9b9ULL, 0xbebe000000bebebeULL, 0xbcbc000000bcbcbcULL,
        0x8b8b0000008b8b8bULL, 0x1616000000161616ULL, 0x3434000000343434ULL,
        0x4d4d0000004d4d4dULL, 0xc3c3000000c3c3c3ULL, 0x7272000000727272ULL,
        0x9595000000959595ULL, 0xabab000000abababULL, 0x8e8e0000008e8e8eULL,
        0xbaba000000bababaULL, 0x7a7a0000007a7a7aULL, 0xb3b3000000b3b3b3ULL,
        0x0202000000020202ULL, 0xb4b4000000b4b4b4ULL, 0xadad000000adadadULL,
        0xa2a2000000a2a2a2ULL, 0xacac000000acacacULL, 0xd8d8000000d8d8d8ULL,
        0x9a9a0000009a9a9aULL, 0x1717000000171717ULL, 0x1a1a0000001a1a1aULL,
        0x3535000000353535ULL, 0xcccc000000ccccccULL, 0xf7f7000000f7f7f7ULL,
        0x9999000000999999ULL, 0x6161000000616161ULL, 0x5a5a0000005a5a5aULL,
        0xe8e8000000e8e8e8ULL, 0x2424000000242424ULL, 0x5656000000565656ULL,
        0x4040000000404040ULL, 0xe1e1000000e1e1e1ULL, 0x6363000000636363ULL,
        0x0909000000090909ULL, 0x3333000000333333ULL, 0xbfbf000000bfbfbfULL,
        0x9898000000989898ULL, 0x9797000000979797ULL, 0x8585000000858585ULL,
        0x6868000000686868ULL, 0xfcfc000000fcfcfcULL, 0xecec000000ecececULL,
        0x0a0a0000000a0a0aULL, 0xdada000000dadadaULL, 0x6f6f0000006f6f6fULL,
        0x5353000000535353ULL, 0x6262000000626262ULL, 0xa3a3000000a3a3a3ULL,
        0x2e2e0000002e2e2eULL, 0x0808000000080808ULL, 0xafaf000000afafafULL,
        0x2828000000282828ULL, 0xb0b0000000b0b0b0ULL, 0x7474000000747474ULL,
        0xc2c2000000c2c2c2ULL, 0xbdbd000000bdbdbdULL, 0x3636000000363636ULL,
        0x2222000000222222ULL, 0x3838000000383838ULL, 0x6464000000646464ULL,
        0x1e1e0000001e1e1eULL, 0x3939000000393939ULL, 0x2c2c0000002c2c2cULL,
        0xa6a6000000a6a6a6ULL, 0x3030000000303030ULL, 0xe5e5000000e5e5e5ULL,
        0x4444000000444444ULL, 0xfdfd000000fdfdfdULL, 0x8888000000888888ULL,
        0x9f9f0000009f9f9fULL, 0x6565000000656565ULL, 0x8787000000878787ULL,
        0x6b6b0000006b6b6bULL, 0xf4f4000000f4f4f4ULL, 0x2323000000232323ULL,
        0x4848000000484848ULL, 0x1010000000101010ULL, 0xd1d1000000d1d1d1ULL,
        0x5151000000515151ULL, 0xc0c0000000c0c0c0ULL, 0xf9f9000000f9f9f9ULL,
        0xd2d2000000d2d2d2ULL, 0xa0a0000000a0a0a0ULL, 0x5555000000555555ULL,
        0xa1a1000000a1a1a1ULL, 0x4141000000414141ULL, 0xfafa000000fafafaULL,
        0x4343000000434343ULL, 0x1313000000131313ULL, 0xc4c4000000c4c4c4ULL,
        0x2f2f0000002f2f2fULL, 0xa8a8000000a8a8a8ULL, 0xb6b6000000b6b6b6ULL,
        0x3c3c0000003c3c3cULL, 0x2b2b0000002b2b2bULL, 0xc1c1000000c1c1c1ULL,
        0xffff000000ffffffULL, 0xc8c8000000c8c8c8ULL, 0xa5a5000000a5a5a5ULL,
        0x2020000000202020ULL, 0x8989000000898989ULL, 0x0000000000000000ULL,
        0x9090000000909090ULL, 0x4747000000474747ULL, 0xefef000000efefefULL,
        0xeaea000000eaeaeaULL, 0xb7b7000000b7b7b7ULL, 0x1515000000151515ULL,
        0x0606000000060606ULL, 0xcdcd000000cdcdcdULL, 0xb5b5000000b5b5b5ULL,
        0x1212000000121212ULL, 0x7e7e0000007e7e7eULL, 0xbbbb000000bbbbbbULL,
        0x2929000000292929ULL, 0x0f0f0000000f0f0fULL, 0xb8b8000000b8b8b8ULL,
        0x0707000000070707ULL, 0x0404000000040404ULL, 0x9b9b0000009b9b9bULL,
        0x9494000000949494ULL, 0x2121000000212121ULL, 0x6666000000666666ULL,
        0xe6e6000000e6e6e6ULL, 0xcece000000cececeULL, 0xeded000000edededULL,
        0xe7e7000000e7e7e7ULL, 0x3b3b0000003b3b3bULL, 0xfefe000000fefefeULL,
        0x7f7f0000007f7f7fULL, 0xc5c5000000c5c5c5ULL, 0xa4a4000000a4a4a4ULL,
        0x3737000000373737ULL, 0xb1b1000000b1b1b1ULL, 0x4c4c0000004c4c4cULL,
        0x9191000000919191ULL, 0x6e6e0000006e6e6eULL, 0x8d8d0000008d8d8dULL,
        0x7676000000767676ULL, 0x0303000000030303ULL, 0x2d2d0000002d2d2dULL,
        0xdede000000dededeULL, 0x9696000000969696ULL, 0x2626000000262626ULL,
        0x7d7d0000007d7d7dULL, 0xc6c6000000c6c6c6ULL, 0x5c5c0000005c5c5cULL,
        0xd3d3000000d3d3d3ULL, 0xf2f2000000f2f2f2ULL, 0x4f4f0000004f4f4fULL,
        0x1919000000191919ULL, 0x3f3f0000003f3f3fULL, 0xdcdc000000dcdcdcULL,
        0x7979000000797979ULL, 0x1d1d0000001d1d1dULL, 0x5252000000525252ULL,
        0xebeb000000ebebebULL, 0xf3f3000000f3f3f3ULL, 0x6d6d0000006d6d6dULL,
        0x5e5e0000005e5e5eULL, 0xfbfb000000fbfbfbULL, 0x6969000000696969ULL,
        0xb2b2000000b2b2b2ULL, 0xf0f0000000f0f0f0ULL, 0x3131000000313131ULL,
        0x0c0c0000000c0c0cULL, 0xd4d4000000d4d4d4ULL, 0xcfcf000000cfcfcfULL,
        0x8c8c0000008c8c8cULL, 0xe2e2000000e2e2e2ULL, 0x7575000000757575ULL,
        0xa9a9000000a9a9a9ULL, 0x4a4a0000004a4a4aULL, 0x5757000000575757ULL,
        0x8484000000848484ULL, 0x1111000000111111ULL, 0x4545000000454545ULL,
        0x1b1b0000001b1b1bULL, 0xf5f5000000f5f5f5ULL, 0xe4e4000000e4e4e4ULL,
        0x0e0e0000000e0e0eULL, 0x7373000000737373ULL, 0xaaaa000000aaaaaaULL,
        0xf1f1000000f1f1f1ULL, 0xdddd000000ddddddULL, 0x5959000000595959ULL,
        0x1414000000141414ULL, 0x6c6c0000006c6c6cULL, 0x9292000000929292ULL,
        0x5454000000545454ULL, 0xd0d0000000d0d0d0ULL, 0x7878000000787878ULL,
        0x7070000000707070ULL, 0xe3e3000000e3e3e3ULL, 0x4949000000494949ULL,
        0x8080000000808080ULL, 0x5050000000505050ULL, 0xa7a7000000a7a7a7ULL,
        0xf6f6000000f6f6f6ULL, 0x7777000000777777ULL, 0x9393000000939393ULL,
        0x8686000000868686ULL, 0x8383000000838383ULL, 0x2a2a0000002a2a2aULL,
        0xc7c7000000c7c7c7ULL, 0x5b5b0000005b5b5bULL, 0xe9e9000000e9e9e9ULL,
        0xeeee000000eeeeeeULL, 0x8f8f0000008f8f8fULL, 0x0101000000010101ULL,
        0x3d3d0000003d3d3dULL,
};

__visible const u64 camellia_sp03303033[256] = {
        0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL,
        0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL,
        0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL,
        0x00c2c200c200c2c2ULL, 0x00abab00ab00ababULL, 0x009a9a009a009a9aULL,
        0x0075750075007575ULL, 0x0006060006000606ULL, 0x0057570057005757ULL,
        0x00a0a000a000a0a0ULL, 0x0091910091009191ULL, 0x00f7f700f700f7f7ULL,
        0x00b5b500b500b5b5ULL, 0x00c9c900c900c9c9ULL, 0x00a2a200a200a2a2ULL,
        0x008c8c008c008c8cULL, 0x00d2d200d200d2d2ULL, 0x0090900090009090ULL,
        0x00f6f600f600f6f6ULL, 0x0007070007000707ULL, 0x00a7a700a700a7a7ULL,
        0x0027270027002727ULL, 0x008e8e008e008e8eULL, 0x00b2b200b200b2b2ULL,
        0x0049490049004949ULL, 0x00dede00de00dedeULL, 0x0043430043004343ULL,
        0x005c5c005c005c5cULL, 0x00d7d700d700d7d7ULL, 0x00c7c700c700c7c7ULL,
        0x003e3e003e003e3eULL, 0x00f5f500f500f5f5ULL, 0x008f8f008f008f8fULL,
        0x0067670067006767ULL, 0x001f1f001f001f1fULL, 0x0018180018001818ULL,
        0x006e6e006e006e6eULL, 0x00afaf00af00afafULL, 0x002f2f002f002f2fULL,
        0x00e2e200e200e2e2ULL, 0x0085850085008585ULL, 0x000d0d000d000d0dULL,
        0x0053530053005353ULL, 0x00f0f000f000f0f0ULL, 0x009c9c009c009c9cULL,
        0x0065650065006565ULL, 0x00eaea00ea00eaeaULL, 0x00a3a300a300a3a3ULL,
        0x00aeae00ae00aeaeULL, 0x009e9e009e009e9eULL, 0x00ecec00ec00ececULL,
        0x0080800080008080ULL, 0x002d2d002d002d2dULL, 0x006b6b006b006b6bULL,
        0x00a8a800a800a8a8ULL, 0x002b2b002b002b2bULL, 0x0036360036003636ULL,
        0x00a6a600a600a6a6ULL, 0x00c5c500c500c5c5ULL, 0x0086860086008686ULL,
        0x004d4d004d004d4dULL, 0x0033330033003333ULL, 0x00fdfd00fd00fdfdULL,
        0x0066660066006666ULL, 0x0058580058005858ULL, 0x0096960096009696ULL,
        0x003a3a003a003a3aULL, 0x0009090009000909ULL, 0x0095950095009595ULL,
        0x0010100010001010ULL, 0x0078780078007878ULL, 0x00d8d800d800d8d8ULL,
        0x0042420042004242ULL, 0x00cccc00cc00ccccULL, 0x00efef00ef00efefULL,
        0x0026260026002626ULL, 0x00e5e500e500e5e5ULL, 0x0061610061006161ULL,
        0x001a1a001a001a1aULL, 0x003f3f003f003f3fULL, 0x003b3b003b003b3bULL,
        0x0082820082008282ULL, 0x00b6b600b600b6b6ULL, 0x00dbdb00db00dbdbULL,
        0x00d4d400d400d4d4ULL, 0x0098980098009898ULL, 0x00e8e800e800e8e8ULL,
        0x008b8b008b008b8bULL, 0x0002020002000202ULL, 0x00ebeb00eb00ebebULL,
        0x000a0a000a000a0aULL, 0x002c2c002c002c2cULL, 0x001d1d001d001d1dULL,
        0x00b0b000b000b0b0ULL, 0x006f6f006f006f6fULL, 0x008d8d008d008d8dULL,
        0x0088880088008888ULL, 0x000e0e000e000e0eULL, 0x0019190019001919ULL,
        0x0087870087008787ULL, 0x004e4e004e004e4eULL, 0x000b0b000b000b0bULL,
        0x00a9a900a900a9a9ULL, 0x000c0c000c000c0cULL, 0x0079790079007979ULL,
        0x0011110011001111ULL, 0x007f7f007f007f7fULL, 0x0022220022002222ULL,
        0x00e7e700e700e7e7ULL, 0x0059590059005959ULL, 0x00e1e100e100e1e1ULL,
        0x00dada00da00dadaULL, 0x003d3d003d003d3dULL, 0x00c8c800c800c8c8ULL,
        0x0012120012001212ULL, 0x0004040004000404ULL, 0x0074740074007474ULL,
        0x0054540054005454ULL, 0x0030300030003030ULL, 0x007e7e007e007e7eULL,
        0x00b4b400b400b4b4ULL, 0x0028280028002828ULL, 0x0055550055005555ULL,
        0x0068680068006868ULL, 0x0050500050005050ULL, 0x00bebe00be00bebeULL,
        0x00d0d000d000d0d0ULL, 0x00c4c400c400c4c4ULL, 0x0031310031003131ULL,
        0x00cbcb00cb00cbcbULL, 0x002a2a002a002a2aULL, 0x00adad00ad00adadULL,
        0x000f0f000f000f0fULL, 0x00caca00ca00cacaULL, 0x0070700070007070ULL,
        0x00ffff00ff00ffffULL, 0x0032320032003232ULL, 0x0069690069006969ULL,
        0x0008080008000808ULL, 0x0062620062006262ULL, 0x0000000000000000ULL,
        0x0024240024002424ULL, 0x00d1d100d100d1d1ULL, 0x00fbfb00fb00fbfbULL,
        0x00baba00ba00babaULL, 0x00eded00ed00ededULL, 0x0045450045004545ULL,
        0x0081810081008181ULL, 0x0073730073007373ULL, 0x006d6d006d006d6dULL,
        0x0084840084008484ULL, 0x009f9f009f009f9fULL, 0x00eeee00ee00eeeeULL,
        0x004a4a004a004a4aULL, 0x00c3c300c300c3c3ULL, 0x002e2e002e002e2eULL,
        0x00c1c100c100c1c1ULL, 0x0001010001000101ULL, 0x00e6e600e600e6e6ULL,
        0x0025250025002525ULL, 0x0048480048004848ULL, 0x0099990099009999ULL,
        0x00b9b900b900b9b9ULL, 0x00b3b300b300b3b3ULL, 0x007b7b007b007b7bULL,
        0x00f9f900f900f9f9ULL, 0x00cece00ce00ceceULL, 0x00bfbf00bf00bfbfULL,
        0x00dfdf00df00dfdfULL, 0x0071710071007171ULL, 0x0029290029002929ULL,
        0x00cdcd00cd00cdcdULL, 0x006c6c006c006c6cULL, 0x0013130013001313ULL,
        0x0064640064006464ULL, 0x009b9b009b009b9bULL, 0x0063630063006363ULL,
        0x009d9d009d009d9dULL, 0x00c0c000c000c0c0ULL, 0x004b4b004b004b4bULL,
        0x00b7b700b700b7b7ULL, 0x00a5a500a500a5a5ULL, 0x0089890089008989ULL,
        0x005f5f005f005f5fULL, 0x00b1b100b100b1b1ULL, 0x0017170017001717ULL,
        0x00f4f400f400f4f4ULL, 0x00bcbc00bc00bcbcULL, 0x00d3d300d300d3d3ULL,
        0x0046460046004646ULL, 0x00cfcf00cf00cfcfULL, 0x0037370037003737ULL,
        0x005e5e005e005e5eULL, 0x0047470047004747ULL, 0x0094940094009494ULL,
        0x00fafa00fa00fafaULL, 0x00fcfc00fc00fcfcULL, 0x005b5b005b005b5bULL,
        0x0097970097009797ULL, 0x00fefe00fe00fefeULL, 0x005a5a005a005a5aULL,
        0x00acac00ac00acacULL, 0x003c3c003c003c3cULL, 0x004c4c004c004c4cULL,
        0x0003030003000303ULL, 0x0035350035003535ULL, 0x00f3f300f300f3f3ULL,
        0x0023230023002323ULL, 0x00b8b800b800b8b8ULL, 0x005d5d005d005d5dULL,
        0x006a6a006a006a6aULL, 0x0092920092009292ULL, 0x00d5d500d500d5d5ULL,
        0x0021210021002121ULL, 0x0044440044004444ULL, 0x0051510051005151ULL,
        0x00c6c600c600c6c6ULL, 0x007d7d007d007d7dULL, 0x0039390039003939ULL,
        0x0083830083008383ULL, 0x00dcdc00dc00dcdcULL, 0x00aaaa00aa00aaaaULL,
        0x007c7c007c007c7cULL, 0x0077770077007777ULL, 0x0056560056005656ULL,
        0x0005050005000505ULL, 0x001b1b001b001b1bULL, 0x00a4a400a400a4a4ULL,
        0x0015150015001515ULL, 0x0034340034003434ULL, 0x001e1e001e001e1eULL,
        0x001c1c001c001c1cULL, 0x00f8f800f800f8f8ULL, 0x0052520052005252ULL,
        0x0020200020002020ULL, 0x0014140014001414ULL, 0x00e9e900e900e9e9ULL,
        0x00bdbd00bd00bdbdULL, 0x00dddd00dd00ddddULL, 0x00e4e400e400e4e4ULL,
        0x00a1a100a100a1a1ULL, 0x00e0e000e000e0e0ULL, 0x008a8a008a008a8aULL,
        0x00f1f100f100f1f1ULL, 0x00d6d600d600d6d6ULL, 0x007a7a007a007a7aULL,
        0x00bbbb00bb00bbbbULL, 0x00e3e300e300e3e3ULL, 0x0040400040004040ULL,
        0x004f4f004f004f4fULL,
};

__visible const u64 camellia_sp00444404[256] = {
        0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL,
        0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL,
        0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL,
        0x00006b6b6b6b006bULL, 0x0000454545450045ULL, 0x0000a5a5a5a500a5ULL,
        0x0000edededed00edULL, 0x00004f4f4f4f004fULL, 0x00001d1d1d1d001dULL,
        0x0000929292920092ULL, 0x0000868686860086ULL, 0x0000afafafaf00afULL,
        0x00007c7c7c7c007cULL, 0x00001f1f1f1f001fULL, 0x00003e3e3e3e003eULL,
        0x0000dcdcdcdc00dcULL, 0x00005e5e5e5e005eULL, 0x00000b0b0b0b000bULL,
        0x0000a6a6a6a600a6ULL, 0x0000393939390039ULL, 0x0000d5d5d5d500d5ULL,
        0x00005d5d5d5d005dULL, 0x0000d9d9d9d900d9ULL, 0x00005a5a5a5a005aULL,
        0x0000515151510051ULL, 0x00006c6c6c6c006cULL, 0x00008b8b8b8b008bULL,
        0x00009a9a9a9a009aULL, 0x0000fbfbfbfb00fbULL, 0x0000b0b0b0b000b0ULL,
        0x0000747474740074ULL, 0x00002b2b2b2b002bULL, 0x0000f0f0f0f000f0ULL,
        0x0000848484840084ULL, 0x0000dfdfdfdf00dfULL, 0x0000cbcbcbcb00cbULL,
        0x0000343434340034ULL, 0x0000767676760076ULL, 0x00006d6d6d6d006dULL,
        0x0000a9a9a9a900a9ULL, 0x0000d1d1d1d100d1ULL, 0x0000040404040004ULL,
        0x0000141414140014ULL, 0x00003a3a3a3a003aULL, 0x0000dededede00deULL,
        0x0000111111110011ULL, 0x0000323232320032ULL, 0x00009c9c9c9c009cULL,
        0x0000535353530053ULL, 0x0000f2f2f2f200f2ULL, 0x0000fefefefe00feULL,
        0x0000cfcfcfcf00cfULL, 0x0000c3c3c3c300c3ULL, 0x00007a7a7a7a007aULL,
        0x0000242424240024ULL, 0x0000e8e8e8e800e8ULL, 0x0000606060600060ULL,
        0x0000696969690069ULL, 0x0000aaaaaaaa00aaULL, 0x0000a0a0a0a000a0ULL,
        0x0000a1a1a1a100a1ULL, 0x0000626262620062ULL, 0x0000545454540054ULL,
        0x00001e1e1e1e001eULL, 0x0000e0e0e0e000e0ULL, 0x0000646464640064ULL,
        0x0000101010100010ULL, 0x0000000000000000ULL, 0x0000a3a3a3a300a3ULL,
        0x0000757575750075ULL, 0x00008a8a8a8a008aULL, 0x0000e6e6e6e600e6ULL,
        0x0000090909090009ULL, 0x0000dddddddd00ddULL, 0x0000878787870087ULL,
        0x0000838383830083ULL, 0x0000cdcdcdcd00cdULL, 0x0000909090900090ULL,
        0x0000737373730073ULL, 0x0000f6f6f6f600f6ULL, 0x00009d9d9d9d009dULL,
        0x0000bfbfbfbf00bfULL, 0x0000525252520052ULL, 0x0000d8d8d8d800d8ULL,
        0x0000c8c8c8c800c8ULL, 0x0000c6c6c6c600c6ULL, 0x0000818181810081ULL,
        0x00006f6f6f6f006fULL, 0x0000131313130013ULL, 0x0000636363630063ULL,
        0x0000e9e9e9e900e9ULL, 0x0000a7a7a7a700a7ULL, 0x00009f9f9f9f009fULL,
        0x0000bcbcbcbc00bcULL, 0x0000292929290029ULL, 0x0000f9f9f9f900f9ULL,
        0x00002f2f2f2f002fULL, 0x0000b4b4b4b400b4ULL, 0x0000787878780078ULL,
        0x0000060606060006ULL, 0x0000e7e7e7e700e7ULL, 0x0000717171710071ULL,
        0x0000d4d4d4d400d4ULL, 0x0000abababab00abULL, 0x0000888888880088ULL,
        0x00008d8d8d8d008dULL, 0x0000727272720072ULL, 0x0000b9b9b9b900b9ULL,
        0x0000f8f8f8f800f8ULL, 0x0000acacacac00acULL, 0x0000363636360036ULL,
        0x00002a2a2a2a002aULL, 0x00003c3c3c3c003cULL, 0x0000f1f1f1f100f1ULL,
        0x0000404040400040ULL, 0x0000d3d3d3d300d3ULL, 0x0000bbbbbbbb00bbULL,
        0x0000434343430043ULL, 0x0000151515150015ULL, 0x0000adadadad00adULL,
        0x0000777777770077ULL, 0x0000808080800080ULL, 0x0000828282820082ULL,
        0x0000ecececec00ecULL, 0x0000272727270027ULL, 0x0000e5e5e5e500e5ULL,
        0x0000858585850085ULL, 0x0000353535350035ULL, 0x00000c0c0c0c000cULL,
        0x0000414141410041ULL, 0x0000efefefef00efULL, 0x0000939393930093ULL,
        0x0000191919190019ULL, 0x0000212121210021ULL, 0x00000e0e0e0e000eULL,
        0x00004e4e4e4e004eULL, 0x0000656565650065ULL, 0x0000bdbdbdbd00bdULL,
        0x0000b8b8b8b800b8ULL, 0x00008f8f8f8f008fULL, 0x0000ebebebeb00ebULL,
        0x0000cececece00ceULL, 0x0000303030300030ULL, 0x00005f5f5f5f005fULL,
        0x0000c5c5c5c500c5ULL, 0x00001a1a1a1a001aULL, 0x0000e1e1e1e100e1ULL,
        0x0000cacacaca00caULL, 0x0000474747470047ULL, 0x00003d3d3d3d003dULL,
        0x0000010101010001ULL, 0x0000d6d6d6d600d6ULL, 0x0000565656560056ULL,
        0x00004d4d4d4d004dULL, 0x00000d0d0d0d000dULL, 0x0000666666660066ULL,
        0x0000cccccccc00ccULL, 0x00002d2d2d2d002dULL, 0x0000121212120012ULL,
        0x0000202020200020ULL, 0x0000b1b1b1b100b1ULL, 0x0000999999990099ULL,
        0x00004c4c4c4c004cULL, 0x0000c2c2c2c200c2ULL, 0x00007e7e7e7e007eULL,
        0x0000050505050005ULL, 0x0000b7b7b7b700b7ULL, 0x0000313131310031ULL,
        0x0000171717170017ULL, 0x0000d7d7d7d700d7ULL, 0x0000585858580058ULL,
        0x0000616161610061ULL, 0x00001b1b1b1b001bULL, 0x00001c1c1c1c001cULL,
        0x00000f0f0f0f000fULL, 0x0000161616160016ULL, 0x0000181818180018ULL,
        0x0000222222220022ULL, 0x0000444444440044ULL, 0x0000b2b2b2b200b2ULL,
        0x0000b5b5b5b500b5ULL, 0x0000919191910091ULL, 0x0000080808080008ULL,
        0x0000a8a8a8a800a8ULL, 0x0000fcfcfcfc00fcULL, 0x0000505050500050ULL,
        0x0000d0d0d0d000d0ULL, 0x00007d7d7d7d007dULL, 0x0000898989890089ULL,
        0x0000979797970097ULL, 0x00005b5b5b5b005bULL, 0x0000959595950095ULL,
        0x0000ffffffff00ffULL, 0x0000d2d2d2d200d2ULL, 0x0000c4c4c4c400c4ULL,
        0x0000484848480048ULL, 0x0000f7f7f7f700f7ULL, 0x0000dbdbdbdb00dbULL,
        0x0000030303030003ULL, 0x0000dadadada00daULL, 0x00003f3f3f3f003fULL,
        0x0000949494940094ULL, 0x00005c5c5c5c005cULL, 0x0000020202020002ULL,
        0x00004a4a4a4a004aULL, 0x0000333333330033ULL, 0x0000676767670067ULL,
        0x0000f3f3f3f300f3ULL, 0x00007f7f7f7f007fULL, 0x0000e2e2e2e200e2ULL,
        0x00009b9b9b9b009bULL, 0x0000262626260026ULL, 0x0000373737370037ULL,
        0x00003b3b3b3b003bULL, 0x0000969696960096ULL, 0x00004b4b4b4b004bULL,
        0x0000bebebebe00beULL, 0x00002e2e2e2e002eULL, 0x0000797979790079ULL,
        0x00008c8c8c8c008cULL, 0x00006e6e6e6e006eULL, 0x00008e8e8e8e008eULL,
        0x0000f5f5f5f500f5ULL, 0x0000b6b6b6b600b6ULL, 0x0000fdfdfdfd00fdULL,
        0x0000595959590059ULL, 0x0000989898980098ULL, 0x00006a6a6a6a006aULL,
        0x0000464646460046ULL, 0x0000babababa00baULL, 0x0000252525250025ULL,
        0x0000424242420042ULL, 0x0000a2a2a2a200a2ULL, 0x0000fafafafa00faULL,
        0x0000070707070007ULL, 0x0000555555550055ULL, 0x0000eeeeeeee00eeULL,
        0x00000a0a0a0a000aULL, 0x0000494949490049ULL, 0x0000686868680068ULL,
        0x0000383838380038ULL, 0x0000a4a4a4a400a4ULL, 0x0000282828280028ULL,
        0x00007b7b7b7b007bULL, 0x0000c9c9c9c900c9ULL, 0x0000c1c1c1c100c1ULL,
        0x0000e3e3e3e300e3ULL, 0x0000f4f4f4f400f4ULL, 0x0000c7c7c7c700c7ULL,
        0x00009e9e9e9e009eULL,
};

__visible const u64 camellia_sp02220222[256] = {
        0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL,
        0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL,
        0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL,
        0x000b0b0b000b0b0bULL, 0x00aeaeae00aeaeaeULL, 0x006a6a6a006a6a6aULL,
        0x00d5d5d500d5d5d5ULL, 0x0018181800181818ULL, 0x005d5d5d005d5d5dULL,
        0x0082828200828282ULL, 0x0046464600464646ULL, 0x00dfdfdf00dfdfdfULL,
        0x00d6d6d600d6d6d6ULL, 0x0027272700272727ULL, 0x008a8a8a008a8a8aULL,
        0x0032323200323232ULL, 0x004b4b4b004b4b4bULL, 0x0042424200424242ULL,
        0x00dbdbdb00dbdbdbULL, 0x001c1c1c001c1c1cULL, 0x009e9e9e009e9e9eULL,
        0x009c9c9c009c9c9cULL, 0x003a3a3a003a3a3aULL, 0x00cacaca00cacacaULL,
        0x0025252500252525ULL, 0x007b7b7b007b7b7bULL, 0x000d0d0d000d0d0dULL,
        0x0071717100717171ULL, 0x005f5f5f005f5f5fULL, 0x001f1f1f001f1f1fULL,
        0x00f8f8f800f8f8f8ULL, 0x00d7d7d700d7d7d7ULL, 0x003e3e3e003e3e3eULL,
        0x009d9d9d009d9d9dULL, 0x007c7c7c007c7c7cULL, 0x0060606000606060ULL,
        0x00b9b9b900b9b9b9ULL, 0x00bebebe00bebebeULL, 0x00bcbcbc00bcbcbcULL,
        0x008b8b8b008b8b8bULL, 0x0016161600161616ULL, 0x0034343400343434ULL,
        0x004d4d4d004d4d4dULL, 0x00c3c3c300c3c3c3ULL, 0x0072727200727272ULL,
        0x0095959500959595ULL, 0x00ababab00abababULL, 0x008e8e8e008e8e8eULL,
        0x00bababa00bababaULL, 0x007a7a7a007a7a7aULL, 0x00b3b3b300b3b3b3ULL,
        0x0002020200020202ULL, 0x00b4b4b400b4b4b4ULL, 0x00adadad00adadadULL,
        0x00a2a2a200a2a2a2ULL, 0x00acacac00acacacULL, 0x00d8d8d800d8d8d8ULL,
        0x009a9a9a009a9a9aULL, 0x0017171700171717ULL, 0x001a1a1a001a1a1aULL,
        0x0035353500353535ULL, 0x00cccccc00ccccccULL, 0x00f7f7f700f7f7f7ULL,
        0x0099999900999999ULL, 0x0061616100616161ULL, 0x005a5a5a005a5a5aULL,
        0x00e8e8e800e8e8e8ULL, 0x0024242400242424ULL, 0x0056565600565656ULL,
        0x0040404000404040ULL, 0x00e1e1e100e1e1e1ULL, 0x0063636300636363ULL,
        0x0009090900090909ULL, 0x0033333300333333ULL, 0x00bfbfbf00bfbfbfULL,
        0x0098989800989898ULL, 0x0097979700979797ULL, 0x0085858500858585ULL,
        0x0068686800686868ULL, 0x00fcfcfc00fcfcfcULL, 0x00ececec00ecececULL,
        0x000a0a0a000a0a0aULL, 0x00dadada00dadadaULL, 0x006f6f6f006f6f6fULL,
        0x0053535300535353ULL, 0x0062626200626262ULL, 0x00a3a3a300a3a3a3ULL,
        0x002e2e2e002e2e2eULL, 0x0008080800080808ULL, 0x00afafaf00afafafULL,
        0x0028282800282828ULL, 0x00b0b0b000b0b0b0ULL, 0x0074747400747474ULL,
        0x00c2c2c200c2c2c2ULL, 0x00bdbdbd00bdbdbdULL, 0x0036363600363636ULL,
        0x0022222200222222ULL, 0x0038383800383838ULL, 0x0064646400646464ULL,
        0x001e1e1e001e1e1eULL, 0x0039393900393939ULL, 0x002c2c2c002c2c2cULL,
        0x00a6a6a600a6a6a6ULL, 0x0030303000303030ULL, 0x00e5e5e500e5e5e5ULL,
        0x0044444400444444ULL, 0x00fdfdfd00fdfdfdULL, 0x0088888800888888ULL,
        0x009f9f9f009f9f9fULL, 0x0065656500656565ULL, 0x0087878700878787ULL,
        0x006b6b6b006b6b6bULL, 0x00f4f4f400f4f4f4ULL, 0x0023232300232323ULL,
        0x0048484800484848ULL, 0x0010101000101010ULL, 0x00d1d1d100d1d1d1ULL,
        0x0051515100515151ULL, 0x00c0c0c000c0c0c0ULL, 0x00f9f9f900f9f9f9ULL,
        0x00d2d2d200d2d2d2ULL, 0x00a0a0a000a0a0a0ULL, 0x0055555500555555ULL,
        0x00a1a1a100a1a1a1ULL, 0x0041414100414141ULL, 0x00fafafa00fafafaULL,
        0x0043434300434343ULL, 0x0013131300131313ULL, 0x00c4c4c400c4c4c4ULL,
        0x002f2f2f002f2f2fULL, 0x00a8a8a800a8a8a8ULL, 0x00b6b6b600b6b6b6ULL,
        0x003c3c3c003c3c3cULL, 0x002b2b2b002b2b2bULL, 0x00c1c1c100c1c1c1ULL,
        0x00ffffff00ffffffULL, 0x00c8c8c800c8c8c8ULL, 0x00a5a5a500a5a5a5ULL,
        0x0020202000202020ULL, 0x0089898900898989ULL, 0x0000000000000000ULL,
        0x0090909000909090ULL, 0x0047474700474747ULL, 0x00efefef00efefefULL,
        0x00eaeaea00eaeaeaULL, 0x00b7b7b700b7b7b7ULL, 0x0015151500151515ULL,
        0x0006060600060606ULL, 0x00cdcdcd00cdcdcdULL, 0x00b5b5b500b5b5b5ULL,
        0x0012121200121212ULL, 0x007e7e7e007e7e7eULL, 0x00bbbbbb00bbbbbbULL,
        0x0029292900292929ULL, 0x000f0f0f000f0f0fULL, 0x00b8b8b800b8b8b8ULL,
        0x0007070700070707ULL, 0x0004040400040404ULL, 0x009b9b9b009b9b9bULL,
        0x0094949400949494ULL, 0x0021212100212121ULL, 0x0066666600666666ULL,
        0x00e6e6e600e6e6e6ULL, 0x00cecece00cececeULL, 0x00ededed00edededULL,
        0x00e7e7e700e7e7e7ULL, 0x003b3b3b003b3b3bULL, 0x00fefefe00fefefeULL,
        0x007f7f7f007f7f7fULL, 0x00c5c5c500c5c5c5ULL, 0x00a4a4a400a4a4a4ULL,
        0x0037373700373737ULL, 0x00b1b1b100b1b1b1ULL, 0x004c4c4c004c4c4cULL,
        0x0091919100919191ULL, 0x006e6e6e006e6e6eULL, 0x008d8d8d008d8d8dULL,
        0x0076767600767676ULL, 0x0003030300030303ULL, 0x002d2d2d002d2d2dULL,
        0x00dedede00dededeULL, 0x0096969600969696ULL, 0x0026262600262626ULL,
        0x007d7d7d007d7d7dULL, 0x00c6c6c600c6c6c6ULL, 0x005c5c5c005c5c5cULL,
        0x00d3d3d300d3d3d3ULL, 0x00f2f2f200f2f2f2ULL, 0x004f4f4f004f4f4fULL,
        0x0019191900191919ULL, 0x003f3f3f003f3f3fULL, 0x00dcdcdc00dcdcdcULL,
        0x0079797900797979ULL, 0x001d1d1d001d1d1dULL, 0x0052525200525252ULL,
        0x00ebebeb00ebebebULL, 0x00f3f3f300f3f3f3ULL, 0x006d6d6d006d6d6dULL,
        0x005e5e5e005e5e5eULL, 0x00fbfbfb00fbfbfbULL, 0x0069696900696969ULL,
        0x00b2b2b200b2b2b2ULL, 0x00f0f0f000f0f0f0ULL, 0x0031313100313131ULL,
        0x000c0c0c000c0c0cULL, 0x00d4d4d400d4d4d4ULL, 0x00cfcfcf00cfcfcfULL,
        0x008c8c8c008c8c8cULL, 0x00e2e2e200e2e2e2ULL, 0x0075757500757575ULL,
        0x00a9a9a900a9a9a9ULL, 0x004a4a4a004a4a4aULL, 0x0057575700575757ULL,
        0x0084848400848484ULL, 0x0011111100111111ULL, 0x0045454500454545ULL,
        0x001b1b1b001b1b1bULL, 0x00f5f5f500f5f5f5ULL, 0x00e4e4e400e4e4e4ULL,
        0x000e0e0e000e0e0eULL, 0x0073737300737373ULL, 0x00aaaaaa00aaaaaaULL,
        0x00f1f1f100f1f1f1ULL, 0x00dddddd00ddddddULL, 0x0059595900595959ULL,
        0x0014141400141414ULL, 0x006c6c6c006c6c6cULL, 0x0092929200929292ULL,
        0x0054545400545454ULL, 0x00d0d0d000d0d0d0ULL, 0x0078787800787878ULL,
        0x0070707000707070ULL, 0x00e3e3e300e3e3e3ULL, 0x0049494900494949ULL,
        0x0080808000808080ULL, 0x0050505000505050ULL, 0x00a7a7a700a7a7a7ULL,
        0x00f6f6f600f6f6f6ULL, 0x0077777700777777ULL, 0x0093939300939393ULL,
        0x0086868600868686ULL, 0x0083838300838383ULL, 0x002a2a2a002a2a2aULL,
        0x00c7c7c700c7c7c7ULL, 0x005b5b5b005b5b5bULL, 0x00e9e9e900e9e9e9ULL,
        0x00eeeeee00eeeeeeULL, 0x008f8f8f008f8f8fULL, 0x0001010100010101ULL,
        0x003d3d3d003d3d3dULL,
};

__visible const u64 camellia_sp30333033[256] = {
        0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL,
        0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL,
        0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL,
        0xc200c2c2c200c2c2ULL, 0xab00ababab00ababULL, 0x9a009a9a9a009a9aULL,
        0x7500757575007575ULL, 0x0600060606000606ULL, 0x5700575757005757ULL,
        0xa000a0a0a000a0a0ULL, 0x9100919191009191ULL, 0xf700f7f7f700f7f7ULL,
        0xb500b5b5b500b5b5ULL, 0xc900c9c9c900c9c9ULL, 0xa200a2a2a200a2a2ULL,
        0x8c008c8c8c008c8cULL, 0xd200d2d2d200d2d2ULL, 0x9000909090009090ULL,
        0xf600f6f6f600f6f6ULL, 0x0700070707000707ULL, 0xa700a7a7a700a7a7ULL,
        0x2700272727002727ULL, 0x8e008e8e8e008e8eULL, 0xb200b2b2b200b2b2ULL,
        0x4900494949004949ULL, 0xde00dedede00dedeULL, 0x4300434343004343ULL,
        0x5c005c5c5c005c5cULL, 0xd700d7d7d700d7d7ULL, 0xc700c7c7c700c7c7ULL,
        0x3e003e3e3e003e3eULL, 0xf500f5f5f500f5f5ULL, 0x8f008f8f8f008f8fULL,
        0x6700676767006767ULL, 0x1f001f1f1f001f1fULL, 0x1800181818001818ULL,
        0x6e006e6e6e006e6eULL, 0xaf00afafaf00afafULL, 0x2f002f2f2f002f2fULL,
        0xe200e2e2e200e2e2ULL, 0x8500858585008585ULL, 0x0d000d0d0d000d0dULL,
        0x5300535353005353ULL, 0xf000f0f0f000f0f0ULL, 0x9c009c9c9c009c9cULL,
        0x6500656565006565ULL, 0xea00eaeaea00eaeaULL, 0xa300a3a3a300a3a3ULL,
        0xae00aeaeae00aeaeULL, 0x9e009e9e9e009e9eULL, 0xec00ececec00ececULL,
        0x8000808080008080ULL, 0x2d002d2d2d002d2dULL, 0x6b006b6b6b006b6bULL,
        0xa800a8a8a800a8a8ULL, 0x2b002b2b2b002b2bULL, 0x3600363636003636ULL,
        0xa600a6a6a600a6a6ULL, 0xc500c5c5c500c5c5ULL, 0x8600868686008686ULL,
        0x4d004d4d4d004d4dULL, 0x3300333333003333ULL, 0xfd00fdfdfd00fdfdULL,
        0x6600666666006666ULL, 0x5800585858005858ULL, 0x9600969696009696ULL,
        0x3a003a3a3a003a3aULL, 0x0900090909000909ULL, 0x9500959595009595ULL,
        0x1000101010001010ULL, 0x7800787878007878ULL, 0xd800d8d8d800d8d8ULL,
        0x4200424242004242ULL, 0xcc00cccccc00ccccULL, 0xef00efefef00efefULL,
        0x2600262626002626ULL, 0xe500e5e5e500e5e5ULL, 0x6100616161006161ULL,
        0x1a001a1a1a001a1aULL, 0x3f003f3f3f003f3fULL, 0x3b003b3b3b003b3bULL,
        0x8200828282008282ULL, 0xb600b6b6b600b6b6ULL, 0xdb00dbdbdb00dbdbULL,
        0xd400d4d4d400d4d4ULL, 0x9800989898009898ULL, 0xe800e8e8e800e8e8ULL,
        0x8b008b8b8b008b8bULL, 0x0200020202000202ULL, 0xeb00ebebeb00ebebULL,
        0x0a000a0a0a000a0aULL, 0x2c002c2c2c002c2cULL, 0x1d001d1d1d001d1dULL,
        0xb000b0b0b000b0b0ULL, 0x6f006f6f6f006f6fULL, 0x8d008d8d8d008d8dULL,
        0x8800888888008888ULL, 0x0e000e0e0e000e0eULL, 0x1900191919001919ULL,
        0x8700878787008787ULL, 0x4e004e4e4e004e4eULL, 0x0b000b0b0b000b0bULL,
        0xa900a9a9a900a9a9ULL, 0x0c000c0c0c000c0cULL, 0x7900797979007979ULL,
        0x1100111111001111ULL, 0x7f007f7f7f007f7fULL, 0x2200222222002222ULL,
        0xe700e7e7e700e7e7ULL, 0x5900595959005959ULL, 0xe100e1e1e100e1e1ULL,
        0xda00dadada00dadaULL, 0x3d003d3d3d003d3dULL, 0xc800c8c8c800c8c8ULL,
        0x1200121212001212ULL, 0x0400040404000404ULL, 0x7400747474007474ULL,
        0x5400545454005454ULL, 0x3000303030003030ULL, 0x7e007e7e7e007e7eULL,
        0xb400b4b4b400b4b4ULL, 0x2800282828002828ULL, 0x5500555555005555ULL,
        0x6800686868006868ULL, 0x5000505050005050ULL, 0xbe00bebebe00bebeULL,
        0xd000d0d0d000d0d0ULL, 0xc400c4c4c400c4c4ULL, 0x3100313131003131ULL,
        0xcb00cbcbcb00cbcbULL, 0x2a002a2a2a002a2aULL, 0xad00adadad00adadULL,
        0x0f000f0f0f000f0fULL, 0xca00cacaca00cacaULL, 0x7000707070007070ULL,
        0xff00ffffff00ffffULL, 0x3200323232003232ULL, 0x6900696969006969ULL,
        0x0800080808000808ULL, 0x6200626262006262ULL, 0x0000000000000000ULL,
        0x2400242424002424ULL, 0xd100d1d1d100d1d1ULL, 0xfb00fbfbfb00fbfbULL,
        0xba00bababa00babaULL, 0xed00ededed00ededULL, 0x4500454545004545ULL,
        0x8100818181008181ULL, 0x7300737373007373ULL, 0x6d006d6d6d006d6dULL,
        0x8400848484008484ULL, 0x9f009f9f9f009f9fULL, 0xee00eeeeee00eeeeULL,
        0x4a004a4a4a004a4aULL, 0xc300c3c3c300c3c3ULL, 0x2e002e2e2e002e2eULL,
        0xc100c1c1c100c1c1ULL, 0x0100010101000101ULL, 0xe600e6e6e600e6e6ULL,
        0x2500252525002525ULL, 0x4800484848004848ULL, 0x9900999999009999ULL,
        0xb900b9b9b900b9b9ULL, 0xb300b3b3b300b3b3ULL, 0x7b007b7b7b007b7bULL,
        0xf900f9f9f900f9f9ULL, 0xce00cecece00ceceULL, 0xbf00bfbfbf00bfbfULL,
        0xdf00dfdfdf00dfdfULL, 0x7100717171007171ULL, 0x2900292929002929ULL,
        0xcd00cdcdcd00cdcdULL, 0x6c006c6c6c006c6cULL, 0x1300131313001313ULL,
        0x6400646464006464ULL, 0x9b009b9b9b009b9bULL, 0x6300636363006363ULL,
        0x9d009d9d9d009d9dULL, 0xc000c0c0c000c0c0ULL, 0x4b004b4b4b004b4bULL,
        0xb700b7b7b700b7b7ULL, 0xa500a5a5a500a5a5ULL, 0x8900898989008989ULL,
        0x5f005f5f5f005f5fULL, 0xb100b1b1b100b1b1ULL, 0x1700171717001717ULL,
        0xf400f4f4f400f4f4ULL, 0xbc00bcbcbc00bcbcULL, 0xd300d3d3d300d3d3ULL,
        0x4600464646004646ULL, 0xcf00cfcfcf00cfcfULL, 0x3700373737003737ULL,
        0x5e005e5e5e005e5eULL, 0x4700474747004747ULL, 0x9400949494009494ULL,
        0xfa00fafafa00fafaULL, 0xfc00fcfcfc00fcfcULL, 0x5b005b5b5b005b5bULL,
        0x9700979797009797ULL, 0xfe00fefefe00fefeULL, 0x5a005a5a5a005a5aULL,
        0xac00acacac00acacULL, 0x3c003c3c3c003c3cULL, 0x4c004c4c4c004c4cULL,
        0x0300030303000303ULL, 0x3500353535003535ULL, 0xf300f3f3f300f3f3ULL,
        0x2300232323002323ULL, 0xb800b8b8b800b8b8ULL, 0x5d005d5d5d005d5dULL,
        0x6a006a6a6a006a6aULL, 0x9200929292009292ULL, 0xd500d5d5d500d5d5ULL,
        0x2100212121002121ULL, 0x4400444444004444ULL, 0x5100515151005151ULL,
        0xc600c6c6c600c6c6ULL, 0x7d007d7d7d007d7dULL, 0x3900393939003939ULL,
        0x8300838383008383ULL, 0xdc00dcdcdc00dcdcULL, 0xaa00aaaaaa00aaaaULL,
        0x7c007c7c7c007c7cULL, 0x7700777777007777ULL, 0x5600565656005656ULL,
        0x0500050505000505ULL, 0x1b001b1b1b001b1bULL, 0xa400a4a4a400a4a4ULL,
        0x1500151515001515ULL, 0x3400343434003434ULL, 0x1e001e1e1e001e1eULL,
        0x1c001c1c1c001c1cULL, 0xf800f8f8f800f8f8ULL, 0x5200525252005252ULL,
        0x2000202020002020ULL, 0x1400141414001414ULL, 0xe900e9e9e900e9e9ULL,
        0xbd00bdbdbd00bdbdULL, 0xdd00dddddd00ddddULL, 0xe400e4e4e400e4e4ULL,
        0xa100a1a1a100a1a1ULL, 0xe000e0e0e000e0e0ULL, 0x8a008a8a8a008a8aULL,
        0xf100f1f1f100f1f1ULL, 0xd600d6d6d600d6d6ULL, 0x7a007a7a7a007a7aULL,
        0xbb00bbbbbb00bbbbULL, 0xe300e3e3e300e3e3ULL, 0x4000404040004040ULL,
        0x4f004f4f4f004f4fULL,
};

__visible const u64 camellia_sp44044404[256] = {
        0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL,
        0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL,
        0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL,
        0x6b6b006b6b6b006bULL, 0x4545004545450045ULL, 0xa5a500a5a5a500a5ULL,
        0xeded00ededed00edULL, 0x4f4f004f4f4f004fULL, 0x1d1d001d1d1d001dULL,
        0x9292009292920092ULL, 0x8686008686860086ULL, 0xafaf00afafaf00afULL,
        0x7c7c007c7c7c007cULL, 0x1f1f001f1f1f001fULL, 0x3e3e003e3e3e003eULL,
        0xdcdc00dcdcdc00dcULL, 0x5e5e005e5e5e005eULL, 0x0b0b000b0b0b000bULL,
        0xa6a600a6a6a600a6ULL, 0x3939003939390039ULL, 0xd5d500d5d5d500d5ULL,
        0x5d5d005d5d5d005dULL, 0xd9d900d9d9d900d9ULL, 0x5a5a005a5a5a005aULL,
        0x5151005151510051ULL, 0x6c6c006c6c6c006cULL, 0x8b8b008b8b8b008bULL,
        0x9a9a009a9a9a009aULL, 0xfbfb00fbfbfb00fbULL, 0xb0b000b0b0b000b0ULL,
        0x7474007474740074ULL, 0x2b2b002b2b2b002bULL, 0xf0f000f0f0f000f0ULL,
        0x8484008484840084ULL, 0xdfdf00dfdfdf00dfULL, 0xcbcb00cbcbcb00cbULL,
        0x3434003434340034ULL, 0x7676007676760076ULL, 0x6d6d006d6d6d006dULL,
        0xa9a900a9a9a900a9ULL, 0xd1d100d1d1d100d1ULL, 0x0404000404040004ULL,
        0x1414001414140014ULL, 0x3a3a003a3a3a003aULL, 0xdede00dedede00deULL,
        0x1111001111110011ULL, 0x3232003232320032ULL, 0x9c9c009c9c9c009cULL,
        0x5353005353530053ULL, 0xf2f200f2f2f200f2ULL, 0xfefe00fefefe00feULL,
        0xcfcf00cfcfcf00cfULL, 0xc3c300c3c3c300c3ULL, 0x7a7a007a7a7a007aULL,
        0x2424002424240024ULL, 0xe8e800e8e8e800e8ULL, 0x6060006060600060ULL,
        0x6969006969690069ULL, 0xaaaa00aaaaaa00aaULL, 0xa0a000a0a0a000a0ULL,
        0xa1a100a1a1a100a1ULL, 0x6262006262620062ULL, 0x5454005454540054ULL,
        0x1e1e001e1e1e001eULL, 0xe0e000e0e0e000e0ULL, 0x6464006464640064ULL,
        0x1010001010100010ULL, 0x0000000000000000ULL, 0xa3a300a3a3a300a3ULL,
        0x7575007575750075ULL, 0x8a8a008a8a8a008aULL, 0xe6e600e6e6e600e6ULL,
        0x0909000909090009ULL, 0xdddd00dddddd00ddULL, 0x8787008787870087ULL,
        0x8383008383830083ULL, 0xcdcd00cdcdcd00cdULL, 0x9090009090900090ULL,
        0x7373007373730073ULL, 0xf6f600f6f6f600f6ULL, 0x9d9d009d9d9d009dULL,
        0xbfbf00bfbfbf00bfULL, 0x5252005252520052ULL, 0xd8d800d8d8d800d8ULL,
        0xc8c800c8c8c800c8ULL, 0xc6c600c6c6c600c6ULL, 0x8181008181810081ULL,
        0x6f6f006f6f6f006fULL, 0x1313001313130013ULL, 0x6363006363630063ULL,
        0xe9e900e9e9e900e9ULL, 0xa7a700a7a7a700a7ULL, 0x9f9f009f9f9f009fULL,
        0xbcbc00bcbcbc00bcULL, 0x2929002929290029ULL, 0xf9f900f9f9f900f9ULL,
        0x2f2f002f2f2f002fULL, 0xb4b400b4b4b400b4ULL, 0x7878007878780078ULL,
        0x0606000606060006ULL, 0xe7e700e7e7e700e7ULL, 0x7171007171710071ULL,
        0xd4d400d4d4d400d4ULL, 0xabab00ababab00abULL, 0x8888008888880088ULL,
        0x8d8d008d8d8d008dULL, 0x7272007272720072ULL, 0xb9b900b9b9b900b9ULL,
        0xf8f800f8f8f800f8ULL, 0xacac00acacac00acULL, 0x3636003636360036ULL,
        0x2a2a002a2a2a002aULL, 0x3c3c003c3c3c003cULL, 0xf1f100f1f1f100f1ULL,
        0x4040004040400040ULL, 0xd3d300d3d3d300d3ULL, 0xbbbb00bbbbbb00bbULL,
        0x4343004343430043ULL, 0x1515001515150015ULL, 0xadad00adadad00adULL,
        0x7777007777770077ULL, 0x8080008080800080ULL, 0x8282008282820082ULL,
        0xecec00ececec00ecULL, 0x2727002727270027ULL, 0xe5e500e5e5e500e5ULL,
        0x8585008585850085ULL, 0x3535003535350035ULL, 0x0c0c000c0c0c000cULL,
        0x4141004141410041ULL, 0xefef00efefef00efULL, 0x9393009393930093ULL,
        0x1919001919190019ULL, 0x2121002121210021ULL, 0x0e0e000e0e0e000eULL,
        0x4e4e004e4e4e004eULL, 0x6565006565650065ULL, 0xbdbd00bdbdbd00bdULL,
        0xb8b800b8b8b800b8ULL, 0x8f8f008f8f8f008fULL, 0xebeb00ebebeb00ebULL,
        0xcece00cecece00ceULL, 0x3030003030300030ULL, 0x5f5f005f5f5f005fULL,
        0xc5c500c5c5c500c5ULL, 0x1a1a001a1a1a001aULL, 0xe1e100e1e1e100e1ULL,
        0xcaca00cacaca00caULL, 0x4747004747470047ULL, 0x3d3d003d3d3d003dULL,
        0x0101000101010001ULL, 0xd6d600d6d6d600d6ULL, 0x5656005656560056ULL,
        0x4d4d004d4d4d004dULL, 0x0d0d000d0d0d000dULL, 0x6666006666660066ULL,
        0xcccc00cccccc00ccULL, 0x2d2d002d2d2d002dULL, 0x1212001212120012ULL,
        0x2020002020200020ULL, 0xb1b100b1b1b100b1ULL, 0x9999009999990099ULL,
        0x4c4c004c4c4c004cULL, 0xc2c200c2c2c200c2ULL, 0x7e7e007e7e7e007eULL,
        0x0505000505050005ULL, 0xb7b700b7b7b700b7ULL, 0x3131003131310031ULL,
        0x1717001717170017ULL, 0xd7d700d7d7d700d7ULL, 0x5858005858580058ULL,
        0x6161006161610061ULL, 0x1b1b001b1b1b001bULL, 0x1c1c001c1c1c001cULL,
        0x0f0f000f0f0f000fULL, 0x1616001616160016ULL, 0x1818001818180018ULL,
        0x2222002222220022ULL, 0x4444004444440044ULL, 0xb2b200b2b2b200b2ULL,
        0xb5b500b5b5b500b5ULL, 0x9191009191910091ULL, 0x0808000808080008ULL,
        0xa8a800a8a8a800a8ULL, 0xfcfc00fcfcfc00fcULL, 0x5050005050500050ULL,
        0xd0d000d0d0d000d0ULL, 0x7d7d007d7d7d007dULL, 0x8989008989890089ULL,
        0x9797009797970097ULL, 0x5b5b005b5b5b005bULL, 0x9595009595950095ULL,
        0xffff00ffffff00ffULL, 0xd2d200d2d2d200d2ULL, 0xc4c400c4c4c400c4ULL,
        0x4848004848480048ULL, 0xf7f700f7f7f700f7ULL, 0xdbdb00dbdbdb00dbULL,
        0x0303000303030003ULL, 0xdada00dadada00daULL, 0x3f3f003f3f3f003fULL,
        0x9494009494940094ULL, 0x5c5c005c5c5c005cULL, 0x0202000202020002ULL,
        0x4a4a004a4a4a004aULL, 0x3333003333330033ULL, 0x6767006767670067ULL,
        0xf3f300f3f3f300f3ULL, 0x7f7f007f7f7f007fULL, 0xe2e200e2e2e200e2ULL,
        0x9b9b009b9b9b009bULL, 0x2626002626260026ULL, 0x3737003737370037ULL,
        0x3b3b003b3b3b003bULL, 0x9696009696960096ULL, 0x4b4b004b4b4b004bULL,
        0xbebe00bebebe00beULL, 0x2e2e002e2e2e002eULL, 0x7979007979790079ULL,
        0x8c8c008c8c8c008cULL, 0x6e6e006e6e6e006eULL, 0x8e8e008e8e8e008eULL,
        0xf5f500f5f5f500f5ULL, 0xb6b600b6b6b600b6ULL, 0xfdfd00fdfdfd00fdULL,
        0x5959005959590059ULL, 0x9898009898980098ULL, 0x6a6a006a6a6a006aULL,
        0x4646004646460046ULL, 0xbaba00bababa00baULL, 0x2525002525250025ULL,
        0x4242004242420042ULL, 0xa2a200a2a2a200a2ULL, 0xfafa00fafafa00faULL,
        0x0707000707070007ULL, 0x5555005555550055ULL, 0xeeee00eeeeee00eeULL,
        0x0a0a000a0a0a000aULL, 0x4949004949490049ULL, 0x6868006868680068ULL,
        0x3838003838380038ULL, 0xa4a400a4a4a400a4ULL, 0x2828002828280028ULL,
        0x7b7b007b7b7b007bULL, 0xc9c900c9c9c900c9ULL, 0xc1c100c1c1c100c1ULL,
        0xe3e300e3e3e300e3ULL, 0xf4f400f4f4f400f4ULL, 0xc7c700c7c7c700c7ULL,
        0x9e9e009e9e9e009eULL,
};

__visible const u64 camellia_sp11101110[256] = {
        0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL,
        0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL,
        0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL,
        0x8585850085858500ULL, 0x5757570057575700ULL, 0x3535350035353500ULL,
        0xeaeaea00eaeaea00ULL, 0x0c0c0c000c0c0c00ULL, 0xaeaeae00aeaeae00ULL,
        0x4141410041414100ULL, 0x2323230023232300ULL, 0xefefef00efefef00ULL,
        0x6b6b6b006b6b6b00ULL, 0x9393930093939300ULL, 0x4545450045454500ULL,
        0x1919190019191900ULL, 0xa5a5a500a5a5a500ULL, 0x2121210021212100ULL,
        0xededed00ededed00ULL, 0x0e0e0e000e0e0e00ULL, 0x4f4f4f004f4f4f00ULL,
        0x4e4e4e004e4e4e00ULL, 0x1d1d1d001d1d1d00ULL, 0x6565650065656500ULL,
        0x9292920092929200ULL, 0xbdbdbd00bdbdbd00ULL, 0x8686860086868600ULL,
        0xb8b8b800b8b8b800ULL, 0xafafaf00afafaf00ULL, 0x8f8f8f008f8f8f00ULL,
        0x7c7c7c007c7c7c00ULL, 0xebebeb00ebebeb00ULL, 0x1f1f1f001f1f1f00ULL,
        0xcecece00cecece00ULL, 0x3e3e3e003e3e3e00ULL, 0x3030300030303000ULL,
        0xdcdcdc00dcdcdc00ULL, 0x5f5f5f005f5f5f00ULL, 0x5e5e5e005e5e5e00ULL,
        0xc5c5c500c5c5c500ULL, 0x0b0b0b000b0b0b00ULL, 0x1a1a1a001a1a1a00ULL,
        0xa6a6a600a6a6a600ULL, 0xe1e1e100e1e1e100ULL, 0x3939390039393900ULL,
        0xcacaca00cacaca00ULL, 0xd5d5d500d5d5d500ULL, 0x4747470047474700ULL,
        0x5d5d5d005d5d5d00ULL, 0x3d3d3d003d3d3d00ULL, 0xd9d9d900d9d9d900ULL,
        0x0101010001010100ULL, 0x5a5a5a005a5a5a00ULL, 0xd6d6d600d6d6d600ULL,
        0x5151510051515100ULL, 0x5656560056565600ULL, 0x6c6c6c006c6c6c00ULL,
        0x4d4d4d004d4d4d00ULL, 0x8b8b8b008b8b8b00ULL, 0x0d0d0d000d0d0d00ULL,
        0x9a9a9a009a9a9a00ULL, 0x6666660066666600ULL, 0xfbfbfb00fbfbfb00ULL,
        0xcccccc00cccccc00ULL, 0xb0b0b000b0b0b000ULL, 0x2d2d2d002d2d2d00ULL,
        0x7474740074747400ULL, 0x1212120012121200ULL, 0x2b2b2b002b2b2b00ULL,
        0x2020200020202000ULL, 0xf0f0f000f0f0f000ULL, 0xb1b1b100b1b1b100ULL,
        0x8484840084848400ULL, 0x9999990099999900ULL, 0xdfdfdf00dfdfdf00ULL,
        0x4c4c4c004c4c4c00ULL, 0xcbcbcb00cbcbcb00ULL, 0xc2c2c200c2c2c200ULL,
        0x3434340034343400ULL, 0x7e7e7e007e7e7e00ULL, 0x7676760076767600ULL,
        0x0505050005050500ULL, 0x6d6d6d006d6d6d00ULL, 0xb7b7b700b7b7b700ULL,
        0xa9a9a900a9a9a900ULL, 0x3131310031313100ULL, 0xd1d1d100d1d1d100ULL,
        0x1717170017171700ULL, 0x0404040004040400ULL, 0xd7d7d700d7d7d700ULL,
        0x1414140014141400ULL, 0x5858580058585800ULL, 0x3a3a3a003a3a3a00ULL,
        0x6161610061616100ULL, 0xdedede00dedede00ULL, 0x1b1b1b001b1b1b00ULL,
        0x1111110011111100ULL, 0x1c1c1c001c1c1c00ULL, 0x3232320032323200ULL,
        0x0f0f0f000f0f0f00ULL, 0x9c9c9c009c9c9c00ULL, 0x1616160016161600ULL,
        0x5353530053535300ULL, 0x1818180018181800ULL, 0xf2f2f200f2f2f200ULL,
        0x2222220022222200ULL, 0xfefefe00fefefe00ULL, 0x4444440044444400ULL,
        0xcfcfcf00cfcfcf00ULL, 0xb2b2b200b2b2b200ULL, 0xc3c3c300c3c3c300ULL,
        0xb5b5b500b5b5b500ULL, 0x7a7a7a007a7a7a00ULL, 0x9191910091919100ULL,
        0x2424240024242400ULL, 0x0808080008080800ULL, 0xe8e8e800e8e8e800ULL,
        0xa8a8a800a8a8a800ULL, 0x6060600060606000ULL, 0xfcfcfc00fcfcfc00ULL,
        0x6969690069696900ULL, 0x5050500050505000ULL, 0xaaaaaa00aaaaaa00ULL,
        0xd0d0d000d0d0d000ULL, 0xa0a0a000a0a0a000ULL, 0x7d7d7d007d7d7d00ULL,
        0xa1a1a100a1a1a100ULL, 0x8989890089898900ULL, 0x6262620062626200ULL,
        0x9797970097979700ULL, 0x5454540054545400ULL, 0x5b5b5b005b5b5b00ULL,
        0x1e1e1e001e1e1e00ULL, 0x9595950095959500ULL, 0xe0e0e000e0e0e000ULL,
        0xffffff00ffffff00ULL, 0x6464640064646400ULL, 0xd2d2d200d2d2d200ULL,
        0x1010100010101000ULL, 0xc4c4c400c4c4c400ULL, 0x0000000000000000ULL,
        0x4848480048484800ULL, 0xa3a3a300a3a3a300ULL, 0xf7f7f700f7f7f700ULL,
        0x7575750075757500ULL, 0xdbdbdb00dbdbdb00ULL, 0x8a8a8a008a8a8a00ULL,
        0x0303030003030300ULL, 0xe6e6e600e6e6e600ULL, 0xdadada00dadada00ULL,
        0x0909090009090900ULL, 0x3f3f3f003f3f3f00ULL, 0xdddddd00dddddd00ULL,
        0x9494940094949400ULL, 0x8787870087878700ULL, 0x5c5c5c005c5c5c00ULL,
        0x8383830083838300ULL, 0x0202020002020200ULL, 0xcdcdcd00cdcdcd00ULL,
        0x4a4a4a004a4a4a00ULL, 0x9090900090909000ULL, 0x3333330033333300ULL,
        0x7373730073737300ULL, 0x6767670067676700ULL, 0xf6f6f600f6f6f600ULL,
        0xf3f3f300f3f3f300ULL, 0x9d9d9d009d9d9d00ULL, 0x7f7f7f007f7f7f00ULL,
        0xbfbfbf00bfbfbf00ULL, 0xe2e2e200e2e2e200ULL, 0x5252520052525200ULL,
        0x9b9b9b009b9b9b00ULL, 0xd8d8d800d8d8d800ULL, 0x2626260026262600ULL,
        0xc8c8c800c8c8c800ULL, 0x3737370037373700ULL, 0xc6c6c600c6c6c600ULL,
        0x3b3b3b003b3b3b00ULL, 0x8181810081818100ULL, 0x9696960096969600ULL,
        0x6f6f6f006f6f6f00ULL, 0x4b4b4b004b4b4b00ULL, 0x1313130013131300ULL,
        0xbebebe00bebebe00ULL, 0x6363630063636300ULL, 0x2e2e2e002e2e2e00ULL,
        0xe9e9e900e9e9e900ULL, 0x7979790079797900ULL, 0xa7a7a700a7a7a700ULL,
        0x8c8c8c008c8c8c00ULL, 0x9f9f9f009f9f9f00ULL, 0x6e6e6e006e6e6e00ULL,
        0xbcbcbc00bcbcbc00ULL, 0x8e8e8e008e8e8e00ULL, 0x2929290029292900ULL,
        0xf5f5f500f5f5f500ULL, 0xf9f9f900f9f9f900ULL, 0xb6b6b600b6b6b600ULL,
        0x2f2f2f002f2f2f00ULL, 0xfdfdfd00fdfdfd00ULL, 0xb4b4b400b4b4b400ULL,
        0x5959590059595900ULL, 0x7878780078787800ULL, 0x9898980098989800ULL,
        0x0606060006060600ULL, 0x6a6a6a006a6a6a00ULL, 0xe7e7e700e7e7e700ULL,
        0x4646460046464600ULL, 0x7171710071717100ULL, 0xbababa00bababa00ULL,
        0xd4d4d400d4d4d400ULL, 0x2525250025252500ULL, 0xababab00ababab00ULL,
        0x4242420042424200ULL, 0x8888880088888800ULL, 0xa2a2a200a2a2a200ULL,
        0x8d8d8d008d8d8d00ULL, 0xfafafa00fafafa00ULL, 0x7272720072727200ULL,
        0x0707070007070700ULL, 0xb9b9b900b9b9b900ULL, 0x5555550055555500ULL,
        0xf8f8f800f8f8f800ULL, 0xeeeeee00eeeeee00ULL, 0xacacac00acacac00ULL,
        0x0a0a0a000a0a0a00ULL, 0x3636360036363600ULL, 0x4949490049494900ULL,
        0x2a2a2a002a2a2a00ULL, 0x6868680068686800ULL, 0x3c3c3c003c3c3c00ULL,
        0x3838380038383800ULL, 0xf1f1f100f1f1f100ULL, 0xa4a4a400a4a4a400ULL,
        0x4040400040404000ULL, 0x2828280028282800ULL, 0xd3d3d300d3d3d300ULL,
        0x7b7b7b007b7b7b00ULL, 0xbbbbbb00bbbbbb00ULL, 0xc9c9c900c9c9c900ULL,
        0x4343430043434300ULL, 0xc1c1c100c1c1c100ULL, 0x1515150015151500ULL,
        0xe3e3e300e3e3e300ULL, 0xadadad00adadad00ULL, 0xf4f4f400f4f4f400ULL,
        0x7777770077777700ULL, 0xc7c7c700c7c7c700ULL, 0x8080800080808000ULL,
        0x9e9e9e009e9e9e00ULL,
};

/* key constants */
#define CAMELLIA_SIGMA1L (0xA09E667FL)
#define CAMELLIA_SIGMA1R (0x3BCC908BL)
#define CAMELLIA_SIGMA2L (0xB67AE858L)
#define CAMELLIA_SIGMA2R (0x4CAA73B2L)
#define CAMELLIA_SIGMA3L (0xC6EF372FL)
#define CAMELLIA_SIGMA3R (0xE94F82BEL)
#define CAMELLIA_SIGMA4L (0x54FF53A5L)
#define CAMELLIA_SIGMA4R (0xF1D36F1CL)
#define CAMELLIA_SIGMA5L (0x10E527FAL)
#define CAMELLIA_SIGMA5R (0xDE682D1DL)
#define CAMELLIA_SIGMA6L (0xB05688C2L)
#define CAMELLIA_SIGMA6R (0xB3E6C1FDL)

/* macros */
#define ROLDQ(l, r, bits) ({ \
        u64 t = l;                                        \
        l = (l << bits) | (r >> (64 - bits));                \
        r = (r << bits) | (t >> (64 - bits));                \
})

#define CAMELLIA_F(x, kl, kr, y) ({ \
        u64 ii = x ^ (((u64)kl << 32) | kr);                                \
        y = camellia_sp11101110[(uint8_t)ii];                                \
        y ^= camellia_sp44044404[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp30333033[(uint8_t)ii];                                \
        y ^= camellia_sp02220222[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp00444404[(uint8_t)ii];                                \
        y ^= camellia_sp03303033[(uint8_t)(ii >> 8)];                        \
        ii >>= 16;                                                        \
        y ^= camellia_sp22000222[(uint8_t)ii];                                \
        y ^= camellia_sp10011110[(uint8_t)(ii >> 8)];                        \
        y = ror64(y, 32);                                                \
})

#define SET_SUBKEY_LR(INDEX, sRL) (subkey[(INDEX)] = ror64((sRL), 32))

static void camellia_setup_tail(u64 *subkey, u64 *subRL, int max)
{
        u64 kw4, tt;
        u32 dw, tl, tr;

        /* absorb kw2 to other subkeys */
        /* round 2 */
        subRL[3] ^= subRL[1]; /*covered*/
        /* round 4 */
        subRL[5] ^= subRL[1];
        /* round 6 */
        subRL[7] ^= subRL[1];

        subRL[1] ^= (subRL[1] & ~subRL[9]) << 32;
        /* modified for FLinv(kl2) */
        dw = (subRL[1] & subRL[9]) >> 32;
        subRL[1] ^= rol32(dw, 1);

        /* round 8 */
        subRL[11] ^= subRL[1];
        /* round 10 */
        subRL[13] ^= subRL[1];
        /* round 12 */
        subRL[15] ^= subRL[1];

        subRL[1] ^= (subRL[1] & ~subRL[17]) << 32;
        /* modified for FLinv(kl4) */
        dw = (subRL[1] & subRL[17]) >> 32;
        subRL[1] ^= rol32(dw, 1);

        /* round 14 */
        subRL[19] ^= subRL[1];
        /* round 16 */
        subRL[21] ^= subRL[1];
        /* round 18 */
        subRL[23] ^= subRL[1];

        if (max == 24) {
                /* kw3 */
                subRL[24] ^= subRL[1]; /*covered*/

                /* absorb kw4 to other subkeys */
                kw4 = subRL[25];
        } else {
                subRL[1] ^= (subRL[1] & ~subRL[25]) << 32; /*covered*/
                /* modified for FLinv(kl6) */
                dw = (subRL[1] & subRL[25]) >> 32;
                subRL[1] ^= rol32(dw, 1);

                /* round 20 */
                subRL[27] ^= subRL[1];
                /* round 22 */
                subRL[29] ^= subRL[1];
                /* round 24 */
                subRL[31] ^= subRL[1];
                /* kw3 */
                subRL[32] ^= subRL[1];

                /* absorb kw4 to other subkeys */
                kw4 = subRL[33];
                /* round 23 */
                subRL[30] ^= kw4;
                /* round 21 */
                subRL[28] ^= kw4;
                /* round 19 */
                subRL[26] ^= kw4;

                kw4 ^= (kw4 & ~subRL[24]) << 32;
                /* modified for FL(kl5) */
                dw = (kw4 & subRL[24]) >> 32;
                kw4 ^= rol32(dw, 1);
        }

        /* round 17 */
        subRL[22] ^= kw4; /*covered*/
        /* round 15 */
        subRL[20] ^= kw4;
        /* round 13 */
        subRL[18] ^= kw4;

        kw4 ^= (kw4 & ~subRL[16]) << 32;
        /* modified for FL(kl3) */
        dw = (kw4 & subRL[16]) >> 32;
        kw4 ^= rol32(dw, 1);

        /* round 11 */
        subRL[14] ^= kw4;
        /* round 9 */
        subRL[12] ^= kw4;
        /* round 7 */
        subRL[10] ^= kw4;

        kw4 ^= (kw4 & ~subRL[8]) << 32;
        /* modified for FL(kl1) */
        dw = (kw4 & subRL[8]) >> 32;
        kw4 ^= rol32(dw, 1);

        /* round 5 */
        subRL[6] ^= kw4;
        /* round 3 */
        subRL[4] ^= kw4;
        /* round 1 */
        subRL[2] ^= kw4;
        /* kw1 */
        subRL[0] ^= kw4;

        /* key XOR is end of F-function */
        SET_SUBKEY_LR(0, subRL[0] ^ subRL[2]);                        /* kw1 */
        SET_SUBKEY_LR(2, subRL[3]);                                /* round 1 */
        SET_SUBKEY_LR(3, subRL[2] ^ subRL[4]);                        /* round 2 */
        SET_SUBKEY_LR(4, subRL[3] ^ subRL[5]);                        /* round 3 */
        SET_SUBKEY_LR(5, subRL[4] ^ subRL[6]);                        /* round 4 */
        SET_SUBKEY_LR(6, subRL[5] ^ subRL[7]);                        /* round 5 */

        tl = (subRL[10] >> 32) ^ (subRL[10] & ~subRL[8]);
        dw = tl & (subRL[8] >> 32);                                /* FL(kl1) */
        tr = subRL[10] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(7, subRL[6] ^ tt);                        /* round 6 */
        SET_SUBKEY_LR(8, subRL[8]);                                /* FL(kl1) */
        SET_SUBKEY_LR(9, subRL[9]);                                /* FLinv(kl2) */

        tl = (subRL[7] >> 32) ^ (subRL[7] & ~subRL[9]);
        dw = tl & (subRL[9] >> 32);                                /* FLinv(kl2) */
        tr = subRL[7] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(10, subRL[11] ^ tt);                        /* round 7 */
        SET_SUBKEY_LR(11, subRL[10] ^ subRL[12]);                /* round 8 */
        SET_SUBKEY_LR(12, subRL[11] ^ subRL[13]);                /* round 9 */
        SET_SUBKEY_LR(13, subRL[12] ^ subRL[14]);                /* round 10 */
        SET_SUBKEY_LR(14, subRL[13] ^ subRL[15]);                /* round 11 */

        tl = (subRL[18] >> 32) ^ (subRL[18] & ~subRL[16]);
        dw = tl & (subRL[16] >> 32);                                /* FL(kl3) */
        tr = subRL[18] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(15, subRL[14] ^ tt);                        /* round 12 */
        SET_SUBKEY_LR(16, subRL[16]);                                /* FL(kl3) */
        SET_SUBKEY_LR(17, subRL[17]);                                /* FLinv(kl4) */

        tl = (subRL[15] >> 32) ^ (subRL[15] & ~subRL[17]);
        dw = tl & (subRL[17] >> 32);                                /* FLinv(kl4) */
        tr = subRL[15] ^ rol32(dw, 1);
        tt = (tr | ((u64)tl << 32));

        SET_SUBKEY_LR(18, subRL[19] ^ tt);                        /* round 13 */
        SET_SUBKEY_LR(19, subRL[18] ^ subRL[20]);                /* round 14 */
        SET_SUBKEY_LR(20, subRL[19] ^ subRL[21]);                /* round 15 */
        SET_SUBKEY_LR(21, subRL[20] ^ subRL[22]);                /* round 16 */
        SET_SUBKEY_LR(22, subRL[21] ^ subRL[23]);                /* round 17 */

        if (max == 24) {
                SET_SUBKEY_LR(23, subRL[22]);                        /* round 18 */ /*covered*/
                SET_SUBKEY_LR(24, subRL[24] ^ subRL[23]);        /* kw3 */
        } else {
                tl = (subRL[26] >> 32) ^ (subRL[26] & ~subRL[24]); /*covered*/
                dw = tl & (subRL[24] >> 32);                        /* FL(kl5) */
                tr = subRL[26] ^ rol32(dw, 1);
                tt = (tr | ((u64)tl << 32));

                SET_SUBKEY_LR(23, subRL[22] ^ tt);                /* round 18 */
                SET_SUBKEY_LR(24, subRL[24]);                        /* FL(kl5) */
                SET_SUBKEY_LR(25, subRL[25]);                        /* FLinv(kl6) */

                tl = (subRL[23] >> 32) ^ (subRL[23] & ~subRL[25]);
                dw = tl & (subRL[25] >> 32);                        /* FLinv(kl6) */
                tr = subRL[23] ^ rol32(dw, 1);
                tt = (tr | ((u64)tl << 32));

                SET_SUBKEY_LR(26, subRL[27] ^ tt);                /* round 19 */
                SET_SUBKEY_LR(27, subRL[26] ^ subRL[28]);        /* round 20 */
                SET_SUBKEY_LR(28, subRL[27] ^ subRL[29]);        /* round 21 */
                SET_SUBKEY_LR(29, subRL[28] ^ subRL[30]);        /* round 22 */
                SET_SUBKEY_LR(30, subRL[29] ^ subRL[31]);        /* round 23 */
                SET_SUBKEY_LR(31, subRL[30]);                        /* round 24 */
                SET_SUBKEY_LR(32, subRL[32] ^ subRL[31]);        /* kw3 */
        }
} /*covered*/

static void camellia_setup128(const unsigned char *key, u64 *subkey)
{
        u64 kl, kr, ww;
        u64 subRL[26]; /*covered*/

        /**
         *  k == kl || kr (|| is concatenation)
         */
        kl = get_unaligned_be64(key);
        kr = get_unaligned_be64(key + 8);

        /* generate KL dependent subkeys */
        /* kw1 */
        subRL[0] = kl;
        /* kw2 */
        subRL[1] = kr;

        /* rotation left shift 15bit */
        ROLDQ(kl, kr, 15);

        /* k3 */
        subRL[4] = kl;
        /* k4 */
        subRL[5] = kr;

        /* rotation left shift 15+30bit */
        ROLDQ(kl, kr, 30);

        /* k7 */
        subRL[10] = kl;
        /* k8 */
        subRL[11] = kr;

        /* rotation left shift 15+30+15bit */
        ROLDQ(kl, kr, 15);

        /* k10 */
        subRL[13] = kr;
        /* rotation left shift 15+30+15+17 bit */
        ROLDQ(kl, kr, 17);

        /* kl3 */
        subRL[16] = kl;
        /* kl4 */
        subRL[17] = kr;

        /* rotation left shift 15+30+15+17+17 bit */
        ROLDQ(kl, kr, 17);

        /* k13 */
        subRL[18] = kl;
        /* k14 */
        subRL[19] = kr;

        /* rotation left shift 15+30+15+17+17+17 bit */
        ROLDQ(kl, kr, 17);

        /* k17 */
        subRL[22] = kl;
        /* k18 */
        subRL[23] = kr;

        /* generate KA */
        kl = subRL[0];
        kr = subRL[1];
        CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl);

        /* current status == (kll, klr, w0, w1) */
        CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww);
        kl ^= ww;

        /* generate KA dependent subkeys */
        /* k1, k2 */
        subRL[2] = kl;
        subRL[3] = kr;
        ROLDQ(kl, kr, 15);
        /* k5,k6 */
        subRL[6] = kl;
        subRL[7] = kr;
        ROLDQ(kl, kr, 15);
        /* kl1, kl2 */
        subRL[8] = kl;
        subRL[9] = kr;
        ROLDQ(kl, kr, 15);
        /* k9 */
        subRL[12] = kl;
        ROLDQ(kl, kr, 15);
        /* k11, k12 */
        subRL[14] = kl;
        subRL[15] = kr;
        ROLDQ(kl, kr, 34);
        /* k15, k16 */
        subRL[20] = kl;
        subRL[21] = kr;
        ROLDQ(kl, kr, 17);
        /* kw3, kw4 */
        subRL[24] = kl;
        subRL[25] = kr;

        camellia_setup_tail(subkey, subRL, 24);
}

static void camellia_setup256(const unsigned char *key, u64 *subkey)
{
        u64 kl, kr;                        /* left half of key */
        u64 krl, krr;                        /* right half of key */
        u64 ww;                                /* temporary variables */
        u64 subRL[34]; /*covered*/

        /**
         *  key = (kl || kr || krl || krr) (|| is concatenation)
         */
        kl = get_unaligned_be64(key);
        kr = get_unaligned_be64(key + 8);
        krl = get_unaligned_be64(key + 16);
        krr = get_unaligned_be64(key + 24);

        /* generate KL dependent subkeys */
        /* kw1 */
        subRL[0] = kl;
        /* kw2 */
        subRL[1] = kr;
        ROLDQ(kl, kr, 45);
        /* k9 */
        subRL[12] = kl;
        /* k10 */
        subRL[13] = kr;
        ROLDQ(kl, kr, 15);
        /* kl3 */
        subRL[16] = kl;
        /* kl4 */
        subRL[17] = kr;
        ROLDQ(kl, kr, 17);
        /* k17 */
        subRL[22] = kl;
        /* k18 */
        subRL[23] = kr;
        ROLDQ(kl, kr, 34);
        /* k23 */
        subRL[30] = kl;
        /* k24 */
        subRL[31] = kr;

        /* generate KR dependent subkeys */
        ROLDQ(krl, krr, 15);
        /* k3 */
        subRL[4] = krl;
        /* k4 */
        subRL[5] = krr;
        ROLDQ(krl, krr, 15);
        /* kl1 */
        subRL[8] = krl;
        /* kl2 */
        subRL[9] = krr;
        ROLDQ(krl, krr, 30);
        /* k13 */
        subRL[18] = krl;
        /* k14 */
        subRL[19] = krr;
        ROLDQ(krl, krr, 34);
        /* k19 */
        subRL[26] = krl;
        /* k20 */
        subRL[27] = krr;
        ROLDQ(krl, krr, 34);

        /* generate KA */
        kl = subRL[0] ^ krl;
        kr = subRL[1] ^ krr;

        CAMELLIA_F(kl, CAMELLIA_SIGMA1L, CAMELLIA_SIGMA1R, ww);
        kr ^= ww;
        CAMELLIA_F(kr, CAMELLIA_SIGMA2L, CAMELLIA_SIGMA2R, kl);
        kl ^= krl;
        CAMELLIA_F(kl, CAMELLIA_SIGMA3L, CAMELLIA_SIGMA3R, kr);
        kr ^= ww ^ krr;
        CAMELLIA_F(kr, CAMELLIA_SIGMA4L, CAMELLIA_SIGMA4R, ww);
        kl ^= ww;

        /* generate KB */
        krl ^= kl;
        krr ^= kr;
        CAMELLIA_F(krl, CAMELLIA_SIGMA5L, CAMELLIA_SIGMA5R, ww);
        krr ^= ww;
        CAMELLIA_F(krr, CAMELLIA_SIGMA6L, CAMELLIA_SIGMA6R, ww);
        krl ^= ww;

        /* generate KA dependent subkeys */
        ROLDQ(kl, kr, 15);
        /* k5 */
        subRL[6] = kl;
        /* k6 */
        subRL[7] = kr;
        ROLDQ(kl, kr, 30);
        /* k11 */
        subRL[14] = kl;
        /* k12 */
        subRL[15] = kr;
        /* rotation left shift 32bit */
        ROLDQ(kl, kr, 32);
        /* kl5 */
        subRL[24] = kl;
        /* kl6 */
        subRL[25] = kr;
        /* rotation left shift 17 from k11,k12 -> k21,k22 */
        ROLDQ(kl, kr, 17);
        /* k21 */
        subRL[28] = kl;
        /* k22 */
        subRL[29] = kr;

        /* generate KB dependent subkeys */
        /* k1 */
        subRL[2] = krl;
        /* k2 */
        subRL[3] = krr;
        ROLDQ(krl, krr, 30);
        /* k7 */
        subRL[10] = krl;
        /* k8 */
        subRL[11] = krr;
        ROLDQ(krl, krr, 30);
        /* k15 */
        subRL[20] = krl;
        /* k16 */
        subRL[21] = krr;
        ROLDQ(krl, krr, 51);
        /* kw3 */
        subRL[32] = krl;
        /* kw4 */
        subRL[33] = krr;

        camellia_setup_tail(subkey, subRL, 32);
}

static void camellia_setup192(const unsigned char *key, u64 *subkey)
{
        unsigned char kk[32];
        u64 krl, krr;

        memcpy(kk, key, 24);
        memcpy((unsigned char *)&krl, key+16, 8);
        krr = ~krl;
        memcpy(kk+24, (unsigned char *)&krr, 8);
        camellia_setup256(kk, subkey);
}

int __camellia_setkey(struct camellia_ctx *cctx, const unsigned char *key,
                      unsigned int key_len, u32 *flags)
{
        if (key_len != 16 && key_len != 24 && key_len != 32) { /*covered*/
                *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; /*covered*/
                return -EINVAL;
        }

        cctx->key_length = key_len; /*covered*/

        switch (key_len) {
        case 16:
                camellia_setup128(key, cctx->key_table);
                break;
        case 24:
                camellia_setup192(key, cctx->key_table);
                break;
        case 32:
                camellia_setup256(key, cctx->key_table);
                break;
        }

        return 0; /*covered*/
}
EXPORT_SYMBOL_GPL(__camellia_setkey);

static int camellia_setkey(struct crypto_tfm *tfm, const u8 *key,
                           unsigned int key_len)
{
        return __camellia_setkey(crypto_tfm_ctx(tfm), key, key_len, /*covered*/
                                 &tfm->crt_flags);
}

static int camellia_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
                                    unsigned int key_len)
{
        return camellia_setkey(&tfm->base, key, key_len); /*covered*/
}

void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
{
        u128 iv = *src; /*covered*/

        camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src);

        u128_xor(&dst[1], &dst[1], &iv);
}
EXPORT_SYMBOL_GPL(camellia_decrypt_cbc_2way);

void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk; /*covered*/

        if (dst != src)
                *dst = *src; /*covered*/

        le128_to_be128(&ctrblk, iv); /*covered*/
        le128_inc(iv); /*covered*/

        camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk);
}
EXPORT_SYMBOL_GPL(camellia_crypt_ctr);

void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblks[2]; /*covered*/

        if (dst != src) {
                dst[0] = src[0]; /*covered*/
                dst[1] = src[1];
        }

        le128_to_be128(&ctrblks[0], iv); /*covered*/
        le128_inc(iv);
        le128_to_be128(&ctrblks[1], iv); /*covered*/
        le128_inc(iv); /*covered*/

        camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks);
}
EXPORT_SYMBOL_GPL(camellia_crypt_ctr_2way);

static const struct common_glue_ctx camellia_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
        } }
};

static const struct common_glue_ctx camellia_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
        } }
};

static const struct common_glue_ctx camellia_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
        } }
};

static const struct common_glue_ctx camellia_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 2,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&camellia_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&camellia_ctr, req); /*covered*/
}

static struct crypto_alg camellia_cipher_alg = {
        .cra_name                = "camellia",
        .cra_driver_name        = "camellia-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = CAMELLIA_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct camellia_ctx),
        .cra_alignmask                = 0,
        .cra_module                = THIS_MODULE,
        .cra_u                        = {
                .cipher = {
                        .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE,
                        .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE,
                        .cia_setkey         = camellia_setkey,
                        .cia_encrypt         = camellia_encrypt,
                        .cia_decrypt         = camellia_decrypt
                }
        }
};

static struct skcipher_alg camellia_skcipher_algs[] = {
        {
                .base.cra_name                = "ecb(camellia)",
                .base.cra_driver_name        = "ecb-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(camellia)",
                .base.cra_driver_name        = "cbc-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = CAMELLIA_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(camellia)",
                .base.cra_driver_name        = "ctr-camellia-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct camellia_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAMELLIA_MIN_KEY_SIZE,
                .max_keysize                = CAMELLIA_MAX_KEY_SIZE,
                .ivsize                        = CAMELLIA_BLOCK_SIZE,
                .chunksize                = CAMELLIA_BLOCK_SIZE,
                .setkey                        = camellia_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, camellia-asm is slower than original assembler
                 * implementation because excessive uses of 64bit rotate and
                 * left-shifts (which are really slow on P4) needed to store and
                 * handle 128bit block in two 64bit registers.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "camellia-x86_64: performance on this CPU "
                        "would be suboptimal: disabling "
                        "camellia-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&camellia_cipher_alg);
        if (err)
                return err;

        err = crypto_register_skciphers(camellia_skcipher_algs,
                                        ARRAY_SIZE(camellia_skcipher_algs));
        if (err)
                crypto_unregister_alg(&camellia_cipher_alg);

        return err;
}

static void __exit fini(void)
{
        crypto_unregister_alg(&camellia_cipher_alg);
        crypto_unregister_skciphers(camellia_skcipher_algs,
                                    ARRAY_SIZE(camellia_skcipher_algs));
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Camellia Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("camellia");
MODULE_ALIAS_CRYPTO("camellia-asm");

/*
 * Glue Code for the AVX assembler implemention of the Cast5 Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <asm/crypto/glue_helper.h>
#include <crypto/algapi.h>
#include <crypto/cast5.h>
#include <crypto/internal/simd.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <linux/module.h>
#include <linux/types.h>

#define CAST5_PARALLEL_BLOCKS 16

asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst,
                                    const u8 *src);
asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst,
                                    const u8 *src);
asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst,
                                    const u8 *src);
asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src,
                                __be64 *iv);

static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
                                 unsigned int keylen)
{
        return cast5_setkey(&tfm->base, key, keylen); /*covered*/
}

static inline bool cast5_fpu_begin(bool fpu_enabled, struct skcipher_walk *walk,
                                   unsigned int nbytes)
{
        return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS, /*covered*/
                              walk, fpu_enabled, nbytes);
}

static inline void cast5_fpu_end(bool fpu_enabled)
{
        return glue_fpu_end(fpu_enabled); /*covered*/
}

static int ecb_crypt(struct skcipher_request *req, bool enc)
{
        bool fpu_enabled = false;
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        unsigned int nbytes;
        void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) { /*covered*/
                u8 *wsrc = walk.src.virt.addr; /*covered*/
                u8 *wdst = walk.dst.virt.addr;

                fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); /*covered*/

                /* Process multi-block batch */
                if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { /*covered*/
                        fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way;
                        do {
                                fn(ctx, wdst, wsrc); /*covered*/

                                wsrc += bsize * CAST5_PARALLEL_BLOCKS;
                                wdst += bsize * CAST5_PARALLEL_BLOCKS;
                                nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
                        } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }

                fn = (enc) ? __cast5_encrypt : __cast5_decrypt;

                /* Handle leftovers */
                do {
                        fn(ctx, wdst, wsrc); /*covered*/

                        wsrc += bsize;
                        wdst += bsize;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

done: /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        cast5_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}

static int ecb_encrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, true);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return ecb_crypt(req, false); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                u64 *src = (u64 *)walk.src.virt.addr; /*covered*/
                u64 *dst = (u64 *)walk.dst.virt.addr;
                u64 *iv = (u64 *)walk.iv;

                do {
                        *dst = *src ^ *iv; /*covered*/
                        __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
                        iv = dst;
                        src++;
                        dst++;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

                *(u64 *)walk.iv = *iv; /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static unsigned int __cbc_decrypt(struct cast5_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes; /*covered*/
        u64 *src = (u64 *)walk->src.virt.addr;
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 last_iv;

        /* Start of the last block. */
        src += nbytes / bsize - 1;
        dst += nbytes / bsize - 1;

        last_iv = *src;

        /* Process multi-block batch */
        if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
                do {
                        nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
                        src -= CAST5_PARALLEL_BLOCKS - 1; /*covered*/
                        dst -= CAST5_PARALLEL_BLOCKS - 1;

                        cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src);

                        nbytes -= bsize;
                        if (nbytes < bsize)
                                goto done;

                        *dst ^= *(src - 1); /*covered*/
                        src -= 1;
                        dst -= 1;
                } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
        }

        /* Handle leftovers */
        for (;;) {
                __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src); /*covered*/

                nbytes -= bsize;
                if (nbytes < bsize)
                        break;

                *dst ^= *(src - 1); /*covered*/
                src -= 1;
                dst -= 1;
        }

done:
        *dst ^= *(u64 *)walk->iv; /*covered*/
        *(u64 *)walk->iv = last_iv;

        return nbytes;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        bool fpu_enabled = false;
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); /*covered*/
                nbytes = __cbc_decrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        cast5_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}

static void ctr_crypt_final(struct skcipher_walk *walk, struct cast5_ctx *ctx)
{
        u8 *ctrblk = walk->iv;
        u8 keystream[CAST5_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        __cast5_encrypt(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
}

static unsigned int __ctr_crypt(struct skcipher_walk *walk,
                                struct cast5_ctx *ctx)
{
        const unsigned int bsize = CAST5_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr;
        u64 *dst = (u64 *)walk->dst.virt.addr;

        /* Process multi-block batch */
        if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
                do {
                        cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src,
                                        (__be64 *)walk->iv);

                        src += CAST5_PARALLEL_BLOCKS;
                        dst += CAST5_PARALLEL_BLOCKS;
                        nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
                } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);

                if (nbytes < bsize)
                        goto done;
        }

        /* Handle leftovers */
        do {
                u64 ctrblk;

                if (dst != src)
                        *dst = *src;

                ctrblk = *(u64 *)walk->iv;
                be64_add_cpu((__be64 *)walk->iv, 1);

                __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
                *dst ^= ctrblk;

                src += 1;
                dst += 1;
                nbytes -= bsize;
        } while (nbytes >= bsize);

done:
        return nbytes;
}

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
        bool fpu_enabled = false;
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
                fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes);
                nbytes = __ctr_crypt(&walk, ctx);
                err = skcipher_walk_done(&walk, nbytes);
        }

        cast5_fpu_end(fpu_enabled);

        if (walk.nbytes) {
                ctr_crypt_final(&walk, ctx);
                err = skcipher_walk_done(&walk, 0);
        }

        return err;
}

static struct skcipher_alg cast5_algs[] = {
        {
                .base.cra_name                = "__ecb(cast5)",
                .base.cra_driver_name        = "__ecb-cast5-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST5_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast5_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST5_MIN_KEY_SIZE,
                .max_keysize                = CAST5_MAX_KEY_SIZE,
                .setkey                        = cast5_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(cast5)",
                .base.cra_driver_name        = "__cbc-cast5-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST5_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast5_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST5_MIN_KEY_SIZE,
                .max_keysize                = CAST5_MAX_KEY_SIZE,
                .ivsize                        = CAST5_BLOCK_SIZE,
                .setkey                        = cast5_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(cast5)",
                .base.cra_driver_name        = "__ctr-cast5-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct cast5_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST5_MIN_KEY_SIZE,
                .max_keysize                = CAST5_MAX_KEY_SIZE,
                .ivsize                        = CAST5_BLOCK_SIZE,
                .chunksize                = CAST5_BLOCK_SIZE,
                .setkey                        = cast5_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }
};

static struct simd_skcipher_alg *cast5_simd_algs[ARRAY_SIZE(cast5_algs)];

static int __init cast5_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(cast5_algs,
                                              ARRAY_SIZE(cast5_algs),
                                              cast5_simd_algs);
}

static void __exit cast5_exit(void)
{
        simd_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs),
                                  cast5_simd_algs);
}

module_init(cast5_init);
module_exit(cast5_exit);

MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("cast5");

/*
 * Glue Code for the AVX assembler implemention of the Cast6 Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/cast6.h>
#include <crypto/internal/simd.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>

#define CAST6_PARALLEL_BLOCKS 8

asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src);
asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src);

asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src);
asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
                               le128 *iv);

asmlinkage void cast6_xts_enc_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);
asmlinkage void cast6_xts_dec_8way(struct cast6_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);

static int cast6_setkey_skcipher(struct crypto_skcipher *tfm,
                                 const u8 *key, unsigned int keylen)
{
        return cast6_setkey(&tfm->base, key, keylen); /*covered*/
}

static void cast6_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(__cast6_encrypt));
}

static void cast6_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, /*covered*/
                                  GLUE_FUNC_CAST(__cast6_decrypt));
}

static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk; /*covered*/

        le128_to_be128(&ctrblk, iv);
        le128_inc(iv); /*covered*/

        __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
}

static const struct common_glue_ctx cast6_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
        } }
};

static const struct common_glue_ctx cast6_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
        } }
};

static const struct common_glue_ctx cast6_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_enc) }
        } }
};

static const struct common_glue_ctx cast6_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
        } }
};

static const struct common_glue_ctx cast6_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
        } }
};

static const struct common_glue_ctx cast6_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = CAST6_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(cast6_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&cast6_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&cast6_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__cast6_encrypt), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&cast6_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&cast6_ctr, req); /*covered*/
}

struct cast6_xts_ctx {
        struct cast6_ctx tweak_ctx;
        struct cast6_ctx crypt_ctx;
};

static int xts_cast6_setkey(struct crypto_skcipher *tfm, const u8 *key,
                            unsigned int keylen)
{
        struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        u32 *flags = &tfm->base.crt_flags; /*covered*/
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        /* first half of xts-key is for crypt */
        err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); /*covered*/
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, /*covered*/
                              flags);
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&cast6_enc_xts, req,
                                   XTS_TWEAK_CAST(__cast6_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&cast6_dec_xts, req,
                                   XTS_TWEAK_CAST(__cast6_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg cast6_algs[] = {
        {
                .base.cra_name                = "__ecb(cast6)",
                .base.cra_driver_name        = "__ecb-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST6_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast6_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST6_MIN_KEY_SIZE,
                .max_keysize                = CAST6_MAX_KEY_SIZE,
                .setkey                        = cast6_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(cast6)",
                .base.cra_driver_name        = "__cbc-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST6_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast6_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST6_MIN_KEY_SIZE,
                .max_keysize                = CAST6_MAX_KEY_SIZE,
                .ivsize                        = CAST6_BLOCK_SIZE,
                .setkey                        = cast6_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(cast6)",
                .base.cra_driver_name        = "__ctr-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct cast6_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = CAST6_MIN_KEY_SIZE,
                .max_keysize                = CAST6_MAX_KEY_SIZE,
                .ivsize                        = CAST6_BLOCK_SIZE,
                .chunksize                = CAST6_BLOCK_SIZE,
                .setkey                        = cast6_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(cast6)",
                .base.cra_driver_name        = "__xts-cast6-avx",
                .base.cra_priority        = 200,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = CAST6_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct cast6_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * CAST6_MIN_KEY_SIZE,
                .max_keysize                = 2 * CAST6_MAX_KEY_SIZE,
                .ivsize                        = CAST6_BLOCK_SIZE,
                .setkey                        = xts_cast6_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *cast6_simd_algs[ARRAY_SIZE(cast6_algs)];

static int __init cast6_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(cast6_algs,
                                              ARRAY_SIZE(cast6_algs),
                                              cast6_simd_algs);
}

static void __exit cast6_exit(void)
{
        simd_unregister_skciphers(cast6_algs, ARRAY_SIZE(cast6_algs),
                                  cast6_simd_algs);
}

module_init(cast6_init);
module_exit(cast6_exit);

MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("cast6");

/*
 * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
 *
 * Copyright (C) 2015 Martin Willi
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <crypto/algapi.h>
#include <crypto/chacha20.h>
#include <crypto/internal/skcipher.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/simd.h>

#define CHACHA20_STATE_ALIGN 16

asmlinkage void chacha20_block_xor_ssse3(u32 *state, u8 *dst, const u8 *src);
asmlinkage void chacha20_4block_xor_ssse3(u32 *state, u8 *dst, const u8 *src);
#ifdef CONFIG_AS_AVX2
asmlinkage void chacha20_8block_xor_avx2(u32 *state, u8 *dst, const u8 *src);
static bool chacha20_use_avx2;
#endif

static void chacha20_dosimd(u32 *state, u8 *dst, const u8 *src,
                            unsigned int bytes)
{
        u8 buf[CHACHA20_BLOCK_SIZE]; /*covered*/

#ifdef CONFIG_AS_AVX2
        if (chacha20_use_avx2) {
                while (bytes >= CHACHA20_BLOCK_SIZE * 8) { /*covered*/
                        chacha20_8block_xor_avx2(state, dst, src); /*covered*/
                        bytes -= CHACHA20_BLOCK_SIZE * 8;
                        src += CHACHA20_BLOCK_SIZE * 8;
                        dst += CHACHA20_BLOCK_SIZE * 8;
                        state[12] += 8;
                }
        }
#endif
        while (bytes >= CHACHA20_BLOCK_SIZE * 4) { /*covered*/
                chacha20_4block_xor_ssse3(state, dst, src); /*covered*/
                bytes -= CHACHA20_BLOCK_SIZE * 4;
                src += CHACHA20_BLOCK_SIZE * 4;
                dst += CHACHA20_BLOCK_SIZE * 4;
                state[12] += 4;
        }
        while (bytes >= CHACHA20_BLOCK_SIZE) { /*covered*/
                chacha20_block_xor_ssse3(state, dst, src); /*covered*/
                bytes -= CHACHA20_BLOCK_SIZE;
                src += CHACHA20_BLOCK_SIZE;
                dst += CHACHA20_BLOCK_SIZE;
                state[12]++;
        }
        if (bytes) { /*covered*/
                memcpy(buf, src, bytes); /*covered*/
                chacha20_block_xor_ssse3(state, buf, buf);
                memcpy(dst, buf, bytes);
        }
}

static int chacha20_simd(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
        u32 *state, state_buf[16 + 2] __aligned(8);
        struct skcipher_walk walk;
        int err;

        BUILD_BUG_ON(CHACHA20_STATE_ALIGN != 16);
        state = PTR_ALIGN(state_buf + 0, CHACHA20_STATE_ALIGN); /*covered*/

        if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd()) /*covered*/
                return crypto_chacha20_crypt(req); /*covered*/

        err = skcipher_walk_virt(&walk, req, true);

        crypto_chacha20_init(state, ctx, walk.iv);

        kernel_fpu_begin();

        while (walk.nbytes >= CHACHA20_BLOCK_SIZE) {
                chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr,
                                rounddown(walk.nbytes, CHACHA20_BLOCK_SIZE)); /*covered*/
                err = skcipher_walk_done(&walk,
                                         walk.nbytes % CHACHA20_BLOCK_SIZE);
        }

        if (walk.nbytes) { /*covered*/
                chacha20_dosimd(state, walk.dst.virt.addr, walk.src.virt.addr, /*covered*/
                                walk.nbytes);
                err = skcipher_walk_done(&walk, 0);
        }

        kernel_fpu_end(); /*covered*/

        return err;
}

static struct skcipher_alg alg = {
        .base.cra_name                = "chacha20",
        .base.cra_driver_name        = "chacha20-simd",
        .base.cra_priority        = 300,
        .base.cra_blocksize        = 1,
        .base.cra_ctxsize        = sizeof(struct chacha20_ctx),
        .base.cra_module        = THIS_MODULE,

        .min_keysize                = CHACHA20_KEY_SIZE,
        .max_keysize                = CHACHA20_KEY_SIZE,
        .ivsize                        = CHACHA20_IV_SIZE,
        .chunksize                = CHACHA20_BLOCK_SIZE,
        .setkey                        = crypto_chacha20_setkey,
        .encrypt                = chacha20_simd,
        .decrypt                = chacha20_simd,
};

static int __init chacha20_simd_mod_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_SSSE3))
                return -ENODEV;

#ifdef CONFIG_AS_AVX2
        chacha20_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) &&
                            boot_cpu_has(X86_FEATURE_AVX2) &&
                            cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
#endif
        return crypto_register_skcipher(&alg);
}

static void __exit chacha20_simd_mod_fini(void)
{
        crypto_unregister_skcipher(&alg);
}

module_init(chacha20_simd_mod_init);
module_exit(chacha20_simd_mod_fini);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
MODULE_DESCRIPTION("chacha20 cipher algorithm, SIMD accelerated");
MODULE_ALIAS_CRYPTO("chacha20");
MODULE_ALIAS_CRYPTO("chacha20-simd");

/* GPL HEADER START
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 only,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License version 2 for more details (a copy is included
 * in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; If not, see http://www.gnu.org/licenses
 *
 * Please  visit http://www.xyratex.com/contact if you need additional
 * information or have any questions.
 *
 * GPL HEADER END
 */

/*
 * Copyright 2012 Xyratex Technology Limited
 *
 * Wrappers for kernel crypto shash api to pclmulqdq crc32 imlementation.
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/crc32.h>
#include <crypto/internal/hash.h>

#include <asm/cpufeatures.h>
#include <asm/cpu_device_id.h>
#include <asm/fpu/api.h>

#define CHKSUM_BLOCK_SIZE        1
#define CHKSUM_DIGEST_SIZE        4

#define PCLMUL_MIN_LEN                64L     /* minimum size of buffer
                                         * for crc32_pclmul_le_16 */
#define SCALE_F                        16L        /* size of xmm register */
#define SCALE_F_MASK                (SCALE_F - 1)

u32 crc32_pclmul_le_16(unsigned char const *buffer, size_t len, u32 crc32);

static u32 __attribute__((pure))
        crc32_pclmul_le(u32 crc, unsigned char const *p, size_t len)
{
        unsigned int iquotient;
        unsigned int iremainder;
        unsigned int prealign;

        if (len < PCLMUL_MIN_LEN + SCALE_F_MASK || !irq_fpu_usable()) /*covered*/
                return crc32_le(crc, p, len); /*covered*/

        if ((long)p & SCALE_F_MASK) { /*covered*/
                /* align p to 16 byte */
                prealign = SCALE_F - ((long)p & SCALE_F_MASK); /*covered*/

                crc = crc32_le(crc, p, prealign);
                len -= prealign;
                p = (unsigned char *)(((unsigned long)p + SCALE_F_MASK) &
                                     ~SCALE_F_MASK);
        }
        iquotient = len & (~SCALE_F_MASK); /*covered*/
        iremainder = len & SCALE_F_MASK;

        kernel_fpu_begin();
        crc = crc32_pclmul_le_16(p, iquotient, crc);
        kernel_fpu_end();

        if (iremainder)
                crc = crc32_le(crc, p + iquotient, iremainder); /*covered*/

        return crc;
}

static int crc32_pclmul_cra_init(struct crypto_tfm *tfm)
{
        u32 *key = crypto_tfm_ctx(tfm);

        *key = 0; /*covered*/

        return 0;
}

static int crc32_pclmul_setkey(struct crypto_shash *hash, const u8 *key,
                        unsigned int keylen)
{
        u32 *mctx = crypto_shash_ctx(hash);

        if (keylen != sizeof(u32)) { /*covered*/
                crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }
        *mctx = le32_to_cpup((__le32 *)key); /*covered*/
        return 0; /*covered*/
}

static int crc32_pclmul_init(struct shash_desc *desc)
{
        u32 *mctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = *mctx;

        return 0;
}

static int crc32_pclmul_update(struct shash_desc *desc, const u8 *data,
                               unsigned int len)
{
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = crc32_pclmul_le(*crcp, data, len); /*covered*/
        return 0;
}

/* No final XOR 0xFFFFFFFF, like crc32_le */
static int __crc32_pclmul_finup(u32 *crcp, const u8 *data, unsigned int len,
                                u8 *out)
{
        *(__le32 *)out = cpu_to_le32(crc32_pclmul_le(*crcp, data, len));
        return 0;
}

static int crc32_pclmul_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return __crc32_pclmul_finup(shash_desc_ctx(desc), data, len, out); /*covered*/
}

static int crc32_pclmul_final(struct shash_desc *desc, u8 *out)
{
        u32 *crcp = shash_desc_ctx(desc);

        *(__le32 *)out = cpu_to_le32p(crcp); /*covered*/
        return 0;
}

static int crc32_pclmul_digest(struct shash_desc *desc, const u8 *data,
                               unsigned int len, u8 *out)
{
        return __crc32_pclmul_finup(crypto_shash_ctx(desc->tfm), data, len,
                                    out);
}

static struct shash_alg alg = {
        .setkey                = crc32_pclmul_setkey,
        .init                = crc32_pclmul_init,
        .update                = crc32_pclmul_update,
        .final                = crc32_pclmul_final,
        .finup                = crc32_pclmul_finup,
        .digest                = crc32_pclmul_digest,
        .descsize        = sizeof(u32),
        .digestsize        = CHKSUM_DIGEST_SIZE,
        .base                = {
                        .cra_name                = "crc32",
                        .cra_driver_name        = "crc32-pclmul",
                        .cra_priority                = 200,
                        .cra_flags                = CRYPTO_ALG_OPTIONAL_KEY,
                        .cra_blocksize                = CHKSUM_BLOCK_SIZE,
                        .cra_ctxsize                = sizeof(u32),
                        .cra_module                = THIS_MODULE,
                        .cra_init                = crc32_pclmul_cra_init,
        }
};

static const struct x86_cpu_id crc32pclmul_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, crc32pclmul_cpu_id);


static int __init crc32_pclmul_mod_init(void)
{

        if (!x86_match_cpu(crc32pclmul_cpu_id)) {
                pr_info("PCLMULQDQ-NI instructions are not detected.\n");
                return -ENODEV;
        }
        return crypto_register_shash(&alg);
}

static void __exit crc32_pclmul_mod_fini(void)
{
        crypto_unregister_shash(&alg);
}

module_init(crc32_pclmul_mod_init);
module_exit(crc32_pclmul_mod_fini);

MODULE_AUTHOR("Alexander Boyko <alexander_boyko@xyratex.com>");
MODULE_LICENSE("GPL");

MODULE_ALIAS_CRYPTO("crc32");
MODULE_ALIAS_CRYPTO("crc32-pclmul");

/*
 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
 * http://www.intel.com/products/processor/manuals/
 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
 * Volume 2A: Instruction Set Reference, A-M
 *
 * Copyright (C) 2008 Intel Corporation
 * Authors: Austin Zhang <austin_zhang@linux.intel.com>
 *          Kent Liu <kent.liu@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */
#include <linux/init.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <crypto/internal/hash.h>

#include <asm/cpufeatures.h>
#include <asm/cpu_device_id.h>
#include <asm/fpu/internal.h>

#define CHKSUM_BLOCK_SIZE        1
#define CHKSUM_DIGEST_SIZE        4

#define SCALE_F        sizeof(unsigned long)

#ifdef CONFIG_X86_64
#define REX_PRE "0x48, "
#else
#define REX_PRE
#endif

#ifdef CONFIG_X86_64
/*
 * use carryless multiply version of crc32c when buffer
 * size is >= 512 to account
 * for fpu state save/restore overhead.
 */
#define CRC32C_PCL_BREAKEVEN        512

asmlinkage unsigned int crc_pcl(const u8 *buffer, int len,
                                unsigned int crc_init);
#endif /* CONFIG_X86_64 */

static u32 crc32c_intel_le_hw_byte(u32 crc, unsigned char const *data, size_t length)
{
        while (length--) {
                __asm__ __volatile__(
                        ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
                        :"=S"(crc)
                        :"0"(crc), "c"(*data) /*covered*/
                );
                data++;
        }

        return crc;
}

static u32 __pure crc32c_intel_le_hw(u32 crc, unsigned char const *p, size_t len)
{
        unsigned int iquotient = len / SCALE_F;
        unsigned int iremainder = len % SCALE_F;
        unsigned long *ptmp = (unsigned long *)p;

        while (iquotient--) {
                __asm__ __volatile__(
                        ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;"
                        :"=S"(crc)
                        :"0"(crc), "c"(*ptmp) /*covered*/
                );
                ptmp++;
        }

        if (iremainder) /*covered*/
                crc = crc32c_intel_le_hw_byte(crc, (unsigned char *)ptmp, /*covered*/
                                 iremainder);

        return crc;
}

/*
 * Setting the seed allows arbitrary accumulators and flexible XOR policy
 * If your algorithm starts with ~0, then XOR with ~0 before you set
 * the seed.
 */
static int crc32c_intel_setkey(struct crypto_shash *hash, const u8 *key,
                        unsigned int keylen)
{
        u32 *mctx = crypto_shash_ctx(hash);

        if (keylen != sizeof(u32)) { /*covered*/
                crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }
        *mctx = le32_to_cpup((__le32 *)key); /*covered*/
        return 0; /*covered*/
}

static int crc32c_intel_init(struct shash_desc *desc)
{
        u32 *mctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = *mctx;

        return 0;
}

static int crc32c_intel_update(struct shash_desc *desc, const u8 *data,
                               unsigned int len)
{
        u32 *crcp = shash_desc_ctx(desc);

        *crcp = crc32c_intel_le_hw(*crcp, data, len);
        return 0;
}

static int __crc32c_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
                                u8 *out)
{
        *(__le32 *)out = ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len));
        return 0;
}

static int crc32c_intel_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return __crc32c_intel_finup(shash_desc_ctx(desc), data, len, out);
}

static int crc32c_intel_final(struct shash_desc *desc, u8 *out)
{
        u32 *crcp = shash_desc_ctx(desc);

        *(__le32 *)out = ~cpu_to_le32p(crcp); /*covered*/
        return 0;
}

static int crc32c_intel_digest(struct shash_desc *desc, const u8 *data,
                               unsigned int len, u8 *out)
{
        return __crc32c_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
                                    out);
}

static int crc32c_intel_cra_init(struct crypto_tfm *tfm)
{
        u32 *key = crypto_tfm_ctx(tfm);

        *key = ~0; /*covered*/

        return 0;
}

#ifdef CONFIG_X86_64
static int crc32c_pcl_intel_update(struct shash_desc *desc, const u8 *data,
                               unsigned int len)
{
        u32 *crcp = shash_desc_ctx(desc);

        /*
         * use faster PCL version if datasize is large enough to
         * overcome kernel fpu state save/restore overhead
         */
        if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                *crcp = crc_pcl(data, len, *crcp);
                kernel_fpu_end();
        } else
                *crcp = crc32c_intel_le_hw(*crcp, data, len); /*covered*/
        return 0; /*covered*/
}

static int __crc32c_pcl_intel_finup(u32 *crcp, const u8 *data, unsigned int len,
                                u8 *out)
{
        if (len >= CRC32C_PCL_BREAKEVEN && irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                *(__le32 *)out = ~cpu_to_le32(crc_pcl(data, len, *crcp));
                kernel_fpu_end();
        } else
                *(__le32 *)out =
                        ~cpu_to_le32(crc32c_intel_le_hw(*crcp, data, len)); /*covered*/
        return 0; /*covered*/
}

static int crc32c_pcl_intel_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return __crc32c_pcl_intel_finup(shash_desc_ctx(desc), data, len, out); /*covered*/
}

static int crc32c_pcl_intel_digest(struct shash_desc *desc, const u8 *data,
                               unsigned int len, u8 *out)
{
        return __crc32c_pcl_intel_finup(crypto_shash_ctx(desc->tfm), data, len,
                                    out);
}
#endif /* CONFIG_X86_64 */

static struct shash_alg alg = {
        .setkey                        =        crc32c_intel_setkey,
        .init                        =        crc32c_intel_init,
        .update                        =        crc32c_intel_update,
        .final                        =        crc32c_intel_final,
        .finup                        =        crc32c_intel_finup,
        .digest                        =        crc32c_intel_digest,
        .descsize                =        sizeof(u32),
        .digestsize                =        CHKSUM_DIGEST_SIZE,
        .base                        =        {
                .cra_name                =        "crc32c",
                .cra_driver_name        =        "crc32c-intel",
                .cra_priority                =        200,
                .cra_flags                =        CRYPTO_ALG_OPTIONAL_KEY,
                .cra_blocksize                =        CHKSUM_BLOCK_SIZE,
                .cra_ctxsize                =        sizeof(u32),
                .cra_module                =        THIS_MODULE,
                .cra_init                =        crc32c_intel_cra_init,
        }
};

static const struct x86_cpu_id crc32c_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_XMM4_2),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, crc32c_cpu_id);

static int __init crc32c_intel_mod_init(void)
{
        if (!x86_match_cpu(crc32c_cpu_id))
                return -ENODEV;
#ifdef CONFIG_X86_64
        if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) {
                alg.update = crc32c_pcl_intel_update;
                alg.finup = crc32c_pcl_intel_finup;
                alg.digest = crc32c_pcl_intel_digest;
        }
#endif
        return crypto_register_shash(&alg);
}

static void __exit crc32c_intel_mod_fini(void)
{
        crypto_unregister_shash(&alg);
}

module_init(crc32c_intel_mod_init);
module_exit(crc32c_intel_mod_fini);

MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
MODULE_LICENSE("GPL");

MODULE_ALIAS_CRYPTO("crc32c");
MODULE_ALIAS_CRYPTO("crc32c-intel");

/*
 * Cryptographic API.
 *
 * T10 Data Integrity Field CRC16 Crypto Transform using PCLMULQDQ Instructions
 *
 * Copyright (C) 2013 Intel Corporation
 * Author: Tim Chen <tim.c.chen@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/crc-t10dif.h>
#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <asm/fpu/api.h>
#include <asm/cpufeatures.h>
#include <asm/cpu_device_id.h>

asmlinkage __u16 crc_t10dif_pcl(__u16 crc, const unsigned char *buf,
                                size_t len);

struct chksum_desc_ctx {
        __u16 crc;
};

/*
 * Steps through buffer one byte at at time, calculates reflected
 * crc using table.
 */

static int chksum_init(struct shash_desc *desc)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        ctx->crc = 0; /*covered*/

        return 0;
}

static int chksum_update(struct shash_desc *desc, const u8 *data,
                         unsigned int length)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        if (irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                ctx->crc = crc_t10dif_pcl(ctx->crc, data, length);
                kernel_fpu_end();
        } else
                ctx->crc = crc_t10dif_generic(ctx->crc, data, length);
        return 0; /*covered*/
}

static int chksum_final(struct shash_desc *desc, u8 *out)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        *(__u16 *)out = ctx->crc; /*covered*/
        return 0;
}

static int __chksum_finup(__u16 *crcp, const u8 *data, unsigned int len,
                        u8 *out)
{
        if (irq_fpu_usable()) { /*covered*/
                kernel_fpu_begin(); /*covered*/
                *(__u16 *)out = crc_t10dif_pcl(*crcp, data, len);
                kernel_fpu_end();
        } else
                *(__u16 *)out = crc_t10dif_generic(*crcp, data, len);
        return 0; /*covered*/
}

static int chksum_finup(struct shash_desc *desc, const u8 *data,
                        unsigned int len, u8 *out)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        return __chksum_finup(&ctx->crc, data, len, out); /*covered*/
}

static int chksum_digest(struct shash_desc *desc, const u8 *data,
                         unsigned int length, u8 *out)
{
        struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);

        return __chksum_finup(&ctx->crc, data, length, out);
}

static struct shash_alg alg = {
        .digestsize                =        CRC_T10DIF_DIGEST_SIZE,
        .init                =        chksum_init,
        .update                =        chksum_update,
        .final                =        chksum_final,
        .finup                =        chksum_finup,
        .digest                =        chksum_digest,
        .descsize                =        sizeof(struct chksum_desc_ctx),
        .base                        =        {
                .cra_name                =        "crct10dif",
                .cra_driver_name        =        "crct10dif-pclmul",
                .cra_priority                =        200,
                .cra_blocksize                =        CRC_T10DIF_BLOCK_SIZE,
                .cra_module                =        THIS_MODULE,
        }
};

static const struct x86_cpu_id crct10dif_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ),
        {}
};
MODULE_DEVICE_TABLE(x86cpu, crct10dif_cpu_id);

static int __init crct10dif_intel_mod_init(void)
{
        if (!x86_match_cpu(crct10dif_cpu_id))
                return -ENODEV;

        return crypto_register_shash(&alg);
}

static void __exit crct10dif_intel_mod_fini(void)
{
        crypto_unregister_shash(&alg);
}

module_init(crct10dif_intel_mod_init);
module_exit(crct10dif_intel_mod_fini);

MODULE_AUTHOR("Tim Chen <tim.c.chen@linux.intel.com>");
MODULE_DESCRIPTION("T10 DIF CRC calculation accelerated with PCLMULQDQ.");
MODULE_LICENSE("GPL");

MODULE_ALIAS_CRYPTO("crct10dif");
MODULE_ALIAS_CRYPTO("crct10dif-pclmul");

/*
 * Glue Code for assembler optimized version of 3DES
 *
 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 */

#include <crypto/algapi.h>
#include <crypto/des.h>
#include <crypto/internal/skcipher.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

struct des3_ede_x86_ctx {
        u32 enc_expkey[DES3_EDE_EXPKEY_WORDS];
        u32 dec_expkey[DES3_EDE_EXPKEY_WORDS];
};

/* regular block cipher functions */
asmlinkage void des3_ede_x86_64_crypt_blk(const u32 *expkey, u8 *dst,
                                          const u8 *src);

/* 3-way parallel cipher functions */
asmlinkage void des3_ede_x86_64_crypt_blk_3way(const u32 *expkey, u8 *dst,
                                               const u8 *src);

static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                    const u8 *src)
{
        u32 *enc_ctx = ctx->enc_expkey; /*covered*/

        des3_ede_x86_64_crypt_blk(enc_ctx, dst, src);
}

static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                    const u8 *src)
{
        u32 *dec_ctx = ctx->dec_expkey;

        des3_ede_x86_64_crypt_blk(dec_ctx, dst, src); /*covered*/
}

static inline void des3_ede_enc_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        u32 *enc_ctx = ctx->enc_expkey;

        des3_ede_x86_64_crypt_blk_3way(enc_ctx, dst, src);
}

static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        u32 *dec_ctx = ctx->dec_expkey;

        des3_ede_x86_64_crypt_blk_3way(dec_ctx, dst, src);
}

static void des3_ede_x86_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        des3_ede_enc_blk(crypto_tfm_ctx(tfm), dst, src); /*covered*/
}

static void des3_ede_x86_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        des3_ede_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static int ecb_crypt(struct skcipher_request *req, const u32 *expkey)
{
        const unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        struct skcipher_walk walk; /*covered*/
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                u8 *wsrc = walk.src.virt.addr; /*covered*/
                u8 *wdst = walk.dst.virt.addr;

                /* Process four block batch */
                if (nbytes >= bsize * 3) {
                        do {
                                des3_ede_x86_64_crypt_blk_3way(expkey, wdst, /*covered*/
                                                               wsrc);

                                wsrc += bsize * 3;
                                wdst += bsize * 3;
                                nbytes -= bsize * 3;
                        } while (nbytes >= bsize * 3);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }

                /* Handle leftovers */
                do {
                        des3_ede_x86_64_crypt_blk(expkey, wdst, wsrc); /*covered*/

                        wsrc += bsize;
                        wdst += bsize;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

done: /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static int ecb_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);

        return ecb_crypt(req, ctx->enc_expkey);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);

        return ecb_crypt(req, ctx->dec_expkey);
}

static unsigned int __cbc_encrypt(struct des3_ede_x86_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 *iv = (u64 *)walk->iv;

        do {
                *dst = *src ^ *iv; /*covered*/
                des3_ede_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
                iv = dst;

                src += 1;
                dst += 1;
                nbytes -= bsize;
        } while (nbytes >= bsize);

        *(u64 *)walk->iv = *iv; /*covered*/
        return nbytes;
}

static int cbc_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) { /*covered*/
                nbytes = __cbc_encrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static unsigned int __cbc_decrypt(struct des3_ede_x86_ctx *ctx,
                                  struct skcipher_walk *walk)
{
        unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        u64 *src = (u64 *)walk->src.virt.addr; /*covered*/
        u64 *dst = (u64 *)walk->dst.virt.addr;
        u64 ivs[3 - 1];
        u64 last_iv;

        /* Start of the last block. */
        src += nbytes / bsize - 1;
        dst += nbytes / bsize - 1;

        last_iv = *src;

        /* Process four block batch */
        if (nbytes >= bsize * 3) {
                do {
                        nbytes -= bsize * 3 - bsize;
                        src -= 3 - 1; /*covered*/
                        dst -= 3 - 1;

                        ivs[0] = src[0];
                        ivs[1] = src[1];

                        des3_ede_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);

                        dst[1] ^= ivs[0];
                        dst[2] ^= ivs[1];

                        nbytes -= bsize;
                        if (nbytes < bsize)
                                goto done;

                        *dst ^= *(src - 1); /*covered*/
                        src -= 1;
                        dst -= 1;
                } while (nbytes >= bsize * 3);
        }

        /* Handle leftovers */
        for (;;) {
                des3_ede_dec_blk(ctx, (u8 *)dst, (u8 *)src); /*covered*/

                nbytes -= bsize;
                if (nbytes < bsize)
                        break;

                *dst ^= *(src - 1); /*covered*/
                src -= 1;
                dst -= 1;
        }

done:
        *dst ^= *(u64 *)walk->iv; /*covered*/
        *(u64 *)walk->iv = last_iv;

        return nbytes;
}

static int cbc_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) { /*covered*/
                nbytes = __cbc_decrypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        return err; /*covered*/
}

static void ctr_crypt_final(struct des3_ede_x86_ctx *ctx,
                            struct skcipher_walk *walk)
{
        u8 *ctrblk = walk->iv; /*covered*/
        u8 keystream[DES3_EDE_BLOCK_SIZE];
        u8 *src = walk->src.virt.addr;
        u8 *dst = walk->dst.virt.addr;
        unsigned int nbytes = walk->nbytes;

        des3_ede_enc_blk(ctx, keystream, ctrblk);
        crypto_xor_cpy(dst, keystream, src, nbytes);

        crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE);
}

static unsigned int __ctr_crypt(struct des3_ede_x86_ctx *ctx,
                                struct skcipher_walk *walk)
{
        unsigned int bsize = DES3_EDE_BLOCK_SIZE;
        unsigned int nbytes = walk->nbytes;
        __be64 *src = (__be64 *)walk->src.virt.addr; /*covered*/
        __be64 *dst = (__be64 *)walk->dst.virt.addr;
        u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
        __be64 ctrblocks[3];

        /* Process four block batch */
        if (nbytes >= bsize * 3) {
                do {
                        /* create ctrblks for parallel encrypt */
                        ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/
                        ctrblocks[1] = cpu_to_be64(ctrblk++);
                        ctrblocks[2] = cpu_to_be64(ctrblk++);

                        des3_ede_enc_blk_3way(ctx, (u8 *)ctrblocks,
                                              (u8 *)ctrblocks);

                        dst[0] = src[0] ^ ctrblocks[0];
                        dst[1] = src[1] ^ ctrblocks[1];
                        dst[2] = src[2] ^ ctrblocks[2];

                        src += 3;
                        dst += 3;
                } while ((nbytes -= bsize * 3) >= bsize * 3);

                if (nbytes < bsize) /*covered*/
                        goto done;
        }

        /* Handle leftovers */
        do {
                ctrblocks[0] = cpu_to_be64(ctrblk++); /*covered*/

                des3_ede_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);

                dst[0] = src[0] ^ ctrblocks[0];

                src += 1;
                dst += 1;
        } while ((nbytes -= bsize) >= bsize);

done: /*covered*/
        *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
        return nbytes;
}

static int ctr_crypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= DES3_EDE_BLOCK_SIZE) { /*covered*/
                nbytes = __ctr_crypt(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        if (nbytes) { /*covered*/
                ctr_crypt_final(ctx, &walk); /*covered*/
                err = skcipher_walk_done(&walk, 0);
        }

        return err; /*covered*/
}

static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key,
                               unsigned int keylen)
{
        struct des3_ede_x86_ctx *ctx = crypto_tfm_ctx(tfm);
        u32 i, j, tmp;
        int err;

        /* Generate encryption context using generic implementation. */
        err = __des3_ede_setkey(ctx->enc_expkey, &tfm->crt_flags, key, keylen); /*covered*/
        if (err < 0)
                return err;

        /* Fix encryption context for this implementation and form decryption
         * context. */
        j = DES3_EDE_EXPKEY_WORDS - 2;
        for (i = 0; i < DES3_EDE_EXPKEY_WORDS; i += 2, j -= 2) { /*covered*/
                tmp = ror32(ctx->enc_expkey[i + 1], 4); /*covered*/
                ctx->enc_expkey[i + 1] = tmp;

                ctx->dec_expkey[j + 0] = ctx->enc_expkey[i + 0];
                ctx->dec_expkey[j + 1] = tmp;
        }

        return 0;
}

static int des3_ede_x86_setkey_skcipher(struct crypto_skcipher *tfm,
                                        const u8 *key,
                                        unsigned int keylen)
{
        return des3_ede_x86_setkey(&tfm->base, key, keylen); /*covered*/
}

static struct crypto_alg des3_ede_cipher = {
        .cra_name                = "des3_ede",
        .cra_driver_name        = "des3_ede-asm",
        .cra_priority                = 200,
        .cra_flags                = CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                = DES3_EDE_BLOCK_SIZE,
        .cra_ctxsize                = sizeof(struct des3_ede_x86_ctx),
        .cra_alignmask                = 0,
        .cra_module                = THIS_MODULE,
        .cra_u = {
                .cipher = {
                        .cia_min_keysize        = DES3_EDE_KEY_SIZE,
                        .cia_max_keysize        = DES3_EDE_KEY_SIZE,
                        .cia_setkey                = des3_ede_x86_setkey,
                        .cia_encrypt                = des3_ede_x86_encrypt,
                        .cia_decrypt                = des3_ede_x86_decrypt,
                }
        }
};

static struct skcipher_alg des3_ede_skciphers[] = {
        {
                .base.cra_name                = "ecb(des3_ede)",
                .base.cra_driver_name        = "ecb-des3_ede-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = DES3_EDE_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct des3_ede_x86_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = DES3_EDE_KEY_SIZE,
                .max_keysize                = DES3_EDE_KEY_SIZE,
                .setkey                        = des3_ede_x86_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(des3_ede)",
                .base.cra_driver_name        = "cbc-des3_ede-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = DES3_EDE_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct des3_ede_x86_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = DES3_EDE_KEY_SIZE,
                .max_keysize                = DES3_EDE_KEY_SIZE,
                .ivsize                        = DES3_EDE_BLOCK_SIZE,
                .setkey                        = des3_ede_x86_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(des3_ede)",
                .base.cra_driver_name        = "ctr-des3_ede-asm",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct des3_ede_x86_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = DES3_EDE_KEY_SIZE,
                .max_keysize                = DES3_EDE_KEY_SIZE,
                .ivsize                        = DES3_EDE_BLOCK_SIZE,
                .chunksize                = DES3_EDE_BLOCK_SIZE,
                .setkey                        = des3_ede_x86_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, des3_ede-x86_64 is slower than generic C
                 * implementation because use of 64bit rotates (which are really
                 * slow on P4). Therefore blacklist P4s.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init des3_ede_x86_init(void)
{
        int err;

        if (!force && is_blacklisted_cpu()) {
                pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n");
                return -ENODEV;
        }

        err = crypto_register_alg(&des3_ede_cipher);
        if (err)
                return err;

        err = crypto_register_skciphers(des3_ede_skciphers,
                                        ARRAY_SIZE(des3_ede_skciphers));
        if (err)
                crypto_unregister_alg(&des3_ede_cipher);

        return err;
}

static void __exit des3_ede_x86_fini(void)
{
        crypto_unregister_alg(&des3_ede_cipher);
        crypto_unregister_skciphers(des3_ede_skciphers,
                                    ARRAY_SIZE(des3_ede_skciphers));
}

module_init(des3_ede_x86_init);
module_exit(des3_ede_x86_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("des3_ede");
MODULE_ALIAS_CRYPTO("des3_ede-asm");
MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");

/*
 * FPU: Wrapper for blkcipher touching fpu
 *
 * Copyright (c) Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#include <crypto/internal/skcipher.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <asm/fpu/api.h>

struct crypto_fpu_ctx {
        struct crypto_skcipher *child;
};

static int crypto_fpu_setkey(struct crypto_skcipher *parent, const u8 *key,
                             unsigned int keylen)
{
        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(parent);
        struct crypto_skcipher *child = ctx->child;
        int err;

        crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
        crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(parent) &
                                         CRYPTO_TFM_REQ_MASK);
        err = crypto_skcipher_setkey(child, key, keylen);
        crypto_skcipher_set_flags(parent, crypto_skcipher_get_flags(child) &
                                          CRYPTO_TFM_RES_MASK);
        return err;
}

static int crypto_fpu_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct crypto_skcipher *child = ctx->child;
        SKCIPHER_REQUEST_ON_STACK(subreq, child);
        int err;

        skcipher_request_set_tfm(subreq, child);
        skcipher_request_set_callback(subreq, 0, NULL, NULL);
        skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
                                   req->iv);

        kernel_fpu_begin();
        err = crypto_skcipher_encrypt(subreq);
        kernel_fpu_end();

        skcipher_request_zero(subreq);
        return err;
}

static int crypto_fpu_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct crypto_skcipher *child = ctx->child;
        SKCIPHER_REQUEST_ON_STACK(subreq, child);
        int err;

        skcipher_request_set_tfm(subreq, child);
        skcipher_request_set_callback(subreq, 0, NULL, NULL);
        skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
                                   req->iv);

        kernel_fpu_begin();
        err = crypto_skcipher_decrypt(subreq);
        kernel_fpu_end();

        skcipher_request_zero(subreq);
        return err;
}

static int crypto_fpu_init_tfm(struct crypto_skcipher *tfm)
{
        struct skcipher_instance *inst = skcipher_alg_instance(tfm); /*covered*/
        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);
        struct crypto_skcipher_spawn *spawn;
        struct crypto_skcipher *cipher;

        spawn = skcipher_instance_ctx(inst);
        cipher = crypto_spawn_skcipher(spawn);
        if (IS_ERR(cipher))
                return PTR_ERR(cipher);

        ctx->child = cipher; /*covered*/

        return 0; /*covered*/
}

static void crypto_fpu_exit_tfm(struct crypto_skcipher *tfm)
{
        struct crypto_fpu_ctx *ctx = crypto_skcipher_ctx(tfm);

        crypto_free_skcipher(ctx->child); /*covered*/
}

static void crypto_fpu_free(struct skcipher_instance *inst)
{
        crypto_drop_skcipher(skcipher_instance_ctx(inst));
        kfree(inst);
}

static int crypto_fpu_create(struct crypto_template *tmpl, struct rtattr **tb)
{
        struct crypto_skcipher_spawn *spawn;
        struct skcipher_instance *inst;
        struct crypto_attr_type *algt;
        struct skcipher_alg *alg;
        const char *cipher_name;
        int err;

        algt = crypto_get_attr_type(tb);
        if (IS_ERR(algt))
                return PTR_ERR(algt);

        if ((algt->type ^ (CRYPTO_ALG_INTERNAL | CRYPTO_ALG_TYPE_SKCIPHER)) &
            algt->mask)
                return -EINVAL;

        if (!(algt->mask & CRYPTO_ALG_INTERNAL))
                return -EINVAL;

        cipher_name = crypto_attr_alg_name(tb[1]);
        if (IS_ERR(cipher_name))
                return PTR_ERR(cipher_name);

        inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
        if (!inst)
                return -ENOMEM;

        spawn = skcipher_instance_ctx(inst);

        crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
        err = crypto_grab_skcipher(spawn, cipher_name, CRYPTO_ALG_INTERNAL,
                                   CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC);
        if (err)
                goto out_free_inst;

        alg = crypto_skcipher_spawn_alg(spawn);

        err = crypto_inst_setname(skcipher_crypto_instance(inst), "fpu",
                                  &alg->base);
        if (err)
                goto out_drop_skcipher;

        inst->alg.base.cra_flags = CRYPTO_ALG_INTERNAL;
        inst->alg.base.cra_priority = alg->base.cra_priority;
        inst->alg.base.cra_blocksize = alg->base.cra_blocksize;
        inst->alg.base.cra_alignmask = alg->base.cra_alignmask;

        inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg);
        inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
        inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);

        inst->alg.base.cra_ctxsize = sizeof(struct crypto_fpu_ctx);

        inst->alg.init = crypto_fpu_init_tfm;
        inst->alg.exit = crypto_fpu_exit_tfm;

        inst->alg.setkey = crypto_fpu_setkey;
        inst->alg.encrypt = crypto_fpu_encrypt;
        inst->alg.decrypt = crypto_fpu_decrypt;

        inst->free = crypto_fpu_free;

        err = skcipher_register_instance(tmpl, inst);
        if (err)
                goto out_drop_skcipher;

out:
        return err;

out_drop_skcipher:
        crypto_drop_skcipher(spawn);
out_free_inst:
        kfree(inst);
        goto out;
}

static struct crypto_template crypto_fpu_tmpl = {
        .name = "fpu",
        .create = crypto_fpu_create,
        .module = THIS_MODULE,
};

int __init crypto_fpu_init(void)
{
        return crypto_register_template(&crypto_fpu_tmpl);
}

void crypto_fpu_exit(void)
{
        crypto_unregister_template(&crypto_fpu_tmpl);
}

MODULE_ALIAS_CRYPTO("fpu");

/*
 * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
 * instructions. This file contains glue code.
 *
 * Copyright (c) 2009 Intel Corp.
 *   Author: Huang Ying <ying.huang@intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <linux/err.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/crypto.h>
#include <crypto/algapi.h>
#include <crypto/cryptd.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/hash.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

#define GHASH_BLOCK_SIZE        16
#define GHASH_DIGEST_SIZE        16

void clmul_ghash_mul(char *dst, const u128 *shash);

void clmul_ghash_update(char *dst, const char *src, unsigned int srclen,
                        const u128 *shash);

struct ghash_async_ctx {
        struct cryptd_ahash *cryptd_tfm;
};

struct ghash_ctx {
        u128 shash;
};

struct ghash_desc_ctx {
        u8 buffer[GHASH_BLOCK_SIZE];
        u32 bytes;
};

static int ghash_init(struct shash_desc *desc)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); /*covered*/

        memset(dctx, 0, sizeof(*dctx));

        return 0;
}

static int ghash_setkey(struct crypto_shash *tfm,
                        const u8 *key, unsigned int keylen)
{
        struct ghash_ctx *ctx = crypto_shash_ctx(tfm);
        be128 *x = (be128 *)key;
        u64 a, b;

        if (keylen != GHASH_BLOCK_SIZE) { /*covered*/
                crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        /* perform multiplication by 'x' in GF(2^128) */
        a = be64_to_cpu(x->a); /*covered*/
        b = be64_to_cpu(x->b);

        ctx->shash.a = (b << 1) | (a >> 63);
        ctx->shash.b = (a << 1) | (b >> 63); /*covered*/

        if (a >> 63)
                ctx->shash.b ^= ((u64)0xc2) << 56; /*covered*/

        return 0; /*covered*/
}

static int ghash_update(struct shash_desc *desc,
                         const u8 *src, unsigned int srclen)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u8 *dst = dctx->buffer;

        kernel_fpu_begin();
        if (dctx->bytes) {
                int n = min(srclen, dctx->bytes); /*covered*/
                u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

                dctx->bytes -= n;
                srclen -= n;

                while (n--)
                        *pos++ ^= *src++; /*covered*/

                if (!dctx->bytes) /*covered*/
                        clmul_ghash_mul(dst, &ctx->shash); /*covered*/
        }

        clmul_ghash_update(dst, src, srclen, &ctx->shash); /*covered*/
        kernel_fpu_end();

        if (srclen & 0xf) {
                src += srclen - (srclen & 0xf); /*covered*/
                srclen &= 0xf;
                dctx->bytes = GHASH_BLOCK_SIZE - srclen;
                while (srclen--)
                        *dst++ ^= *src++; /*covered*/
        }

        return 0; /*covered*/
}

static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx)
{
        u8 *dst = dctx->buffer;

        if (dctx->bytes) { /*covered*/
                u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes);

                while (dctx->bytes--) /*covered*/
                        *tmp++ ^= 0;

                kernel_fpu_begin();
                clmul_ghash_mul(dst, &ctx->shash);
                kernel_fpu_end();
        }

        dctx->bytes = 0; /*covered*/
}

static int ghash_final(struct shash_desc *desc, u8 *dst)
{
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);
        struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); /*covered*/
        u8 *buf = dctx->buffer;

        ghash_flush(ctx, dctx); /*covered*/
        memcpy(dst, buf, GHASH_BLOCK_SIZE);

        return 0;
}

static struct shash_alg ghash_alg = {
        .digestsize        = GHASH_DIGEST_SIZE,
        .init                = ghash_init,
        .update                = ghash_update,
        .final                = ghash_final,
        .setkey                = ghash_setkey,
        .descsize        = sizeof(struct ghash_desc_ctx),
        .base                = {
                .cra_name                = "__ghash",
                .cra_driver_name        = "__ghash-pclmulqdqni",
                .cra_priority                = 0,
                .cra_flags                = CRYPTO_ALG_INTERNAL,
                .cra_blocksize                = GHASH_BLOCK_SIZE,
                .cra_ctxsize                = sizeof(struct ghash_ctx),
                .cra_module                = THIS_MODULE,
        },
};

static int ghash_async_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *cryptd_req = ahash_request_ctx(req);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
        struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
        struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

        desc->tfm = child;
        desc->flags = req->base.flags;
        return crypto_shash_init(desc); /*covered*/
}

static int ghash_async_update(struct ahash_request *req)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

        if (!irq_fpu_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { /*covered*/
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
                return crypto_ahash_update(cryptd_req);
        } else {
                struct shash_desc *desc = cryptd_shash_desc(cryptd_req); /*covered*/
                return shash_ahash_update(req, desc); /*covered*/
        }
}

static int ghash_async_final(struct ahash_request *req)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

        if (!irq_fpu_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { /*covered*/
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
                return crypto_ahash_final(cryptd_req); /*covered*/
        } else {
                struct shash_desc *desc = cryptd_shash_desc(cryptd_req); /*covered*/
                return crypto_shash_final(desc, req->result);
        }
}

static int ghash_async_import(struct ahash_request *req, const void *in)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

        ghash_async_init(req);
        memcpy(dctx, in, sizeof(*dctx));
        return 0;

}

static int ghash_async_export(struct ahash_request *req, void *out)
{
        struct ahash_request *cryptd_req = ahash_request_ctx(req); /*covered*/
        struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
        struct ghash_desc_ctx *dctx = shash_desc_ctx(desc);

        memcpy(out, dctx, sizeof(*dctx));
        return 0;

}

static int ghash_async_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *cryptd_req = ahash_request_ctx(req);
        struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;

        if (!irq_fpu_usable() ||
            (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
                memcpy(cryptd_req, req, sizeof(*req));
                ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
                return crypto_ahash_digest(cryptd_req);
        } else {
                struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
                struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);

                desc->tfm = child;
                desc->flags = req->base.flags;
                return shash_ahash_digest(req, desc);
        }
}

static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
                              unsigned int keylen)
{
        struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
        struct crypto_ahash *child = &ctx->cryptd_tfm->base; /*covered*/
        int err;

        crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
        crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
                               & CRYPTO_TFM_REQ_MASK);
        err = crypto_ahash_setkey(child, key, keylen);
        crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
                               & CRYPTO_TFM_RES_MASK);

        return err;
}

static int ghash_async_init_tfm(struct crypto_tfm *tfm)
{
        struct cryptd_ahash *cryptd_tfm;
        struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

        cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", /*covered*/
                                        CRYPTO_ALG_INTERNAL,
                                        CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);
        ctx->cryptd_tfm = cryptd_tfm; /*covered*/
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
                                 sizeof(struct ahash_request) +
                                 crypto_ahash_reqsize(&cryptd_tfm->base));

        return 0; /*covered*/
}

static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
{
        struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);

        cryptd_free_ahash(ctx->cryptd_tfm); /*covered*/
}

static struct ahash_alg ghash_async_alg = {
        .init                = ghash_async_init,
        .update                = ghash_async_update,
        .final                = ghash_async_final,
        .setkey                = ghash_async_setkey,
        .digest                = ghash_async_digest,
        .export                = ghash_async_export,
        .import                = ghash_async_import,
        .halg = {
                .digestsize        = GHASH_DIGEST_SIZE,
                .statesize = sizeof(struct ghash_desc_ctx),
                .base = {
                        .cra_name                = "ghash",
                        .cra_driver_name        = "ghash-clmulni",
                        .cra_priority                = 400,
                        .cra_ctxsize                = sizeof(struct ghash_async_ctx),
                        .cra_flags                = CRYPTO_ALG_ASYNC,
                        .cra_blocksize                = GHASH_BLOCK_SIZE,
                        .cra_module                = THIS_MODULE,
                        .cra_init                = ghash_async_init_tfm,
                        .cra_exit                = ghash_async_exit_tfm,
                },
        },
};

static const struct x86_cpu_id pcmul_cpu_id[] = {
        X86_FEATURE_MATCH(X86_FEATURE_PCLMULQDQ), /* Pickle-Mickle-Duck */
        {}
};
MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);

static int __init ghash_pclmulqdqni_mod_init(void)
{
        int err;

        if (!x86_match_cpu(pcmul_cpu_id))
                return -ENODEV;

        err = crypto_register_shash(&ghash_alg);
        if (err)
                goto err_out;
        err = crypto_register_ahash(&ghash_async_alg);
        if (err)
                goto err_shash;

        return 0;

err_shash:
        crypto_unregister_shash(&ghash_alg);
err_out:
        return err;
}

static void __exit ghash_pclmulqdqni_mod_exit(void)
{
        crypto_unregister_ahash(&ghash_async_alg);
        crypto_unregister_shash(&ghash_alg);
}

module_init(ghash_pclmulqdqni_mod_init);
module_exit(ghash_pclmulqdqni_mod_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("GHASH Message Digest Algorithm, "
                   "accelerated by PCLMULQDQ-NI");
MODULE_ALIAS_CRYPTO("ghash");

/*
 * Shared glue code for 128bit block ciphers
 *
 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <crypto/b128ops.h>
#include <crypto/gf128mul.h>
#include <crypto/internal/skcipher.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>

int glue_ecb_req_128bit(const struct common_glue_ctx *gctx,
                        struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                const u8 *src = walk.src.virt.addr; /*covered*/
                u8 *dst = walk.dst.virt.addr;
                unsigned int func_bytes;
                unsigned int i;

                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                             &walk, fpu_enabled, nbytes);
                for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                        func_bytes = bsize * gctx->funcs[i].num_blocks; /*covered*/

                        if (nbytes < func_bytes) /*covered*/
                                continue;

                        /* Process multi-block batch */
                        do {
                                gctx->funcs[i].fn_u.ecb(ctx, dst, src); /*covered*/
                                src += func_bytes;
                                dst += func_bytes;
                                nbytes -= func_bytes;
                        } while (nbytes >= func_bytes);

                        if (nbytes < bsize) /*covered*/
                                break;
                }
                err = skcipher_walk_done(&walk, nbytes); /*covered*/
        }

        glue_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_ecb_req_128bit);

int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn,
                                struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                const u128 *src = (u128 *)walk.src.virt.addr; /*covered*/
                u128 *dst = (u128 *)walk.dst.virt.addr;
                u128 *iv = (u128 *)walk.iv;

                do {
                        u128_xor(dst, src, iv); /*covered*/
                        fn(ctx, (u8 *)dst, (u8 *)dst);
                        iv = dst;
                        src++;
                        dst++;
                        nbytes -= bsize;
                } while (nbytes >= bsize);

                *(u128 *)walk.iv = *iv; /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }
        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_cbc_encrypt_req_128bit);

int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx,
                                struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes)) {
                const u128 *src = walk.src.virt.addr; /*covered*/
                u128 *dst = walk.dst.virt.addr;
                unsigned int func_bytes, num_blocks;
                unsigned int i;
                u128 last_iv;

                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                             &walk, fpu_enabled, nbytes);
                /* Start of the last block. */
                src += nbytes / bsize - 1; /*covered*/
                dst += nbytes / bsize - 1;

                last_iv = *src;

                for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                        num_blocks = gctx->funcs[i].num_blocks; /*covered*/
                        func_bytes = bsize * num_blocks;

                        if (nbytes < func_bytes)
                                continue;

                        /* Process multi-block batch */
                        do {
                                src -= num_blocks - 1; /*covered*/
                                dst -= num_blocks - 1;

                                gctx->funcs[i].fn_u.cbc(ctx, dst, src);

                                nbytes -= func_bytes;
                                if (nbytes < bsize)
                                        goto done;

                                u128_xor(dst, dst, --src); /*covered*/
                                dst--;
                        } while (nbytes >= func_bytes);
                }
done: /*covered*/
                u128_xor(dst, dst, (u128 *)walk.iv);
                *(u128 *)walk.iv = last_iv;
                err = skcipher_walk_done(&walk, nbytes);
        }

        glue_fpu_end(fpu_enabled); /*covered*/
        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_cbc_decrypt_req_128bit);

int glue_ctr_req_128bit(const struct common_glue_ctx *gctx,
                        struct skcipher_request *req)
{
        void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); /*covered*/
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk;
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);

        while ((nbytes = walk.nbytes) >= bsize) {
                const u128 *src = walk.src.virt.addr; /*covered*/
                u128 *dst = walk.dst.virt.addr;
                unsigned int func_bytes, num_blocks;
                unsigned int i;
                le128 ctrblk;

                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                             &walk, fpu_enabled, nbytes);

                be128_to_le128(&ctrblk, (be128 *)walk.iv); /*covered*/

                for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                        num_blocks = gctx->funcs[i].num_blocks; /*covered*/
                        func_bytes = bsize * num_blocks;

                        if (nbytes < func_bytes)
                                continue;

                        /* Process multi-block batch */
                        do {
                                gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk); /*covered*/
                                src += num_blocks; /*covered*/
                                dst += num_blocks;
                                nbytes -= func_bytes;
                        } while (nbytes >= func_bytes);

                        if (nbytes < bsize) /*covered*/
                                break;
                }

                le128_to_be128((be128 *)walk.iv, &ctrblk); /*covered*/
                err = skcipher_walk_done(&walk, nbytes);
        }

        glue_fpu_end(fpu_enabled); /*covered*/

        if (nbytes) { /*covered*/
                le128 ctrblk; /*covered*/
                u128 tmp;

                be128_to_le128(&ctrblk, (be128 *)walk.iv);
                memcpy(&tmp, walk.src.virt.addr, nbytes);
                gctx->funcs[gctx->num_funcs - 1].fn_u.ctr(ctx, &tmp, &tmp,
                                                          &ctrblk);
                memcpy(walk.dst.virt.addr, &tmp, nbytes);
                le128_to_be128((be128 *)walk.iv, &ctrblk);

                err = skcipher_walk_done(&walk, 0);
        }

        return err; /*covered*/
}
EXPORT_SYMBOL_GPL(glue_ctr_req_128bit);

static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx,
                                          void *ctx,
                                          struct skcipher_walk *walk)
{
        const unsigned int bsize = 128 / 8;
        unsigned int nbytes = walk->nbytes;
        u128 *src = walk->src.virt.addr; /*covered*/
        u128 *dst = walk->dst.virt.addr;
        unsigned int num_blocks, func_bytes;
        unsigned int i;

        /* Process multi-block batch */
        for (i = 0; i < gctx->num_funcs; i++) { /*covered*/
                num_blocks = gctx->funcs[i].num_blocks; /*covered*/
                func_bytes = bsize * num_blocks;

                if (nbytes >= func_bytes) {
                        do {
                                gctx->funcs[i].fn_u.xts(ctx, dst, src, /*covered*/
                                                        walk->iv);

                                src += num_blocks; /*covered*/
                                dst += num_blocks;
                                nbytes -= func_bytes;
                        } while (nbytes >= func_bytes);

                        if (nbytes < bsize) /*covered*/
                                goto done;
                }
        }

done:
        return nbytes;
}

int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
                        struct skcipher_request *req,
                        common_glue_func_t tweak_fn, void *tweak_ctx,
                        void *crypt_ctx)
{
        const unsigned int bsize = 128 / 8;
        struct skcipher_walk walk; /*covered*/
        bool fpu_enabled = false;
        unsigned int nbytes;
        int err;

        err = skcipher_walk_virt(&walk, req, false);
        nbytes = walk.nbytes;
        if (!nbytes)
                return err;

        /* set minimum length to bsize, for tweak_fn */
        fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, /*covered*/
                                     &walk, fpu_enabled,
                                     nbytes < bsize ? bsize : nbytes);

        /* calculate first value of T */
        tweak_fn(tweak_ctx, walk.iv, walk.iv); /*covered*/

        while (nbytes) {
                nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk); /*covered*/

                err = skcipher_walk_done(&walk, nbytes); /*covered*/
                nbytes = walk.nbytes;
        }

        glue_fpu_end(fpu_enabled); /*covered*/

        return err;
}
EXPORT_SYMBOL_GPL(glue_xts_req_128bit);

void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
                               common_glue_func_t fn)
{
        le128 ivblk = *iv; /*covered*/

        /* generate next IV */
        gf128mul_x_ble(iv, &ivblk);

        /* CC <- T xor C */
        u128_xor(dst, src, (u128 *)&ivblk);

        /* PP <- D(Key2,CC) */
        fn(ctx, (u8 *)dst, (u8 *)dst);

        /* P <- T xor PP */
        u128_xor(dst, dst, (u128 *)&ivblk);
}
EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);

MODULE_LICENSE("GPL");

/*
 * The MORUS-1280 Authenticated-Encryption Algorithm
 *   Glue for AVX2 implementation
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/internal/aead.h>
#include <crypto/morus1280_glue.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

asmlinkage void crypto_morus1280_avx2_init(void *state, const void *key,
                                           const void *iv);
asmlinkage void crypto_morus1280_avx2_ad(void *state, const void *data,
                                         unsigned int length);

asmlinkage void crypto_morus1280_avx2_enc(void *state, const void *src,
                                          void *dst, unsigned int length);
asmlinkage void crypto_morus1280_avx2_dec(void *state, const void *src,
                                          void *dst, unsigned int length);

asmlinkage void crypto_morus1280_avx2_enc_tail(void *state, const void *src,
                                               void *dst, unsigned int length);
asmlinkage void crypto_morus1280_avx2_dec_tail(void *state, const void *src,
                                               void *dst, unsigned int length);

asmlinkage void crypto_morus1280_avx2_final(void *state, void *tag_xor,
                                            u64 assoclen, u64 cryptlen);

MORUS1280_DECLARE_ALGS(avx2, "morus1280-avx2", 400); /*covered*/

static int __init crypto_morus1280_avx2_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
            !boot_cpu_has(X86_FEATURE_OSXSAVE) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_morus1280_avx2_algs,
                                     ARRAY_SIZE(crypto_morus1280_avx2_algs));
}

static void __exit crypto_morus1280_avx2_module_exit(void)
{
        crypto_unregister_aeads(crypto_morus1280_avx2_algs,
                                ARRAY_SIZE(crypto_morus1280_avx2_algs));
}

module_init(crypto_morus1280_avx2_module_init);
module_exit(crypto_morus1280_avx2_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- AVX2 implementation");
MODULE_ALIAS_CRYPTO("morus1280");
MODULE_ALIAS_CRYPTO("morus1280-avx2");

/*
 * The MORUS-1280 Authenticated-Encryption Algorithm
 *   Glue for SSE2 implementation
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/internal/aead.h>
#include <crypto/morus1280_glue.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

asmlinkage void crypto_morus1280_sse2_init(void *state, const void *key,
                                           const void *iv);
asmlinkage void crypto_morus1280_sse2_ad(void *state, const void *data,
                                         unsigned int length);

asmlinkage void crypto_morus1280_sse2_enc(void *state, const void *src,
                                          void *dst, unsigned int length);
asmlinkage void crypto_morus1280_sse2_dec(void *state, const void *src,
                                          void *dst, unsigned int length);

asmlinkage void crypto_morus1280_sse2_enc_tail(void *state, const void *src,
                                               void *dst, unsigned int length);
asmlinkage void crypto_morus1280_sse2_dec_tail(void *state, const void *src,
                                               void *dst, unsigned int length);

asmlinkage void crypto_morus1280_sse2_final(void *state, void *tag_xor,
                                            u64 assoclen, u64 cryptlen);

MORUS1280_DECLARE_ALGS(sse2, "morus1280-sse2", 350); /*covered*/

static int __init crypto_morus1280_sse2_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_morus1280_sse2_algs,
                                     ARRAY_SIZE(crypto_morus1280_sse2_algs));
}

static void __exit crypto_morus1280_sse2_module_exit(void)
{
        crypto_unregister_aeads(crypto_morus1280_sse2_algs,
                                ARRAY_SIZE(crypto_morus1280_sse2_algs));
}

module_init(crypto_morus1280_sse2_module_init);
module_exit(crypto_morus1280_sse2_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-1280 AEAD algorithm -- SSE2 implementation");
MODULE_ALIAS_CRYPTO("morus1280");
MODULE_ALIAS_CRYPTO("morus1280-sse2");

/*
 * The MORUS-1280 Authenticated-Encryption Algorithm
 *   Common x86 SIMD glue skeleton
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/morus1280_glue.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <asm/fpu/api.h>

struct morus1280_state {
        struct morus1280_block s[MORUS_STATE_BLOCKS];
};

struct morus1280_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, const void *src, void *dst,
                             unsigned int length);
        void (*crypt_tail)(void *state, const void *src, void *dst,
                           unsigned int length);
};

static void crypto_morus1280_glue_process_ad(
                struct morus1280_state *state,
                const struct morus1280_glue_ops *ops,
                struct scatterlist *sg_src, unsigned int assoclen)
{
        struct scatter_walk walk;
        struct morus1280_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) {
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= MORUS1280_BLOCK_SIZE) {
                        if (pos > 0) { /*covered*/
                                unsigned int fill = MORUS1280_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        ops->ad(state, src, left); /*covered*/
                        src += left & ~(MORUS1280_BLOCK_SIZE - 1);
                        left &= MORUS1280_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/

                pos += left;
                assoclen -= size;
                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, MORUS1280_BLOCK_SIZE - pos);
                ops->ad(state, buf.bytes, MORUS1280_BLOCK_SIZE); /*covered*/
        }
}

static void crypto_morus1280_glue_process_crypt(struct morus1280_state *state,
                                                struct morus1280_ops ops,
                                                struct aead_request *req)
{
        struct skcipher_walk walk; /*covered*/
        u8 *cursor_src, *cursor_dst;
        unsigned int chunksize, base;

        ops.skcipher_walk_init(&walk, req, false);

        while (walk.nbytes) { /*covered*/
                cursor_src = walk.src.virt.addr; /*covered*/
                cursor_dst = walk.dst.virt.addr;
                chunksize = walk.nbytes;

                ops.crypt_blocks(state, cursor_src, cursor_dst, chunksize);

                base = chunksize & ~(MORUS1280_BLOCK_SIZE - 1); /*covered*/
                cursor_src += base;
                cursor_dst += base;
                chunksize &= MORUS1280_BLOCK_SIZE - 1;

                if (chunksize > 0)
                        ops.crypt_tail(state, cursor_src, cursor_dst,
                                       chunksize);

                skcipher_walk_done(&walk, 0); /*covered*/
        }
}

int crypto_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                 unsigned int keylen)
{
        struct morus1280_ctx *ctx = crypto_aead_ctx(aead);

        if (keylen == MORUS1280_BLOCK_SIZE) { /*covered*/
                memcpy(ctx->key.bytes, key, MORUS1280_BLOCK_SIZE);
        } else if (keylen == MORUS1280_BLOCK_SIZE / 2) { /*covered*/
                memcpy(ctx->key.bytes, key, keylen);
                memcpy(ctx->key.bytes + keylen, key, keylen);
        } else {
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL; /*covered*/
        }

        return 0;
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setkey);

int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm,
                                      unsigned int authsize)
{
        return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_setauthsize);

static void crypto_morus1280_glue_crypt(struct aead_request *req,
                                        struct morus1280_ops ops,
                                        unsigned int cryptlen,
                                        struct morus1280_block *tag_xor)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus1280_state state;

        kernel_fpu_begin();

        ctx->ops->init(&state, &ctx->key, req->iv);
        crypto_morus1280_glue_process_ad(&state, ctx->ops, req->src, req->assoclen); /*covered*/
        crypto_morus1280_glue_process_crypt(&state, ops, req);
        ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen);

        kernel_fpu_end();
}

int crypto_morus1280_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus1280_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = ctx->ops->enc,
                .crypt_tail = ctx->ops->enc_tail,
        };

        struct morus1280_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_encrypt);

int crypto_morus1280_glue_decrypt(struct aead_request *req)
{
        static const u8 zeros[MORUS1280_BLOCK_SIZE] = {};

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus1280_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus1280_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = ctx->ops->dec,
                .crypt_tail = ctx->ops->dec_tail,
        };

        struct morus1280_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_morus1280_glue_crypt(req, OPS, cryptlen, &tag);

        return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_decrypt);

void crypto_morus1280_glue_init_ops(struct crypto_aead *aead,
                                    const struct morus1280_glue_ops *ops)
{
        struct morus1280_ctx *ctx = crypto_aead_ctx(aead);
        ctx->ops = ops; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus1280_glue_init_ops);

int cryptd_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                 unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setkey);

int cryptd_morus1280_glue_setauthsize(struct crypto_aead *aead,
                                      unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_setauthsize);

int cryptd_morus1280_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_encrypt);

int cryptd_morus1280_glue_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_decrypt);

int cryptd_morus1280_glue_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        const char *name = crypto_aead_alg(aead)->base.cra_driver_name; /*covered*/
        char internal_name[CRYPTO_MAX_ALG_NAME];

        if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name)
                        >= CRYPTO_MAX_ALG_NAME)
                return -ENAMETOOLONG;

        cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_init_tfm);

void cryptd_morus1280_glue_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus1280_glue_exit_tfm);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-1280 AEAD mode -- glue for x86 optimizations");

/*
 * The MORUS-640 Authenticated-Encryption Algorithm
 *   Glue for SSE2 implementation
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/internal/aead.h>
#include <crypto/morus640_glue.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/cpu_device_id.h>

asmlinkage void crypto_morus640_sse2_init(void *state, const void *key,
                                          const void *iv);
asmlinkage void crypto_morus640_sse2_ad(void *state, const void *data,
                                        unsigned int length);

asmlinkage void crypto_morus640_sse2_enc(void *state, const void *src,
                                         void *dst, unsigned int length);
asmlinkage void crypto_morus640_sse2_dec(void *state, const void *src,
                                         void *dst, unsigned int length);

asmlinkage void crypto_morus640_sse2_enc_tail(void *state, const void *src,
                                              void *dst, unsigned int length);
asmlinkage void crypto_morus640_sse2_dec_tail(void *state, const void *src,
                                              void *dst, unsigned int length);

asmlinkage void crypto_morus640_sse2_final(void *state, void *tag_xor,
                                           u64 assoclen, u64 cryptlen);

MORUS640_DECLARE_ALGS(sse2, "morus640-sse2", 400); /*covered*/

static int __init crypto_morus640_sse2_module_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2) ||
            !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL))
                return -ENODEV;

        return crypto_register_aeads(crypto_morus640_sse2_algs,
                                     ARRAY_SIZE(crypto_morus640_sse2_algs));
}

static void __exit crypto_morus640_sse2_module_exit(void)
{
        crypto_unregister_aeads(crypto_morus640_sse2_algs,
                                ARRAY_SIZE(crypto_morus640_sse2_algs));
}

module_init(crypto_morus640_sse2_module_init);
module_exit(crypto_morus640_sse2_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-640 AEAD algorithm -- SSE2 implementation");
MODULE_ALIAS_CRYPTO("morus640");
MODULE_ALIAS_CRYPTO("morus640-sse2");

/*
 * The MORUS-640 Authenticated-Encryption Algorithm
 *   Common x86 SIMD glue skeleton
 *
 * Copyright (c) 2016-2018 Ondrej Mosnacek <omosnacek@gmail.com>
 * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 */

#include <crypto/cryptd.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/skcipher.h>
#include <crypto/morus640_glue.h>
#include <crypto/scatterwalk.h>
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <asm/fpu/api.h>

struct morus640_state {
        struct morus640_block s[MORUS_STATE_BLOCKS];
};

struct morus640_ops {
        int (*skcipher_walk_init)(struct skcipher_walk *walk,
                                  struct aead_request *req, bool atomic);

        void (*crypt_blocks)(void *state, const void *src, void *dst,
                             unsigned int length);
        void (*crypt_tail)(void *state, const void *src, void *dst,
                           unsigned int length);
};

static void crypto_morus640_glue_process_ad(
                struct morus640_state *state,
                const struct morus640_glue_ops *ops,
                struct scatterlist *sg_src, unsigned int assoclen)
{
        struct scatter_walk walk;
        struct morus640_block buf;
        unsigned int pos = 0;

        scatterwalk_start(&walk, sg_src);
        while (assoclen != 0) {
                unsigned int size = scatterwalk_clamp(&walk, assoclen); /*covered*/
                unsigned int left = size;
                void *mapped = scatterwalk_map(&walk); /*covered*/
                const u8 *src = (const u8 *)mapped;

                if (pos + size >= MORUS640_BLOCK_SIZE) { /*covered*/
                        if (pos > 0) { /*covered*/
                                unsigned int fill = MORUS640_BLOCK_SIZE - pos; /*covered*/
                                memcpy(buf.bytes + pos, src, fill);
                                ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE);
                                pos = 0;
                                left -= fill;
                                src += fill;
                        }

                        ops->ad(state, src, left); /*covered*/
                        src += left & ~(MORUS640_BLOCK_SIZE - 1);
                        left &= MORUS640_BLOCK_SIZE - 1;
                }

                memcpy(buf.bytes + pos, src, left); /*covered*/

                pos += left;
                assoclen -= size;
                scatterwalk_unmap(mapped);
                scatterwalk_advance(&walk, size); /*covered*/
                scatterwalk_done(&walk, 0, assoclen); /*covered*/
        }

        if (pos > 0) { /*covered*/
                memset(buf.bytes + pos, 0, MORUS640_BLOCK_SIZE - pos); /*covered*/
                ops->ad(state, buf.bytes, MORUS640_BLOCK_SIZE); /*covered*/
        }
}

static void crypto_morus640_glue_process_crypt(struct morus640_state *state,
                                               struct morus640_ops ops,
                                               struct aead_request *req)
{
        struct skcipher_walk walk; /*covered*/
        u8 *cursor_src, *cursor_dst;
        unsigned int chunksize, base;

        ops.skcipher_walk_init(&walk, req, false);

        while (walk.nbytes) { /*covered*/
                cursor_src = walk.src.virt.addr; /*covered*/
                cursor_dst = walk.dst.virt.addr;
                chunksize = walk.nbytes;

                ops.crypt_blocks(state, cursor_src, cursor_dst, chunksize);

                base = chunksize & ~(MORUS640_BLOCK_SIZE - 1); /*covered*/
                cursor_src += base;
                cursor_dst += base;
                chunksize &= MORUS640_BLOCK_SIZE - 1;

                if (chunksize > 0)
                        ops.crypt_tail(state, cursor_src, cursor_dst,
                                       chunksize);

                skcipher_walk_done(&walk, 0); /*covered*/
        }
}

int crypto_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                unsigned int keylen)
{
        struct morus640_ctx *ctx = crypto_aead_ctx(aead);

        if (keylen != MORUS640_BLOCK_SIZE) { /*covered*/
                crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); /*covered*/
                return -EINVAL;
        }

        memcpy(ctx->key.bytes, key, MORUS640_BLOCK_SIZE);
        return 0; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_setkey);

int crypto_morus640_glue_setauthsize(struct crypto_aead *tfm,
                                     unsigned int authsize)
{
        return (authsize <= MORUS_MAX_AUTH_SIZE) ? 0 : -EINVAL; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_setauthsize);

static void crypto_morus640_glue_crypt(struct aead_request *req,
                                       struct morus640_ops ops,
                                       unsigned int cryptlen,
                                       struct morus640_block *tag_xor)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus640_state state;

        kernel_fpu_begin();

        ctx->ops->init(&state, &ctx->key, req->iv);
        crypto_morus640_glue_process_ad(&state, ctx->ops, req->src, req->assoclen); /*covered*/
        crypto_morus640_glue_process_crypt(&state, ops, req);
        ctx->ops->final(&state, tag_xor, req->assoclen, cryptlen);

        kernel_fpu_end();
}

int crypto_morus640_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus640_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_encrypt,
                .crypt_blocks = ctx->ops->enc,
                .crypt_tail = ctx->ops->enc_tail,
        };

        struct morus640_block tag = {};
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen;

        crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);

        scatterwalk_map_and_copy(tag.bytes, req->dst,
                                 req->assoclen + cryptlen, authsize, 1);
        return 0;
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_encrypt);

int crypto_morus640_glue_decrypt(struct aead_request *req)
{
        static const u8 zeros[MORUS640_BLOCK_SIZE] = {};

        struct crypto_aead *tfm = crypto_aead_reqtfm(req); /*covered*/
        struct morus640_ctx *ctx = crypto_aead_ctx(tfm);
        struct morus640_ops OPS = {
                .skcipher_walk_init = skcipher_walk_aead_decrypt,
                .crypt_blocks = ctx->ops->dec,
                .crypt_tail = ctx->ops->dec_tail,
        };

        struct morus640_block tag;
        unsigned int authsize = crypto_aead_authsize(tfm);
        unsigned int cryptlen = req->cryptlen - authsize;

        scatterwalk_map_and_copy(tag.bytes, req->src,
                                 req->assoclen + cryptlen, authsize, 0);

        crypto_morus640_glue_crypt(req, OPS, cryptlen, &tag);

        return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_decrypt);

void crypto_morus640_glue_init_ops(struct crypto_aead *aead,
                                   const struct morus640_glue_ops *ops)
{
        struct morus640_ctx *ctx = crypto_aead_ctx(aead);
        ctx->ops = ops; /*covered*/
}
EXPORT_SYMBOL_GPL(crypto_morus640_glue_init_ops);

int cryptd_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key,
                                unsigned int keylen)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setkey(&cryptd_tfm->base, key, keylen);
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setkey);

int cryptd_morus640_glue_setauthsize(struct crypto_aead *aead,
                                     unsigned int authsize)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx; /*covered*/

        return crypto_aead_setauthsize(&cryptd_tfm->base, authsize);
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_setauthsize);

int cryptd_morus640_glue_encrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_encrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_encrypt);

int cryptd_morus640_glue_decrypt(struct aead_request *req)
{
        struct crypto_aead *aead = crypto_aead_reqtfm(req); /*covered*/
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        struct cryptd_aead *cryptd_tfm = *ctx;

        aead = &cryptd_tfm->base;
        if (irq_fpu_usable() && (!in_atomic() || /*covered*/
                                 !cryptd_aead_queued(cryptd_tfm)))
                aead = cryptd_aead_child(cryptd_tfm); /*covered*/

        aead_request_set_tfm(req, aead); /*covered*/

        return crypto_aead_decrypt(req); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_decrypt);

int cryptd_morus640_glue_init_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead *cryptd_tfm;
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);
        const char *name = crypto_aead_alg(aead)->base.cra_driver_name; /*covered*/
        char internal_name[CRYPTO_MAX_ALG_NAME];

        if (snprintf(internal_name, CRYPTO_MAX_ALG_NAME, "__%s", name)
                        >= CRYPTO_MAX_ALG_NAME)
                return -ENAMETOOLONG;

        cryptd_tfm = cryptd_alloc_aead(internal_name, CRYPTO_ALG_INTERNAL, /*covered*/
                                       CRYPTO_ALG_INTERNAL);
        if (IS_ERR(cryptd_tfm))
                return PTR_ERR(cryptd_tfm);

        *ctx = cryptd_tfm; /*covered*/
        crypto_aead_set_reqsize(aead, crypto_aead_reqsize(&cryptd_tfm->base));
        return 0; /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_init_tfm);

void cryptd_morus640_glue_exit_tfm(struct crypto_aead *aead)
{
        struct cryptd_aead **ctx = crypto_aead_ctx(aead);

        cryptd_free_aead(*ctx); /*covered*/
}
EXPORT_SYMBOL_GPL(cryptd_morus640_glue_exit_tfm);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>");
MODULE_DESCRIPTION("MORUS-640 AEAD mode -- glue for x86 optimizations");

/*
 * Poly1305 authenticator algorithm, RFC7539, SIMD glue code
 *
 * Copyright (C) 2015 Martin Willi
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <crypto/algapi.h>
#include <crypto/internal/hash.h>
#include <crypto/poly1305.h>
#include <linux/crypto.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <asm/fpu/api.h>
#include <asm/simd.h>

struct poly1305_simd_desc_ctx {
        struct poly1305_desc_ctx base;
        /* derived key u set? */
        bool uset;
#ifdef CONFIG_AS_AVX2
        /* derived keys r^3, r^4 set? */
        bool wset;
#endif
        /* derived Poly1305 key r^2 */
        u32 u[5];
        /* ... silently appended r^3 and r^4 when using AVX2 */
};

asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src,
                                    const u32 *r, unsigned int blocks);
asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r,
                                     unsigned int blocks, const u32 *u);
#ifdef CONFIG_AS_AVX2
asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r,
                                     unsigned int blocks, const u32 *u);
static bool poly1305_use_avx2;
#endif

static int poly1305_simd_init(struct shash_desc *desc)
{
        struct poly1305_simd_desc_ctx *sctx = shash_desc_ctx(desc);

        sctx->uset = false; /*covered*/
#ifdef CONFIG_AS_AVX2
        sctx->wset = false;
#endif

        return crypto_poly1305_init(desc);
}

static void poly1305_simd_mult(u32 *a, const u32 *b)
{
        u8 m[POLY1305_BLOCK_SIZE]; /*covered*/

        memset(m, 0, sizeof(m));
        /* The poly1305 block function adds a hi-bit to the accumulator which
         * we don't need for key multiplication; compensate for it. */
        a[4] -= 1 << 24;
        poly1305_block_sse2(a, m, b, 1);
}

static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx,
                                         const u8 *src, unsigned int srclen)
{
        struct poly1305_simd_desc_ctx *sctx;
        unsigned int blocks, datalen;

        BUILD_BUG_ON(offsetof(struct poly1305_simd_desc_ctx, base));
        sctx = container_of(dctx, struct poly1305_simd_desc_ctx, base);

        if (unlikely(!dctx->sset)) { /*covered*/
                datalen = crypto_poly1305_setdesckey(dctx, src, srclen); /*covered*/
                src += srclen - datalen;
                srclen = datalen;
        }

#ifdef CONFIG_AS_AVX2
        if (poly1305_use_avx2 && srclen >= POLY1305_BLOCK_SIZE * 4) { /*covered*/
                if (unlikely(!sctx->wset)) { /*covered*/
                        if (!sctx->uset) { /*covered*/
                                memcpy(sctx->u, dctx->r, sizeof(sctx->u)); /*covered*/
                                poly1305_simd_mult(sctx->u, dctx->r);
                                sctx->uset = true;
                        }
                        memcpy(sctx->u + 5, sctx->u, sizeof(sctx->u)); /*covered*/
                        poly1305_simd_mult(sctx->u + 5, dctx->r);
                        memcpy(sctx->u + 10, sctx->u + 5, sizeof(sctx->u));
                        poly1305_simd_mult(sctx->u + 10, dctx->r);
                        sctx->wset = true;
                }
                blocks = srclen / (POLY1305_BLOCK_SIZE * 4); /*covered*/
                poly1305_4block_avx2(dctx->h, src, dctx->r, blocks, sctx->u);
                src += POLY1305_BLOCK_SIZE * 4 * blocks;
                srclen -= POLY1305_BLOCK_SIZE * 4 * blocks;
        }
#endif
        if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) { /*covered*/
                if (unlikely(!sctx->uset)) { /*covered*/
                        memcpy(sctx->u, dctx->r, sizeof(sctx->u));
                        poly1305_simd_mult(sctx->u, dctx->r);
                        sctx->uset = true;
                }
                blocks = srclen / (POLY1305_BLOCK_SIZE * 2); /*covered*/
                poly1305_2block_sse2(dctx->h, src, dctx->r, blocks, sctx->u);
                src += POLY1305_BLOCK_SIZE * 2 * blocks;
                srclen -= POLY1305_BLOCK_SIZE * 2 * blocks;
        }
        if (srclen >= POLY1305_BLOCK_SIZE) { /*covered*/
                poly1305_block_sse2(dctx->h, src, dctx->r, 1); /*covered*/
                srclen -= POLY1305_BLOCK_SIZE;
        }
        return srclen; /*covered*/
}

static int poly1305_simd_update(struct shash_desc *desc,
                                const u8 *src, unsigned int srclen)
{
        struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); /*covered*/
        unsigned int bytes;

        /* kernel_fpu_begin/end is costly, use fallback for small updates */
        if (srclen <= 288 || !may_use_simd()) /*covered*/
                return crypto_poly1305_update(desc, src, srclen); /*covered*/

        kernel_fpu_begin();

        if (unlikely(dctx->buflen)) {
                bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); /*covered*/
                memcpy(dctx->buf + dctx->buflen, src, bytes);
                src += bytes;
                srclen -= bytes;
                dctx->buflen += bytes;

                if (dctx->buflen == POLY1305_BLOCK_SIZE) {
                        poly1305_simd_blocks(dctx, dctx->buf, /*covered*/
                                             POLY1305_BLOCK_SIZE);
                        dctx->buflen = 0;
                }
        }

        if (likely(srclen >= POLY1305_BLOCK_SIZE)) { /*covered*/
                bytes = poly1305_simd_blocks(dctx, src, srclen); /*covered*/
                src += srclen - bytes;
                srclen = bytes;
        }

        kernel_fpu_end(); /*covered*/

        if (unlikely(srclen)) {
                dctx->buflen = srclen; /*covered*/
                memcpy(dctx->buf, src, srclen);
        }

        return 0;
}

static struct shash_alg alg = {
        .digestsize        = POLY1305_DIGEST_SIZE,
        .init                = poly1305_simd_init,
        .update                = poly1305_simd_update,
        .final                = crypto_poly1305_final,
        .descsize        = sizeof(struct poly1305_simd_desc_ctx),
        .base                = {
                .cra_name                = "poly1305",
                .cra_driver_name        = "poly1305-simd",
                .cra_priority                = 300,
                .cra_blocksize                = POLY1305_BLOCK_SIZE,
                .cra_module                = THIS_MODULE,
        },
};

static int __init poly1305_simd_mod_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2))
                return -ENODEV;

#ifdef CONFIG_AS_AVX2
        poly1305_use_avx2 = boot_cpu_has(X86_FEATURE_AVX) &&
                            boot_cpu_has(X86_FEATURE_AVX2) &&
                            cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL);
        alg.descsize = sizeof(struct poly1305_simd_desc_ctx);
        if (poly1305_use_avx2)
                alg.descsize += 10 * sizeof(u32);
#endif
        return crypto_register_shash(&alg);
}

static void __exit poly1305_simd_mod_exit(void)
{
        crypto_unregister_shash(&alg);
}

module_init(poly1305_simd_mod_init);
module_exit(poly1305_simd_mod_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
MODULE_DESCRIPTION("Poly1305 authenticator");
MODULE_ALIAS_CRYPTO("poly1305");
MODULE_ALIAS_CRYPTO("poly1305-simd");

/*
 * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
 *
 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/serpent.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <asm/crypto/serpent-avx.h>

#define SERPENT_AVX2_PARALLEL_BLOCKS 16

/* 16-way AVX2 parallel cipher functions */
asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src);
asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src);
asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);

asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
                                  le128 *iv);
asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src, le128 *iv);
asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
                                      const u8 *src, le128 *iv);

static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); /*covered*/
}

static const struct common_glue_ctx serpent_enc = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
        } }
};

static const struct common_glue_ctx serpent_ctr = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
        },  {
                .num_blocks = 8,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
        } }
};

static const struct common_glue_ctx serpent_enc_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
        } }
};

static const struct common_glue_ctx serpent_dec = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_cbc = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_xts = {
        .num_funcs = 3,
        .fpu_blocks_limit = 8,

        .funcs = { {
                .num_blocks = 16,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
        }, {
                .num_blocks = 8,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&serpent_ctr, req); /*covered*/
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_enc_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); /*covered*/
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_dec_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg serpent_algs[] = {
        {
                .base.cra_name                = "__ecb(serpent)",
                .base.cra_driver_name        = "__ecb-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(serpent)",
                .base.cra_driver_name        = "__cbc-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(serpent)",
                .base.cra_driver_name        = "__ctr-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .chunksize                = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(serpent)",
                .base.cra_driver_name        = "__xts-serpent-avx2",
                .base.cra_priority        = 600,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * SERPENT_MIN_KEY_SIZE,
                .max_keysize                = 2 * SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = xts_serpent_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];

static int __init init(void)
{
        const char *feature_name;

        if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
                pr_info("AVX2 instructions are not detected.\n");
                return -ENODEV;
        }
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(serpent_algs,
                                              ARRAY_SIZE(serpent_algs),
                                              serpent_simd_algs);
}

static void __exit fini(void)
{
        simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
                                  serpent_simd_algs);
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
MODULE_ALIAS_CRYPTO("serpent");
MODULE_ALIAS_CRYPTO("serpent-asm");

/*
 * Glue Code for AVX assembler versions of Serpent Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2011-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/serpent.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <asm/crypto/serpent-avx.h>

/* 8-way parallel cipher functions */
asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src);
EXPORT_SYMBOL_GPL(serpent_ecb_enc_8way_avx);

asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src);
EXPORT_SYMBOL_GPL(serpent_ecb_dec_8way_avx);

asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src);
EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx);

asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                     const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx);

asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx);

asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
                                         const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx);

void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk; /*covered*/

        le128_to_be128(&ctrblk, iv);
        le128_inc(iv); /*covered*/

        __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
}
EXPORT_SYMBOL_GPL(__serpent_crypt_ctr);

void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, /*covered*/
                                  GLUE_FUNC_CAST(__serpent_encrypt));
}
EXPORT_SYMBOL_GPL(serpent_xts_enc);

void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv, /*covered*/
                                  GLUE_FUNC_CAST(__serpent_decrypt));
}
EXPORT_SYMBOL_GPL(serpent_xts_dec);

static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); /*covered*/
}

int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key,
                       unsigned int keylen)
{
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        /* first half of xts-key is for crypt */
        err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); /*covered*/
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); /*covered*/
}
EXPORT_SYMBOL_GPL(xts_serpent_setkey);

static const struct common_glue_ctx serpent_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
        } }
};

static const struct common_glue_ctx serpent_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
        } }
};

static const struct common_glue_ctx serpent_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
        } }
};

static const struct common_glue_ctx serpent_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&serpent_ctr, req);
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_enc_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&serpent_dec_xts, req,
                                   XTS_TWEAK_CAST(__serpent_encrypt),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg serpent_algs[] = {
        {
                .base.cra_name                = "__ecb(serpent)",
                .base.cra_driver_name        = "__ecb-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(serpent)",
                .base.cra_driver_name        = "__cbc-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(serpent)",
                .base.cra_driver_name        = "__ctr-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .chunksize                = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(serpent)",
                .base.cra_driver_name        = "__xts-serpent-avx",
                .base.cra_priority        = 500,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * SERPENT_MIN_KEY_SIZE,
                .max_keysize                = 2 * SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = xts_serpent_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];

static int __init serpent_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
                                &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(serpent_algs,
                                              ARRAY_SIZE(serpent_algs),
                                              serpent_simd_algs);
}

static void __exit serpent_exit(void)
{
        simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
                                  serpent_simd_algs);
}

module_init(serpent_init);
module_exit(serpent_exit);

MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("serpent");

/*
 * Glue Code for SSE2 assembler versions of Serpent Cipher
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * Glue code based on aesni-intel_glue.c by:
 *  Copyright (C) 2008, Intel Corp.
 *    Author: Huang Ying <ying.huang@intel.com>
 *
 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
 * CTR part based on code (crypto/ctr.c) by:
 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/b128ops.h>
#include <crypto/internal/simd.h>
#include <crypto/serpent.h>
#include <asm/crypto/serpent-sse2.h>
#include <asm/crypto/glue_helper.h>

static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); /*covered*/
}

static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
{
        u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
        unsigned int j;

        for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
                ivs[j] = src[j];

        serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);

        for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
                u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
}

static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk;

        le128_to_be128(&ctrblk, iv);
        le128_inc(iv);

        __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, src, (u128 *)&ctrblk);
}

static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
                                   le128 *iv)
{
        be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
        unsigned int i;

        for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
                if (dst != src)
                        dst[i] = src[i];

                le128_to_be128(&ctrblks[i], iv);
                le128_inc(iv);
        }

        serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
}

static const struct common_glue_ctx serpent_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
        } }
};

static const struct common_glue_ctx serpent_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
        } }
};

static const struct common_glue_ctx serpent_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
        } }
};

static const struct common_glue_ctx serpent_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = SERPENT_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&serpent_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&serpent_ctr, req);
}

static struct skcipher_alg serpent_algs[] = {
        {
                .base.cra_name                = "__ecb(serpent)",
                .base.cra_driver_name        = "__ecb-serpent-sse2",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(serpent)",
                .base.cra_driver_name        = "__cbc-serpent-sse2",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = SERPENT_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(serpent)",
                .base.cra_driver_name        = "__ctr-serpent-sse2",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct serpent_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = SERPENT_MIN_KEY_SIZE,
                .max_keysize                = SERPENT_MAX_KEY_SIZE,
                .ivsize                        = SERPENT_BLOCK_SIZE,
                .chunksize                = SERPENT_BLOCK_SIZE,
                .setkey                        = serpent_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        },
};

static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];

static int __init serpent_sse2_init(void)
{
        if (!boot_cpu_has(X86_FEATURE_XMM2)) {
                printk(KERN_INFO "SSE2 instructions are not detected.\n");
                return -ENODEV;
        }

        return simd_register_skciphers_compat(serpent_algs,
                                              ARRAY_SIZE(serpent_algs),
                                              serpent_simd_algs);
}

static void __exit serpent_sse2_exit(void)
{
        simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
                                  serpent_simd_algs);
}

module_init(serpent_sse2_init);
module_exit(serpent_sse2_exit);

MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("serpent");

/*
 * Multi buffer SHA1 algorithm Glue Code
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 *  Copyright(c) 2014 Intel Corporation.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of version 2 of the GNU General Public License as
 *  published by the Free Software Foundation.
 *
 *  This program is distributed in the hope that it will be useful, but
 *  WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  General Public License for more details.
 *
 *  Contact Information:
 *        Tim Chen <tim.c.chen@linux.intel.com>
 *
 *  BSD LICENSE
 *
 *  Copyright(c) 2014 Intel Corporation.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *    * Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in
 *      the documentation and/or other materials provided with the
 *      distribution.
 *    * Neither the name of Intel Corporation nor the names of its
 *      contributors may be used to endorse or promote products derived
 *      from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <linux/list.h>
#include <crypto/scatterwalk.h>
#include <crypto/sha.h>
#include <crypto/mcryptd.h>
#include <crypto/crypto_wq.h>
#include <asm/byteorder.h>
#include <linux/hardirq.h>
#include <asm/fpu/api.h>
#include "sha1_mb_ctx.h"

#define FLUSH_INTERVAL 1000 /* in usec */

static struct mcryptd_alg_state sha1_mb_alg_state;

struct sha1_mb_ctx {
        struct mcryptd_ahash *mcryptd_tfm;
};

static inline struct mcryptd_hash_request_ctx
                *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx)
{
        struct ahash_request *areq;

        areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
        return container_of(areq, struct mcryptd_hash_request_ctx, areq);
}

static inline struct ahash_request
                *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
{
        return container_of((void *) ctx, struct ahash_request, __ctx);
}

static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
                                struct ahash_request *areq)
{
        rctx->flag = HASH_UPDATE;
}

static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state);
static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)
                        (struct sha1_mb_mgr *state, struct job_sha1 *job);
static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)
                                                (struct sha1_mb_mgr *state);
static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)
                                                (struct sha1_mb_mgr *state);

static inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2],
                         uint64_t total_len)
{
        uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1);

        memset(&padblock[i], 0, SHA1_BLOCK_SIZE);
        padblock[i] = 0x80;

        i += ((SHA1_BLOCK_SIZE - 1) &
              (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1)))
             + 1 + SHA1_PADLENGTHFIELD_SIZE;

#if SHA1_PADLENGTHFIELD_SIZE == 16
        *((uint64_t *) &padblock[i - 16]) = 0;
#endif

        *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);

        /* Number of extra blocks to hash */
        return i >> SHA1_LOG2_BLOCK_SIZE;
}

static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr,
                                                struct sha1_hash_ctx *ctx)
{
        while (ctx) {
                if (ctx->status & HASH_CTX_STS_COMPLETE) {
                        /* Clear PROCESSING bit */
                        ctx->status = HASH_CTX_STS_COMPLETE;
                        return ctx;
                }

                /*
                 * If the extra blocks are empty, begin hashing what remains
                 * in the user's buffer.
                 */
                if (ctx->partial_block_buffer_length == 0 &&
                    ctx->incoming_buffer_length) {

                        const void *buffer = ctx->incoming_buffer;
                        uint32_t len = ctx->incoming_buffer_length;
                        uint32_t copy_len;

                        /*
                         * Only entire blocks can be hashed.
                         * Copy remainder to extra blocks buffer.
                         */
                        copy_len = len & (SHA1_BLOCK_SIZE-1);

                        if (copy_len) {
                                len -= copy_len;
                                memcpy(ctx->partial_block_buffer,
                                       ((const char *) buffer + len),
                                       copy_len);
                                ctx->partial_block_buffer_length = copy_len;
                        }

                        ctx->incoming_buffer_length = 0;

                        /* len should be a multiple of the block size now */
                        assert((len % SHA1_BLOCK_SIZE) == 0);

                        /* Set len to the number of blocks to be hashed */
                        len >>= SHA1_LOG2_BLOCK_SIZE;

                        if (len) {

                                ctx->job.buffer = (uint8_t *) buffer;
                                ctx->job.len = len;
                                ctx = (struct sha1_hash_ctx *)sha1_job_mgr_submit(&mgr->mgr,
                                                                                &ctx->job);
                                continue;
                        }
                }

                /*
                 * If the extra blocks are not empty, then we are
                 * either on the last block(s) or we need more
                 * user input before continuing.
                 */
                if (ctx->status & HASH_CTX_STS_LAST) {

                        uint8_t *buf = ctx->partial_block_buffer;
                        uint32_t n_extra_blocks =
                                        sha1_pad(buf, ctx->total_length);

                        ctx->status = (HASH_CTX_STS_PROCESSING |
                                       HASH_CTX_STS_COMPLETE);
                        ctx->job.buffer = buf;
                        ctx->job.len = (uint32_t) n_extra_blocks;
                        ctx = (struct sha1_hash_ctx *)
                                sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
                        continue;
                }

                ctx->status = HASH_CTX_STS_IDLE;
                return ctx;
        }

        return NULL;
}

static struct sha1_hash_ctx
                        *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr)
{
        /*
         * If get_comp_job returns NULL, there are no jobs complete.
         * If get_comp_job returns a job, verify that it is safe to return to
         * the user.
         * If it is not ready, resubmit the job to finish processing.
         * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
         * Otherwise, all jobs currently being managed by the hash_ctx_mgr
         * still need processing.
         */
        struct sha1_hash_ctx *ctx;

        ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr);
        return sha1_ctx_mgr_resubmit(mgr, ctx);
}

static void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr)
{
        sha1_job_mgr_init(&mgr->mgr);
}

static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr,
                                          struct sha1_hash_ctx *ctx,
                                          const void *buffer,
                                          uint32_t len,
                                          int flags)
{
        if (flags & ~(HASH_UPDATE | HASH_LAST)) {
                /* User should not pass anything other than UPDATE or LAST */
                ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
                return ctx;
        }

        if (ctx->status & HASH_CTX_STS_PROCESSING) {
                /* Cannot submit to a currently processing job. */
                ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
                return ctx;
        }

        if (ctx->status & HASH_CTX_STS_COMPLETE) {
                /* Cannot update a finished job. */
                ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
                return ctx;
        }

        /*
         * If we made it here, there were no errors during this call to
         * submit
         */
        ctx->error = HASH_CTX_ERROR_NONE;

        /* Store buffer ptr info from user */
        ctx->incoming_buffer = buffer;
        ctx->incoming_buffer_length = len;

        /*
         * Store the user's request flags and mark this ctx as currently
         * being processed.
         */
        ctx->status = (flags & HASH_LAST) ?
                        (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
                        HASH_CTX_STS_PROCESSING;

        /* Advance byte counter */
        ctx->total_length += len;

        /*
         * If there is anything currently buffered in the extra blocks,
         * append to it until it contains a whole block.
         * Or if the user's buffer contains less than a whole block,
         * append as much as possible to the extra block.
         */
        if (ctx->partial_block_buffer_length || len < SHA1_BLOCK_SIZE) {
                /*
                 * Compute how many bytes to copy from user buffer into
                 * extra block
                 */
                uint32_t copy_len = SHA1_BLOCK_SIZE -
                                        ctx->partial_block_buffer_length;
                if (len < copy_len)
                        copy_len = len;

                if (copy_len) {
                        /* Copy and update relevant pointers and counters */
                        memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
                                buffer, copy_len);

                        ctx->partial_block_buffer_length += copy_len;
                        ctx->incoming_buffer = (const void *)
                                        ((const char *)buffer + copy_len);
                        ctx->incoming_buffer_length = len - copy_len;
                }

                /*
                 * The extra block should never contain more than 1 block
                 * here
                 */
                assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE);

                /*
                 * If the extra block buffer contains exactly 1 block, it can
                 * be hashed.
                 */
                if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) {
                        ctx->partial_block_buffer_length = 0;

                        ctx->job.buffer = ctx->partial_block_buffer;
                        ctx->job.len = 1;
                        ctx = (struct sha1_hash_ctx *)
                                sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
                }
        }

        return sha1_ctx_mgr_resubmit(mgr, ctx);
}

static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr)
{
        struct sha1_hash_ctx *ctx;

        while (1) {
                ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr);

                /* If flush returned 0, there are no more jobs in flight. */
                if (!ctx)
                        return NULL;

                /*
                 * If flush returned a job, resubmit the job to finish
                 * processing.
                 */
                ctx = sha1_ctx_mgr_resubmit(mgr, ctx);

                /*
                 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be
                 * returned. Otherwise, all jobs currently being managed by the
                 * sha1_ctx_mgr still need processing. Loop.
                 */
                if (ctx)
                        return ctx;
        }
}

static int sha1_mb_init(struct ahash_request *areq)
{
        struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);

        hash_ctx_init(sctx);
        sctx->job.result_digest[0] = SHA1_H0;
        sctx->job.result_digest[1] = SHA1_H1;
        sctx->job.result_digest[2] = SHA1_H2;
        sctx->job.result_digest[3] = SHA1_H3;
        sctx->job.result_digest[4] = SHA1_H4;
        sctx->total_length = 0;
        sctx->partial_block_buffer_length = 0;
        sctx->status = HASH_CTX_STS_IDLE;

        return 0;
}

static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
{
        int        i;
        struct        sha1_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
        __be32        *dst = (__be32 *) rctx->out;

        for (i = 0; i < 5; ++i)
                dst[i] = cpu_to_be32(sctx->job.result_digest[i]);

        return 0;
}

static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
                        struct mcryptd_alg_cstate *cstate, bool flush)
{
        int        flag = HASH_UPDATE;
        int        nbytes, err = 0;
        struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
        struct sha1_hash_ctx *sha_ctx;

        /* more work ? */
        while (!(rctx->flag & HASH_DONE)) {
                nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
                if (nbytes < 0) {
                        err = nbytes;
                        goto out;
                }
                /* check if the walk is done */
                if (crypto_ahash_walk_last(&rctx->walk)) {
                        rctx->flag |= HASH_DONE;
                        if (rctx->flag & HASH_FINAL)
                                flag |= HASH_LAST;

                }
                sha_ctx = (struct sha1_hash_ctx *)
                                                ahash_request_ctx(&rctx->areq);
                kernel_fpu_begin();
                sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx,
                                                rctx->walk.data, nbytes, flag);
                if (!sha_ctx) {
                        if (flush)
                                sha_ctx = sha1_ctx_mgr_flush(cstate->mgr);
                }
                kernel_fpu_end();
                if (sha_ctx)
                        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                else {
                        rctx = NULL;
                        goto out;
                }
        }

        /* copy the results */
        if (rctx->flag & HASH_FINAL)
                sha1_mb_set_results(rctx);

out:
        *ret_rctx = rctx;
        return err;
}

static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
                            struct mcryptd_alg_cstate *cstate,
                            int err)
{
        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha1_hash_ctx *sha_ctx;
        struct mcryptd_hash_request_ctx *req_ctx;
        int ret;

        /* remove from work list */
        spin_lock(&cstate->work_lock);
        list_del(&rctx->waiter);
        spin_unlock(&cstate->work_lock);

        if (irqs_disabled())
                rctx->complete(&req->base, err);
        else {
                local_bh_disable();
                rctx->complete(&req->base, err);
                local_bh_enable();
        }

        /* check to see if there are other jobs that are done */
        sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
        while (sha_ctx) {
                req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                ret = sha_finish_walk(&req_ctx, cstate, false);
                if (req_ctx) {
                        spin_lock(&cstate->work_lock);
                        list_del(&req_ctx->waiter);
                        spin_unlock(&cstate->work_lock);

                        req = cast_mcryptd_ctx_to_req(req_ctx);
                        if (irqs_disabled())
                                req_ctx->complete(&req->base, ret);
                        else {
                                local_bh_disable();
                                req_ctx->complete(&req->base, ret);
                                local_bh_enable();
                        }
                }
                sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
        }

        return 0;
}

static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
                             struct mcryptd_alg_cstate *cstate)
{
        unsigned long next_flush;
        unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);

        /* initialize tag */
        rctx->tag.arrival = jiffies;    /* tag the arrival time */
        rctx->tag.seq_num = cstate->next_seq_num++;
        next_flush = rctx->tag.arrival + delay;
        rctx->tag.expire = next_flush;

        spin_lock(&cstate->work_lock);
        list_add_tail(&rctx->waiter, &cstate->work_list);
        spin_unlock(&cstate->work_lock);

        mcryptd_arm_flusher(cstate, delay);
}

static int sha1_mb_update(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                container_of(areq, struct mcryptd_hash_request_ctx, areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha1_mb_alg_state.alg_cstate);

        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha1_hash_ctx *sha_ctx;
        int ret = 0, nbytes;


        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        nbytes = crypto_ahash_walk_first(req, &rctx->walk);

        if (nbytes < 0) {
                ret = nbytes;
                goto done;
        }

        if (crypto_ahash_walk_last(&rctx->walk))
                rctx->flag |= HASH_DONE;

        /* submit */
        sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
        sha1_mb_add_list(rctx, cstate);
        kernel_fpu_begin();
        sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
                                                        nbytes, HASH_UPDATE);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);

        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha1_mb_finup(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                container_of(areq, struct mcryptd_hash_request_ctx, areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha1_mb_alg_state.alg_cstate);

        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha1_hash_ctx *sha_ctx;
        int ret = 0, flag = HASH_UPDATE, nbytes;

        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        nbytes = crypto_ahash_walk_first(req, &rctx->walk);

        if (nbytes < 0) {
                ret = nbytes;
                goto done;
        }

        if (crypto_ahash_walk_last(&rctx->walk)) {
                rctx->flag |= HASH_DONE;
                flag = HASH_LAST;
        }

        /* submit */
        rctx->flag |= HASH_FINAL;
        sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
        sha1_mb_add_list(rctx, cstate);

        kernel_fpu_begin();
        sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
                                                                nbytes, flag);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);
        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha1_mb_final(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                container_of(areq, struct mcryptd_hash_request_ctx, areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha1_mb_alg_state.alg_cstate);

        struct sha1_hash_ctx *sha_ctx;
        int ret = 0;
        u8 data;

        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        rctx->flag |= HASH_DONE | HASH_FINAL;

        sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
        /* flag HASH_FINAL and 0 data size */
        sha1_mb_add_list(rctx, cstate);
        kernel_fpu_begin();
        sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
                                                                HASH_LAST);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);
        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha1_mb_export(struct ahash_request *areq, void *out)
{
        struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);

        memcpy(out, sctx, sizeof(*sctx));

        return 0;
}

static int sha1_mb_import(struct ahash_request *areq, const void *in)
{
        struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);

        memcpy(sctx, in, sizeof(*sctx));

        return 0;
}

static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm)
{
        struct mcryptd_ahash *mcryptd_tfm;
        struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
        struct mcryptd_hash_ctx *mctx;

        mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", /*covered*/
                                                CRYPTO_ALG_INTERNAL,
                                                CRYPTO_ALG_INTERNAL);
        if (IS_ERR(mcryptd_tfm))
                return PTR_ERR(mcryptd_tfm);
        mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
        mctx->alg_state = &sha1_mb_alg_state; /*covered*/
        ctx->mcryptd_tfm = mcryptd_tfm;
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
                                sizeof(struct ahash_request) +
                                crypto_ahash_reqsize(&mcryptd_tfm->base));

        return 0; /*covered*/
}

static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm)
{
        struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);

        mcryptd_free_ahash(ctx->mcryptd_tfm); /*covered*/
}

static int sha1_mb_areq_init_tfm(struct crypto_tfm *tfm)
{
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), /*covered*/
                                sizeof(struct ahash_request) +
                                sizeof(struct sha1_hash_ctx));

        return 0;
}

static void sha1_mb_areq_exit_tfm(struct crypto_tfm *tfm)
{
        struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);

        mcryptd_free_ahash(ctx->mcryptd_tfm);
}

static struct ahash_alg sha1_mb_areq_alg = {
        .init                =        sha1_mb_init,
        .update                =        sha1_mb_update,
        .final                =        sha1_mb_final,
        .finup                =        sha1_mb_finup,
        .export                =        sha1_mb_export,
        .import                =        sha1_mb_import,
        .halg                =        {
                .digestsize        =        SHA1_DIGEST_SIZE,
                .statesize        =        sizeof(struct sha1_hash_ctx),
                .base                =        {
                        .cra_name         = "__sha1-mb",
                        .cra_driver_name = "__intel_sha1-mb",
                        .cra_priority         = 100,
                        /*
                         * use ASYNC flag as some buffers in multi-buffer
                         * algo may not have completed before hashing thread
                         * sleep
                         */
                        .cra_flags        = CRYPTO_ALG_ASYNC |
                                          CRYPTO_ALG_INTERNAL,
                        .cra_blocksize        = SHA1_BLOCK_SIZE,
                        .cra_module        = THIS_MODULE,
                        .cra_list        = LIST_HEAD_INIT
                                        (sha1_mb_areq_alg.halg.base.cra_list),
                        .cra_init        = sha1_mb_areq_init_tfm,
                        .cra_exit        = sha1_mb_areq_exit_tfm,
                        .cra_ctxsize        = sizeof(struct sha1_hash_ctx),
                }
        }
};

static int sha1_mb_async_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_init(mcryptd_req); /*covered*/
}

static int sha1_mb_async_update(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_update(mcryptd_req);
}

static int sha1_mb_async_finup(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_finup(mcryptd_req);
}

static int sha1_mb_async_final(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_final(mcryptd_req);
}

static int sha1_mb_async_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_digest(mcryptd_req);
}

static int sha1_mb_async_export(struct ahash_request *req, void *out)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_export(mcryptd_req, out);
}

static int sha1_mb_async_import(struct ahash_request *req, const void *in)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
        struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
        struct mcryptd_hash_request_ctx *rctx;
        struct ahash_request *areq;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        rctx = ahash_request_ctx(mcryptd_req);
        areq = &rctx->areq;

        ahash_request_set_tfm(areq, child);
        ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
                                        rctx->complete, req);

        return crypto_ahash_import(mcryptd_req, in);
}

static struct ahash_alg sha1_mb_async_alg = {
        .init           = sha1_mb_async_init,
        .update         = sha1_mb_async_update,
        .final          = sha1_mb_async_final,
        .finup          = sha1_mb_async_finup,
        .digest         = sha1_mb_async_digest,
        .export                = sha1_mb_async_export,
        .import                = sha1_mb_async_import,
        .halg = {
                .digestsize     = SHA1_DIGEST_SIZE,
                .statesize        = sizeof(struct sha1_hash_ctx),
                .base = {
                        .cra_name               = "sha1",
                        .cra_driver_name        = "sha1_mb",
                        /*
                         * Low priority, since with few concurrent hash requests
                         * this is extremely slow due to the flush delay.  Users
                         * whose workloads would benefit from this can request
                         * it explicitly by driver name, or can increase its
                         * priority at runtime using NETLINK_CRYPTO.
                         */
                        .cra_priority           = 50,
                        .cra_flags              = CRYPTO_ALG_ASYNC,
                        .cra_blocksize          = SHA1_BLOCK_SIZE,
                        .cra_module             = THIS_MODULE,
                        .cra_list               = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list),
                        .cra_init               = sha1_mb_async_init_tfm,
                        .cra_exit               = sha1_mb_async_exit_tfm,
                        .cra_ctxsize                = sizeof(struct sha1_mb_ctx),
                        .cra_alignmask                = 0,
                },
        },
};

static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate)
{
        struct mcryptd_hash_request_ctx *rctx;
        unsigned long cur_time;
        unsigned long next_flush = 0;
        struct sha1_hash_ctx *sha_ctx;


        cur_time = jiffies;

        while (!list_empty(&cstate->work_list)) {
                rctx = list_entry(cstate->work_list.next,
                                struct mcryptd_hash_request_ctx, waiter);
                if (time_before(cur_time, rctx->tag.expire))
                        break;
                kernel_fpu_begin();
                sha_ctx = (struct sha1_hash_ctx *)
                                        sha1_ctx_mgr_flush(cstate->mgr);
                kernel_fpu_end();
                if (!sha_ctx) {
                        pr_err("sha1_mb error: nothing got flushed for non-empty list\n");
                        break;
                }
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                sha_finish_walk(&rctx, cstate, true);
                sha_complete_job(rctx, cstate, 0);
        }

        if (!list_empty(&cstate->work_list)) {
                rctx = list_entry(cstate->work_list.next,
                                struct mcryptd_hash_request_ctx, waiter);
                /* get the hash context and then flush time */
                next_flush = rctx->tag.expire;
                mcryptd_arm_flusher(cstate, get_delay(next_flush));
        }
        return next_flush;
}

static int __init sha1_mb_mod_init(void)
{

        int cpu;
        int err;
        struct mcryptd_alg_cstate *cpu_state;

        /* check for dependent cpu features */
        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
            !boot_cpu_has(X86_FEATURE_BMI2))
                return -ENODEV;

        /* initialize multibuffer structures */
        sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate);

        sha1_job_mgr_init = sha1_mb_mgr_init_avx2;
        sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2;
        sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2;
        sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2;

        if (!sha1_mb_alg_state.alg_cstate)
                return -ENOMEM;
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
                cpu_state->next_flush = 0;
                cpu_state->next_seq_num = 0;
                cpu_state->flusher_engaged = false;
                INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
                cpu_state->cpu = cpu;
                cpu_state->alg_state = &sha1_mb_alg_state;
                cpu_state->mgr = kzalloc(sizeof(struct sha1_ctx_mgr),
                                        GFP_KERNEL);
                if (!cpu_state->mgr)
                        goto err2;
                sha1_ctx_mgr_init(cpu_state->mgr);
                INIT_LIST_HEAD(&cpu_state->work_list);
                spin_lock_init(&cpu_state->work_lock);
        }
        sha1_mb_alg_state.flusher = &sha1_mb_flusher;

        err = crypto_register_ahash(&sha1_mb_areq_alg);
        if (err)
                goto err2;
        err = crypto_register_ahash(&sha1_mb_async_alg);
        if (err)
                goto err1;


        return 0;
err1:
        crypto_unregister_ahash(&sha1_mb_areq_alg);
err2:
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
                kfree(cpu_state->mgr);
        }
        free_percpu(sha1_mb_alg_state.alg_cstate);
        return -ENODEV;
}

static void __exit sha1_mb_mod_fini(void)
{
        int cpu;
        struct mcryptd_alg_cstate *cpu_state;

        crypto_unregister_ahash(&sha1_mb_async_alg);
        crypto_unregister_ahash(&sha1_mb_areq_alg);
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
                kfree(cpu_state->mgr);
        }
        free_percpu(sha1_mb_alg_state.alg_cstate);
}

module_init(sha1_mb_mod_init);
module_exit(sha1_mb_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated");

MODULE_ALIAS_CRYPTO("sha1");

/*
 * Cryptographic API.
 *
 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
 * Supplemental SSE3 instructions.
 *
 * This file is based on sha1_generic.c
 *
 * Copyright (c) Alan Smithee.
 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
 * Copyright (c) Mathias Krause <minipli@googlemail.com>
 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 */

#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha1_base.h>
#include <asm/fpu/api.h>

typedef void (sha1_transform_fn)(u32 *digest, const char *data,
                                unsigned int rounds);

static int sha1_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len, sha1_transform_fn *sha1_xform)
{
        struct sha1_state *sctx = shash_desc_ctx(desc);

        if (!irq_fpu_usable() || /*covered*/
            (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) /*covered*/
                return crypto_sha1_update(desc, data, len); /*covered*/

        /* make sure casting to sha1_block_fn() is safe */
        BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);

        kernel_fpu_begin(); /*covered*/
        sha1_base_do_update(desc, data, len, /*covered*/
                            (sha1_block_fn *)sha1_xform);
        kernel_fpu_end(); /*covered*/

        return 0;
}

static int sha1_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
{
        if (!irq_fpu_usable()) /*covered*/
                return crypto_sha1_finup(desc, data, len, out);

        kernel_fpu_begin(); /*covered*/
        if (len)
                sha1_base_do_update(desc, data, len, /*covered*/
                                    (sha1_block_fn *)sha1_xform);
        sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform); /*covered*/
        kernel_fpu_end();

        return sha1_base_finish(desc, out); /*covered*/
}

asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
                                     unsigned int rounds);

static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len, /*covered*/
                        (sha1_transform_fn *) sha1_transform_ssse3);
}

static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out, /*covered*/
                        (sha1_transform_fn *) sha1_transform_ssse3);
}

/* Add padding and return the message digest. */
static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
{
        return sha1_ssse3_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha1_ssse3_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_ssse3_update,
        .final                =        sha1_ssse3_final,
        .finup                =        sha1_ssse3_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static int register_sha1_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                return crypto_register_shash(&sha1_ssse3_alg);
        return 0;
}

static void unregister_sha1_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                crypto_unregister_shash(&sha1_ssse3_alg);
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
                                   unsigned int rounds);

static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len, /*covered*/
                        (sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out, /*covered*/
                        (sha1_transform_fn *) sha1_transform_avx);
}

static int sha1_avx_final(struct shash_desc *desc, u8 *out)
{
        return sha1_avx_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha1_avx_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_avx_update,
        .final                =        sha1_avx_final,
        .finup                =        sha1_avx_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static bool avx_usable(void)
{
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
                if (boot_cpu_has(X86_FEATURE_AVX))
                        pr_info("AVX detected but unusable.\n");
                return false;
        }

        return true;
}

static int register_sha1_avx(void)
{
        if (avx_usable())
                return crypto_register_shash(&sha1_avx_alg);
        return 0;
}

static void unregister_sha1_avx(void)
{
        if (avx_usable())
                crypto_unregister_shash(&sha1_avx_alg);
}

#else  /* CONFIG_AS_AVX */
static inline int register_sha1_avx(void) { return 0; }
static inline void unregister_sha1_avx(void) { }
#endif /* CONFIG_AS_AVX */


#if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
#define SHA1_AVX2_BLOCK_OPTSIZE        4        /* optimal 4*64 bytes of SHA1 blocks */

asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
                                    unsigned int rounds);

static bool avx2_usable(void)
{
        if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
                && boot_cpu_has(X86_FEATURE_BMI1)
                && boot_cpu_has(X86_FEATURE_BMI2))
                return true;

        return false;
}

static void sha1_apply_transform_avx2(u32 *digest, const char *data,
                                unsigned int rounds)
{
        /* Select the optimal transform based on data block size */
        if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE) /*covered*/
                sha1_transform_avx2(digest, data, rounds); /*covered*/
        else
                sha1_transform_avx(digest, data, rounds); /*covered*/
} /*covered*/

static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len, /*covered*/
                (sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out, /*covered*/
                (sha1_transform_fn *) sha1_apply_transform_avx2);
}

static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
{
        return sha1_avx2_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha1_avx2_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_avx2_update,
        .final                =        sha1_avx2_final,
        .finup                =        sha1_avx2_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static int register_sha1_avx2(void)
{
        if (avx2_usable())
                return crypto_register_shash(&sha1_avx2_alg);
        return 0;
}

static void unregister_sha1_avx2(void)
{
        if (avx2_usable())
                crypto_unregister_shash(&sha1_avx2_alg);
}

#else
static inline int register_sha1_avx2(void) { return 0; }
static inline void unregister_sha1_avx2(void) { }
#endif

#ifdef CONFIG_AS_SHA1_NI
asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
                                   unsigned int rounds);

static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
                             unsigned int len)
{
        return sha1_update(desc, data, len,
                (sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
                              unsigned int len, u8 *out)
{
        return sha1_finup(desc, data, len, out,
                (sha1_transform_fn *) sha1_ni_transform);
}

static int sha1_ni_final(struct shash_desc *desc, u8 *out)
{
        return sha1_ni_finup(desc, NULL, 0, out);
}

static struct shash_alg sha1_ni_alg = {
        .digestsize        =        SHA1_DIGEST_SIZE,
        .init                =        sha1_base_init,
        .update                =        sha1_ni_update,
        .final                =        sha1_ni_final,
        .finup                =        sha1_ni_finup,
        .descsize        =        sizeof(struct sha1_state),
        .base                =        {
                .cra_name        =        "sha1",
                .cra_driver_name =        "sha1-ni",
                .cra_priority        =        250,
                .cra_blocksize        =        SHA1_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
};

static int register_sha1_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                return crypto_register_shash(&sha1_ni_alg);
        return 0;
}

static void unregister_sha1_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                crypto_unregister_shash(&sha1_ni_alg);
}

#else
static inline int register_sha1_ni(void) { return 0; }
static inline void unregister_sha1_ni(void) { }
#endif

static int __init sha1_ssse3_mod_init(void)
{
        if (register_sha1_ssse3())
                goto fail;

        if (register_sha1_avx()) {
                unregister_sha1_ssse3();
                goto fail;
        }

        if (register_sha1_avx2()) {
                unregister_sha1_avx();
                unregister_sha1_ssse3();
                goto fail;
        }

        if (register_sha1_ni()) {
                unregister_sha1_avx2();
                unregister_sha1_avx();
                unregister_sha1_ssse3();
                goto fail;
        }

        return 0;
fail:
        return -ENODEV;
}

static void __exit sha1_ssse3_mod_fini(void)
{
        unregister_sha1_ni();
        unregister_sha1_avx2();
        unregister_sha1_avx();
        unregister_sha1_ssse3();
}

module_init(sha1_ssse3_mod_init);
module_exit(sha1_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha1");
MODULE_ALIAS_CRYPTO("sha1-ssse3");
MODULE_ALIAS_CRYPTO("sha1-avx");
MODULE_ALIAS_CRYPTO("sha1-avx2");
#ifdef CONFIG_AS_SHA1_NI
MODULE_ALIAS_CRYPTO("sha1-ni");
#endif

/*
 * Multi buffer SHA256 algorithm Glue Code
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 *  Copyright(c) 2016 Intel Corporation.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of version 2 of the GNU General Public License as
 *  published by the Free Software Foundation.
 *
 *  This program is distributed in the hope that it will be useful, but
 *  WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  General Public License for more details.
 *
 *  Contact Information:
 *        Megha Dey <megha.dey@linux.intel.com>
 *
 *  BSD LICENSE
 *
 *  Copyright(c) 2016 Intel Corporation.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions
 *  are met:
 *
 *    * Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in
 *      the documentation and/or other materials provided with the
 *      distribution.
 *    * Neither the name of Intel Corporation nor the names of its
 *      contributors may be used to endorse or promote products derived
 *      from this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <linux/list.h>
#include <crypto/scatterwalk.h>
#include <crypto/sha.h>
#include <crypto/mcryptd.h>
#include <crypto/crypto_wq.h>
#include <asm/byteorder.h>
#include <linux/hardirq.h>
#include <asm/fpu/api.h>
#include "sha256_mb_ctx.h"

#define FLUSH_INTERVAL 1000 /* in usec */

static struct mcryptd_alg_state sha256_mb_alg_state;

struct sha256_mb_ctx {
        struct mcryptd_ahash *mcryptd_tfm;
};

static inline struct mcryptd_hash_request_ctx
                *cast_hash_to_mcryptd_ctx(struct sha256_hash_ctx *hash_ctx)
{
        struct ahash_request *areq;

        areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
        return container_of(areq, struct mcryptd_hash_request_ctx, areq);
}

static inline struct ahash_request
                *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
{
        return container_of((void *) ctx, struct ahash_request, __ctx);
}

static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
                                struct ahash_request *areq)
{
        rctx->flag = HASH_UPDATE;
}

static asmlinkage void (*sha256_job_mgr_init)(struct sha256_mb_mgr *state);
static asmlinkage struct job_sha256* (*sha256_job_mgr_submit)
                        (struct sha256_mb_mgr *state, struct job_sha256 *job);
static asmlinkage struct job_sha256* (*sha256_job_mgr_flush)
                        (struct sha256_mb_mgr *state);
static asmlinkage struct job_sha256* (*sha256_job_mgr_get_comp_job)
                        (struct sha256_mb_mgr *state);

inline uint32_t sha256_pad(uint8_t padblock[SHA256_BLOCK_SIZE * 2],
                         uint64_t total_len)
{
        uint32_t i = total_len & (SHA256_BLOCK_SIZE - 1);

        memset(&padblock[i], 0, SHA256_BLOCK_SIZE);
        padblock[i] = 0x80;

        i += ((SHA256_BLOCK_SIZE - 1) &
              (0 - (total_len + SHA256_PADLENGTHFIELD_SIZE + 1)))
             + 1 + SHA256_PADLENGTHFIELD_SIZE;

#if SHA256_PADLENGTHFIELD_SIZE == 16
        *((uint64_t *) &padblock[i - 16]) = 0;
#endif

        *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);

        /* Number of extra blocks to hash */
        return i >> SHA256_LOG2_BLOCK_SIZE;
}

static struct sha256_hash_ctx
                *sha256_ctx_mgr_resubmit(struct sha256_ctx_mgr *mgr,
                                        struct sha256_hash_ctx *ctx)
{
        while (ctx) {
                if (ctx->status & HASH_CTX_STS_COMPLETE) {
                        /* Clear PROCESSING bit */
                        ctx->status = HASH_CTX_STS_COMPLETE;
                        return ctx;
                }

                /*
                 * If the extra blocks are empty, begin hashing what remains
                 * in the user's buffer.
                 */
                if (ctx->partial_block_buffer_length == 0 &&
                    ctx->incoming_buffer_length) {

                        const void *buffer = ctx->incoming_buffer;
                        uint32_t len = ctx->incoming_buffer_length;
                        uint32_t copy_len;

                        /*
                         * Only entire blocks can be hashed.
                         * Copy remainder to extra blocks buffer.
                         */
                        copy_len = len & (SHA256_BLOCK_SIZE-1);

                        if (copy_len) {
                                len -= copy_len;
                                memcpy(ctx->partial_block_buffer,
                                       ((const char *) buffer + len),
                                       copy_len);
                                ctx->partial_block_buffer_length = copy_len;
                        }

                        ctx->incoming_buffer_length = 0;

                        /* len should be a multiple of the block size now */
                        assert((len % SHA256_BLOCK_SIZE) == 0);

                        /* Set len to the number of blocks to be hashed */
                        len >>= SHA256_LOG2_BLOCK_SIZE;

                        if (len) {

                                ctx->job.buffer = (uint8_t *) buffer;
                                ctx->job.len = len;
                                ctx = (struct sha256_hash_ctx *)
                                sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
                                continue;
                        }
                }

                /*
                 * If the extra blocks are not empty, then we are
                 * either on the last block(s) or we need more
                 * user input before continuing.
                 */
                if (ctx->status & HASH_CTX_STS_LAST) {

                        uint8_t *buf = ctx->partial_block_buffer;
                        uint32_t n_extra_blocks =
                                sha256_pad(buf, ctx->total_length);

                        ctx->status = (HASH_CTX_STS_PROCESSING |
                                       HASH_CTX_STS_COMPLETE);
                        ctx->job.buffer = buf;
                        ctx->job.len = (uint32_t) n_extra_blocks;
                        ctx = (struct sha256_hash_ctx *)
                                sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
                        continue;
                }

                ctx->status = HASH_CTX_STS_IDLE;
                return ctx;
        }

        return NULL;
}

static struct sha256_hash_ctx
                *sha256_ctx_mgr_get_comp_ctx(struct sha256_ctx_mgr *mgr)
{
        /*
         * If get_comp_job returns NULL, there are no jobs complete.
         * If get_comp_job returns a job, verify that it is safe to return to
         * the user. If it is not ready, resubmit the job to finish processing.
         * If sha256_ctx_mgr_resubmit returned a job, it is ready to be
         * returned. Otherwise, all jobs currently being managed by the
         * hash_ctx_mgr still need processing.
         */
        struct sha256_hash_ctx *ctx;

        ctx = (struct sha256_hash_ctx *) sha256_job_mgr_get_comp_job(&mgr->mgr);
        return sha256_ctx_mgr_resubmit(mgr, ctx);
}

static void sha256_ctx_mgr_init(struct sha256_ctx_mgr *mgr)
{
        sha256_job_mgr_init(&mgr->mgr);
}

static struct sha256_hash_ctx *sha256_ctx_mgr_submit(struct sha256_ctx_mgr *mgr,
                                          struct sha256_hash_ctx *ctx,
                                          const void *buffer,
                                          uint32_t len,
                                          int flags)
{
        if (flags & ~(HASH_UPDATE | HASH_LAST)) {
                /* User should not pass anything other than UPDATE or LAST */
                ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
                return ctx;
        }

        if (ctx->status & HASH_CTX_STS_PROCESSING) {
                /* Cannot submit to a currently processing job. */
                ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
                return ctx;
        }

        if (ctx->status & HASH_CTX_STS_COMPLETE) {
                /* Cannot update a finished job. */
                ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
                return ctx;
        }

        /* If we made it here, there was no error during this call to submit */
        ctx->error = HASH_CTX_ERROR_NONE;

        /* Store buffer ptr info from user */
        ctx->incoming_buffer = buffer;
        ctx->incoming_buffer_length = len;

        /*
         * Store the user's request flags and mark this ctx as currently
         * being processed.
         */
        ctx->status = (flags & HASH_LAST) ?
                        (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
                        HASH_CTX_STS_PROCESSING;

        /* Advance byte counter */
        ctx->total_length += len;

        /*
         * If there is anything currently buffered in the extra blocks,
         * append to it until it contains a whole block.
         * Or if the user's buffer contains less than a whole block,
         * append as much as possible to the extra block.
         */
        if (ctx->partial_block_buffer_length || len < SHA256_BLOCK_SIZE) {
                /*
                 * Compute how many bytes to copy from user buffer into
                 * extra block
                 */
                uint32_t copy_len = SHA256_BLOCK_SIZE -
                                        ctx->partial_block_buffer_length;
                if (len < copy_len)
                        copy_len = len;

                if (copy_len) {
                        /* Copy and update relevant pointers and counters */
                        memcpy(
                &ctx->partial_block_buffer[ctx->partial_block_buffer_length],
                                buffer, copy_len);

                        ctx->partial_block_buffer_length += copy_len;
                        ctx->incoming_buffer = (const void *)
                                        ((const char *)buffer + copy_len);
                        ctx->incoming_buffer_length = len - copy_len;
                }

                /* The extra block should never contain more than 1 block */
                assert(ctx->partial_block_buffer_length <= SHA256_BLOCK_SIZE);

                /*
                 * If the extra block buffer contains exactly 1 block,
                 * it can be hashed.
                 */
                if (ctx->partial_block_buffer_length >= SHA256_BLOCK_SIZE) {
                        ctx->partial_block_buffer_length = 0;

                        ctx->job.buffer = ctx->partial_block_buffer;
                        ctx->job.len = 1;
                        ctx = (struct sha256_hash_ctx *)
                                sha256_job_mgr_submit(&mgr->mgr, &ctx->job);
                }
        }

        return sha256_ctx_mgr_resubmit(mgr, ctx);
}

static struct sha256_hash_ctx *sha256_ctx_mgr_flush(struct sha256_ctx_mgr *mgr)
{
        struct sha256_hash_ctx *ctx;

        while (1) {
                ctx = (struct sha256_hash_ctx *)
                                        sha256_job_mgr_flush(&mgr->mgr);

                /* If flush returned 0, there are no more jobs in flight. */
                if (!ctx)
                        return NULL;

                /*
                 * If flush returned a job, resubmit the job to finish
                 * processing.
                 */
                ctx = sha256_ctx_mgr_resubmit(mgr, ctx);

                /*
                 * If sha256_ctx_mgr_resubmit returned a job, it is ready to
                 * be returned. Otherwise, all jobs currently being managed by
                 * the sha256_ctx_mgr still need processing. Loop.
                 */
                if (ctx)
                        return ctx;
        }
}

static int sha256_mb_init(struct ahash_request *areq)
{
        struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);

        hash_ctx_init(sctx);
        sctx->job.result_digest[0] = SHA256_H0;
        sctx->job.result_digest[1] = SHA256_H1;
        sctx->job.result_digest[2] = SHA256_H2;
        sctx->job.result_digest[3] = SHA256_H3;
        sctx->job.result_digest[4] = SHA256_H4;
        sctx->job.result_digest[5] = SHA256_H5;
        sctx->job.result_digest[6] = SHA256_H6;
        sctx->job.result_digest[7] = SHA256_H7;
        sctx->total_length = 0;
        sctx->partial_block_buffer_length = 0;
        sctx->status = HASH_CTX_STS_IDLE;

        return 0;
}

static int sha256_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
{
        int        i;
        struct        sha256_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
        __be32        *dst = (__be32 *) rctx->out;

        for (i = 0; i < 8; ++i)
                dst[i] = cpu_to_be32(sctx->job.result_digest[i]);

        return 0;
}

static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
                        struct mcryptd_alg_cstate *cstate, bool flush)
{
        int        flag = HASH_UPDATE;
        int        nbytes, err = 0;
        struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
        struct sha256_hash_ctx *sha_ctx;

        /* more work ? */
        while (!(rctx->flag & HASH_DONE)) {
                nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
                if (nbytes < 0) {
                        err = nbytes;
                        goto out;
                }
                /* check if the walk is done */
                if (crypto_ahash_walk_last(&rctx->walk)) {
                        rctx->flag |= HASH_DONE;
                        if (rctx->flag & HASH_FINAL)
                                flag |= HASH_LAST;

                }
                sha_ctx = (struct sha256_hash_ctx *)
                                                ahash_request_ctx(&rctx->areq);
                kernel_fpu_begin();
                sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx,
                                                rctx->walk.data, nbytes, flag);
                if (!sha_ctx) {
                        if (flush)
                                sha_ctx = sha256_ctx_mgr_flush(cstate->mgr);
                }
                kernel_fpu_end();
                if (sha_ctx)
                        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                else {
                        rctx = NULL;
                        goto out;
                }
        }

        /* copy the results */
        if (rctx->flag & HASH_FINAL)
                sha256_mb_set_results(rctx);

out:
        *ret_rctx = rctx;
        return err;
}

static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
                            struct mcryptd_alg_cstate *cstate,
                            int err)
{
        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha256_hash_ctx *sha_ctx;
        struct mcryptd_hash_request_ctx *req_ctx;
        int ret;

        /* remove from work list */
        spin_lock(&cstate->work_lock);
        list_del(&rctx->waiter);
        spin_unlock(&cstate->work_lock);

        if (irqs_disabled())
                rctx->complete(&req->base, err);
        else {
                local_bh_disable();
                rctx->complete(&req->base, err);
                local_bh_enable();
        }

        /* check to see if there are other jobs that are done */
        sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr);
        while (sha_ctx) {
                req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                ret = sha_finish_walk(&req_ctx, cstate, false);
                if (req_ctx) {
                        spin_lock(&cstate->work_lock);
                        list_del(&req_ctx->waiter);
                        spin_unlock(&cstate->work_lock);

                        req = cast_mcryptd_ctx_to_req(req_ctx);
                        if (irqs_disabled())
                                req_ctx->complete(&req->base, ret);
                        else {
                                local_bh_disable();
                                req_ctx->complete(&req->base, ret);
                                local_bh_enable();
                        }
                }
                sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr);
        }

        return 0;
}

static void sha256_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
                             struct mcryptd_alg_cstate *cstate)
{
        unsigned long next_flush;
        unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);

        /* initialize tag */
        rctx->tag.arrival = jiffies;    /* tag the arrival time */
        rctx->tag.seq_num = cstate->next_seq_num++;
        next_flush = rctx->tag.arrival + delay;
        rctx->tag.expire = next_flush;

        spin_lock(&cstate->work_lock);
        list_add_tail(&rctx->waiter, &cstate->work_list);
        spin_unlock(&cstate->work_lock);

        mcryptd_arm_flusher(cstate, delay);
}

static int sha256_mb_update(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                container_of(areq, struct mcryptd_hash_request_ctx, areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha256_mb_alg_state.alg_cstate);

        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha256_hash_ctx *sha_ctx;
        int ret = 0, nbytes;

        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        nbytes = crypto_ahash_walk_first(req, &rctx->walk);

        if (nbytes < 0) {
                ret = nbytes;
                goto done;
        }

        if (crypto_ahash_walk_last(&rctx->walk))
                rctx->flag |= HASH_DONE;

        /* submit */
        sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
        sha256_mb_add_list(rctx, cstate);
        kernel_fpu_begin();
        sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
                                                        nbytes, HASH_UPDATE);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);

        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha256_mb_finup(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                container_of(areq, struct mcryptd_hash_request_ctx, areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha256_mb_alg_state.alg_cstate);

        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha256_hash_ctx *sha_ctx;
        int ret = 0, flag = HASH_UPDATE, nbytes;

        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        nbytes = crypto_ahash_walk_first(req, &rctx->walk);

        if (nbytes < 0) {
                ret = nbytes;
                goto done;
        }

        if (crypto_ahash_walk_last(&rctx->walk)) {
                rctx->flag |= HASH_DONE;
                flag = HASH_LAST;
        }

        /* submit */
        rctx->flag |= HASH_FINAL;
        sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
        sha256_mb_add_list(rctx, cstate);

        kernel_fpu_begin();
        sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
                                                                nbytes, flag);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);
        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha256_mb_final(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                        container_of(areq, struct mcryptd_hash_request_ctx,
                        areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha256_mb_alg_state.alg_cstate);

        struct sha256_hash_ctx *sha_ctx;
        int ret = 0;
        u8 data;

        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        rctx->flag |= HASH_DONE | HASH_FINAL;

        sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq);
        /* flag HASH_FINAL and 0 data size */
        sha256_mb_add_list(rctx, cstate);
        kernel_fpu_begin();
        sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
                                                                HASH_LAST);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);
        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha256_mb_export(struct ahash_request *areq, void *out)
{
        struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);

        memcpy(out, sctx, sizeof(*sctx));

        return 0;
}

static int sha256_mb_import(struct ahash_request *areq, const void *in)
{
        struct sha256_hash_ctx *sctx = ahash_request_ctx(areq);

        memcpy(sctx, in, sizeof(*sctx));

        return 0;
}

static int sha256_mb_async_init_tfm(struct crypto_tfm *tfm)
{
        struct mcryptd_ahash *mcryptd_tfm;
        struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);
        struct mcryptd_hash_ctx *mctx;

        mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha256-mb", /*covered*/
                                                CRYPTO_ALG_INTERNAL,
                                                CRYPTO_ALG_INTERNAL);
        if (IS_ERR(mcryptd_tfm))
                return PTR_ERR(mcryptd_tfm);
        mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
        mctx->alg_state = &sha256_mb_alg_state; /*covered*/
        ctx->mcryptd_tfm = mcryptd_tfm;
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
                                sizeof(struct ahash_request) +
                                crypto_ahash_reqsize(&mcryptd_tfm->base));

        return 0; /*covered*/
}

static void sha256_mb_async_exit_tfm(struct crypto_tfm *tfm)
{
        struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);

        mcryptd_free_ahash(ctx->mcryptd_tfm); /*covered*/
}

static int sha256_mb_areq_init_tfm(struct crypto_tfm *tfm)
{
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), /*covered*/
                                sizeof(struct ahash_request) +
                                sizeof(struct sha256_hash_ctx));

        return 0;
}

static void sha256_mb_areq_exit_tfm(struct crypto_tfm *tfm)
{
        struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm);

        mcryptd_free_ahash(ctx->mcryptd_tfm);
}

static struct ahash_alg sha256_mb_areq_alg = {
        .init                =        sha256_mb_init,
        .update                =        sha256_mb_update,
        .final                =        sha256_mb_final,
        .finup                =        sha256_mb_finup,
        .export                =        sha256_mb_export,
        .import                =        sha256_mb_import,
        .halg                =        {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .statesize        =        sizeof(struct sha256_hash_ctx),
                .base                =        {
                        .cra_name         = "__sha256-mb",
                        .cra_driver_name = "__intel_sha256-mb",
                        .cra_priority         = 100,
                        /*
                         * use ASYNC flag as some buffers in multi-buffer
                         * algo may not have completed before hashing thread
                         * sleep
                         */
                        .cra_flags        = CRYPTO_ALG_ASYNC |
                                          CRYPTO_ALG_INTERNAL,
                        .cra_blocksize        = SHA256_BLOCK_SIZE,
                        .cra_module        = THIS_MODULE,
                        .cra_list        = LIST_HEAD_INIT
                                        (sha256_mb_areq_alg.halg.base.cra_list),
                        .cra_init        = sha256_mb_areq_init_tfm,
                        .cra_exit        = sha256_mb_areq_exit_tfm,
                        .cra_ctxsize        = sizeof(struct sha256_hash_ctx),
                }
        }
};

static int sha256_mb_async_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_init(mcryptd_req); /*covered*/
}

static int sha256_mb_async_update(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_update(mcryptd_req);
}

static int sha256_mb_async_finup(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_finup(mcryptd_req);
}

static int sha256_mb_async_final(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_final(mcryptd_req);
}

static int sha256_mb_async_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_digest(mcryptd_req);
}

static int sha256_mb_async_export(struct ahash_request *req, void *out)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_export(mcryptd_req, out);
}

static int sha256_mb_async_import(struct ahash_request *req, const void *in)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
        struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
        struct mcryptd_hash_request_ctx *rctx;
        struct ahash_request *areq;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        rctx = ahash_request_ctx(mcryptd_req);
        areq = &rctx->areq;

        ahash_request_set_tfm(areq, child);
        ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
                                        rctx->complete, req);

        return crypto_ahash_import(mcryptd_req, in);
}

static struct ahash_alg sha256_mb_async_alg = {
        .init           = sha256_mb_async_init,
        .update         = sha256_mb_async_update,
        .final          = sha256_mb_async_final,
        .finup          = sha256_mb_async_finup,
        .export         = sha256_mb_async_export,
        .import         = sha256_mb_async_import,
        .digest         = sha256_mb_async_digest,
        .halg = {
                .digestsize     = SHA256_DIGEST_SIZE,
                .statesize      = sizeof(struct sha256_hash_ctx),
                .base = {
                        .cra_name               = "sha256",
                        .cra_driver_name        = "sha256_mb",
                        /*
                         * Low priority, since with few concurrent hash requests
                         * this is extremely slow due to the flush delay.  Users
                         * whose workloads would benefit from this can request
                         * it explicitly by driver name, or can increase its
                         * priority at runtime using NETLINK_CRYPTO.
                         */
                        .cra_priority           = 50,
                        .cra_flags              = CRYPTO_ALG_ASYNC,
                        .cra_blocksize          = SHA256_BLOCK_SIZE,
                        .cra_module             = THIS_MODULE,
                        .cra_list               = LIST_HEAD_INIT
                                (sha256_mb_async_alg.halg.base.cra_list),
                        .cra_init               = sha256_mb_async_init_tfm,
                        .cra_exit               = sha256_mb_async_exit_tfm,
                        .cra_ctxsize                = sizeof(struct sha256_mb_ctx),
                        .cra_alignmask                = 0,
                },
        },
};

static unsigned long sha256_mb_flusher(struct mcryptd_alg_cstate *cstate)
{
        struct mcryptd_hash_request_ctx *rctx;
        unsigned long cur_time;
        unsigned long next_flush = 0;
        struct sha256_hash_ctx *sha_ctx;


        cur_time = jiffies;

        while (!list_empty(&cstate->work_list)) {
                rctx = list_entry(cstate->work_list.next,
                                struct mcryptd_hash_request_ctx, waiter);
                if (time_before(cur_time, rctx->tag.expire))
                        break;
                kernel_fpu_begin();
                sha_ctx = (struct sha256_hash_ctx *)
                                        sha256_ctx_mgr_flush(cstate->mgr);
                kernel_fpu_end();
                if (!sha_ctx) {
                        pr_err("sha256_mb error: nothing got"
                                        " flushed for non-empty list\n");
                        break;
                }
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                sha_finish_walk(&rctx, cstate, true);
                sha_complete_job(rctx, cstate, 0);
        }

        if (!list_empty(&cstate->work_list)) {
                rctx = list_entry(cstate->work_list.next,
                                struct mcryptd_hash_request_ctx, waiter);
                /* get the hash context and then flush time */
                next_flush = rctx->tag.expire;
                mcryptd_arm_flusher(cstate, get_delay(next_flush));
        }
        return next_flush;
}

static int __init sha256_mb_mod_init(void)
{

        int cpu;
        int err;
        struct mcryptd_alg_cstate *cpu_state;

        /* check for dependent cpu features */
        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
            !boot_cpu_has(X86_FEATURE_BMI2))
                return -ENODEV;

        /* initialize multibuffer structures */
        sha256_mb_alg_state.alg_cstate = alloc_percpu
                                                (struct mcryptd_alg_cstate);

        sha256_job_mgr_init = sha256_mb_mgr_init_avx2;
        sha256_job_mgr_submit = sha256_mb_mgr_submit_avx2;
        sha256_job_mgr_flush = sha256_mb_mgr_flush_avx2;
        sha256_job_mgr_get_comp_job = sha256_mb_mgr_get_comp_job_avx2;

        if (!sha256_mb_alg_state.alg_cstate)
                return -ENOMEM;
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
                cpu_state->next_flush = 0;
                cpu_state->next_seq_num = 0;
                cpu_state->flusher_engaged = false;
                INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
                cpu_state->cpu = cpu;
                cpu_state->alg_state = &sha256_mb_alg_state;
                cpu_state->mgr = kzalloc(sizeof(struct sha256_ctx_mgr),
                                        GFP_KERNEL);
                if (!cpu_state->mgr)
                        goto err2;
                sha256_ctx_mgr_init(cpu_state->mgr);
                INIT_LIST_HEAD(&cpu_state->work_list);
                spin_lock_init(&cpu_state->work_lock);
        }
        sha256_mb_alg_state.flusher = &sha256_mb_flusher;

        err = crypto_register_ahash(&sha256_mb_areq_alg);
        if (err)
                goto err2;
        err = crypto_register_ahash(&sha256_mb_async_alg);
        if (err)
                goto err1;


        return 0;
err1:
        crypto_unregister_ahash(&sha256_mb_areq_alg);
err2:
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
                kfree(cpu_state->mgr);
        }
        free_percpu(sha256_mb_alg_state.alg_cstate);
        return -ENODEV;
}

static void __exit sha256_mb_mod_fini(void)
{
        int cpu;
        struct mcryptd_alg_cstate *cpu_state;

        crypto_unregister_ahash(&sha256_mb_async_alg);
        crypto_unregister_ahash(&sha256_mb_areq_alg);
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu);
                kfree(cpu_state->mgr);
        }
        free_percpu(sha256_mb_alg_state.alg_cstate);
}

module_init(sha256_mb_mod_init);
module_exit(sha256_mb_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, multi buffer accelerated");

MODULE_ALIAS_CRYPTO("sha256");

/*
 * Cryptographic API.
 *
 * Glue code for the SHA256 Secure Hash Algorithm assembler
 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
 *
 * This file is based on sha256_generic.c
 *
 * Copyright (C) 2013 Intel Corporation.
 *
 * Author:
 *     Tim Chen <tim.c.chen@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */


#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha256_base.h>
#include <asm/fpu/api.h>
#include <linux/string.h>

asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data,
                                       u64 rounds);
typedef void (sha256_transform_fn)(u32 *digest, const char *data, u64 rounds);

static int sha256_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len, sha256_transform_fn *sha256_xform)
{
        struct sha256_state *sctx = shash_desc_ctx(desc);

        if (!irq_fpu_usable() || /*covered*/
            (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) /*covered*/
                return crypto_sha256_update(desc, data, len); /*covered*/

        /* make sure casting to sha256_block_fn() is safe */
        BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);

        kernel_fpu_begin(); /*covered*/
        sha256_base_do_update(desc, data, len, /*covered*/
                              (sha256_block_fn *)sha256_xform);
        kernel_fpu_end(); /*covered*/

        return 0;
}

static int sha256_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out, sha256_transform_fn *sha256_xform)
{
        if (!irq_fpu_usable()) /*covered*/
                return crypto_sha256_finup(desc, data, len, out);

        kernel_fpu_begin(); /*covered*/
        if (len)
                sha256_base_do_update(desc, data, len, /*covered*/
                                      (sha256_block_fn *)sha256_xform);
        sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_xform); /*covered*/
        kernel_fpu_end();

        return sha256_base_finish(desc, out); /*covered*/
}

static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_transform_ssse3); /*covered*/
}

static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_transform_ssse3); /*covered*/
}

/* Add padding and return the message digest. */
static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
{
        return sha256_ssse3_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha256_ssse3_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_ssse3_update,
        .final                =        sha256_ssse3_final,
        .finup                =        sha256_ssse3_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_ssse3_update,
        .final                =        sha256_ssse3_final,
        .finup                =        sha256_ssse3_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha256_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                return crypto_register_shashes(sha256_ssse3_algs,
                                ARRAY_SIZE(sha256_ssse3_algs));
        return 0;
}

static void unregister_sha256_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                crypto_unregister_shashes(sha256_ssse3_algs,
                                ARRAY_SIZE(sha256_ssse3_algs));
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha256_transform_avx(u32 *digest, const char *data,
                                     u64 rounds);

static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_transform_avx); /*covered*/
}

static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_transform_avx); /*covered*/
}

static int sha256_avx_final(struct shash_desc *desc, u8 *out)
{
        return sha256_avx_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha256_avx_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_avx_update,
        .final                =        sha256_avx_final,
        .finup                =        sha256_avx_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_avx_update,
        .final                =        sha256_avx_final,
        .finup                =        sha256_avx_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static bool avx_usable(void)
{
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
                if (boot_cpu_has(X86_FEATURE_AVX))
                        pr_info("AVX detected but unusable.\n");
                return false;
        }

        return true;
}

static int register_sha256_avx(void)
{
        if (avx_usable())
                return crypto_register_shashes(sha256_avx_algs,
                                ARRAY_SIZE(sha256_avx_algs));
        return 0;
}

static void unregister_sha256_avx(void)
{
        if (avx_usable())
                crypto_unregister_shashes(sha256_avx_algs,
                                ARRAY_SIZE(sha256_avx_algs));
}

#else
static inline int register_sha256_avx(void) { return 0; }
static inline void unregister_sha256_avx(void) { }
#endif

#if defined(CONFIG_AS_AVX2) && defined(CONFIG_AS_AVX)
asmlinkage void sha256_transform_rorx(u32 *digest, const char *data,
                                      u64 rounds);

static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_transform_rorx); /*covered*/
}

static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_transform_rorx); /*covered*/
}

static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
{
        return sha256_avx2_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha256_avx2_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_avx2_update,
        .final                =        sha256_avx2_final,
        .finup                =        sha256_avx2_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_avx2_update,
        .final                =        sha256_avx2_final,
        .finup                =        sha256_avx2_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static bool avx2_usable(void)
{
        if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
                    boot_cpu_has(X86_FEATURE_BMI2))
                return true;

        return false;
}

static int register_sha256_avx2(void)
{
        if (avx2_usable())
                return crypto_register_shashes(sha256_avx2_algs,
                                ARRAY_SIZE(sha256_avx2_algs));
        return 0;
}

static void unregister_sha256_avx2(void)
{
        if (avx2_usable())
                crypto_unregister_shashes(sha256_avx2_algs,
                                ARRAY_SIZE(sha256_avx2_algs));
}

#else
static inline int register_sha256_avx2(void) { return 0; }
static inline void unregister_sha256_avx2(void) { }
#endif

#ifdef CONFIG_AS_SHA256_NI
asmlinkage void sha256_ni_transform(u32 *digest, const char *data,
                                   u64 rounds); /*unsigned int rounds);*/

static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
                         unsigned int len)
{
        return sha256_update(desc, data, len, sha256_ni_transform);
}

static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
                      unsigned int len, u8 *out)
{
        return sha256_finup(desc, data, len, out, sha256_ni_transform);
}

static int sha256_ni_final(struct shash_desc *desc, u8 *out)
{
        return sha256_ni_finup(desc, NULL, 0, out);
}

static struct shash_alg sha256_ni_algs[] = { {
        .digestsize        =        SHA256_DIGEST_SIZE,
        .init                =        sha256_base_init,
        .update                =        sha256_ni_update,
        .final                =        sha256_ni_final,
        .finup                =        sha256_ni_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha256",
                .cra_driver_name =        "sha256-ni",
                .cra_priority        =        250,
                .cra_blocksize        =        SHA256_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
}, {
        .digestsize        =        SHA224_DIGEST_SIZE,
        .init                =        sha224_base_init,
        .update                =        sha256_ni_update,
        .final                =        sha256_ni_final,
        .finup                =        sha256_ni_finup,
        .descsize        =        sizeof(struct sha256_state),
        .base                =        {
                .cra_name        =        "sha224",
                .cra_driver_name =        "sha224-ni",
                .cra_priority        =        250,
                .cra_blocksize        =        SHA224_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha256_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                return crypto_register_shashes(sha256_ni_algs,
                                ARRAY_SIZE(sha256_ni_algs));
        return 0;
}

static void unregister_sha256_ni(void)
{
        if (boot_cpu_has(X86_FEATURE_SHA_NI))
                crypto_unregister_shashes(sha256_ni_algs,
                                ARRAY_SIZE(sha256_ni_algs));
}

#else
static inline int register_sha256_ni(void) { return 0; }
static inline void unregister_sha256_ni(void) { }
#endif

static int __init sha256_ssse3_mod_init(void)
{
        if (register_sha256_ssse3())
                goto fail;

        if (register_sha256_avx()) {
                unregister_sha256_ssse3();
                goto fail;
        }

        if (register_sha256_avx2()) {
                unregister_sha256_avx();
                unregister_sha256_ssse3();
                goto fail;
        }

        if (register_sha256_ni()) {
                unregister_sha256_avx2();
                unregister_sha256_avx();
                unregister_sha256_ssse3();
                goto fail;
        }

        return 0;
fail:
        return -ENODEV;
}

static void __exit sha256_ssse3_mod_fini(void)
{
        unregister_sha256_ni();
        unregister_sha256_avx2();
        unregister_sha256_avx();
        unregister_sha256_ssse3();
}

module_init(sha256_ssse3_mod_init);
module_exit(sha256_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha256");
MODULE_ALIAS_CRYPTO("sha256-ssse3");
MODULE_ALIAS_CRYPTO("sha256-avx");
MODULE_ALIAS_CRYPTO("sha256-avx2");
MODULE_ALIAS_CRYPTO("sha224");
MODULE_ALIAS_CRYPTO("sha224-ssse3");
MODULE_ALIAS_CRYPTO("sha224-avx");
MODULE_ALIAS_CRYPTO("sha224-avx2");
#ifdef CONFIG_AS_SHA256_NI
MODULE_ALIAS_CRYPTO("sha256-ni");
MODULE_ALIAS_CRYPTO("sha224-ni");
#endif

/*
 * Multi buffer SHA512 algorithm Glue Code
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
 * Copyright(c) 2016 Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * Contact Information:
 *        Megha Dey <megha.dey@linux.intel.com>
 *
 * BSD LICENSE
 *
 * Copyright(c) 2016 Intel Corporation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *   * Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in
 *     the documentation and/or other materials provided with the
 *     distribution.
 *   * Neither the name of Intel Corporation nor the names of its
 *     contributors may be used to endorse or promote products derived
 *     from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <linux/list.h>
#include <crypto/scatterwalk.h>
#include <crypto/sha.h>
#include <crypto/mcryptd.h>
#include <crypto/crypto_wq.h>
#include <asm/byteorder.h>
#include <linux/hardirq.h>
#include <asm/fpu/api.h>
#include "sha512_mb_ctx.h"

#define FLUSH_INTERVAL 1000 /* in usec */

static struct mcryptd_alg_state sha512_mb_alg_state;

struct sha512_mb_ctx {
        struct mcryptd_ahash *mcryptd_tfm;
};

static inline struct mcryptd_hash_request_ctx
                *cast_hash_to_mcryptd_ctx(struct sha512_hash_ctx *hash_ctx)
{
        struct ahash_request *areq;

        areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
        return container_of(areq, struct mcryptd_hash_request_ctx, areq);
}

static inline struct ahash_request
                *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
{
        return container_of((void *) ctx, struct ahash_request, __ctx);
}

static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
                                struct ahash_request *areq)
{
        rctx->flag = HASH_UPDATE;
}

static asmlinkage void (*sha512_job_mgr_init)(struct sha512_mb_mgr *state);
static asmlinkage struct job_sha512* (*sha512_job_mgr_submit)
                                                (struct sha512_mb_mgr *state,
                                                struct job_sha512 *job);
static asmlinkage struct job_sha512* (*sha512_job_mgr_flush)
                                                (struct sha512_mb_mgr *state);
static asmlinkage struct job_sha512* (*sha512_job_mgr_get_comp_job)
                                                (struct sha512_mb_mgr *state);

inline uint32_t sha512_pad(uint8_t padblock[SHA512_BLOCK_SIZE * 2],
                         uint64_t total_len)
{
        uint32_t i = total_len & (SHA512_BLOCK_SIZE - 1);

        memset(&padblock[i], 0, SHA512_BLOCK_SIZE);
        padblock[i] = 0x80;

        i += ((SHA512_BLOCK_SIZE - 1) &
              (0 - (total_len + SHA512_PADLENGTHFIELD_SIZE + 1)))
             + 1 + SHA512_PADLENGTHFIELD_SIZE;

#if SHA512_PADLENGTHFIELD_SIZE == 16
        *((uint64_t *) &padblock[i - 16]) = 0;
#endif

        *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);

        /* Number of extra blocks to hash */
        return i >> SHA512_LOG2_BLOCK_SIZE;
}

static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit
                (struct sha512_ctx_mgr *mgr, struct sha512_hash_ctx *ctx)
{
        while (ctx) {
                if (ctx->status & HASH_CTX_STS_COMPLETE) {
                        /* Clear PROCESSING bit */
                        ctx->status = HASH_CTX_STS_COMPLETE;
                        return ctx;
                }

                /*
                 * If the extra blocks are empty, begin hashing what remains
                 * in the user's buffer.
                 */
                if (ctx->partial_block_buffer_length == 0 &&
                    ctx->incoming_buffer_length) {

                        const void *buffer = ctx->incoming_buffer;
                        uint32_t len = ctx->incoming_buffer_length;
                        uint32_t copy_len;

                        /*
                         * Only entire blocks can be hashed.
                         * Copy remainder to extra blocks buffer.
                         */
                        copy_len = len & (SHA512_BLOCK_SIZE-1);

                        if (copy_len) {
                                len -= copy_len;
                                memcpy(ctx->partial_block_buffer,
                                       ((const char *) buffer + len),
                                       copy_len);
                                ctx->partial_block_buffer_length = copy_len;
                        }

                        ctx->incoming_buffer_length = 0;

                        /* len should be a multiple of the block size now */
                        assert((len % SHA512_BLOCK_SIZE) == 0);

                        /* Set len to the number of blocks to be hashed */
                        len >>= SHA512_LOG2_BLOCK_SIZE;

                        if (len) {

                                ctx->job.buffer = (uint8_t *) buffer;
                                ctx->job.len = len;
                                ctx = (struct sha512_hash_ctx *)
                                        sha512_job_mgr_submit(&mgr->mgr,
                                        &ctx->job);
                                continue;
                        }
                }

                /*
                 * If the extra blocks are not empty, then we are
                 * either on the last block(s) or we need more
                 * user input before continuing.
                 */
                if (ctx->status & HASH_CTX_STS_LAST) {

                        uint8_t *buf = ctx->partial_block_buffer;
                        uint32_t n_extra_blocks =
                                        sha512_pad(buf, ctx->total_length);

                        ctx->status = (HASH_CTX_STS_PROCESSING |
                                       HASH_CTX_STS_COMPLETE);
                        ctx->job.buffer = buf;
                        ctx->job.len = (uint32_t) n_extra_blocks;
                        ctx = (struct sha512_hash_ctx *)
                                sha512_job_mgr_submit(&mgr->mgr, &ctx->job);
                        continue;
                }

                if (ctx)
                        ctx->status = HASH_CTX_STS_IDLE;
                return ctx;
        }

        return NULL;
}

static struct sha512_hash_ctx
                *sha512_ctx_mgr_get_comp_ctx(struct mcryptd_alg_cstate *cstate)
{
        /*
         * If get_comp_job returns NULL, there are no jobs complete.
         * If get_comp_job returns a job, verify that it is safe to return to
         * the user.
         * If it is not ready, resubmit the job to finish processing.
         * If sha512_ctx_mgr_resubmit returned a job, it is ready to be
         * returned.
         * Otherwise, all jobs currently being managed by the hash_ctx_mgr
         * still need processing.
         */
        struct sha512_ctx_mgr *mgr;
        struct sha512_hash_ctx *ctx;
        unsigned long flags;

        mgr = cstate->mgr;
        spin_lock_irqsave(&cstate->work_lock, flags);
        ctx = (struct sha512_hash_ctx *)
                                sha512_job_mgr_get_comp_job(&mgr->mgr);
        ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
        spin_unlock_irqrestore(&cstate->work_lock, flags);
        return ctx;
}

static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr)
{
        sha512_job_mgr_init(&mgr->mgr);
}

static struct sha512_hash_ctx
                        *sha512_ctx_mgr_submit(struct mcryptd_alg_cstate *cstate,
                                          struct sha512_hash_ctx *ctx,
                                          const void *buffer,
                                          uint32_t len,
                                          int flags)
{
        struct sha512_ctx_mgr *mgr;
        unsigned long irqflags;

        mgr = cstate->mgr;
        spin_lock_irqsave(&cstate->work_lock, irqflags);
        if (flags & ~(HASH_UPDATE | HASH_LAST)) {
                /* User should not pass anything other than UPDATE or LAST */
                ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
                goto unlock;
        }

        if (ctx->status & HASH_CTX_STS_PROCESSING) {
                /* Cannot submit to a currently processing job. */
                ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
                goto unlock;
        }

        if (ctx->status & HASH_CTX_STS_COMPLETE) {
                /* Cannot update a finished job. */
                ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
                goto unlock;
        }

        /*
         * If we made it here, there were no errors during this call to
         * submit
         */
        ctx->error = HASH_CTX_ERROR_NONE;

        /* Store buffer ptr info from user */
        ctx->incoming_buffer = buffer;
        ctx->incoming_buffer_length = len;

        /*
         * Store the user's request flags and mark this ctx as currently being
         * processed.
         */
        ctx->status = (flags & HASH_LAST) ?
                        (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
                        HASH_CTX_STS_PROCESSING;

        /* Advance byte counter */
        ctx->total_length += len;

        /*
         * If there is anything currently buffered in the extra blocks,
         * append to it until it contains a whole block.
         * Or if the user's buffer contains less than a whole block,
         * append as much as possible to the extra block.
         */
        if (ctx->partial_block_buffer_length || len < SHA512_BLOCK_SIZE) {
                /* Compute how many bytes to copy from user buffer into extra
                 * block
                 */
                uint32_t copy_len = SHA512_BLOCK_SIZE -
                                        ctx->partial_block_buffer_length;
                if (len < copy_len)
                        copy_len = len;

                if (copy_len) {
                        /* Copy and update relevant pointers and counters */
                        memcpy
                (&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
                                buffer, copy_len);

                        ctx->partial_block_buffer_length += copy_len;
                        ctx->incoming_buffer = (const void *)
                                        ((const char *)buffer + copy_len);
                        ctx->incoming_buffer_length = len - copy_len;
                }

                /* The extra block should never contain more than 1 block
                 * here
                 */
                assert(ctx->partial_block_buffer_length <= SHA512_BLOCK_SIZE);

                /* If the extra block buffer contains exactly 1 block, it can
                 * be hashed.
                 */
                if (ctx->partial_block_buffer_length >= SHA512_BLOCK_SIZE) {
                        ctx->partial_block_buffer_length = 0;

                        ctx->job.buffer = ctx->partial_block_buffer;
                        ctx->job.len = 1;
                        ctx = (struct sha512_hash_ctx *)
                                sha512_job_mgr_submit(&mgr->mgr, &ctx->job);
                }
        }

        ctx = sha512_ctx_mgr_resubmit(mgr, ctx);
unlock:
        spin_unlock_irqrestore(&cstate->work_lock, irqflags);
        return ctx;
}

static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct mcryptd_alg_cstate *cstate)
{
        struct sha512_ctx_mgr *mgr;
        struct sha512_hash_ctx *ctx;
        unsigned long flags;

        mgr = cstate->mgr;
        spin_lock_irqsave(&cstate->work_lock, flags);
        while (1) {
                ctx = (struct sha512_hash_ctx *)
                                        sha512_job_mgr_flush(&mgr->mgr);

                /* If flush returned 0, there are no more jobs in flight. */
                if (!ctx)
                        break;

                /*
                 * If flush returned a job, resubmit the job to finish
                 * processing.
                 */
                ctx = sha512_ctx_mgr_resubmit(mgr, ctx);

                /*
                 * If sha512_ctx_mgr_resubmit returned a job, it is ready to
                 * be returned. Otherwise, all jobs currently being managed by
                 * the sha512_ctx_mgr still need processing. Loop.
                 */
                if (ctx)
                        break;
        }
        spin_unlock_irqrestore(&cstate->work_lock, flags);
        return ctx;
}

static int sha512_mb_init(struct ahash_request *areq)
{
        struct sha512_hash_ctx *sctx = ahash_request_ctx(areq);

        hash_ctx_init(sctx);
        sctx->job.result_digest[0] = SHA512_H0;
        sctx->job.result_digest[1] = SHA512_H1;
        sctx->job.result_digest[2] = SHA512_H2;
        sctx->job.result_digest[3] = SHA512_H3;
        sctx->job.result_digest[4] = SHA512_H4;
        sctx->job.result_digest[5] = SHA512_H5;
        sctx->job.result_digest[6] = SHA512_H6;
        sctx->job.result_digest[7] = SHA512_H7;
        sctx->total_length = 0;
        sctx->partial_block_buffer_length = 0;
        sctx->status = HASH_CTX_STS_IDLE;

        return 0;
}

static int sha512_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
{
        int        i;
        struct        sha512_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
        __be64        *dst = (__be64 *) rctx->out;

        for (i = 0; i < 8; ++i)
                dst[i] = cpu_to_be64(sctx->job.result_digest[i]);

        return 0;
}

static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
                        struct mcryptd_alg_cstate *cstate, bool flush)
{
        int        flag = HASH_UPDATE;
        int        nbytes, err = 0;
        struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
        struct sha512_hash_ctx *sha_ctx;

        /* more work ? */
        while (!(rctx->flag & HASH_DONE)) {
                nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
                if (nbytes < 0) {
                        err = nbytes;
                        goto out;
                }
                /* check if the walk is done */
                if (crypto_ahash_walk_last(&rctx->walk)) {
                        rctx->flag |= HASH_DONE;
                        if (rctx->flag & HASH_FINAL)
                                flag |= HASH_LAST;

                }
                sha_ctx = (struct sha512_hash_ctx *)
                                                ahash_request_ctx(&rctx->areq);
                kernel_fpu_begin();
                sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx,
                                                rctx->walk.data, nbytes, flag);
                if (!sha_ctx) {
                        if (flush)
                                sha_ctx = sha512_ctx_mgr_flush(cstate);
                }
                kernel_fpu_end();
                if (sha_ctx)
                        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                else {
                        rctx = NULL;
                        goto out;
                }
        }

        /* copy the results */
        if (rctx->flag & HASH_FINAL)
                sha512_mb_set_results(rctx);

out:
        *ret_rctx = rctx;
        return err;
}

static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
                            struct mcryptd_alg_cstate *cstate,
                            int err)
{
        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha512_hash_ctx *sha_ctx;
        struct mcryptd_hash_request_ctx *req_ctx;
        int ret;
        unsigned long flags;

        /* remove from work list */
        spin_lock_irqsave(&cstate->work_lock, flags);
        list_del(&rctx->waiter);
        spin_unlock_irqrestore(&cstate->work_lock, flags);

        if (irqs_disabled())
                rctx->complete(&req->base, err);
        else {
                local_bh_disable();
                rctx->complete(&req->base, err);
                local_bh_enable();
        }

        /* check to see if there are other jobs that are done */
        sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
        while (sha_ctx) {
                req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                ret = sha_finish_walk(&req_ctx, cstate, false);
                if (req_ctx) {
                        spin_lock_irqsave(&cstate->work_lock, flags);
                        list_del(&req_ctx->waiter);
                        spin_unlock_irqrestore(&cstate->work_lock, flags);

                        req = cast_mcryptd_ctx_to_req(req_ctx);
                        if (irqs_disabled())
                                req_ctx->complete(&req->base, ret);
                        else {
                                local_bh_disable();
                                req_ctx->complete(&req->base, ret);
                                local_bh_enable();
                        }
                }
                sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate);
        }

        return 0;
}

static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
                             struct mcryptd_alg_cstate *cstate)
{
        unsigned long next_flush;
        unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
        unsigned long flags;

        /* initialize tag */
        rctx->tag.arrival = jiffies;    /* tag the arrival time */
        rctx->tag.seq_num = cstate->next_seq_num++;
        next_flush = rctx->tag.arrival + delay;
        rctx->tag.expire = next_flush;

        spin_lock_irqsave(&cstate->work_lock, flags);
        list_add_tail(&rctx->waiter, &cstate->work_list);
        spin_unlock_irqrestore(&cstate->work_lock, flags);

        mcryptd_arm_flusher(cstate, delay);
}

static int sha512_mb_update(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                        container_of(areq, struct mcryptd_hash_request_ctx,
                                                                        areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha512_mb_alg_state.alg_cstate);

        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha512_hash_ctx *sha_ctx;
        int ret = 0, nbytes;


        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        nbytes = crypto_ahash_walk_first(req, &rctx->walk);

        if (nbytes < 0) {
                ret = nbytes;
                goto done;
        }

        if (crypto_ahash_walk_last(&rctx->walk))
                rctx->flag |= HASH_DONE;

        /* submit */
        sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
        sha512_mb_add_list(rctx, cstate);
        kernel_fpu_begin();
        sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
                                                        nbytes, HASH_UPDATE);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);

        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha512_mb_finup(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                        container_of(areq, struct mcryptd_hash_request_ctx,
                                                                        areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha512_mb_alg_state.alg_cstate);

        struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
        struct sha512_hash_ctx *sha_ctx;
        int ret = 0, flag = HASH_UPDATE, nbytes;

        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        nbytes = crypto_ahash_walk_first(req, &rctx->walk);

        if (nbytes < 0) {
                ret = nbytes;
                goto done;
        }

        if (crypto_ahash_walk_last(&rctx->walk)) {
                rctx->flag |= HASH_DONE;
                flag = HASH_LAST;
        }

        /* submit */
        rctx->flag |= HASH_FINAL;
        sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
        sha512_mb_add_list(rctx, cstate);

        kernel_fpu_begin();
        sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data,
                                                                nbytes, flag);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);
        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha512_mb_final(struct ahash_request *areq)
{
        struct mcryptd_hash_request_ctx *rctx =
                        container_of(areq, struct mcryptd_hash_request_ctx,
                                                                        areq);
        struct mcryptd_alg_cstate *cstate =
                                this_cpu_ptr(sha512_mb_alg_state.alg_cstate);

        struct sha512_hash_ctx *sha_ctx;
        int ret = 0;
        u8 data;

        /* sanity check */
        if (rctx->tag.cpu != smp_processor_id()) {
                pr_err("mcryptd error: cpu clash\n");
                goto done;
        }

        /* need to init context */
        req_ctx_init(rctx, areq);

        rctx->flag |= HASH_DONE | HASH_FINAL;

        sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq);
        /* flag HASH_FINAL and 0 data size */
        sha512_mb_add_list(rctx, cstate);
        kernel_fpu_begin();
        sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, &data, 0, HASH_LAST);
        kernel_fpu_end();

        /* check if anything is returned */
        if (!sha_ctx)
                return -EINPROGRESS;

        if (sha_ctx->error) {
                ret = sha_ctx->error;
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                goto done;
        }

        rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
        ret = sha_finish_walk(&rctx, cstate, false);
        if (!rctx)
                return -EINPROGRESS;
done:
        sha_complete_job(rctx, cstate, ret);
        return ret;
}

static int sha512_mb_export(struct ahash_request *areq, void *out)
{
        struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); /*covered*/

        memcpy(out, sctx, sizeof(*sctx));

        return 0;
}

static int sha512_mb_import(struct ahash_request *areq, const void *in)
{
        struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); /*covered*/

        memcpy(sctx, in, sizeof(*sctx));

        return 0;
}

static int sha512_mb_async_init_tfm(struct crypto_tfm *tfm)
{
        struct mcryptd_ahash *mcryptd_tfm;
        struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);
        struct mcryptd_hash_ctx *mctx;

        mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha512-mb", /*covered*/
                                                CRYPTO_ALG_INTERNAL,
                                                CRYPTO_ALG_INTERNAL);
        if (IS_ERR(mcryptd_tfm))
                return PTR_ERR(mcryptd_tfm);
        mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
        mctx->alg_state = &sha512_mb_alg_state; /*covered*/
        ctx->mcryptd_tfm = mcryptd_tfm;
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
                                sizeof(struct ahash_request) +
                                crypto_ahash_reqsize(&mcryptd_tfm->base));

        return 0; /*covered*/
}

static void sha512_mb_async_exit_tfm(struct crypto_tfm *tfm)
{
        struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);

        mcryptd_free_ahash(ctx->mcryptd_tfm); /*covered*/
}

static int sha512_mb_areq_init_tfm(struct crypto_tfm *tfm)
{
        crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), /*covered*/
                                sizeof(struct ahash_request) +
                                sizeof(struct sha512_hash_ctx));

        return 0;
}

static void sha512_mb_areq_exit_tfm(struct crypto_tfm *tfm)
{
        struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm);

        mcryptd_free_ahash(ctx->mcryptd_tfm);
}

static struct ahash_alg sha512_mb_areq_alg = {
        .init                =        sha512_mb_init,
        .update                =        sha512_mb_update,
        .final                =        sha512_mb_final,
        .finup                =        sha512_mb_finup,
        .export                =        sha512_mb_export,
        .import                =        sha512_mb_import,
        .halg                =        {
        .digestsize        =        SHA512_DIGEST_SIZE,
        .statesize        =        sizeof(struct sha512_hash_ctx),
        .base                =        {
                        .cra_name         = "__sha512-mb",
                        .cra_driver_name = "__intel_sha512-mb",
                        .cra_priority         = 100,
                        /*
                         * use ASYNC flag as some buffers in multi-buffer
                         * algo may not have completed before hashing thread
                         * sleep
                         */
                        .cra_flags        = CRYPTO_ALG_ASYNC |
                                          CRYPTO_ALG_INTERNAL,
                        .cra_blocksize        = SHA512_BLOCK_SIZE,
                        .cra_module        = THIS_MODULE,
                        .cra_list        = LIST_HEAD_INIT
                                        (sha512_mb_areq_alg.halg.base.cra_list),
                        .cra_init        = sha512_mb_areq_init_tfm,
                        .cra_exit        = sha512_mb_areq_exit_tfm,
                        .cra_ctxsize        = sizeof(struct sha512_hash_ctx),
                }
        }
};

static int sha512_mb_async_init(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_init(mcryptd_req); /*covered*/
}

static int sha512_mb_async_update(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_update(mcryptd_req);
}

static int sha512_mb_async_finup(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_finup(mcryptd_req);
}

static int sha512_mb_async_final(struct ahash_request *req)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/

        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_final(mcryptd_req);
}

static int sha512_mb_async_digest(struct ahash_request *req)
{
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); /*covered*/
        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct ahash_request *mcryptd_req = ahash_request_ctx(req);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_digest(mcryptd_req);
}

static int sha512_mb_async_export(struct ahash_request *req, void *out)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        return crypto_ahash_export(mcryptd_req, out);
}

static int sha512_mb_async_import(struct ahash_request *req, const void *in)
{
        struct ahash_request *mcryptd_req = ahash_request_ctx(req); /*covered*/
        struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
        struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm);
        struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
        struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
        struct mcryptd_hash_request_ctx *rctx;
        struct ahash_request *areq;

        memcpy(mcryptd_req, req, sizeof(*req));
        ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
        rctx = ahash_request_ctx(mcryptd_req);

        areq = &rctx->areq;

        ahash_request_set_tfm(areq, child);
        ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
                                        rctx->complete, req);

        return crypto_ahash_import(mcryptd_req, in); /*covered*/
}

static struct ahash_alg sha512_mb_async_alg = {
        .init           = sha512_mb_async_init,
        .update         = sha512_mb_async_update,
        .final          = sha512_mb_async_final,
        .finup          = sha512_mb_async_finup,
        .digest         = sha512_mb_async_digest,
        .export                = sha512_mb_async_export,
        .import                = sha512_mb_async_import,
        .halg = {
                .digestsize     = SHA512_DIGEST_SIZE,
                .statesize      = sizeof(struct sha512_hash_ctx),
                .base = {
                        .cra_name               = "sha512",
                        .cra_driver_name        = "sha512_mb",
                        /*
                         * Low priority, since with few concurrent hash requests
                         * this is extremely slow due to the flush delay.  Users
                         * whose workloads would benefit from this can request
                         * it explicitly by driver name, or can increase its
                         * priority at runtime using NETLINK_CRYPTO.
                         */
                        .cra_priority           = 50,
                        .cra_flags              = CRYPTO_ALG_ASYNC,
                        .cra_blocksize          = SHA512_BLOCK_SIZE,
                        .cra_module             = THIS_MODULE,
                        .cra_list               = LIST_HEAD_INIT
                                (sha512_mb_async_alg.halg.base.cra_list),
                        .cra_init               = sha512_mb_async_init_tfm,
                        .cra_exit               = sha512_mb_async_exit_tfm,
                        .cra_ctxsize                = sizeof(struct sha512_mb_ctx),
                        .cra_alignmask                = 0,
                },
        },
};

static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate)
{
        struct mcryptd_hash_request_ctx *rctx;
        unsigned long cur_time;
        unsigned long next_flush = 0;
        struct sha512_hash_ctx *sha_ctx;


        cur_time = jiffies;

        while (!list_empty(&cstate->work_list)) {
                rctx = list_entry(cstate->work_list.next,
                                struct mcryptd_hash_request_ctx, waiter);
                if time_before(cur_time, rctx->tag.expire)
                        break;
                kernel_fpu_begin();
                sha_ctx = (struct sha512_hash_ctx *)
                                        sha512_ctx_mgr_flush(cstate);
                kernel_fpu_end();
                if (!sha_ctx) {
                        pr_err("sha512_mb error: nothing got flushed for"
                                                        " non-empty list\n");
                        break;
                }
                rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
                sha_finish_walk(&rctx, cstate, true);
                sha_complete_job(rctx, cstate, 0);
        }

        if (!list_empty(&cstate->work_list)) {
                rctx = list_entry(cstate->work_list.next,
                                struct mcryptd_hash_request_ctx, waiter);
                /* get the hash context and then flush time */
                next_flush = rctx->tag.expire;
                mcryptd_arm_flusher(cstate, get_delay(next_flush));
        }
        return next_flush;
}

static int __init sha512_mb_mod_init(void)
{

        int cpu;
        int err;
        struct mcryptd_alg_cstate *cpu_state;

        /* check for dependent cpu features */
        if (!boot_cpu_has(X86_FEATURE_AVX2) ||
            !boot_cpu_has(X86_FEATURE_BMI2))
                return -ENODEV;

        /* initialize multibuffer structures */
        sha512_mb_alg_state.alg_cstate =
                                alloc_percpu(struct mcryptd_alg_cstate);

        sha512_job_mgr_init = sha512_mb_mgr_init_avx2;
        sha512_job_mgr_submit = sha512_mb_mgr_submit_avx2;
        sha512_job_mgr_flush = sha512_mb_mgr_flush_avx2;
        sha512_job_mgr_get_comp_job = sha512_mb_mgr_get_comp_job_avx2;

        if (!sha512_mb_alg_state.alg_cstate)
                return -ENOMEM;
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
                cpu_state->next_flush = 0;
                cpu_state->next_seq_num = 0;
                cpu_state->flusher_engaged = false;
                INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
                cpu_state->cpu = cpu;
                cpu_state->alg_state = &sha512_mb_alg_state;
                cpu_state->mgr = kzalloc(sizeof(struct sha512_ctx_mgr),
                                                                GFP_KERNEL);
                if (!cpu_state->mgr)
                        goto err2;
                sha512_ctx_mgr_init(cpu_state->mgr);
                INIT_LIST_HEAD(&cpu_state->work_list);
                spin_lock_init(&cpu_state->work_lock);
        }
        sha512_mb_alg_state.flusher = &sha512_mb_flusher;

        err = crypto_register_ahash(&sha512_mb_areq_alg);
        if (err)
                goto err2;
        err = crypto_register_ahash(&sha512_mb_async_alg);
        if (err)
                goto err1;


        return 0;
err1:
        crypto_unregister_ahash(&sha512_mb_areq_alg);
err2:
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
                kfree(cpu_state->mgr);
        }
        free_percpu(sha512_mb_alg_state.alg_cstate);
        return -ENODEV;
}

static void __exit sha512_mb_mod_fini(void)
{
        int cpu;
        struct mcryptd_alg_cstate *cpu_state;

        crypto_unregister_ahash(&sha512_mb_async_alg);
        crypto_unregister_ahash(&sha512_mb_areq_alg);
        for_each_possible_cpu(cpu) {
                cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu);
                kfree(cpu_state->mgr);
        }
        free_percpu(sha512_mb_alg_state.alg_cstate);
}

module_init(sha512_mb_mod_init);
module_exit(sha512_mb_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, multi buffer accelerated");

MODULE_ALIAS("sha512");

/*
 * Cryptographic API.
 *
 * Glue code for the SHA512 Secure Hash Algorithm assembler
 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
 *
 * This file is based on sha512_generic.c
 *
 * Copyright (C) 2013 Intel Corporation
 * Author: Tim Chen <tim.c.chen@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; either version 2 of the License, or (at your option)
 * any later version.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#define pr_fmt(fmt)        KBUILD_MODNAME ": " fmt

#include <crypto/internal/hash.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/cryptohash.h>
#include <linux/types.h>
#include <crypto/sha.h>
#include <crypto/sha512_base.h>
#include <asm/fpu/api.h>

#include <linux/string.h>

asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data,
                                       u64 rounds);

typedef void (sha512_transform_fn)(u64 *digest, const char *data, u64 rounds);

static int sha512_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len, sha512_transform_fn *sha512_xform)
{
        struct sha512_state *sctx = shash_desc_ctx(desc);

        if (!irq_fpu_usable() || /*covered*/
            (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) /*covered*/
                return crypto_sha512_update(desc, data, len); /*covered*/

        /* make sure casting to sha512_block_fn() is safe */
        BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);

        kernel_fpu_begin(); /*covered*/
        sha512_base_do_update(desc, data, len, /*covered*/
                              (sha512_block_fn *)sha512_xform);
        kernel_fpu_end(); /*covered*/

        return 0;
}

static int sha512_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out, sha512_transform_fn *sha512_xform)
{
        if (!irq_fpu_usable()) /*covered*/
                return crypto_sha512_finup(desc, data, len, out);

        kernel_fpu_begin(); /*covered*/
        if (len)
                sha512_base_do_update(desc, data, len, /*covered*/
                                      (sha512_block_fn *)sha512_xform);
        sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_xform); /*covered*/
        kernel_fpu_end();

        return sha512_base_finish(desc, out); /*covered*/
}

static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len)
{
        return sha512_update(desc, data, len, sha512_transform_ssse3); /*covered*/
}

static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha512_finup(desc, data, len, out, sha512_transform_ssse3); /*covered*/
}

/* Add padding and return the message digest. */
static int sha512_ssse3_final(struct shash_desc *desc, u8 *out)
{
        return sha512_ssse3_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha512_ssse3_algs[] = { {
        .digestsize        =        SHA512_DIGEST_SIZE,
        .init                =        sha512_base_init,
        .update                =        sha512_ssse3_update,
        .final                =        sha512_ssse3_final,
        .finup                =        sha512_ssse3_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha512",
                .cra_driver_name =        "sha512-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA512_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
},  {
        .digestsize        =        SHA384_DIGEST_SIZE,
        .init                =        sha384_base_init,
        .update                =        sha512_ssse3_update,
        .final                =        sha512_ssse3_final,
        .finup                =        sha512_ssse3_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha384",
                .cra_driver_name =        "sha384-ssse3",
                .cra_priority        =        150,
                .cra_blocksize        =        SHA384_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha512_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                return crypto_register_shashes(sha512_ssse3_algs,
                        ARRAY_SIZE(sha512_ssse3_algs));
        return 0;
}

static void unregister_sha512_ssse3(void)
{
        if (boot_cpu_has(X86_FEATURE_SSSE3))
                crypto_unregister_shashes(sha512_ssse3_algs,
                        ARRAY_SIZE(sha512_ssse3_algs));
}

#ifdef CONFIG_AS_AVX
asmlinkage void sha512_transform_avx(u64 *digest, const char *data,
                                     u64 rounds);
static bool avx_usable(void)
{
        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
                if (boot_cpu_has(X86_FEATURE_AVX))
                        pr_info("AVX detected but unusable.\n");
                return false;
        }

        return true;
}

static int sha512_avx_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len)
{
        return sha512_update(desc, data, len, sha512_transform_avx); /*covered*/
}

static int sha512_avx_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha512_finup(desc, data, len, out, sha512_transform_avx); /*covered*/
}

/* Add padding and return the message digest. */
static int sha512_avx_final(struct shash_desc *desc, u8 *out)
{
        return sha512_avx_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha512_avx_algs[] = { {
        .digestsize        =        SHA512_DIGEST_SIZE,
        .init                =        sha512_base_init,
        .update                =        sha512_avx_update,
        .final                =        sha512_avx_final,
        .finup                =        sha512_avx_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha512",
                .cra_driver_name =        "sha512-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA512_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
},  {
        .digestsize        =        SHA384_DIGEST_SIZE,
        .init                =        sha384_base_init,
        .update                =        sha512_avx_update,
        .final                =        sha512_avx_final,
        .finup                =        sha512_avx_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha384",
                .cra_driver_name =        "sha384-avx",
                .cra_priority        =        160,
                .cra_blocksize        =        SHA384_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static int register_sha512_avx(void)
{
        if (avx_usable())
                return crypto_register_shashes(sha512_avx_algs,
                        ARRAY_SIZE(sha512_avx_algs));
        return 0;
}

static void unregister_sha512_avx(void)
{
        if (avx_usable())
                crypto_unregister_shashes(sha512_avx_algs,
                        ARRAY_SIZE(sha512_avx_algs));
}
#else
static inline int register_sha512_avx(void) { return 0; }
static inline void unregister_sha512_avx(void) { }
#endif

#if defined(CONFIG_AS_AVX2) && defined(CONFIG_AS_AVX)
asmlinkage void sha512_transform_rorx(u64 *digest, const char *data,
                                      u64 rounds);

static int sha512_avx2_update(struct shash_desc *desc, const u8 *data,
                       unsigned int len)
{
        return sha512_update(desc, data, len, sha512_transform_rorx); /*covered*/
}

static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data,
              unsigned int len, u8 *out)
{
        return sha512_finup(desc, data, len, out, sha512_transform_rorx); /*covered*/
}

/* Add padding and return the message digest. */
static int sha512_avx2_final(struct shash_desc *desc, u8 *out)
{
        return sha512_avx2_finup(desc, NULL, 0, out); /*covered*/
}

static struct shash_alg sha512_avx2_algs[] = { {
        .digestsize        =        SHA512_DIGEST_SIZE,
        .init                =        sha512_base_init,
        .update                =        sha512_avx2_update,
        .final                =        sha512_avx2_final,
        .finup                =        sha512_avx2_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha512",
                .cra_driver_name =        "sha512-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA512_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
},  {
        .digestsize        =        SHA384_DIGEST_SIZE,
        .init                =        sha384_base_init,
        .update                =        sha512_avx2_update,
        .final                =        sha512_avx2_final,
        .finup                =        sha512_avx2_finup,
        .descsize        =        sizeof(struct sha512_state),
        .base                =        {
                .cra_name        =        "sha384",
                .cra_driver_name =        "sha384-avx2",
                .cra_priority        =        170,
                .cra_blocksize        =        SHA384_BLOCK_SIZE,
                .cra_module        =        THIS_MODULE,
        }
} };

static bool avx2_usable(void)
{
        if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
                    boot_cpu_has(X86_FEATURE_BMI2))
                return true;

        return false;
}

static int register_sha512_avx2(void)
{
        if (avx2_usable())
                return crypto_register_shashes(sha512_avx2_algs,
                        ARRAY_SIZE(sha512_avx2_algs));
        return 0;
}

static void unregister_sha512_avx2(void)
{
        if (avx2_usable())
                crypto_unregister_shashes(sha512_avx2_algs,
                        ARRAY_SIZE(sha512_avx2_algs));
}
#else
static inline int register_sha512_avx2(void) { return 0; }
static inline void unregister_sha512_avx2(void) { }
#endif

static int __init sha512_ssse3_mod_init(void)
{

        if (register_sha512_ssse3())
                goto fail;

        if (register_sha512_avx()) {
                unregister_sha512_ssse3();
                goto fail;
        }

        if (register_sha512_avx2()) {
                unregister_sha512_avx();
                unregister_sha512_ssse3();
                goto fail;
        }

        return 0;
fail:
        return -ENODEV;
}

static void __exit sha512_ssse3_mod_fini(void)
{
        unregister_sha512_avx2();
        unregister_sha512_avx();
        unregister_sha512_ssse3();
}

module_init(sha512_ssse3_mod_init);
module_exit(sha512_ssse3_mod_fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated");

MODULE_ALIAS_CRYPTO("sha512");
MODULE_ALIAS_CRYPTO("sha512-ssse3");
MODULE_ALIAS_CRYPTO("sha512-avx");
MODULE_ALIAS_CRYPTO("sha512-avx2");
MODULE_ALIAS_CRYPTO("sha384");
MODULE_ALIAS_CRYPTO("sha384-ssse3");
MODULE_ALIAS_CRYPTO("sha384-avx");
MODULE_ALIAS_CRYPTO("sha384-avx2");

/*
 * Glue Code for AVX assembler version of Twofish Cipher
 *
 * Copyright (C) 2012 Johannes Goetzfried
 *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
 *
 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/crypto.h>
#include <linux/err.h>
#include <crypto/algapi.h>
#include <crypto/internal/simd.h>
#include <crypto/twofish.h>
#include <crypto/xts.h>
#include <asm/crypto/glue_helper.h>
#include <asm/crypto/twofish.h>

#define TWOFISH_PARALLEL_BLOCKS 8

/* 8-way parallel cipher functions */
asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src);
asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src);

asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src);
asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
                                 const u8 *src, le128 *iv);

asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src, le128 *iv);
asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
                                     const u8 *src, le128 *iv);

static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return twofish_setkey(&tfm->base, key, keylen); /*covered*/
}

static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __twofish_enc_blk_3way(ctx, dst, src, false);
}

static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(twofish_enc_blk));
}

static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        glue_xts_crypt_128bit_one(ctx, dst, src, iv,
                                  GLUE_FUNC_CAST(twofish_dec_blk));
}

struct twofish_xts_ctx {
        struct twofish_ctx tweak_ctx;
        struct twofish_ctx crypt_ctx;
};

static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
                              unsigned int keylen)
{
        struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
        u32 *flags = &tfm->base.crt_flags; /*covered*/
        int err;

        err = xts_verify_key(tfm, key, keylen); /*covered*/
        if (err)
                return err;

        /* first half of xts-key is for crypt */
        err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags); /*covered*/
        if (err) /*covered*/
                return err;

        /* second half of xts-key is for tweak */
        return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, /*covered*/
                                flags);
}

static const struct common_glue_ctx twofish_enc = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
        } }
};

static const struct common_glue_ctx twofish_ctr = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
        } }
};

static const struct common_glue_ctx twofish_enc_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
        } }
};

static const struct common_glue_ctx twofish_dec = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
        } }
};

static const struct common_glue_ctx twofish_dec_cbc = {
        .num_funcs = 3,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
        }, {
                .num_blocks = 3,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
        } }
};

static const struct common_glue_ctx twofish_dec_xts = {
        .num_funcs = 2,
        .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,

        .funcs = { {
                .num_blocks = TWOFISH_PARALLEL_BLOCKS,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_enc, req);
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_dec, req);
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&twofish_ctr, req); /*covered*/
}

static int xts_encrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&twofish_enc_xts, req,
                                   XTS_TWEAK_CAST(twofish_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static int xts_decrypt(struct skcipher_request *req)
{
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);

        return glue_xts_req_128bit(&twofish_dec_xts, req,
                                   XTS_TWEAK_CAST(twofish_enc_blk),
                                   &ctx->tweak_ctx, &ctx->crypt_ctx);
}

static struct skcipher_alg twofish_algs[] = {
        {
                .base.cra_name                = "__ecb(twofish)",
                .base.cra_driver_name        = "__ecb-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "__cbc(twofish)",
                .base.cra_driver_name        = "__cbc-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "__ctr(twofish)",
                .base.cra_driver_name        = "__ctr-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .chunksize                = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        }, {
                .base.cra_name                = "__xts(twofish)",
                .base.cra_driver_name        = "__xts-twofish-avx",
                .base.cra_priority        = 400,
                .base.cra_flags                = CRYPTO_ALG_INTERNAL,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_xts_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = 2 * TF_MIN_KEY_SIZE,
                .max_keysize                = 2 * TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .setkey                        = xts_twofish_setkey,
                .encrypt                = xts_encrypt,
                .decrypt                = xts_decrypt,
        },
};

static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];

static int __init twofish_init(void)
{
        const char *feature_name;

        if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
                pr_info("CPU feature '%s' is not supported.\n", feature_name);
                return -ENODEV;
        }

        return simd_register_skciphers_compat(twofish_algs,
                                              ARRAY_SIZE(twofish_algs),
                                              twofish_simd_algs);
}

static void __exit twofish_exit(void)
{
        simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
                                  twofish_simd_algs);
}

module_init(twofish_init);
module_exit(twofish_exit);

MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("twofish");

/*
 * Glue Code for assembler optimized version of TWOFISH
 *
 * Originally Twofish for GPG
 * By Matthew Skala <mskala@ansuz.sooke.bc.ca>, July 26, 1998
 * 256-bit key length added March 20, 1999
 * Some modifications to reduce the text size by Werner Koch, April, 1998
 * Ported to the kerneli patch by Marc Mutz <Marc@Mutz.com>
 * Ported to CryptoAPI by Colin Slater <hoho@tacomeat.net>
 *
 * The original author has disclaimed all copyright interest in this
 * code and thus put it in the public domain. The subsequent authors
 * have put this under the GNU General Public License.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 * This code is a "clean room" implementation, written from the paper
 * _Twofish: A 128-Bit Block Cipher_ by Bruce Schneier, John Kelsey,
 * Doug Whiting, David Wagner, Chris Hall, and Niels Ferguson, available
 * through http://www.counterpane.com/twofish.html
 *
 * For background information on multiplication in finite fields, used for
 * the matrix operations in the key schedule, see the book _Contemporary
 * Abstract Algebra_ by Joseph A. Gallian, especially chapter 22 in the
 * Third Edition.
 */

#include <crypto/twofish.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
                                const u8 *src);
EXPORT_SYMBOL_GPL(twofish_enc_blk);
asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
                                const u8 *src);
EXPORT_SYMBOL_GPL(twofish_dec_blk);

static void twofish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        twofish_enc_blk(crypto_tfm_ctx(tfm), dst, src); /*covered*/
}

static void twofish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
{
        twofish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
}

static struct crypto_alg alg = {
        .cra_name                =        "twofish",
        .cra_driver_name        =        "twofish-asm",
        .cra_priority                =        200,
        .cra_flags                =        CRYPTO_ALG_TYPE_CIPHER,
        .cra_blocksize                =        TF_BLOCK_SIZE,
        .cra_ctxsize                =        sizeof(struct twofish_ctx),
        .cra_alignmask                =        0,
        .cra_module                =        THIS_MODULE,
        .cra_u                        =        {
                .cipher = {
                        .cia_min_keysize        =        TF_MIN_KEY_SIZE,
                        .cia_max_keysize        =        TF_MAX_KEY_SIZE,
                        .cia_setkey                =        twofish_setkey,
                        .cia_encrypt                =        twofish_encrypt,
                        .cia_decrypt                =        twofish_decrypt
                }
        }
};

static int __init init(void)
{
        return crypto_register_alg(&alg);
}

static void __exit fini(void)
{
        crypto_unregister_alg(&alg);
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION ("Twofish Cipher Algorithm, asm optimized");
MODULE_ALIAS_CRYPTO("twofish");
MODULE_ALIAS_CRYPTO("twofish-asm");

/*
 * Glue Code for 3-way parallel assembler optimized version of Twofish
 *
 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 * USA
 *
 */

#include <asm/crypto/glue_helper.h>
#include <asm/crypto/twofish.h>
#include <crypto/algapi.h>
#include <crypto/b128ops.h>
#include <crypto/internal/skcipher.h>
#include <crypto/twofish.h>
#include <linux/crypto.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/types.h>

EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);

static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
                                   const u8 *key, unsigned int keylen)
{
        return twofish_setkey(&tfm->base, key, keylen); /*covered*/
}

static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __twofish_enc_blk_3way(ctx, dst, src, false); /*covered*/
}

static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
                                            const u8 *src)
{
        __twofish_enc_blk_3way(ctx, dst, src, true);
}

void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
{
        u128 ivs[2]; /*covered*/

        ivs[0] = src[0];
        ivs[1] = src[1];

        twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);

        u128_xor(&dst[1], &dst[1], &ivs[0]);
        u128_xor(&dst[2], &dst[2], &ivs[1]);
}
EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);

void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
        be128 ctrblk; /*covered*/

        if (dst != src)
                *dst = *src; /*covered*/

        le128_to_be128(&ctrblk, iv); /*covered*/
        le128_inc(iv); /*covered*/

        twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
        u128_xor(dst, dst, (u128 *)&ctrblk);
}
EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);

void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
                              le128 *iv)
{
        be128 ctrblks[3]; /*covered*/

        if (dst != src) {
                dst[0] = src[0]; /*covered*/
                dst[1] = src[1];
                dst[2] = src[2];
        }

        le128_to_be128(&ctrblks[0], iv); /*covered*/
        le128_inc(iv);
        le128_to_be128(&ctrblks[1], iv); /*covered*/
        le128_inc(iv);
        le128_to_be128(&ctrblks[2], iv); /*covered*/
        le128_inc(iv); /*covered*/

        twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
}
EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);

static const struct common_glue_ctx twofish_enc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
        } }
};

static const struct common_glue_ctx twofish_ctr = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
        } }
};

static const struct common_glue_ctx twofish_dec = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
        } }
};

static const struct common_glue_ctx twofish_dec_cbc = {
        .num_funcs = 2,
        .fpu_blocks_limit = -1,

        .funcs = { {
                .num_blocks = 3,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
        }, {
                .num_blocks = 1,
                .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
        } }
};

static int ecb_encrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_enc, req); /*covered*/
}

static int ecb_decrypt(struct skcipher_request *req)
{
        return glue_ecb_req_128bit(&twofish_dec, req); /*covered*/
}

static int cbc_encrypt(struct skcipher_request *req)
{
        return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk), /*covered*/
                                           req);
}

static int cbc_decrypt(struct skcipher_request *req)
{
        return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); /*covered*/
}

static int ctr_crypt(struct skcipher_request *req)
{
        return glue_ctr_req_128bit(&twofish_ctr, req); /*covered*/
}

static struct skcipher_alg tf_skciphers[] = {
        {
                .base.cra_name                = "ecb(twofish)",
                .base.cra_driver_name        = "ecb-twofish-3way",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ecb_encrypt,
                .decrypt                = ecb_decrypt,
        }, {
                .base.cra_name                = "cbc(twofish)",
                .base.cra_driver_name        = "cbc-twofish-3way",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = TF_BLOCK_SIZE,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = cbc_encrypt,
                .decrypt                = cbc_decrypt,
        }, {
                .base.cra_name                = "ctr(twofish)",
                .base.cra_driver_name        = "ctr-twofish-3way",
                .base.cra_priority        = 300,
                .base.cra_blocksize        = 1,
                .base.cra_ctxsize        = sizeof(struct twofish_ctx),
                .base.cra_module        = THIS_MODULE,
                .min_keysize                = TF_MIN_KEY_SIZE,
                .max_keysize                = TF_MAX_KEY_SIZE,
                .ivsize                        = TF_BLOCK_SIZE,
                .chunksize                = TF_BLOCK_SIZE,
                .setkey                        = twofish_setkey_skcipher,
                .encrypt                = ctr_crypt,
                .decrypt                = ctr_crypt,
        },
};

static bool is_blacklisted_cpu(void)
{
        if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
                return false;

        if (boot_cpu_data.x86 == 0x06 &&
                (boot_cpu_data.x86_model == 0x1c ||
                 boot_cpu_data.x86_model == 0x26 ||
                 boot_cpu_data.x86_model == 0x36)) {
                /*
                 * On Atom, twofish-3way is slower than original assembler
                 * implementation. Twofish-3way trades off some performance in
                 * storing blocks in 64bit registers to allow three blocks to
                 * be processed parallel. Parallel operation then allows gaining
                 * more performance than was trade off, on out-of-order CPUs.
                 * However Atom does not benefit from this parallellism and
                 * should be blacklisted.
                 */
                return true;
        }

        if (boot_cpu_data.x86 == 0x0f) {
                /*
                 * On Pentium 4, twofish-3way is slower than original assembler
                 * implementation because excessive uses of 64bit rotate and
                 * left-shifts (which are really slow on P4) needed to store and
                 * handle 128bit block in two 64bit registers.
                 */
                return true;
        }

        return false;
}

static int force;
module_param(force, int, 0);
MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");

static int __init init(void)
{
        if (!force && is_blacklisted_cpu()) {
                printk(KERN_INFO
                        "twofish-x86_64-3way: performance on this CPU "
                        "would be suboptimal: disabling "
                        "twofish-x86_64-3way.\n");
                return -ENODEV;
        }

        return crypto_register_skciphers(tf_skciphers,
                                         ARRAY_SIZE(tf_skciphers));
}

static void __exit fini(void)
{
        crypto_unregister_skciphers(tf_skciphers, ARRAY_SIZE(tf_skciphers));
}

module_init(init);
module_exit(fini);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
MODULE_ALIAS_CRYPTO("twofish");
MODULE_ALIAS_CRYPTO("twofish-asm");

/*
 * common.c - C code for kernel entry and exit
 * Copyright (c) 2015 Andrew Lutomirski
 * GPL v2
 *
 * Based on asm and ptrace code by many authors.  The code here originated
 * in ptrace.c and signal.c.
 */

#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/errno.h>
#include <linux/ptrace.h>
#include <linux/tracehook.h>
#include <linux/audit.h>
#include <linux/seccomp.h>
#include <linux/signal.h>
#include <linux/export.h>
#include <linux/context_tracking.h>
#include <linux/user-return-notifier.h>
#include <linux/nospec.h>
#include <linux/uprobes.h>
#include <linux/livepatch.h>
#include <linux/syscalls.h>

#include <asm/desc.h>
#include <asm/traps.h>
#include <asm/vdso.h>
#include <linux/uaccess.h>
#include <asm/cpufeature.h>

#define CREATE_TRACE_POINTS
#include <trace/events/syscalls.h>

#ifdef CONFIG_CONTEXT_TRACKING
/* Called on entry from user mode with IRQs off. */
__visible inline void enter_from_user_mode(void)
{
        CT_WARN_ON(ct_state() != CONTEXT_USER);
        user_exit_irqoff();
}
#else
static inline void enter_from_user_mode(void) {}
#endif

static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)
{
#ifdef CONFIG_X86_64
        if (arch == AUDIT_ARCH_X86_64) {
                audit_syscall_entry(regs->orig_ax, regs->di,
                                    regs->si, regs->dx, regs->r10);
        } else
#endif
        {
                audit_syscall_entry(regs->orig_ax, regs->bx,
                                    regs->cx, regs->dx, regs->si);
        }
}

/*
 * Returns the syscall nr to run (which should match regs->orig_ax) or -1
 * to skip the syscall.
 */
static long syscall_trace_enter(struct pt_regs *regs)
{
        u32 arch = in_ia32_syscall() ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64;

        struct thread_info *ti = current_thread_info();
        unsigned long ret = 0;
        bool emulated = false;
        u32 work;

        if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
                BUG_ON(regs != task_pt_regs(current));

        work = READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY;

        if (unlikely(work & _TIF_SYSCALL_EMU))
                emulated = true;

        if ((emulated || (work & _TIF_SYSCALL_TRACE)) &&
            tracehook_report_syscall_entry(regs))
                return -1L;

        if (emulated)
                return -1L;

#ifdef CONFIG_SECCOMP
        /*
         * Do seccomp after ptrace, to catch any tracer changes.
         */
        if (work & _TIF_SECCOMP) {
                struct seccomp_data sd;

                sd.arch = arch;
                sd.nr = regs->orig_ax;
                sd.instruction_pointer = regs->ip;
#ifdef CONFIG_X86_64
                if (arch == AUDIT_ARCH_X86_64) {
                        sd.args[0] = regs->di;
                        sd.args[1] = regs->si;
                        sd.args[2] = regs->dx;
                        sd.args[3] = regs->r10;
                        sd.args[4] = regs->r8;
                        sd.args[5] = regs->r9;
                } else
#endif
                {
                        sd.args[0] = regs->bx;
                        sd.args[1] = regs->cx;
                        sd.args[2] = regs->dx;
                        sd.args[3] = regs->si;
                        sd.args[4] = regs->di;
                        sd.args[5] = regs->bp;
                }

                ret = __secure_computing(&sd);
                if (ret == -1)
                        return ret;
        }
#endif

        if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT)))
                trace_sys_enter(regs, regs->orig_ax);

        do_audit_syscall_entry(regs, arch);

        return ret ?: regs->orig_ax;
}

#define EXIT_TO_USERMODE_LOOP_FLAGS                                \
        (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE |        \
         _TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY | _TIF_PATCH_PENDING)

static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
{
        /*
         * In order to return to user mode, we need to have IRQs off with
         * none of EXIT_TO_USERMODE_LOOP_FLAGS set.  Several of these flags
         * can be set at any time on preemptable kernels if we have IRQs on,
         * so we need to loop.  Disabling preemption wouldn't help: doing the
         * work to clear some of the flags can sleep.
         */
        while (true) {
                /* We have work to do. */
                local_irq_enable(); /*covered*/

                if (cached_flags & _TIF_NEED_RESCHED)
                        schedule(); /*covered*/

                if (cached_flags & _TIF_UPROBE) /*covered*/
                        uprobe_notify_resume(regs);

                if (cached_flags & _TIF_PATCH_PENDING)
                        klp_update_patch_state(current); /*covered*/

                /* deal with pending signal delivery */
                if (cached_flags & _TIF_SIGPENDING) /*covered*/
                        do_signal(regs); /*covered*/

                if (cached_flags & _TIF_NOTIFY_RESUME) { /*covered*/
                        clear_thread_flag(TIF_NOTIFY_RESUME); /*covered*/
                        tracehook_notify_resume(regs); /*covered*/
                        rseq_handle_notify_resume(NULL, regs);
                }

                if (cached_flags & _TIF_USER_RETURN_NOTIFY) /*covered*/
                        fire_user_return_notifiers();

                /* Disable IRQs and retry */
                local_irq_disable(); /*covered*/

                cached_flags = READ_ONCE(current_thread_info()->flags);

                if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
                        break;
        }
} /*covered*/

/* Called with IRQs disabled. */
__visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
{
        struct thread_info *ti = current_thread_info(); /*covered*/
        u32 cached_flags;

        addr_limit_user_check(); /*covered*/

        lockdep_assert_irqs_disabled(); /*covered*/
        lockdep_sys_exit(); /*covered*/

        cached_flags = READ_ONCE(ti->flags);

        if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
                exit_to_usermode_loop(regs, cached_flags); /*covered*/

#ifdef CONFIG_COMPAT
        /*
         * Compat syscalls set TS_COMPAT.  Make sure we clear it before
         * returning to user mode.  We need to clear it *after* signal
         * handling, because syscall restart has a fixup for compat
         * syscalls.  The fixup is exercised by the ptrace_syscall_32
         * selftest.
         *
         * We also need to clear TS_REGS_POKED_I386: the 32-bit tracer
         * special case only applies after poking regs and before the
         * very next return to user mode.
         */
        ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED); /*covered*/
#endif

        user_enter_irqoff();
}

#define SYSCALL_EXIT_WORK_FLAGS                                \
        (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT |        \
         _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)

static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
{
        bool step;

        audit_syscall_exit(regs);

        if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
                trace_sys_exit(regs, regs->ax);

        /*
         * If TIF_SYSCALL_EMU is set, we only get here because of
         * TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
         * We already reported this syscall instruction in
         * syscall_trace_enter().
         */
        step = unlikely(
                (cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
                == _TIF_SINGLESTEP);
        if (step || cached_flags & _TIF_SYSCALL_TRACE)
                tracehook_report_syscall_exit(regs, step);
}

/*
 * Called with IRQs on and fully valid regs.  Returns with IRQs off in a
 * state such that we can immediately switch to user mode.
 */
__visible inline void syscall_return_slowpath(struct pt_regs *regs)
{
        struct thread_info *ti = current_thread_info(); /*covered*/
        u32 cached_flags = READ_ONCE(ti->flags);

        CT_WARN_ON(ct_state() != CONTEXT_KERNEL);

        if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
            WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax)) /*covered*/
                local_irq_enable();

        rseq_syscall(regs);

        /*
         * First do one-time work.  If these work items are enabled, we
         * want to run them exactly once per syscall exit with IRQs on.
         */
        if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS)) /*covered*/
                syscall_slow_exit_work(regs, cached_flags);

        local_irq_disable(); /*covered*/
        prepare_exit_to_usermode(regs); /*covered*/
}

#ifdef CONFIG_X86_64
__visible void do_syscall_64(unsigned long nr, struct pt_regs *regs)
{
        struct thread_info *ti;

        enter_from_user_mode();
        local_irq_enable(); /*covered*/
        ti = current_thread_info();
        if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY)
                nr = syscall_trace_enter(regs);

        /*
         * NB: Native and x32 syscalls are dispatched from the same
         * table.  The only functional difference is the x32 bit in
         * regs->orig_ax, which changes the behavior of some syscalls.
         */
        nr &= __SYSCALL_MASK; /*covered*/
        if (likely(nr < NR_syscalls)) {
                nr = array_index_nospec(nr, NR_syscalls); /*covered*/
                regs->ax = sys_call_table[nr](regs);
        }

        syscall_return_slowpath(regs); /*covered*/
}
#endif

#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
/*
 * Does a 32-bit syscall.  Called with IRQs on in CONTEXT_KERNEL.  Does
 * all entry and exit work and returns with IRQs off.  This function is
 * extremely hot in workloads that use it, and it's usually called from
 * do_fast_syscall_32, so forcibly inline it to improve performance.
 */
static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
{
        struct thread_info *ti = current_thread_info();
        unsigned int nr = (unsigned int)regs->orig_ax;

#ifdef CONFIG_IA32_EMULATION
        ti->status |= TS_COMPAT;
#endif

        if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
                /*
                 * Subtlety here: if ptrace pokes something larger than
                 * 2^32-1 into orig_ax, this truncates it.  This may or
                 * may not be necessary, but it matches the old asm
                 * behavior.
                 */
                nr = syscall_trace_enter(regs);
        }

        if (likely(nr < IA32_NR_syscalls)) {
                nr = array_index_nospec(nr, IA32_NR_syscalls);
#ifdef CONFIG_IA32_EMULATION
                regs->ax = ia32_sys_call_table[nr](regs);
#else
                /*
                 * It's possible that a 32-bit syscall implementation
                 * takes a 64-bit parameter but nonetheless assumes that
                 * the high bits are zero.  Make sure we zero-extend all
                 * of the args.
                 */
                regs->ax = ia32_sys_call_table[nr](
                        (unsigned int)regs->bx, (unsigned int)regs->cx,
                        (unsigned int)regs->dx, (unsigned int)regs->si,
                        (unsigned int)regs->di, (unsigned int)regs->bp);
#endif /* CONFIG_IA32_EMULATION */
        }

        syscall_return_slowpath(regs);
}

/* Handles int $0x80 */
__visible void do_int80_syscall_32(struct pt_regs *regs)
{
        enter_from_user_mode();
        local_irq_enable();
        do_syscall_32_irqs_on(regs);
}

/* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */
__visible long do_fast_syscall_32(struct pt_regs *regs)
{
        /*
         * Called using the internal vDSO SYSENTER/SYSCALL32 calling
         * convention.  Adjust regs so it looks like we entered using int80.
         */

        unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
                vdso_image_32.sym_int80_landing_pad;

        /*
         * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
         * so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
         * Fix it up.
         */
        regs->ip = landing_pad;

        enter_from_user_mode();

        local_irq_enable();

        /* Fetch EBP from where the vDSO stashed it. */
        if (
#ifdef CONFIG_X86_64
                /*
                 * Micro-optimization: the pointer we're following is explicitly
                 * 32 bits, so it can't be out of range.
                 */
                __get_user(*(u32 *)&regs->bp,
                            (u32 __user __force *)(unsigned long)(u32)regs->sp)
#else
                get_user(*(u32 *)&regs->bp,
                         (u32 __user __force *)(unsigned long)(u32)regs->sp)
#endif
                ) {

                /* User code screwed up. */
                local_irq_disable();
                regs->ax = -EFAULT;
                prepare_exit_to_usermode(regs);
                return 0;        /* Keep it simple: use IRET. */
        }

        /* Now this is just like a normal syscall. */
        do_syscall_32_irqs_on(regs);

#ifdef CONFIG_X86_64
        /*
         * Opportunistic SYSRETL: if possible, try to return using SYSRETL.
         * SYSRETL is available on all 64-bit CPUs, so we don't need to
         * bother with SYSEXIT.
         *
         * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
         * because the ECX fixup above will ensure that this is essentially
         * never the case.
         */
        return regs->cs == __USER32_CS && regs->ss == __USER_DS &&
                regs->ip == landing_pad &&
                (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)) == 0;
#else
        /*
         * Opportunistic SYSEXIT: if possible, try to return using SYSEXIT.
         *
         * Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
         * because the ECX fixup above will ensure that this is essentially
         * never the case.
         *
         * We don't allow syscalls at all from VM86 mode, but we still
         * need to check VM, because we might be returning from sys_vm86.
         */
        return static_cpu_has(X86_FEATURE_SEP) &&
                regs->cs == __USER_CS && regs->ss == __USER_DS &&
                regs->ip == landing_pad &&
                (regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)) == 0;
#endif
}
#endif

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2012-2014 Andy Lutomirski <luto@amacapital.net>
 *
 * Based on the original implementation which is:
 *  Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
 *  Copyright 2003 Andi Kleen, SuSE Labs.
 *
 *  Parts of the original code have been moved to arch/x86/vdso/vma.c
 *
 * This file implements vsyscall emulation.  vsyscalls are a legacy ABI:
 * Userspace can request certain kernel services by calling fixed
 * addresses.  This concept is problematic:
 *
 * - It interferes with ASLR.
 * - It's awkward to write code that lives in kernel addresses but is
 *   callable by userspace at fixed addresses.
 * - The whole concept is impossible for 32-bit compat userspace.
 * - UML cannot easily virtualize a vsyscall.
 *
 * As of mid-2014, I believe that there is no new userspace code that
 * will use a vsyscall if the vDSO is present.  I hope that there will
 * soon be no new userspace code that will ever use a vsyscall.
 *
 * The code in this file emulates vsyscalls when notified of a page
 * fault to a vsyscall address.
 */

#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/sched/signal.h>
#include <linux/mm_types.h>
#include <linux/syscalls.h>
#include <linux/ratelimit.h>

#include <asm/vsyscall.h>
#include <asm/unistd.h>
#include <asm/fixmap.h>
#include <asm/traps.h>
#include <asm/paravirt.h>

#define CREATE_TRACE_POINTS
#include "vsyscall_trace.h"

static enum { EMULATE, NONE } vsyscall_mode =
#ifdef CONFIG_LEGACY_VSYSCALL_NONE
        NONE;
#else
        EMULATE;
#endif

static int __init vsyscall_setup(char *str)
{
        if (str) {
                if (!strcmp("emulate", str))
                        vsyscall_mode = EMULATE;
                else if (!strcmp("none", str))
                        vsyscall_mode = NONE;
                else
                        return -EINVAL;

                return 0;
        }

        return -EINVAL;
}
early_param("vsyscall", vsyscall_setup);

static void warn_bad_vsyscall(const char *level, struct pt_regs *regs,
                              const char *message)
{
        if (!show_unhandled_signals)
                return;

        printk_ratelimited("%s%s[%d] %s ip:%lx cs:%lx sp:%lx ax:%lx si:%lx di:%lx\n",
                           level, current->comm, task_pid_nr(current),
                           message, regs->ip, regs->cs,
                           regs->sp, regs->ax, regs->si, regs->di);
}

static int addr_to_vsyscall_nr(unsigned long addr)
{
        int nr;

        if ((addr & ~0xC00UL) != VSYSCALL_ADDR)
                return -EINVAL;

        nr = (addr & 0xC00UL) >> 10;
        if (nr >= 3)
                return -EINVAL;

        return nr;
}

static bool write_ok_or_segv(unsigned long ptr, size_t size)
{
        /*
         * XXX: if access_ok, get_user, and put_user handled
         * sig_on_uaccess_err, this could go away.
         */

        if (!access_ok(VERIFY_WRITE, (void __user *)ptr, size)) {
                siginfo_t info;
                struct thread_struct *thread = &current->thread;

                thread->error_code        = 6;  /* user fault, no page, write */
                thread->cr2                = ptr;
                thread->trap_nr                = X86_TRAP_PF;

                clear_siginfo(&info);
                info.si_signo                = SIGSEGV;
                info.si_errno                = 0;
                info.si_code                = SEGV_MAPERR;
                info.si_addr                = (void __user *)ptr;

                force_sig_info(SIGSEGV, &info, current);
                return false;
        } else {
                return true;
        }
}

bool emulate_vsyscall(struct pt_regs *regs, unsigned long address)
{
        struct task_struct *tsk;
        unsigned long caller;
        int vsyscall_nr, syscall_nr, tmp;
        int prev_sig_on_uaccess_err;
        long ret;
        unsigned long orig_dx;

        /*
         * No point in checking CS -- the only way to get here is a user mode
         * trap to a high address, which means that we're in 64-bit user code.
         */

        WARN_ON_ONCE(address != regs->ip);

        if (vsyscall_mode == NONE) {
                warn_bad_vsyscall(KERN_INFO, regs,
                                  "vsyscall attempted with vsyscall=none");
                return false;
        }

        vsyscall_nr = addr_to_vsyscall_nr(address);

        trace_emulate_vsyscall(vsyscall_nr);

        if (vsyscall_nr < 0) {
                warn_bad_vsyscall(KERN_WARNING, regs,
                                  "misaligned vsyscall (exploit attempt or buggy program) -- look up the vsyscall kernel parameter if you need a workaround");
                goto sigsegv;
        }

        if (get_user(caller, (unsigned long __user *)regs->sp) != 0) {
                warn_bad_vsyscall(KERN_WARNING, regs,
                                  "vsyscall with bad stack (exploit attempt?)");
                goto sigsegv;
        }

        tsk = current;

        /*
         * Check for access_ok violations and find the syscall nr.
         *
         * NULL is a valid user pointer (in the access_ok sense) on 32-bit and
         * 64-bit, so we don't need to special-case it here.  For all the
         * vsyscalls, NULL means "don't write anything" not "write it at
         * address 0".
         */
        switch (vsyscall_nr) {
        case 0:
                if (!write_ok_or_segv(regs->di, sizeof(struct timeval)) ||
                    !write_ok_or_segv(regs->si, sizeof(struct timezone))) {
                        ret = -EFAULT;
                        goto check_fault;
                }

                syscall_nr = __NR_gettimeofday;
                break;

        case 1:
                if (!write_ok_or_segv(regs->di, sizeof(time_t))) {
                        ret = -EFAULT;
                        goto check_fault;
                }

                syscall_nr = __NR_time;
                break;

        case 2:
                if (!write_ok_or_segv(regs->di, sizeof(unsigned)) ||
                    !write_ok_or_segv(regs->si, sizeof(unsigned))) {
                        ret = -EFAULT;
                        goto check_fault;
                }

                syscall_nr = __NR_getcpu;
                break;
        }

        /*
         * Handle seccomp.  regs->ip must be the original value.
         * See seccomp_send_sigsys and Documentation/userspace-api/seccomp_filter.rst.
         *
         * We could optimize the seccomp disabled case, but performance
         * here doesn't matter.
         */
        regs->orig_ax = syscall_nr;
        regs->ax = -ENOSYS;
        tmp = secure_computing(NULL);
        if ((!tmp && regs->orig_ax != syscall_nr) || regs->ip != address) {
                warn_bad_vsyscall(KERN_DEBUG, regs,
                                  "seccomp tried to change syscall nr or ip");
                do_exit(SIGSYS);
        }
        regs->orig_ax = -1;
        if (tmp)
                goto do_ret;  /* skip requested */

        /*
         * With a real vsyscall, page faults cause SIGSEGV.  We want to
         * preserve that behavior to make writing exploits harder.
         */
        prev_sig_on_uaccess_err = current->thread.sig_on_uaccess_err;
        current->thread.sig_on_uaccess_err = 1;

        ret = -EFAULT;
        switch (vsyscall_nr) {
        case 0:
                /* this decodes regs->di and regs->si on its own */
                ret = __x64_sys_gettimeofday(regs);
                break;

        case 1:
                /* this decodes regs->di on its own */
                ret = __x64_sys_time(regs);
                break;

        case 2:
                /* while we could clobber regs->dx, we didn't in the past... */
                orig_dx = regs->dx;
                regs->dx = 0;
                /* this decodes regs->di, regs->si and regs->dx on its own */
                ret = __x64_sys_getcpu(regs);
                regs->dx = orig_dx;
                break;
        }

        current->thread.sig_on_uaccess_err = prev_sig_on_uaccess_err;

check_fault:
        if (ret == -EFAULT) {
                /* Bad news -- userspace fed a bad pointer to a vsyscall. */
                warn_bad_vsyscall(KERN_INFO, regs,
                                  "vsyscall fault (exploit attempt?)");

                /*
                 * If we failed to generate a signal for any reason,
                 * generate one here.  (This should be impossible.)
                 */
                if (WARN_ON_ONCE(!sigismember(&tsk->pending.signal, SIGBUS) &&
                                 !sigismember(&tsk->pending.signal, SIGSEGV)))
                        goto sigsegv;

                return true;  /* Don't emulate the ret. */
        }

        regs->ax = ret;

do_ret:
        /* Emulate a ret instruction. */
        regs->ip = caller;
        regs->sp += 8;
        return true;

sigsegv:
        force_sig(SIGSEGV, current);
        return true;
}

/*
 * A pseudo VMA to allow ptrace access for the vsyscall page.  This only
 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does
 * not need special handling anymore:
 */
static const char *gate_vma_name(struct vm_area_struct *vma)
{
        return "[vsyscall]";
}
static const struct vm_operations_struct gate_vma_ops = {
        .name = gate_vma_name,
};
static struct vm_area_struct gate_vma = {
        .vm_start        = VSYSCALL_ADDR,
        .vm_end                = VSYSCALL_ADDR + PAGE_SIZE,
        .vm_page_prot        = PAGE_READONLY_EXEC,
        .vm_flags        = VM_READ | VM_EXEC,
        .vm_ops                = &gate_vma_ops,
};

struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
#ifdef CONFIG_COMPAT
        if (!mm || mm->context.ia32_compat) /*covered*/
                return NULL;
#endif
        if (vsyscall_mode == NONE) /*covered*/
                return NULL;
        return &gate_vma;
}

int in_gate_area(struct mm_struct *mm, unsigned long addr)
{
        struct vm_area_struct *vma = get_gate_vma(mm); /*covered*/

        if (!vma)
                return 0;

        return (addr >= vma->vm_start) && (addr < vma->vm_end); /*covered*/
}

/*
 * Use this when you have no reliable mm, typically from interrupt
 * context. It is less reliable than using a task's mm and may give
 * false positives.
 */
int in_gate_area_no_mm(unsigned long addr)
{
        return vsyscall_mode != NONE && (addr & PAGE_MASK) == VSYSCALL_ADDR;
}

/*
 * The VSYSCALL page is the only user-accessible page in the kernel address
 * range.  Normally, the kernel page tables can have _PAGE_USER clear, but
 * the tables covering VSYSCALL_ADDR need _PAGE_USER set if vsyscalls
 * are enabled.
 *
 * Some day we may create a "minimal" vsyscall mode in which we emulate
 * vsyscalls but leave the page not present.  If so, we skip calling
 * this.
 */
void __init set_vsyscall_pgtable_user_bits(pgd_t *root)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;

        pgd = pgd_offset_pgd(root, VSYSCALL_ADDR);
        set_pgd(pgd, __pgd(pgd_val(*pgd) | _PAGE_USER));
        p4d = p4d_offset(pgd, VSYSCALL_ADDR);
#if CONFIG_PGTABLE_LEVELS >= 5
        set_p4d(p4d, __p4d(p4d_val(*p4d) | _PAGE_USER));
#endif
        pud = pud_offset(p4d, VSYSCALL_ADDR);
        set_pud(pud, __pud(pud_val(*pud) | _PAGE_USER));
        pmd = pmd_offset(pud, VSYSCALL_ADDR);
        set_pmd(pmd, __pmd(pmd_val(*pmd) | _PAGE_USER));
}

void __init map_vsyscall(void)
{
        extern char __vsyscall_page;
        unsigned long physaddr_vsyscall = __pa_symbol(&__vsyscall_page);

        if (vsyscall_mode != NONE) {
                __set_fixmap(VSYSCALL_PAGE, physaddr_vsyscall,
                             PAGE_KERNEL_VVAR);
                set_vsyscall_pgtable_user_bits(swapper_pg_dir);
        }

        BUILD_BUG_ON((unsigned long)__fix_to_virt(VSYSCALL_PAGE) !=
                     (unsigned long)VSYSCALL_ADDR);
}

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_HWEIGHT_H
#define _ASM_X86_HWEIGHT_H

#include <asm/cpufeatures.h>

#ifdef CONFIG_64BIT
/* popcnt %edi, %eax */
#define POPCNT32 ".byte 0xf3,0x0f,0xb8,0xc7"
/* popcnt %rdi, %rax */
#define POPCNT64 ".byte 0xf3,0x48,0x0f,0xb8,0xc7"
#define REG_IN "D"
#define REG_OUT "a"
#else
/* popcnt %eax, %eax */
#define POPCNT32 ".byte 0xf3,0x0f,0xb8,0xc0"
#define REG_IN "a"
#define REG_OUT "a"
#endif

#define __HAVE_ARCH_SW_HWEIGHT

static __always_inline unsigned int __arch_hweight32(unsigned int w)
{
        unsigned int res;

        asm (ALTERNATIVE("call __sw_hweight32", POPCNT32, X86_FEATURE_POPCNT)
                         : "="REG_OUT (res)
                         : REG_IN (w));

        return res;
}

static inline unsigned int __arch_hweight16(unsigned int w)
{
        return __arch_hweight32(w & 0xffff);
}

static inline unsigned int __arch_hweight8(unsigned int w)
{
        return __arch_hweight32(w & 0xff);
}

#ifdef CONFIG_X86_32
static inline unsigned long __arch_hweight64(__u64 w)
{
        return  __arch_hweight32((u32)w) +
                __arch_hweight32((u32)(w >> 32));
}
#else
static __always_inline unsigned long __arch_hweight64(__u64 w)
{
        unsigned long res;

        asm (ALTERNATIVE("call __sw_hweight64", POPCNT64, X86_FEATURE_POPCNT) /*covered*/
                         : "="REG_OUT (res)
                         : REG_IN (w));

        return res;
}
#endif /* CONFIG_X86_32 */

#endif

/*
 * This file is part of the Linux kernel.
 *
 * Copyright (c) 2011-2014, Intel Corporation
 * Authors: Fenghua Yu <fenghua.yu@intel.com>,
 *          H. Peter Anvin <hpa@linux.intel.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */

#ifndef ASM_X86_ARCHRANDOM_H
#define ASM_X86_ARCHRANDOM_H

#include <asm/processor.h>
#include <asm/cpufeature.h>

#define RDRAND_RETRY_LOOPS        10

#define RDRAND_INT        ".byte 0x0f,0xc7,0xf0"
#define RDSEED_INT        ".byte 0x0f,0xc7,0xf8"
#ifdef CONFIG_X86_64
# define RDRAND_LONG        ".byte 0x48,0x0f,0xc7,0xf0"
# define RDSEED_LONG        ".byte 0x48,0x0f,0xc7,0xf8"
#else
# define RDRAND_LONG        RDRAND_INT
# define RDSEED_LONG        RDSEED_INT
#endif

/* Unconditional execution of RDRAND and RDSEED */

static inline bool rdrand_long(unsigned long *v)
{
        bool ok;
        unsigned int retry = RDRAND_RETRY_LOOPS;
        do {
                asm volatile(RDRAND_LONG /*covered*/
                             CC_SET(c)
                             : CC_OUT(c) (ok), "=a" (*v));
                if (ok)
                        return true;
        } while (--retry);
        return false;
}

static inline bool rdrand_int(unsigned int *v)
{
        bool ok;
        unsigned int retry = RDRAND_RETRY_LOOPS;
        do {
                asm volatile(RDRAND_INT /*covered*/
                             CC_SET(c)
                             : CC_OUT(c) (ok), "=a" (*v));
                if (ok)
                        return true;
        } while (--retry);
        return false;
}

static inline bool rdseed_long(unsigned long *v)
{
        bool ok;
        asm volatile(RDSEED_LONG
                     CC_SET(c)
                     : CC_OUT(c) (ok), "=a" (*v));
        return ok;
}

static inline bool rdseed_int(unsigned int *v)
{
        bool ok;
        asm volatile(RDSEED_INT
                     CC_SET(c)
                     : CC_OUT(c) (ok), "=a" (*v));
        return ok;
}

/* Conditional execution based on CPU type */
#define arch_has_random()        static_cpu_has(X86_FEATURE_RDRAND)
#define arch_has_random_seed()        static_cpu_has(X86_FEATURE_RDSEED)

/*
 * These are the generic interfaces; they must not be declared if the
 * stubs in <linux/random.h> are to be invoked,
 * i.e. CONFIG_ARCH_RANDOM is not defined.
 */
#ifdef CONFIG_ARCH_RANDOM

static inline bool arch_get_random_long(unsigned long *v)
{
        return arch_has_random() ? rdrand_long(v) : false; /*covered*/
}

static inline bool arch_get_random_int(unsigned int *v)
{
        return arch_has_random() ? rdrand_int(v) : false; /*covered*/
}

static inline bool arch_get_random_seed_long(unsigned long *v)
{
        return arch_has_random_seed() ? rdseed_long(v) : false;
}

static inline bool arch_get_random_seed_int(unsigned int *v)
{
        return arch_has_random_seed() ? rdseed_int(v) : false;
}

extern void x86_init_rdrand(struct cpuinfo_x86 *c);

#else  /* !CONFIG_ARCH_RANDOM */

static inline void x86_init_rdrand(struct cpuinfo_x86 *c) { }

#endif  /* !CONFIG_ARCH_RANDOM */

#endif /* ASM_X86_ARCHRANDOM_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_ATOMIC_H
#define _ASM_X86_ATOMIC_H

#include <linux/compiler.h>
#include <linux/types.h>
#include <asm/alternative.h>
#include <asm/cmpxchg.h>
#include <asm/rmwcc.h>
#include <asm/barrier.h>

/*
 * Atomic operations that C can't guarantee us.  Useful for
 * resource counting etc..
 */

#define ATOMIC_INIT(i)        { (i) }

/**
 * arch_atomic_read - read atomic variable
 * @v: pointer of type atomic_t
 *
 * Atomically reads the value of @v.
 */
static __always_inline int arch_atomic_read(const atomic_t *v)
{
        /*
         * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here,
         * it's non-inlined function that increases binary size and stack usage.
         */
        return READ_ONCE((v)->counter); /*covered*/
}

/**
 * arch_atomic_set - set atomic variable
 * @v: pointer of type atomic_t
 * @i: required value
 *
 * Atomically sets the value of @v to @i.
 */
static __always_inline void arch_atomic_set(atomic_t *v, int i)
{
        WRITE_ONCE(v->counter, i); /*covered*/
}

/**
 * arch_atomic_add - add integer to atomic variable
 * @i: integer value to add
 * @v: pointer of type atomic_t
 *
 * Atomically adds @i to @v.
 */
static __always_inline void arch_atomic_add(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "addl %1,%0"
                     : "+m" (v->counter)
                     : "ir" (i));
}

/**
 * arch_atomic_sub - subtract integer from atomic variable
 * @i: integer value to subtract
 * @v: pointer of type atomic_t
 *
 * Atomically subtracts @i from @v.
 */
static __always_inline void arch_atomic_sub(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "subl %1,%0"
                     : "+m" (v->counter)
                     : "ir" (i));
}

/**
 * arch_atomic_sub_and_test - subtract value from variable and test result
 * @i: integer value to subtract
 * @v: pointer of type atomic_t
 *
 * Atomically subtracts @i from @v and returns
 * true if the result is zero, or false for all
 * other cases.
 */
static __always_inline bool arch_atomic_sub_and_test(int i, atomic_t *v)
{
        GEN_BINARY_RMWcc(LOCK_PREFIX "subl", v->counter, "er", i, "%0", e);
}
#define arch_atomic_sub_and_test arch_atomic_sub_and_test

/**
 * arch_atomic_inc - increment atomic variable
 * @v: pointer of type atomic_t
 *
 * Atomically increments @v by 1.
 */
static __always_inline void arch_atomic_inc(atomic_t *v)
{
        asm volatile(LOCK_PREFIX "incl %0" /*covered*/
                     : "+m" (v->counter));
}
#define arch_atomic_inc arch_atomic_inc

/**
 * arch_atomic_dec - decrement atomic variable
 * @v: pointer of type atomic_t
 *
 * Atomically decrements @v by 1.
 */
static __always_inline void arch_atomic_dec(atomic_t *v)
{
        asm volatile(LOCK_PREFIX "decl %0" /*covered*/
                     : "+m" (v->counter));
}
#define arch_atomic_dec arch_atomic_dec

/**
 * arch_atomic_dec_and_test - decrement and test
 * @v: pointer of type atomic_t
 *
 * Atomically decrements @v by 1 and
 * returns true if the result is 0, or false for all other
 * cases.
 */
static __always_inline bool arch_atomic_dec_and_test(atomic_t *v)
{
        GEN_UNARY_RMWcc(LOCK_PREFIX "decl", v->counter, "%0", e);
}
#define arch_atomic_dec_and_test arch_atomic_dec_and_test

/**
 * arch_atomic_inc_and_test - increment and test
 * @v: pointer of type atomic_t
 *
 * Atomically increments @v by 1
 * and returns true if the result is zero, or false for all
 * other cases.
 */
static __always_inline bool arch_atomic_inc_and_test(atomic_t *v)
{
        GEN_UNARY_RMWcc(LOCK_PREFIX "incl", v->counter, "%0", e);
}
#define arch_atomic_inc_and_test arch_atomic_inc_and_test

/**
 * arch_atomic_add_negative - add and test if negative
 * @i: integer value to add
 * @v: pointer of type atomic_t
 *
 * Atomically adds @i to @v and returns true
 * if the result is negative, or false when
 * result is greater than or equal to zero.
 */
static __always_inline bool arch_atomic_add_negative(int i, atomic_t *v)
{
        GEN_BINARY_RMWcc(LOCK_PREFIX "addl", v->counter, "er", i, "%0", s);
}
#define arch_atomic_add_negative arch_atomic_add_negative

/**
 * arch_atomic_add_return - add integer and return
 * @i: integer value to add
 * @v: pointer of type atomic_t
 *
 * Atomically adds @i to @v and returns @i + @v
 */
static __always_inline int arch_atomic_add_return(int i, atomic_t *v)
{
        return i + xadd(&v->counter, i);
}

/**
 * arch_atomic_sub_return - subtract integer and return
 * @v: pointer of type atomic_t
 * @i: integer value to subtract
 *
 * Atomically subtracts @i from @v and returns @v - @i
 */
static __always_inline int arch_atomic_sub_return(int i, atomic_t *v)
{
        return arch_atomic_add_return(-i, v);
}

static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v)
{
        return xadd(&v->counter, i);
}

static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v)
{
        return xadd(&v->counter, -i);
}

static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
{
        return arch_cmpxchg(&v->counter, old, new);
}

#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
{
        return try_cmpxchg(&v->counter, old, new); /*covered*/
}

static inline int arch_atomic_xchg(atomic_t *v, int new)
{
        return arch_xchg(&v->counter, new);
}

static inline void arch_atomic_and(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "andl %1,%0"
                        : "+m" (v->counter)
                        : "ir" (i)
                        : "memory");
}

static inline int arch_atomic_fetch_and(int i, atomic_t *v)
{
        int val = arch_atomic_read(v);

        do { } while (!arch_atomic_try_cmpxchg(v, &val, val & i));

        return val;
}

static inline void arch_atomic_or(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "orl %1,%0"
                        : "+m" (v->counter)
                        : "ir" (i)
                        : "memory");
}

static inline int arch_atomic_fetch_or(int i, atomic_t *v)
{
        int val = arch_atomic_read(v);

        do { } while (!arch_atomic_try_cmpxchg(v, &val, val | i));

        return val;
}

static inline void arch_atomic_xor(int i, atomic_t *v)
{
        asm volatile(LOCK_PREFIX "xorl %1,%0"
                        : "+m" (v->counter)
                        : "ir" (i)
                        : "memory");
}

static inline int arch_atomic_fetch_xor(int i, atomic_t *v)
{
        int val = arch_atomic_read(v);

        do { } while (!arch_atomic_try_cmpxchg(v, &val, val ^ i));

        return val;
}

#ifdef CONFIG_X86_32
# include <asm/atomic64_32.h>
#else
# include <asm/atomic64_64.h>
#endif

#include <asm-generic/atomic-instrumented.h>

#endif /* _ASM_X86_ATOMIC_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_ATOMIC64_64_H
#define _ASM_X86_ATOMIC64_64_H

#include <linux/types.h>
#include <asm/alternative.h>
#include <asm/cmpxchg.h>

/* The 64-bit atomic type */

#define ATOMIC64_INIT(i)        { (i) }

/**
 * arch_atomic64_read - read atomic64 variable
 * @v: pointer of type atomic64_t
 *
 * Atomically reads the value of @v.
 * Doesn't imply a read memory barrier.
 */
static inline long arch_atomic64_read(const atomic64_t *v)
{
        return READ_ONCE((v)->counter);
}

/**
 * arch_atomic64_set - set atomic64 variable
 * @v: pointer to type atomic64_t
 * @i: required value
 *
 * Atomically sets the value of @v to @i.
 */
static inline void arch_atomic64_set(atomic64_t *v, long i)
{
        WRITE_ONCE(v->counter, i);
}

/**
 * arch_atomic64_add - add integer to atomic64 variable
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v.
 */
static __always_inline void arch_atomic64_add(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "addq %1,%0" /*covered*/
                     : "=m" (v->counter)
                     : "er" (i), "m" (v->counter));
}

/**
 * arch_atomic64_sub - subtract the atomic64 variable
 * @i: integer value to subtract
 * @v: pointer to type atomic64_t
 *
 * Atomically subtracts @i from @v.
 */
static inline void arch_atomic64_sub(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "subq %1,%0"
                     : "=m" (v->counter)
                     : "er" (i), "m" (v->counter));
}

/**
 * arch_atomic64_sub_and_test - subtract value from variable and test result
 * @i: integer value to subtract
 * @v: pointer to type atomic64_t
 *
 * Atomically subtracts @i from @v and returns
 * true if the result is zero, or false for all
 * other cases.
 */
static inline bool arch_atomic64_sub_and_test(long i, atomic64_t *v)
{
        GEN_BINARY_RMWcc(LOCK_PREFIX "subq", v->counter, "er", i, "%0", e);
}
#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test

/**
 * arch_atomic64_inc - increment atomic64 variable
 * @v: pointer to type atomic64_t
 *
 * Atomically increments @v by 1.
 */
static __always_inline void arch_atomic64_inc(atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "incq %0"
                     : "=m" (v->counter)
                     : "m" (v->counter));
}
#define arch_atomic64_inc arch_atomic64_inc

/**
 * arch_atomic64_dec - decrement atomic64 variable
 * @v: pointer to type atomic64_t
 *
 * Atomically decrements @v by 1.
 */
static __always_inline void arch_atomic64_dec(atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "decq %0"
                     : "=m" (v->counter)
                     : "m" (v->counter));
}
#define arch_atomic64_dec arch_atomic64_dec

/**
 * arch_atomic64_dec_and_test - decrement and test
 * @v: pointer to type atomic64_t
 *
 * Atomically decrements @v by 1 and
 * returns true if the result is 0, or false for all other
 * cases.
 */
static inline bool arch_atomic64_dec_and_test(atomic64_t *v)
{
        GEN_UNARY_RMWcc(LOCK_PREFIX "decq", v->counter, "%0", e);
}
#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test

/**
 * arch_atomic64_inc_and_test - increment and test
 * @v: pointer to type atomic64_t
 *
 * Atomically increments @v by 1
 * and returns true if the result is zero, or false for all
 * other cases.
 */
static inline bool arch_atomic64_inc_and_test(atomic64_t *v)
{
        GEN_UNARY_RMWcc(LOCK_PREFIX "incq", v->counter, "%0", e);
}
#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test

/**
 * arch_atomic64_add_negative - add and test if negative
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v and returns true
 * if the result is negative, or false when
 * result is greater than or equal to zero.
 */
static inline bool arch_atomic64_add_negative(long i, atomic64_t *v)
{
        GEN_BINARY_RMWcc(LOCK_PREFIX "addq", v->counter, "er", i, "%0", s);
}
#define arch_atomic64_add_negative arch_atomic64_add_negative

/**
 * arch_atomic64_add_return - add and return
 * @i: integer value to add
 * @v: pointer to type atomic64_t
 *
 * Atomically adds @i to @v and returns @i + @v
 */
static __always_inline long arch_atomic64_add_return(long i, atomic64_t *v)
{
        return i + xadd(&v->counter, i);
}

static inline long arch_atomic64_sub_return(long i, atomic64_t *v)
{
        return arch_atomic64_add_return(-i, v);
}

static inline long arch_atomic64_fetch_add(long i, atomic64_t *v)
{
        return xadd(&v->counter, i);
}

static inline long arch_atomic64_fetch_sub(long i, atomic64_t *v)
{
        return xadd(&v->counter, -i);
}

static inline long arch_atomic64_cmpxchg(atomic64_t *v, long old, long new)
{
        return arch_cmpxchg(&v->counter, old, new);
}

#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, long new)
{
        return try_cmpxchg(&v->counter, old, new);
}

static inline long arch_atomic64_xchg(atomic64_t *v, long new)
{
        return arch_xchg(&v->counter, new);
}

static inline void arch_atomic64_and(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "andq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static inline long arch_atomic64_fetch_and(long i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val & i));
        return val;
}

static inline void arch_atomic64_or(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "orq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static inline long arch_atomic64_fetch_or(long i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val | i));
        return val;
}

static inline void arch_atomic64_xor(long i, atomic64_t *v)
{
        asm volatile(LOCK_PREFIX "xorq %1,%0"
                        : "+m" (v->counter)
                        : "er" (i)
                        : "memory");
}

static inline long arch_atomic64_fetch_xor(long i, atomic64_t *v)
{
        s64 val = arch_atomic64_read(v);

        do {
        } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i));
        return val;
}

#endif /* _ASM_X86_ATOMIC64_64_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_BARRIER_H
#define _ASM_X86_BARRIER_H

#include <asm/alternative.h>
#include <asm/nops.h>

/*
 * Force strict CPU ordering.
 * And yes, this might be required on UP too when we're talking
 * to devices.
 */

#ifdef CONFIG_X86_32
#define mb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "mfence", \
                                      X86_FEATURE_XMM2) ::: "memory", "cc")
#define rmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "lfence", \
                                       X86_FEATURE_XMM2) ::: "memory", "cc")
#define wmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "sfence", \
                                       X86_FEATURE_XMM2) ::: "memory", "cc")
#else
#define mb()         asm volatile("mfence":::"memory")
#define rmb()        asm volatile("lfence":::"memory")
#define wmb()        asm volatile("sfence" ::: "memory")
#endif

/**
 * array_index_mask_nospec() - generate a mask that is ~0UL when the
 *         bounds check succeeds and 0 otherwise
 * @index: array element index
 * @size: number of elements in array
 *
 * Returns:
 *     0 - (index < size)
 */
static inline unsigned long array_index_mask_nospec(unsigned long index,
                unsigned long size)
{
        unsigned long mask;

        asm volatile ("cmp %1,%2; sbb %0,%0;" /*covered*/
                        :"=r" (mask)
                        :"g"(size),"r" (index)
                        :"cc");
        return mask;
}

/* Override the default implementation from linux/nospec.h. */
#define array_index_mask_nospec array_index_mask_nospec

/* Prevent speculative execution past this barrier. */
#define barrier_nospec() alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC, \
                                           "lfence", X86_FEATURE_LFENCE_RDTSC)

#define dma_rmb()        barrier()
#define dma_wmb()        barrier()

#ifdef CONFIG_X86_32
#define __smp_mb()        asm volatile("lock; addl $0,-4(%%esp)" ::: "memory", "cc")
#else
#define __smp_mb()        asm volatile("lock; addl $0,-4(%%rsp)" ::: "memory", "cc")
#endif
#define __smp_rmb()        dma_rmb()
#define __smp_wmb()        barrier()
#define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)

#define __smp_store_release(p, v)                                        \
do {                                                                        \
        compiletime_assert_atomic_type(*p);                                \
        barrier();                                                        \
        WRITE_ONCE(*p, v);                                                \
} while (0)

#define __smp_load_acquire(p)                                                \
({                                                                        \
        typeof(*p) ___p1 = READ_ONCE(*p);                                \
        compiletime_assert_atomic_type(*p);                                \
        barrier();                                                        \
        ___p1;                                                                \
})

/* Atomic operations are already serializing on x86 */
#define __smp_mb__before_atomic()        barrier()
#define __smp_mb__after_atomic()        barrier()

#include <asm-generic/barrier.h>

#endif /* _ASM_X86_BARRIER_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_BITOPS_H
#define _ASM_X86_BITOPS_H

/*
 * Copyright 1992, Linus Torvalds.
 *
 * Note: inlines with more than a single statement should be marked
 * __always_inline to avoid problems with older gcc's inlining heuristics.
 */

#ifndef _LINUX_BITOPS_H
#error only <linux/bitops.h> can be included directly
#endif

#include <linux/compiler.h>
#include <asm/alternative.h>
#include <asm/rmwcc.h>
#include <asm/barrier.h>

#if BITS_PER_LONG == 32
# define _BITOPS_LONG_SHIFT 5
#elif BITS_PER_LONG == 64
# define _BITOPS_LONG_SHIFT 6
#else
# error "Unexpected BITS_PER_LONG"
#endif

#define BIT_64(n)                        (U64_C(1) << (n))

/*
 * These have to be done with inline assembly: that way the bit-setting
 * is guaranteed to be atomic. All bit operations return 0 if the bit
 * was cleared before the operation and != 0 if it was not.
 *
 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
 */

#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
/* Technically wrong, but this avoids compilation errors on some gcc
   versions. */
#define BITOP_ADDR(x) "=m" (*(volatile long *) (x))
#else
#define BITOP_ADDR(x) "+m" (*(volatile long *) (x))
#endif

#define ADDR                                BITOP_ADDR(addr)

/*
 * We do the locked ops that don't return the old value as
 * a mask operation on a byte.
 */
#define IS_IMMEDIATE(nr)                (__builtin_constant_p(nr))
#define CONST_MASK_ADDR(nr, addr)        BITOP_ADDR((void *)(addr) + ((nr)>>3))
#define CONST_MASK(nr)                        (1 << ((nr) & 7))

/**
 * set_bit - Atomically set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * This function is atomic and may not be reordered.  See __set_bit()
 * if you do not require the atomic guarantees.
 *
 * Note: there are no guarantees that this function will not be reordered
 * on non x86 architectures, so if you are writing portable code,
 * make sure not to rely on its reordering guarantees.
 *
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static __always_inline void
set_bit(long nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "orb %1,%0" /*covered*/
                        : CONST_MASK_ADDR(nr, addr) /*covered*/
                        : "iq" ((u8)CONST_MASK(nr))
                        : "memory");
        } else {
                asm volatile(LOCK_PREFIX __ASM_SIZE(bts) " %1,%0" /*covered*/
                        : BITOP_ADDR(addr) : "Ir" (nr) : "memory");
        }
}

/**
 * __set_bit - Set a bit in memory
 * @nr: the bit to set
 * @addr: the address to start counting from
 *
 * Unlike set_bit(), this function is non-atomic and may be reordered.
 * If it's called on the same region of memory simultaneously, the effect
 * may be that only one operation succeeds.
 */
static __always_inline void __set_bit(long nr, volatile unsigned long *addr)
{
        asm volatile(__ASM_SIZE(bts) " %1,%0" : ADDR : "Ir" (nr) : "memory"); /*covered*/
}

/**
 * clear_bit - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * clear_bit() is atomic and may not be reordered.  However, it does
 * not contain a memory barrier, so if it is used for locking purposes,
 * you should call smp_mb__before_atomic() and/or smp_mb__after_atomic()
 * in order to ensure changes are visible on other processors.
 */
static __always_inline void
clear_bit(long nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "andb %1,%0" /*covered*/
                        : CONST_MASK_ADDR(nr, addr) /*covered*/
                        : "iq" ((u8)~CONST_MASK(nr)));
        } else {
                asm volatile(LOCK_PREFIX __ASM_SIZE(btr) " %1,%0" /*covered*/
                        : BITOP_ADDR(addr)
                        : "Ir" (nr));
        }
}

/*
 * clear_bit_unlock - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * clear_bit() is atomic and implies release semantics before the memory
 * operation. It can be used for an unlock.
 */
static __always_inline void clear_bit_unlock(long nr, volatile unsigned long *addr)
{
        barrier();
        clear_bit(nr, addr);
}

static __always_inline void __clear_bit(long nr, volatile unsigned long *addr)
{
        asm volatile(__ASM_SIZE(btr) " %1,%0" : ADDR : "Ir" (nr)); /*covered*/
}

static __always_inline bool clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
{
        bool negative;
        asm volatile(LOCK_PREFIX "andb %2,%1"
                CC_SET(s)
                : CC_OUT(s) (negative), ADDR
                : "ir" ((char) ~(1 << nr)) : "memory");
        return negative;
}

// Let everybody know we have it
#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte

/*
 * __clear_bit_unlock - Clears a bit in memory
 * @nr: Bit to clear
 * @addr: Address to start counting from
 *
 * __clear_bit() is non-atomic and implies release semantics before the memory
 * operation. It can be used for an unlock if no other CPUs can concurrently
 * modify other bits in the word.
 *
 * No memory barrier is required here, because x86 cannot reorder stores past
 * older loads. Same principle as spin_unlock.
 */
static __always_inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
{
        barrier(); /*covered*/
        __clear_bit(nr, addr);
}

/**
 * __change_bit - Toggle a bit in memory
 * @nr: the bit to change
 * @addr: the address to start counting from
 *
 * Unlike change_bit(), this function is non-atomic and may be reordered.
 * If it's called on the same region of memory simultaneously, the effect
 * may be that only one operation succeeds.
 */
static __always_inline void __change_bit(long nr, volatile unsigned long *addr)
{
        asm volatile(__ASM_SIZE(btc) " %1,%0" : ADDR : "Ir" (nr));
}

/**
 * change_bit - Toggle a bit in memory
 * @nr: Bit to change
 * @addr: Address to start counting from
 *
 * change_bit() is atomic and may not be reordered.
 * Note that @nr may be almost arbitrarily large; this function is not
 * restricted to acting on a single-word quantity.
 */
static __always_inline void change_bit(long nr, volatile unsigned long *addr)
{
        if (IS_IMMEDIATE(nr)) {
                asm volatile(LOCK_PREFIX "xorb %1,%0"
                        : CONST_MASK_ADDR(nr, addr)
                        : "iq" ((u8)CONST_MASK(nr)));
        } else {
                asm volatile(LOCK_PREFIX __ASM_SIZE(btc) " %1,%0"
                        : BITOP_ADDR(addr)
                        : "Ir" (nr));
        }
}

/**
 * test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static __always_inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
{
        GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(bts), /*covered*/
                         *addr, "Ir", nr, "%0", c);
}

/**
 * test_and_set_bit_lock - Set a bit and return its old value for lock
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This is the same as test_and_set_bit on x86.
 */
static __always_inline bool
test_and_set_bit_lock(long nr, volatile unsigned long *addr)
{
        return test_and_set_bit(nr, addr); /*covered*/
}

/**
 * __test_and_set_bit - Set a bit and return its old value
 * @nr: Bit to set
 * @addr: Address to count from
 *
 * This operation is non-atomic and can be reordered.
 * If two examples of this operation race, one can appear to succeed
 * but actually fail.  You must protect multiple accesses with a lock.
 */
static __always_inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
{
        bool oldbit;

        asm(__ASM_SIZE(bts) " %2,%1"
            CC_SET(c)
            : CC_OUT(c) (oldbit), ADDR
            : "Ir" (nr));
        return oldbit;
}

/**
 * test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static __always_inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
{
        GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btr), /*covered*/
                         *addr, "Ir", nr, "%0", c);
}

/**
 * __test_and_clear_bit - Clear a bit and return its old value
 * @nr: Bit to clear
 * @addr: Address to count from
 *
 * This operation is non-atomic and can be reordered.
 * If two examples of this operation race, one can appear to succeed
 * but actually fail.  You must protect multiple accesses with a lock.
 *
 * Note: the operation is performed atomically with respect to
 * the local CPU, but not other CPUs. Portable code should not
 * rely on this behaviour.
 * KVM relies on this behaviour on x86 for modifying memory that is also
 * accessed from a hypervisor on the same CPU if running in a VM: don't change
 * this without also updating arch/x86/kernel/kvm.c
 */
static __always_inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
{
        bool oldbit;

        asm volatile(__ASM_SIZE(btr) " %2,%1"
                     CC_SET(c)
                     : CC_OUT(c) (oldbit), ADDR
                     : "Ir" (nr));
        return oldbit;
}

/* WARNING: non atomic and it can be reordered! */
static __always_inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
{
        bool oldbit;

        asm volatile(__ASM_SIZE(btc) " %2,%1"
                     CC_SET(c)
                     : CC_OUT(c) (oldbit), ADDR
                     : "Ir" (nr) : "memory");

        return oldbit;
}

/**
 * test_and_change_bit - Change a bit and return its old value
 * @nr: Bit to change
 * @addr: Address to count from
 *
 * This operation is atomic and cannot be reordered.
 * It also implies a memory barrier.
 */
static __always_inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
{
        GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc),
                         *addr, "Ir", nr, "%0", c);
}

static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr)
{
        return ((1UL << (nr & (BITS_PER_LONG-1))) &
                (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; /*covered*/
}

static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)
{
        bool oldbit;

        asm volatile(__ASM_SIZE(bt) " %2,%1" /*covered*/
                     CC_SET(c)
                     : CC_OUT(c) (oldbit)
                     : "m" (*(unsigned long *)addr), "Ir" (nr));

        return oldbit; /*covered*/
}

#if 0 /* Fool kernel-doc since it doesn't do macros yet */
/**
 * test_bit - Determine whether a bit is set
 * @nr: bit number to test
 * @addr: Address to start counting from
 */
static bool test_bit(int nr, const volatile unsigned long *addr);
#endif

#define test_bit(nr, addr)                        \
        (__builtin_constant_p((nr))                \
         ? constant_test_bit((nr), (addr))        \
         : variable_test_bit((nr), (addr)))

/**
 * __ffs - find first set bit in word
 * @word: The word to search
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
static __always_inline unsigned long __ffs(unsigned long word)
{
        asm("rep; bsf %1,%0" /*covered*/
                : "=r" (word)
                : "rm" (word));
        return word;
}

/**
 * ffz - find first zero bit in word
 * @word: The word to search
 *
 * Undefined if no zero exists, so code should check against ~0UL first.
 */
static __always_inline unsigned long ffz(unsigned long word)
{
        asm("rep; bsf %1,%0" /*covered*/
                : "=r" (word)
                : "r" (~word)); /*covered*/
        return word; /*covered*/
}

/*
 * __fls: find last set bit in word
 * @word: The word to search
 *
 * Undefined if no set bit exists, so code should check against 0 first.
 */
static __always_inline unsigned long __fls(unsigned long word)
{
        asm("bsr %1,%0"
            : "=r" (word)
            : "rm" (word));
        return word;
}

#undef ADDR

#ifdef __KERNEL__
/**
 * ffs - find first set bit in word
 * @x: the word to search
 *
 * This is defined the same way as the libc and compiler builtin ffs
 * routines, therefore differs in spirit from the other bitops.
 *
 * ffs(value) returns 0 if value is 0 or the position of the first
 * set bit if value is nonzero. The first (least significant) bit
 * is at position 1.
 */
static __always_inline int ffs(int x)
{
        int r;

#ifdef CONFIG_X86_64
        /*
         * AMD64 says BSFL won't clobber the dest reg if x==0; Intel64 says the
         * dest reg is undefined if x==0, but their CPU architect says its
         * value is written to set it to the same as before, except that the
         * top 32 bits will be cleared.
         *
         * We cannot do this on 32 bits because at the very least some
         * 486 CPUs did not behave this way.
         */
        asm("bsfl %1,%0"
            : "=r" (r)
            : "rm" (x), "0" (-1));
#elif defined(CONFIG_X86_CMOV)
        asm("bsfl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "r" (-1));
#else
        asm("bsfl %1,%0\n\t"
            "jnz 1f\n\t"
            "movl $-1,%0\n"
            "1:" : "=r" (r) : "rm" (x));
#endif
        return r + 1;
}

/**
 * fls - find last set bit in word
 * @x: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffs, but returns the position of the most significant set bit.
 *
 * fls(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 32.
 */
static __always_inline int fls(int x)
{
        int r;

#ifdef CONFIG_X86_64
        /*
         * AMD64 says BSRL won't clobber the dest reg if x==0; Intel64 says the
         * dest reg is undefined if x==0, but their CPU architect says its
         * value is written to set it to the same as before, except that the
         * top 32 bits will be cleared.
         *
         * We cannot do this on 32 bits because at the very least some
         * 486 CPUs did not behave this way.
         */
        asm("bsrl %1,%0" /*covered*/
            : "=r" (r)
            : "rm" (x), "0" (-1));
#elif defined(CONFIG_X86_CMOV)
        asm("bsrl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "rm" (-1));
#else
        asm("bsrl %1,%0\n\t"
            "jnz 1f\n\t"
            "movl $-1,%0\n"
            "1:" : "=r" (r) : "rm" (x));
#endif
        return r + 1;
}

/**
 * fls64 - find last set bit in a 64-bit word
 * @x: the word to search
 *
 * This is defined in a similar way as the libc and compiler builtin
 * ffsll, but returns the position of the most significant set bit.
 *
 * fls64(value) returns 0 if value is 0 or the position of the last
 * set bit if value is nonzero. The last (most significant) bit is
 * at position 64.
 */
#ifdef CONFIG_X86_64
static __always_inline int fls64(__u64 x)
{
        int bitpos = -1;
        /*
         * AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the
         * dest reg is undefined if x==0, but their CPU architect says its
         * value is written to set it to the same as before.
         */
        asm("bsrq %1,%q0"
            : "+r" (bitpos)
            : "rm" (x));
        return bitpos + 1;
}
#else
#include <asm-generic/bitops/fls64.h>
#endif

#include <asm-generic/bitops/find.h>

#include <asm-generic/bitops/sched.h>

#include <asm/arch_hweight.h>

#include <asm-generic/bitops/const_hweight.h>

#include <asm-generic/bitops/le.h>

#include <asm-generic/bitops/ext2-atomic-setbit.h>

#endif /* __KERNEL__ */
#endif /* _ASM_X86_BITOPS_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CHECKSUM_64_H
#define _ASM_X86_CHECKSUM_64_H

/*
 * Checksums for x86-64
 * Copyright 2002 by Andi Kleen, SuSE Labs
 * with some code from asm-x86/checksum.h
 */

#include <linux/compiler.h>
#include <linux/uaccess.h>
#include <asm/byteorder.h>

/**
 * csum_fold - Fold and invert a 32bit checksum.
 * sum: 32bit unfolded sum
 *
 * Fold a 32bit running checksum to 16bit and invert it. This is usually
 * the last step before putting a checksum into a packet.
 * Make sure not to mix with 64bit checksums.
 */
static inline __sum16 csum_fold(__wsum sum)
{
        asm("  addl %1,%0\n"
            "  adcl $0xffff,%0"
            : "=r" (sum)
            : "r" ((__force u32)sum << 16), /*covered*/
              "0" ((__force u32)sum & 0xffff0000));
        return (__force __sum16)(~(__force u32)sum >> 16); /*covered*/
}

/*
 *        This is a version of ip_compute_csum() optimized for IP headers,
 *        which always checksum on 4 octet boundaries.
 *
 *        By Jorge Cwik <jorge@laser.satlink.net>, adapted for linux by
 *        Arnt Gulbrandsen.
 */

/**
 * ip_fast_csum - Compute the IPv4 header checksum efficiently.
 * iph: ipv4 header
 * ihl: length of header / 4
 */
static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
{
        unsigned int sum;

        asm("  movl (%1), %0\n" /*covered*/
            "  subl $4, %2\n"
            "  jbe 2f\n"
            "  addl 4(%1), %0\n"
            "  adcl 8(%1), %0\n"
            "  adcl 12(%1), %0\n"
            "1: adcl 16(%1), %0\n"
            "  lea 4(%1), %1\n"
            "  decl %2\n"
            "  jne        1b\n"
            "  adcl $0, %0\n"
            "  movl %0, %2\n"
            "  shrl $16, %0\n"
            "  addw %w2, %w0\n"
            "  adcl $0, %0\n"
            "  notl %0\n"
            "2:"
        /* Since the input registers which are loaded with iph and ihl
           are modified, we must also specify them as outputs, or gcc
           will assume they contain their original values. */
            : "=r" (sum), "=r" (iph), "=r" (ihl)
            : "1" (iph), "2" (ihl)
            : "memory");
        return (__force __sum16)sum;
}

/**
 * csum_tcpup_nofold - Compute an IPv4 pseudo header checksum.
 * @saddr: source address
 * @daddr: destination address
 * @len: length of packet
 * @proto: ip protocol of packet
 * @sum: initial sum to be added in (32bit unfolded)
 *
 * Returns the pseudo header checksum the input data. Result is
 * 32bit unfolded.
 */
static inline __wsum
csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len,
                   __u8 proto, __wsum sum)
{
        asm("  addl %1, %0\n"
            "  adcl %2, %0\n"
            "  adcl %3, %0\n"
            "  adcl $0, %0\n"
            : "=r" (sum)
            : "g" (daddr), "g" (saddr),
              "g" ((len + proto)<<8), "0" (sum));
        return sum; /*covered*/
}


/**
 * csum_tcpup_magic - Compute an IPv4 pseudo header checksum.
 * @saddr: source address
 * @daddr: destination address
 * @len: length of packet
 * @proto: ip protocol of packet
 * @sum: initial sum to be added in (32bit unfolded)
 *
 * Returns the 16bit pseudo header checksum the input data already
 * complemented and ready to be filled in.
 */
static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
                                        __u32 len, __u8 proto,
                                        __wsum sum)
{
        return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); /*covered*/
}

/**
 * csum_partial - Compute an internet checksum.
 * @buff: buffer to be checksummed
 * @len: length of buffer.
 * @sum: initial sum to be added in (32bit unfolded)
 *
 * Returns the 32bit unfolded internet checksum of the buffer.
 * Before filling it in it needs to be csum_fold()'ed.
 * buff should be aligned to a 64bit boundary if possible.
 */
extern __wsum csum_partial(const void *buff, int len, __wsum sum);

#define  _HAVE_ARCH_COPY_AND_CSUM_FROM_USER 1
#define HAVE_CSUM_COPY_USER 1


/* Do not call this directly. Use the wrappers below */
extern __visible __wsum csum_partial_copy_generic(const void *src, const void *dst,
                                        int len, __wsum sum,
                                        int *src_err_ptr, int *dst_err_ptr);


extern __wsum csum_partial_copy_from_user(const void __user *src, void *dst,
                                          int len, __wsum isum, int *errp);
extern __wsum csum_partial_copy_to_user(const void *src, void __user *dst,
                                        int len, __wsum isum, int *errp);
extern __wsum csum_partial_copy_nocheck(const void *src, void *dst,
                                        int len, __wsum sum);

/* Old names. To be removed. */
#define csum_and_copy_to_user csum_partial_copy_to_user
#define csum_and_copy_from_user csum_partial_copy_from_user

/**
 * ip_compute_csum - Compute an 16bit IP checksum.
 * @buff: buffer address.
 * @len: length of buffer.
 *
 * Returns the 16bit folded/inverted checksum of the passed buffer.
 * Ready to fill in.
 */
extern __sum16 ip_compute_csum(const void *buff, int len);

/**
 * csum_ipv6_magic - Compute checksum of an IPv6 pseudo header.
 * @saddr: source address
 * @daddr: destination address
 * @len: length of packet
 * @proto: protocol of packet
 * @sum: initial sum (32bit unfolded) to be added in
 *
 * Computes an IPv6 pseudo header checksum. This sum is added the checksum
 * into UDP/TCP packets and contains some link layer information.
 * Returns the unfolded 32bit checksum.
 */

struct in6_addr;

#define _HAVE_ARCH_IPV6_CSUM 1
extern __sum16
csum_ipv6_magic(const struct in6_addr *saddr, const struct in6_addr *daddr,
                __u32 len, __u8 proto, __wsum sum);

static inline unsigned add32_with_carry(unsigned a, unsigned b)
{
        asm("addl %2,%0\n\t" /*covered*/
            "adcl $0,%0"
            : "=r" (a)
            : "0" (a), "rm" (b));
        return a;
}

#define HAVE_ARCH_CSUM_ADD
static inline __wsum csum_add(__wsum csum, __wsum addend)
{
        return (__force __wsum)add32_with_carry((__force unsigned)csum, /*covered*/
                                                (__force unsigned)addend);
}

#endif /* _ASM_X86_CHECKSUM_64_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_COMPAT_H
#define _ASM_X86_COMPAT_H

/*
 * Architecture specific compatibility types
 */
#include <linux/types.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <asm/processor.h>
#include <asm/user32.h>
#include <asm/unistd.h>

#define COMPAT_USER_HZ                100
#define COMPAT_UTS_MACHINE        "i686\0\0"

typedef u32                compat_size_t;
typedef s32                compat_ssize_t;
typedef s32                compat_clock_t;
typedef s32                compat_pid_t;
typedef u16                __compat_uid_t;
typedef u16                __compat_gid_t;
typedef u32                __compat_uid32_t;
typedef u32                __compat_gid32_t;
typedef u16                compat_mode_t;
typedef u32                compat_ino_t;
typedef u16                compat_dev_t;
typedef s32                compat_off_t;
typedef s64                compat_loff_t;
typedef u16                compat_nlink_t;
typedef u16                compat_ipc_pid_t;
typedef s32                compat_daddr_t;
typedef u32                compat_caddr_t;
typedef __kernel_fsid_t        compat_fsid_t;
typedef s32                compat_timer_t;
typedef s32                compat_key_t;

typedef s32                compat_int_t;
typedef s32                compat_long_t;
typedef s64 __attribute__((aligned(4))) compat_s64;
typedef u32                compat_uint_t;
typedef u32                compat_ulong_t;
typedef u32                compat_u32;
typedef u64 __attribute__((aligned(4))) compat_u64;
typedef u32                compat_uptr_t;

struct compat_stat {
        compat_dev_t        st_dev;
        u16                __pad1;
        compat_ino_t        st_ino;
        compat_mode_t        st_mode;
        compat_nlink_t        st_nlink;
        __compat_uid_t        st_uid;
        __compat_gid_t        st_gid;
        compat_dev_t        st_rdev;
        u16                __pad2;
        u32                st_size;
        u32                st_blksize;
        u32                st_blocks;
        u32                st_atime;
        u32                st_atime_nsec;
        u32                st_mtime;
        u32                st_mtime_nsec;
        u32                st_ctime;
        u32                st_ctime_nsec;
        u32                __unused4;
        u32                __unused5;
};

struct compat_flock {
        short                l_type;
        short                l_whence;
        compat_off_t        l_start;
        compat_off_t        l_len;
        compat_pid_t        l_pid;
};

#define F_GETLK64        12        /*  using 'struct flock64' */
#define F_SETLK64        13
#define F_SETLKW64        14

/*
 * IA32 uses 4 byte alignment for 64 bit quantities,
 * so we need to pack this structure.
 */
struct compat_flock64 {
        short                l_type;
        short                l_whence;
        compat_loff_t        l_start;
        compat_loff_t        l_len;
        compat_pid_t        l_pid;
} __attribute__((packed));

struct compat_statfs {
        int                f_type;
        int                f_bsize;
        int                f_blocks;
        int                f_bfree;
        int                f_bavail;
        int                f_files;
        int                f_ffree;
        compat_fsid_t        f_fsid;
        int                f_namelen;        /* SunOS ignores this field. */
        int                f_frsize;
        int                f_flags;
        int                f_spare[4];
};

#define COMPAT_RLIM_INFINITY                0xffffffff

typedef u32                compat_old_sigset_t;        /* at least 32 bits */

#define _COMPAT_NSIG                64
#define _COMPAT_NSIG_BPW        32

typedef u32               compat_sigset_word;

#define COMPAT_OFF_T_MAX        0x7fffffff

struct compat_ipc64_perm {
        compat_key_t key;
        __compat_uid32_t uid;
        __compat_gid32_t gid;
        __compat_uid32_t cuid;
        __compat_gid32_t cgid;
        unsigned short mode;
        unsigned short __pad1;
        unsigned short seq;
        unsigned short __pad2;
        compat_ulong_t unused1;
        compat_ulong_t unused2;
};

struct compat_semid64_ds {
        struct compat_ipc64_perm sem_perm;
        compat_ulong_t sem_otime;
        compat_ulong_t sem_otime_high;
        compat_ulong_t sem_ctime;
        compat_ulong_t sem_ctime_high;
        compat_ulong_t sem_nsems;
        compat_ulong_t __unused3;
        compat_ulong_t __unused4;
};

struct compat_msqid64_ds {
        struct compat_ipc64_perm msg_perm;
        compat_ulong_t msg_stime;
        compat_ulong_t msg_stime_high;
        compat_ulong_t msg_rtime;
        compat_ulong_t msg_rtime_high;
        compat_ulong_t msg_ctime;
        compat_ulong_t msg_ctime_high;
        compat_ulong_t msg_cbytes;
        compat_ulong_t msg_qnum;
        compat_ulong_t msg_qbytes;
        compat_pid_t   msg_lspid;
        compat_pid_t   msg_lrpid;
        compat_ulong_t __unused4;
        compat_ulong_t __unused5;
};

struct compat_shmid64_ds {
        struct compat_ipc64_perm shm_perm;
        compat_size_t  shm_segsz;
        compat_ulong_t shm_atime;
        compat_ulong_t shm_atime_high;
        compat_ulong_t shm_dtime;
        compat_ulong_t shm_dtime_high;
        compat_ulong_t shm_ctime;
        compat_ulong_t shm_ctime_high;
        compat_pid_t   shm_cpid;
        compat_pid_t   shm_lpid;
        compat_ulong_t shm_nattch;
        compat_ulong_t __unused4;
        compat_ulong_t __unused5;
};

/*
 * The type of struct elf_prstatus.pr_reg in compatible core dumps.
 */
typedef struct user_regs_struct compat_elf_gregset_t;

/* Full regset -- prstatus on x32, otherwise on ia32 */
#define PRSTATUS_SIZE(S, R) (R != sizeof(S.pr_reg) ? 144 : 296)
#define SET_PR_FPVALID(S, V, R) \
  do { *(int *) (((void *) &((S)->pr_reg)) + R) = (V); } \
  while (0)

#ifdef CONFIG_X86_X32_ABI
#define COMPAT_USE_64BIT_TIME \
        (!!(task_pt_regs(current)->orig_ax & __X32_SYSCALL_BIT))
#endif

/*
 * A pointer passed in from user mode. This should not
 * be used for syscall parameters, just declare them
 * as pointers because the syscall entry code will have
 * appropriately converted them already.
 */

static inline void __user *compat_ptr(compat_uptr_t uptr)
{
        return (void __user *)(unsigned long)uptr;
}

static inline compat_uptr_t ptr_to_compat(void __user *uptr)
{
        return (u32)(unsigned long)uptr;
}

static inline void __user *arch_compat_alloc_user_space(long len)
{
        compat_uptr_t sp;

        if (test_thread_flag(TIF_IA32)) {
                sp = task_pt_regs(current)->sp;
        } else {
                /* -128 for the x32 ABI redzone */
                sp = task_pt_regs(current)->sp - 128;
        }

        return (void __user *)round_down(sp - len, 16);
}

static inline bool in_x32_syscall(void)
{
#ifdef CONFIG_X86_X32_ABI
        if (task_pt_regs(current)->orig_ax & __X32_SYSCALL_BIT) /*covered*/
                return true;
#endif
        return false;
}

static inline bool in_compat_syscall(void)
{
        return in_ia32_syscall() || in_x32_syscall(); /*covered*/
}
#define in_compat_syscall in_compat_syscall        /* override the generic impl */

struct compat_siginfo;
int __copy_siginfo_to_user32(struct compat_siginfo __user *to,
                const siginfo_t *from, bool x32_ABI);

#endif /* _ASM_X86_COMPAT_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CPUFEATURE_H
#define _ASM_X86_CPUFEATURE_H

#include <asm/processor.h>

#if defined(__KERNEL__) && !defined(__ASSEMBLY__)

#include <asm/asm.h>
#include <linux/bitops.h>

enum cpuid_leafs
{
        CPUID_1_EDX                = 0,
        CPUID_8000_0001_EDX,
        CPUID_8086_0001_EDX,
        CPUID_LNX_1,
        CPUID_1_ECX,
        CPUID_C000_0001_EDX,
        CPUID_8000_0001_ECX,
        CPUID_LNX_2,
        CPUID_LNX_3,
        CPUID_7_0_EBX,
        CPUID_D_1_EAX,
        CPUID_F_0_EDX,
        CPUID_F_1_EDX,
        CPUID_8000_0008_EBX,
        CPUID_6_EAX,
        CPUID_8000_000A_EDX,
        CPUID_7_ECX,
        CPUID_8000_0007_EBX,
        CPUID_7_EDX,
};

#ifdef CONFIG_X86_FEATURE_NAMES
extern const char * const x86_cap_flags[NCAPINTS*32];
extern const char * const x86_power_flags[32];
#define X86_CAP_FMT "%s"
#define x86_cap_flag(flag) x86_cap_flags[flag]
#else
#define X86_CAP_FMT "%d:%d"
#define x86_cap_flag(flag) ((flag) >> 5), ((flag) & 31)
#endif

/*
 * In order to save room, we index into this array by doing
 * X86_BUG_<name> - NCAPINTS*32.
 */
extern const char * const x86_bug_flags[NBUGINTS*32];

#define test_cpu_cap(c, bit)                                                \
         test_bit(bit, (unsigned long *)((c)->x86_capability))

/*
 * There are 32 bits/features in each mask word.  The high bits
 * (selected with (bit>>5) give us the word number and the low 5
 * bits give us the bit/feature number inside the word.
 * (1UL<<((bit)&31) gives us a mask for the feature_bit so we can
 * see if it is set in the mask word.
 */
#define CHECK_BIT_IN_MASK_WORD(maskname, word, bit)        \
        (((bit)>>5)==(word) && (1UL<<((bit)&31) & maskname##word ))

#define REQUIRED_MASK_BIT_SET(feature_bit)                \
         ( CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  0, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  1, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  2, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  3, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  4, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  5, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  6, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  7, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  8, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  9, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 10, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 11, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 12, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 13, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 14, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) ||        \
           REQUIRED_MASK_CHECK                                          ||        \
           BUILD_BUG_ON_ZERO(NCAPINTS != 19))

#define DISABLED_MASK_BIT_SET(feature_bit)                                \
         ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  1, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  2, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  3, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  4, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  5, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  6, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  7, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  8, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  9, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 10, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 11, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 12, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 13, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 14, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) ||        \
           DISABLED_MASK_CHECK                                          ||        \
           BUILD_BUG_ON_ZERO(NCAPINTS != 19))

#define cpu_has(c, bit)                                                        \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :        \
         test_cpu_cap(c, bit))

#define this_cpu_has(bit)                                                \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :         \
         x86_this_cpu_test_bit(bit, (unsigned long *)&cpu_info.x86_capability))

/*
 * This macro is for detection of features which need kernel
 * infrastructure to be used.  It may *not* directly test the CPU
 * itself.  Use the cpu_has() family if you want true runtime
 * testing of CPU features, like in hypervisor code where you are
 * supporting a possible guest feature where host support for it
 * is not relevant.
 */
#define cpu_feature_enabled(bit)        \
        (__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))

#define boot_cpu_has(bit)        cpu_has(&boot_cpu_data, bit)

#define set_cpu_cap(c, bit)        set_bit(bit, (unsigned long *)((c)->x86_capability))

extern void setup_clear_cpu_cap(unsigned int bit);
extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);

#define setup_force_cpu_cap(bit) do { \
        set_cpu_cap(&boot_cpu_data, bit);        \
        set_bit(bit, (unsigned long *)cpu_caps_set);        \
} while (0)

#define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)

#if defined(__clang__) && !defined(CC_HAVE_ASM_GOTO)

/*
 * Workaround for the sake of BPF compilation which utilizes kernel
 * headers, but clang does not support ASM GOTO and fails the build.
 */
#ifndef __BPF_TRACING__
#warning "Compiler lacks ASM_GOTO support. Add -D __BPF_TRACING__ to your compiler arguments"
#endif

#define static_cpu_has(bit)            boot_cpu_has(bit)

#else

/*
 * Static testing of CPU features.  Used the same as boot_cpu_has().
 * These will statically patch the target code for additional
 * performance.
 */
static __always_inline __pure bool _static_cpu_has(u16 bit)
{
        asm_volatile_goto("1: jmp 6f\n" /*covered*/
                 "2:\n"
                 ".skip -(((5f-4f) - (2b-1b)) > 0) * "
                         "((5f-4f) - (2b-1b)),0x90\n"
                 "3:\n"
                 ".section .altinstructions,\"a\"\n"
                 " .long 1b - .\n"                /* src offset */
                 " .long 4f - .\n"                /* repl offset */
                 " .word %P[always]\n"                /* always replace */
                 " .byte 3b - 1b\n"                /* src len */
                 " .byte 5f - 4f\n"                /* repl len */
                 " .byte 3b - 2b\n"                /* pad len */
                 ".previous\n"
                 ".section .altinstr_replacement,\"ax\"\n"
                 "4: jmp %l[t_no]\n"
                 "5:\n"
                 ".previous\n"
                 ".section .altinstructions,\"a\"\n"
                 " .long 1b - .\n"                /* src offset */
                 " .long 0\n"                        /* no replacement */
                 " .word %P[feature]\n"                /* feature bit */
                 " .byte 3b - 1b\n"                /* src len */
                 " .byte 0\n"                        /* repl len */
                 " .byte 0\n"                        /* pad len */
                 ".previous\n"
                 ".section .altinstr_aux,\"ax\"\n"
                 "6:\n"
                 " testb %[bitnum],%[cap_byte]\n"
                 " jnz %l[t_yes]\n"
                 " jmp %l[t_no]\n"
                 ".previous\n"
                 : : [feature]  "i" (bit),
                     [always]   "i" (X86_FEATURE_ALWAYS),
                     [bitnum]   "i" (1 << (bit & 7)),
                     [cap_byte] "m" (((const char *)boot_cpu_data.x86_capability)[bit >> 3])
                 : : t_yes, t_no);
t_yes:
        return true;
t_no:
        return false;
}

#define static_cpu_has(bit)                                        \
(                                                                \
        __builtin_constant_p(boot_cpu_has(bit)) ?                \
                boot_cpu_has(bit) :                                \
                _static_cpu_has(bit)                                \
)
#endif

#define cpu_has_bug(c, bit)                cpu_has(c, (bit))
#define set_cpu_bug(c, bit)                set_cpu_cap(c, (bit))
#define clear_cpu_bug(c, bit)                clear_cpu_cap(c, (bit))

#define static_cpu_has_bug(bit)                static_cpu_has((bit))
#define boot_cpu_has_bug(bit)                cpu_has_bug(&boot_cpu_data, (bit))
#define boot_cpu_set_bug(bit)                set_cpu_cap(&boot_cpu_data, (bit))

#define MAX_CPU_FEATURES                (NCAPINTS * 32)
#define cpu_have_feature                boot_cpu_has

#define CPU_FEATURE_TYPEFMT                "x86,ven%04Xfam%04Xmod%04X"
#define CPU_FEATURE_TYPEVAL                boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
                                        boot_cpu_data.x86_model

#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
#endif /* _ASM_X86_CPUFEATURE_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ASM_X86_CAMELLIA_H
#define ASM_X86_CAMELLIA_H

#include <crypto/b128ops.h>
#include <linux/crypto.h>
#include <linux/kernel.h>

#define CAMELLIA_MIN_KEY_SIZE        16
#define CAMELLIA_MAX_KEY_SIZE        32
#define CAMELLIA_BLOCK_SIZE        16
#define CAMELLIA_TABLE_BYTE_LEN        272
#define CAMELLIA_PARALLEL_BLOCKS 2

struct crypto_skcipher;

struct camellia_ctx {
        u64 key_table[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
        u32 key_length;
};

struct camellia_xts_ctx {
        struct camellia_ctx tweak_ctx;
        struct camellia_ctx crypt_ctx;
};

extern int __camellia_setkey(struct camellia_ctx *cctx,
                             const unsigned char *key,
                             unsigned int key_len, u32 *flags);

extern int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
                               unsigned int keylen);

/* regular block cipher functions */
asmlinkage void __camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, bool xor);
asmlinkage void camellia_dec_blk(struct camellia_ctx *ctx, u8 *dst,
                                 const u8 *src);

/* 2-way parallel cipher functions */
asmlinkage void __camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                        const u8 *src, bool xor);
asmlinkage void camellia_dec_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                      const u8 *src);

/* 16-way parallel cipher functions (avx/aes-ni) */
asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);

asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src);
asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst,
                                   const u8 *src, le128 *iv);

asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);
asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst,
                                       const u8 *src, le128 *iv);

static inline void camellia_enc_blk(struct camellia_ctx *ctx, u8 *dst,
                                    const u8 *src)
{
        __camellia_enc_blk(ctx, dst, src, false); /*covered*/
}

static inline void camellia_enc_blk_xor(struct camellia_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __camellia_enc_blk(ctx, dst, src, true);
}

static inline void camellia_enc_blk_2way(struct camellia_ctx *ctx, u8 *dst,
                                         const u8 *src)
{
        __camellia_enc_blk_2way(ctx, dst, src, false); /*covered*/
}

static inline void camellia_enc_blk_xor_2way(struct camellia_ctx *ctx, u8 *dst,
                                             const u8 *src)
{
        __camellia_enc_blk_2way(ctx, dst, src, true);
}

/* glue helpers */
extern void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src);
extern void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src,
                               le128 *iv);
extern void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
                                    le128 *iv);

extern void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv);
extern void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv);

#endif /* ASM_X86_CAMELLIA_H */

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Shared glue code for 128bit block ciphers
 */

#ifndef _CRYPTO_GLUE_HELPER_H
#define _CRYPTO_GLUE_HELPER_H

#include <crypto/internal/skcipher.h>
#include <linux/kernel.h>
#include <asm/fpu/api.h>
#include <crypto/b128ops.h>

typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src);
typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src);
typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src,
                                       le128 *iv);
typedef void (*common_glue_xts_func_t)(void *ctx, u128 *dst, const u128 *src,
                                       le128 *iv);

#define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn))
#define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn))
#define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn))
#define GLUE_XTS_FUNC_CAST(fn) ((common_glue_xts_func_t)(fn))

struct common_glue_func_entry {
        unsigned int num_blocks; /* number of blocks that @fn will process */
        union {
                common_glue_func_t ecb;
                common_glue_cbc_func_t cbc;
                common_glue_ctr_func_t ctr;
                common_glue_xts_func_t xts;
        } fn_u;
};

struct common_glue_ctx {
        unsigned int num_funcs;
        int fpu_blocks_limit; /* -1 means fpu not needed at all */

        /*
         * First funcs entry must have largest num_blocks and last funcs entry
         * must have num_blocks == 1!
         */
        struct common_glue_func_entry funcs[];
};

static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit,
                                  struct skcipher_walk *walk,
                                  bool fpu_enabled, unsigned int nbytes)
{
        if (likely(fpu_blocks_limit < 0))
                return false;

        if (fpu_enabled) /*covered*/
                return true;

        /*
         * Vector-registers are only used when chunk to be processed is large
         * enough, so do not enable FPU until it is necessary.
         */
        if (nbytes < bsize * (unsigned int)fpu_blocks_limit) /*covered*/
                return false;

        /* prevent sleeping if FPU is in use */
        skcipher_walk_atomise(walk); /*covered*/

        kernel_fpu_begin();
        return true;
}

static inline void glue_fpu_end(bool fpu_enabled)
{
        if (fpu_enabled) /*covered*/
                kernel_fpu_end(); /*covered*/
}

static inline void le128_to_be128(be128 *dst, const le128 *src)
{
        dst->a = cpu_to_be64(le64_to_cpu(src->a)); /*covered*/
        dst->b = cpu_to_be64(le64_to_cpu(src->b));
}

static inline void be128_to_le128(le128 *dst, const be128 *src)
{
        dst->a = cpu_to_le64(be64_to_cpu(src->a));
        dst->b = cpu_to_le64(be64_to_cpu(src->b));
}

static inline void le128_inc(le128 *i)
{
        u64 a = le64_to_cpu(i->a);
        u64 b = le64_to_cpu(i->b);

        b++;
        if (!b)
                a++;

        i->a = cpu_to_le64(a); /*covered*/
        i->b = cpu_to_le64(b);
}

extern int glue_ecb_req_128bit(const struct common_glue_ctx *gctx,
                               struct skcipher_request *req);

extern int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn,
                                       struct skcipher_request *req);

extern int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx,
                                       struct skcipher_request *req);

extern int glue_ctr_req_128bit(const struct common_glue_ctx *gctx,
                               struct skcipher_request *req);

extern int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
                               struct skcipher_request *req,
                               common_glue_func_t tweak_fn, void *tweak_ctx,
                               void *crypt_ctx);

extern void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src,
                                      le128 *iv, common_glue_func_t fn);

#endif /* _CRYPTO_GLUE_HELPER_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef ASM_X86_SERPENT_SSE2_H
#define ASM_X86_SERPENT_SSE2_H

#include <linux/crypto.h>
#include <crypto/serpent.h>

#ifdef CONFIG_X86_32

#define SERPENT_PARALLEL_BLOCKS 4

asmlinkage void __serpent_enc_blk_4way(struct serpent_ctx *ctx, u8 *dst,
                                       const u8 *src, bool xor);
asmlinkage void serpent_dec_blk_4way(struct serpent_ctx *ctx, u8 *dst,
                                     const u8 *src);

static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        __serpent_enc_blk_4way(ctx, dst, src, false);
}

static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst,
                                            const u8 *src)
{
        __serpent_enc_blk_4way(ctx, dst, src, true);
}

static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                        const u8 *src)
{
        serpent_dec_blk_4way(ctx, dst, src);
}

#else

#define SERPENT_PARALLEL_BLOCKS 8

asmlinkage void __serpent_enc_blk_8way(struct serpent_ctx *ctx, u8 *dst,
                                       const u8 *src, bool xor);
asmlinkage void serpent_dec_blk_8way(struct serpent_ctx *ctx, u8 *dst,
                                     const u8 *src);

static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                   const u8 *src)
{
        __serpent_enc_blk_8way(ctx, dst, src, false); /*covered*/
}

static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst,
                                       const u8 *src)
{
        __serpent_enc_blk_8way(ctx, dst, src, true);
}

static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst,
                                   const u8 *src)
{
        serpent_dec_blk_8way(ctx, dst, src);
}

#endif

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CURRENT_H
#define _ASM_X86_CURRENT_H

#include <linux/compiler.h>
#include <asm/percpu.h>

#ifndef __ASSEMBLY__
struct task_struct;

DECLARE_PER_CPU(struct task_struct *, current_task);

static __always_inline struct task_struct *get_current(void)
{
        return this_cpu_read_stable(current_task); /*covered*/
}

#define current get_current()

#endif /* __ASSEMBLY__ */

#endif /* _ASM_X86_CURRENT_H */

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 1994 Linus Torvalds
 *
 * Pentium III FXSR, SSE support
 * General FPU state handling cleanups
 *        Gareth Hughes <gareth@valinux.com>, May 2000
 * x86-64 work by Andi Kleen 2002
 */

#ifndef _ASM_X86_FPU_INTERNAL_H
#define _ASM_X86_FPU_INTERNAL_H

#include <linux/compat.h>
#include <linux/sched.h>
#include <linux/slab.h>

#include <asm/user.h>
#include <asm/fpu/api.h>
#include <asm/fpu/xstate.h>
#include <asm/cpufeature.h>
#include <asm/trace/fpu.h>

/*
 * High level FPU state handling functions:
 */
extern void fpu__initialize(struct fpu *fpu);
extern void fpu__prepare_read(struct fpu *fpu);
extern void fpu__prepare_write(struct fpu *fpu);
extern void fpu__save(struct fpu *fpu);
extern void fpu__restore(struct fpu *fpu);
extern int  fpu__restore_sig(void __user *buf, int ia32_frame);
extern void fpu__drop(struct fpu *fpu);
extern int  fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu);
extern void fpu__clear(struct fpu *fpu);
extern int  fpu__exception_code(struct fpu *fpu, int trap_nr);
extern int  dump_fpu(struct pt_regs *ptregs, struct user_i387_struct *fpstate);

/*
 * Boot time FPU initialization functions:
 */
extern void fpu__init_cpu(void);
extern void fpu__init_system_xstate(void);
extern void fpu__init_cpu_xstate(void);
extern void fpu__init_system(struct cpuinfo_x86 *c);
extern void fpu__init_check_bugs(void);
extern void fpu__resume_cpu(void);
extern u64 fpu__get_supported_xfeatures_mask(void);

/*
 * Debugging facility:
 */
#ifdef CONFIG_X86_DEBUG_FPU
# define WARN_ON_FPU(x) WARN_ON_ONCE(x)
#else
# define WARN_ON_FPU(x) ({ (void)(x); 0; })
#endif

/*
 * FPU related CPU feature flag helper routines:
 */
static __always_inline __pure bool use_xsaveopt(void)
{
        return static_cpu_has(X86_FEATURE_XSAVEOPT);
}

static __always_inline __pure bool use_xsave(void)
{
        return static_cpu_has(X86_FEATURE_XSAVE); /*covered*/
}

static __always_inline __pure bool use_fxsr(void)
{
        return static_cpu_has(X86_FEATURE_FXSR);
}

/*
 * fpstate handling functions:
 */

extern union fpregs_state init_fpstate;

extern void fpstate_init(union fpregs_state *state);
#ifdef CONFIG_MATH_EMULATION
extern void fpstate_init_soft(struct swregs_state *soft);
#else
static inline void fpstate_init_soft(struct swregs_state *soft) {}
#endif

static inline void fpstate_init_xstate(struct xregs_state *xsave)
{
        /*
         * XRSTORS requires these bits set in xcomp_bv, or it will
         * trigger #GP:
         */
        xsave->header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT | xfeatures_mask;
}

static inline void fpstate_init_fxstate(struct fxregs_state *fx)
{
        fx->cwd = 0x37f; /*covered*/
        fx->mxcsr = MXCSR_DEFAULT;
}
extern void fpstate_sanitize_xstate(struct fpu *fpu);

#define user_insn(insn, output, input...)                                \
({                                                                        \
        int err;                                                        \
        asm volatile(ASM_STAC "\n"                                        \
                     "1:" #insn "\n\t"                                        \
                     "2: " ASM_CLAC "\n"                                \
                     ".section .fixup,\"ax\"\n"                                \
                     "3:  movl $-1,%[err]\n"                                \
                     "    jmp  2b\n"                                        \
                     ".previous\n"                                        \
                     _ASM_EXTABLE(1b, 3b)                                \
                     : [err] "=r" (err), output                                \
                     : "0"(0), input);                                        \
        err;                                                                \
})

#define kernel_insn(insn, output, input...)                                \
        asm volatile("1:" #insn "\n\t"                                        \
                     "2:\n"                                                \
                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_fprestore)        \
                     : output : input)

static inline int copy_fregs_to_user(struct fregs_state __user *fx)
{
        return user_insn(fnsave %[fx]; fwait,  [fx] "=m" (*fx), "m" (*fx));
}

static inline int copy_fxregs_to_user(struct fxregs_state __user *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
        else if (IS_ENABLED(CONFIG_AS_FXSAVEQ))
                return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));

        /* See comment in copy_fxregs_to_kernel() below. */
        return user_insn(rex64/fxsave (%[fx]), "=m" (*fx), [fx] "R" (fx));
}

static inline void copy_kernel_to_fxregs(struct fxregs_state *fx)
{
        if (IS_ENABLED(CONFIG_X86_32)) {
                kernel_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
        } else {
                if (IS_ENABLED(CONFIG_AS_FXSAVEQ)) {
                        kernel_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
                } else {
                        /* See comment in copy_fxregs_to_kernel() below. */
                        kernel_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx), "m" (*fx));
                }
        }
}

static inline int copy_user_to_fxregs(struct fxregs_state __user *fx)
{
        if (IS_ENABLED(CONFIG_X86_32))
                return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
        else if (IS_ENABLED(CONFIG_AS_FXSAVEQ))
                return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));

        /* See comment in copy_fxregs_to_kernel() below. */
        return user_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx),
                          "m" (*fx));
}

static inline void copy_kernel_to_fregs(struct fregs_state *fx)
{
        kernel_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline int copy_user_to_fregs(struct fregs_state __user *fx)
{
        return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
}

static inline void copy_fxregs_to_kernel(struct fpu *fpu)
{
        if (IS_ENABLED(CONFIG_X86_32))
                asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state.fxsave));
        else if (IS_ENABLED(CONFIG_AS_FXSAVEQ))
                asm volatile("fxsaveq %[fx]" : [fx] "=m" (fpu->state.fxsave));
        else {
                /* Using "rex64; fxsave %0" is broken because, if the memory
                 * operand uses any extended registers for addressing, a second
                 * REX prefix will be generated (to the assembler, rex64
                 * followed by semicolon is a separate instruction), and hence
                 * the 64-bitness is lost.
                 *
                 * Using "fxsaveq %0" would be the ideal choice, but is only
                 * supported starting with gas 2.16.
                 *
                 * Using, as a workaround, the properly prefixed form below
                 * isn't accepted by any binutils version so far released,
                 * complaining that the same type of prefix is used twice if
                 * an extended register is needed for addressing (fix submitted
                 * to mainline 2005-11-21).
                 *
                 *  asm volatile("rex64/fxsave %0" : "=m" (fpu->state.fxsave));
                 *
                 * This, however, we can work around by forcing the compiler to
                 * select an addressing mode that doesn't require extended
                 * registers.
                 */
                asm volatile( "rex64/fxsave (%[fx])"
                             : "=m" (fpu->state.fxsave)
                             : [fx] "R" (&fpu->state.fxsave));
        }
}

/* These macros all use (%edi)/(%rdi) as the single memory argument. */
#define XSAVE                ".byte " REX_PREFIX "0x0f,0xae,0x27"
#define XSAVEOPT        ".byte " REX_PREFIX "0x0f,0xae,0x37"
#define XSAVES                ".byte " REX_PREFIX "0x0f,0xc7,0x2f"
#define XRSTOR                ".byte " REX_PREFIX "0x0f,0xae,0x2f"
#define XRSTORS                ".byte " REX_PREFIX "0x0f,0xc7,0x1f"

#define XSTATE_OP(op, st, lmask, hmask, err)                                \
        asm volatile("1:" op "\n\t"                                        \
                     "xor %[err], %[err]\n"                                \
                     "2:\n\t"                                                \
                     ".pushsection .fixup,\"ax\"\n\t"                        \
                     "3: movl $-2,%[err]\n\t"                                \
                     "jmp 2b\n\t"                                        \
                     ".popsection\n\t"                                        \
                     _ASM_EXTABLE(1b, 3b)                                \
                     : [err] "=r" (err)                                        \
                     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)        \
                     : "memory")

/*
 * If XSAVES is enabled, it replaces XSAVEOPT because it supports a compact
 * format and supervisor states in addition to modified optimization in
 * XSAVEOPT.
 *
 * Otherwise, if XSAVEOPT is enabled, XSAVEOPT replaces XSAVE because XSAVEOPT
 * supports modified optimization which is not supported by XSAVE.
 *
 * We use XSAVE as a fallback.
 *
 * The 661 label is defined in the ALTERNATIVE* macros as the address of the
 * original instruction which gets replaced. We need to use it here as the
 * address of the instruction where we might get an exception at.
 */
#define XSTATE_XSAVE(st, lmask, hmask, err)                                \
        asm volatile(ALTERNATIVE_2(XSAVE,                                \
                                   XSAVEOPT, X86_FEATURE_XSAVEOPT,        \
                                   XSAVES,   X86_FEATURE_XSAVES)        \
                     "\n"                                                \
                     "xor %[err], %[err]\n"                                \
                     "3:\n"                                                \
                     ".pushsection .fixup,\"ax\"\n"                        \
                     "4: movl $-2, %[err]\n"                                \
                     "jmp 3b\n"                                                \
                     ".popsection\n"                                        \
                     _ASM_EXTABLE(661b, 4b)                                \
                     : [err] "=r" (err)                                        \
                     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)        \
                     : "memory")

/*
 * Use XRSTORS to restore context if it is enabled. XRSTORS supports compact
 * XSAVE area format.
 */
#define XSTATE_XRESTORE(st, lmask, hmask)                                \
        asm volatile(ALTERNATIVE(XRSTOR,                                \
                                 XRSTORS, X86_FEATURE_XSAVES)                \
                     "\n"                                                \
                     "3:\n"                                                \
                     _ASM_EXTABLE_HANDLE(661b, 3b, ex_handler_fprestore)\
                     :                                                        \
                     : "D" (st), "m" (*st), "a" (lmask), "d" (hmask)        \
                     : "memory")

/*
 * This function is called only during boot time when x86 caps are not set
 * up and alternative can not be used yet.
 */
static inline void copy_xregs_to_kernel_booting(struct xregs_state *xstate)
{
        u64 mask = -1;
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        WARN_ON(system_state != SYSTEM_BOOTING);

        if (static_cpu_has(X86_FEATURE_XSAVES))
                XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
        else
                XSTATE_OP(XSAVE, xstate, lmask, hmask, err);

        /* We should never fault when copying to a kernel buffer: */
        WARN_ON_FPU(err);
}

/*
 * This function is called only during boot time when x86 caps are not set
 * up and alternative can not be used yet.
 */
static inline void copy_kernel_to_xregs_booting(struct xregs_state *xstate)
{
        u64 mask = -1;
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        WARN_ON(system_state != SYSTEM_BOOTING);

        if (static_cpu_has(X86_FEATURE_XSAVES))
                XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
        else
                XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);

        /*
         * We should never fault when copying from a kernel buffer, and the FPU
         * state we set at boot time should be valid.
         */
        WARN_ON_FPU(err);
}

/*
 * Save processor xstate to xsave area.
 */
static inline void copy_xregs_to_kernel(struct xregs_state *xstate)
{
        u64 mask = -1;
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        WARN_ON_FPU(!alternatives_patched);

        XSTATE_XSAVE(xstate, lmask, hmask, err); /*covered*/

        /* We should never fault when copying to a kernel buffer: */
        WARN_ON_FPU(err);
}

/*
 * Restore processor xstate from xsave area.
 */
static inline void copy_kernel_to_xregs(struct xregs_state *xstate, u64 mask)
{
        u32 lmask = mask;
        u32 hmask = mask >> 32;

        XSTATE_XRESTORE(xstate, lmask, hmask); /*covered*/
}

/*
 * Save xstate to user space xsave area.
 *
 * We don't use modified optimization because xrstor/xrstors might track
 * a different application.
 *
 * We don't use compacted format xsave area for
 * backward compatibility for old applications which don't understand
 * compacted format of xsave area.
 */
static inline int copy_xregs_to_user(struct xregs_state __user *buf)
{
        int err;

        /*
         * Clear the xsave header first, so that reserved fields are
         * initialized to zero.
         */
        err = __clear_user(&buf->header, sizeof(buf->header));
        if (unlikely(err))
                return -EFAULT;

        stac(); /*covered*/
        XSTATE_OP(XSAVE, buf, -1, -1, err);
        clac();

        return err;
}

/*
 * Restore xstate from user space xsave area.
 */
static inline int copy_user_to_xregs(struct xregs_state __user *buf, u64 mask)
{
        struct xregs_state *xstate = ((__force struct xregs_state *)buf);
        u32 lmask = mask;
        u32 hmask = mask >> 32;
        int err;

        stac();
        XSTATE_OP(XRSTOR, xstate, lmask, hmask, err);
        clac();

        return err;
}

/*
 * These must be called with preempt disabled. Returns
 * 'true' if the FPU state is still intact and we can
 * keep registers active.
 *
 * The legacy FNSAVE instruction cleared all FPU state
 * unconditionally, so registers are essentially destroyed.
 * Modern FPU state can be kept in registers, if there are
 * no pending FP exceptions.
 */
static inline int copy_fpregs_to_fpstate(struct fpu *fpu)
{
        if (likely(use_xsave())) { /*covered*/
                copy_xregs_to_kernel(&fpu->state.xsave); /*covered*/
                return 1;
        }

        if (likely(use_fxsr())) {
                copy_fxregs_to_kernel(fpu);
                return 1;
        }

        /*
         * Legacy FPU register saving, FNSAVE always clears FPU registers,
         * so we have to mark them inactive:
         */
        asm volatile("fnsave %[fp]; fwait" : [fp] "=m" (fpu->state.fsave));

        return 0;
}

static inline void __copy_kernel_to_fpregs(union fpregs_state *fpstate, u64 mask)
{
        if (use_xsave()) { /*covered*/
                copy_kernel_to_xregs(&fpstate->xsave, mask); /*covered*/
        } else {
                if (use_fxsr())
                        copy_kernel_to_fxregs(&fpstate->fxsave);
                else
                        copy_kernel_to_fregs(&fpstate->fsave);
        }
}

static inline void copy_kernel_to_fpregs(union fpregs_state *fpstate)
{
        /*
         * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is
         * pending. Clear the x87 state here by setting it to fixed values.
         * "m" is a random variable that should be in L1.
         */
        if (unlikely(static_cpu_has_bug(X86_BUG_FXSAVE_LEAK))) {
                asm volatile(
                        "fnclex\n\t"
                        "emms\n\t"
                        "fildl %P[addr]"        /* set F?P to defined value */
                        : : [addr] "m" (fpstate));
        }

        __copy_kernel_to_fpregs(fpstate, -1); /*covered*/
}

extern int copy_fpstate_to_sigframe(void __user *buf, void __user *fp, int size);

/*
 * FPU context switch related helper methods:
 */

DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);

/*
 * The in-register FPU state for an FPU context on a CPU is assumed to be
 * valid if the fpu->last_cpu matches the CPU, and the fpu_fpregs_owner_ctx
 * matches the FPU.
 *
 * If the FPU register state is valid, the kernel can skip restoring the
 * FPU state from memory.
 *
 * Any code that clobbers the FPU registers or updates the in-memory
 * FPU state for a task MUST let the rest of the kernel know that the
 * FPU registers are no longer valid for this task.
 *
 * Either one of these invalidation functions is enough. Invalidate
 * a resource you control: CPU if using the CPU for something else
 * (with preemption disabled), FPU for the current task, or a task that
 * is prevented from running by the current task.
 */
static inline void __cpu_invalidate_fpregs_state(void)
{
        __this_cpu_write(fpu_fpregs_owner_ctx, NULL);
}

static inline void __fpu_invalidate_fpregs_state(struct fpu *fpu)
{
        fpu->last_cpu = -1;
}

static inline int fpregs_state_valid(struct fpu *fpu, unsigned int cpu)
{
        return fpu == this_cpu_read_stable(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu;
}

/*
 * These generally need preemption protection to work,
 * do try to avoid using these on their own:
 */
static inline void fpregs_deactivate(struct fpu *fpu)
{
        this_cpu_write(fpu_fpregs_owner_ctx, NULL);
        trace_x86_fpu_regs_deactivated(fpu); /*covered*/
}

static inline void fpregs_activate(struct fpu *fpu)
{
        this_cpu_write(fpu_fpregs_owner_ctx, fpu);
        trace_x86_fpu_regs_activated(fpu); /*covered*/
}

/*
 * FPU state switching for scheduling.
 *
 * This is a two-stage process:
 *
 *  - switch_fpu_prepare() saves the old state.
 *    This is done within the context of the old process.
 *
 *  - switch_fpu_finish() restores the new state as
 *    necessary.
 */
static inline void
switch_fpu_prepare(struct fpu *old_fpu, int cpu)
{
        if (old_fpu->initialized) {
                if (!copy_fpregs_to_fpstate(old_fpu))
                        old_fpu->last_cpu = -1;
                else
                        old_fpu->last_cpu = cpu;

                /* But leave fpu_fpregs_owner_ctx! */
                trace_x86_fpu_regs_deactivated(old_fpu);
        } else
                old_fpu->last_cpu = -1;
}

/*
 * Misc helper functions:
 */

/*
 * Set up the userspace FPU context for the new task, if the task
 * has used the FPU.
 */
static inline void switch_fpu_finish(struct fpu *new_fpu, int cpu)
{
        bool preload = static_cpu_has(X86_FEATURE_FPU) &&
                       new_fpu->initialized;

        if (preload) {
                if (!fpregs_state_valid(new_fpu, cpu))
                        copy_kernel_to_fpregs(&new_fpu->state);
                fpregs_activate(new_fpu);
        }
}

/*
 * Needs to be preemption-safe.
 *
 * NOTE! user_fpu_begin() must be used only immediately before restoring
 * the save state. It does not do any saving/restoring on its own. In
 * lazy FPU mode, it is just an optimization to avoid a #NM exception,
 * the task can lose the FPU right after preempt_enable().
 */
static inline void user_fpu_begin(void)
{
        struct fpu *fpu = &current->thread.fpu;

        preempt_disable();
        fpregs_activate(fpu); /*covered*/
        preempt_enable(); /*covered*/
}

/*
 * MXCSR and XCR definitions:
 */

extern unsigned int mxcsr_feature_mask;

#define XCR_XFEATURE_ENABLED_MASK        0x00000000

static inline u64 xgetbv(u32 index)
{
        u32 eax, edx;

        asm volatile(".byte 0x0f,0x01,0xd0" /* xgetbv */
                     : "=a" (eax), "=d" (edx)
                     : "c" (index));
        return eax + ((u64)edx << 32);
}

static inline void xsetbv(u32 index, u64 value)
{
        u32 eax = value;
        u32 edx = value >> 32;

        asm volatile(".byte 0x0f,0x01,0xd1" /* xsetbv */
                     : : "a" (eax), "d" (edx), "c" (index));
}

#endif /* _ASM_X86_FPU_INTERNAL_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_HUGETLB_H
#define _ASM_X86_HUGETLB_H

#include <asm/page.h>
#include <asm-generic/hugetlb.h>

#define hugepages_supported() boot_cpu_has(X86_FEATURE_PSE)

static inline int is_hugepage_only_range(struct mm_struct *mm,
                                         unsigned long addr,
                                         unsigned long len) {
        return 0;
}

/*
 * If the arch doesn't supply something else, assume that hugepage
 * size aligned regions are ok without further preparation.
 */
static inline int prepare_hugepage_range(struct file *file,
                        unsigned long addr, unsigned long len)
{
        struct hstate *h = hstate_file(file); /*covered*/
        if (len & ~huge_page_mask(h))
                return -EINVAL;
        if (addr & ~huge_page_mask(h)) /*covered*/
                return -EINVAL;
        return 0;
}

static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
                                          unsigned long addr, unsigned long end,
                                          unsigned long floor,
                                          unsigned long ceiling)
{
        free_pgd_range(tlb, addr, end, floor, ceiling);
}

static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
                                   pte_t *ptep, pte_t pte)
{
        set_pte_at(mm, addr, ptep, pte);
}

static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
                                            unsigned long addr, pte_t *ptep)
{
        return ptep_get_and_clear(mm, addr, ptep);
}

static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
                                         unsigned long addr, pte_t *ptep)
{
        ptep_clear_flush(vma, addr, ptep);
}

static inline int huge_pte_none(pte_t pte)
{
        return pte_none(pte); /*covered*/
}

static inline pte_t huge_pte_wrprotect(pte_t pte)
{
        return pte_wrprotect(pte);
}

static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
                                           unsigned long addr, pte_t *ptep)
{
        ptep_set_wrprotect(mm, addr, ptep);
}

static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
                                             unsigned long addr, pte_t *ptep,
                                             pte_t pte, int dirty)
{
        return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
}

static inline pte_t huge_ptep_get(pte_t *ptep)
{
        return *ptep;
}

static inline void arch_clear_hugepage_flags(struct page *page)
{
}

#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
static inline bool gigantic_page_supported(void) { return true; }
#endif

#endif /* _ASM_X86_HUGETLB_H */

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Per-cpu current frame pointer - the location of the last exception frame on
 * the stack, stored in the per-cpu area.
 *
 * Jeremy Fitzhardinge <jeremy@goop.org>
 */
#ifndef _ASM_X86_IRQ_REGS_H
#define _ASM_X86_IRQ_REGS_H

#include <asm/percpu.h>

#define ARCH_HAS_OWN_IRQ_REGS

DECLARE_PER_CPU(struct pt_regs *, irq_regs);

static inline struct pt_regs *get_irq_regs(void)
{
        return this_cpu_read(irq_regs); /*covered*/
}

static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
{
        struct pt_regs *old_regs;

        old_regs = get_irq_regs();
        this_cpu_write(irq_regs, new_regs);

        return old_regs;
}

#endif /* _ASM_X86_IRQ_REGS_32_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _X86_IRQFLAGS_H_
#define _X86_IRQFLAGS_H_

#include <asm/processor-flags.h>

#ifndef __ASSEMBLY__

/* Provide __cpuidle; we can't safely include <linux/cpu.h> */
#define __cpuidle __attribute__((__section__(".cpuidle.text")))

/*
 * Interrupt control:
 */

/* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */
extern inline unsigned long native_save_fl(void);
extern inline unsigned long native_save_fl(void)
{
        unsigned long flags;

        /*
         * "=rm" is safe here, because "pop" adjusts the stack before
         * it evaluates its effective address -- this is part of the
         * documented behavior of the "pop" instruction.
         */
        asm volatile("# __raw_save_flags\n\t"
                     "pushf ; pop %0"
                     : "=rm" (flags)
                     : /* no input */
                     : "memory");

        return flags;
}

extern inline void native_restore_fl(unsigned long flags);
extern inline void native_restore_fl(unsigned long flags)
{
        asm volatile("push %0 ; popf"
                     : /* no output */
                     :"g" (flags)
                     :"memory", "cc");
}

static inline void native_irq_disable(void)
{
        asm volatile("cli": : :"memory");
}

static inline void native_irq_enable(void)
{
        asm volatile("sti": : :"memory");
}

static inline __cpuidle void native_safe_halt(void)
{
        asm volatile("sti; hlt": : :"memory");
}

static inline __cpuidle void native_halt(void)
{
        asm volatile("hlt": : :"memory");
}

#endif

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#ifndef __ASSEMBLY__
#include <linux/types.h>

static inline notrace unsigned long arch_local_save_flags(void)
{
        return native_save_fl();
}

static inline notrace void arch_local_irq_restore(unsigned long flags)
{
        native_restore_fl(flags);
}

static inline notrace void arch_local_irq_disable(void)
{
        native_irq_disable();
}

static inline notrace void arch_local_irq_enable(void)
{
        native_irq_enable();
}

/*
 * Used in the idle loop; sti takes one instruction cycle
 * to complete:
 */
static inline __cpuidle void arch_safe_halt(void)
{
        native_safe_halt();
}

/*
 * Used when interrupts are already enabled or to
 * shutdown the processor:
 */
static inline __cpuidle void halt(void)
{
        native_halt();
}

/*
 * For spinlocks, etc:
 */
static inline notrace unsigned long arch_local_irq_save(void)
{
        unsigned long flags = arch_local_save_flags();
        arch_local_irq_disable();
        return flags;
}
#else

#define ENABLE_INTERRUPTS(x)        sti
#define DISABLE_INTERRUPTS(x)        cli

#ifdef CONFIG_X86_64
#define SWAPGS        swapgs
/*
 * Currently paravirt can't handle swapgs nicely when we
 * don't have a stack we can rely on (such as a user space
 * stack).  So we either find a way around these or just fault
 * and emulate if a guest tries to call swapgs directly.
 *
 * Either way, this is a good way to document that we don't
 * have a reliable stack. x86_64 only.
 */
#define SWAPGS_UNSAFE_STACK        swapgs

#define PARAVIRT_ADJUST_EXCEPTION_FRAME        /*  */

#define INTERRUPT_RETURN        jmp native_iret
#define USERGS_SYSRET64                                \
        swapgs;                                        \
        sysretq;
#define USERGS_SYSRET32                                \
        swapgs;                                        \
        sysretl

#ifdef CONFIG_DEBUG_ENTRY
#define SAVE_FLAGS(x)                pushfq; popq %rax
#endif
#else
#define INTERRUPT_RETURN                iret
#define ENABLE_INTERRUPTS_SYSEXIT        sti; sysexit
#define GET_CR0_INTO_EAX                movl %cr0, %eax
#endif


#endif /* __ASSEMBLY__ */
#endif /* CONFIG_PARAVIRT */

#ifndef __ASSEMBLY__
static inline int arch_irqs_disabled_flags(unsigned long flags)
{
        return !(flags & X86_EFLAGS_IF); /*covered*/
}

static inline int arch_irqs_disabled(void)
{
        unsigned long flags = arch_local_save_flags();

        return arch_irqs_disabled_flags(flags);
}
#endif /* !__ASSEMBLY__ */

#ifdef __ASSEMBLY__
#ifdef CONFIG_TRACE_IRQFLAGS
#  define TRACE_IRQS_ON                call trace_hardirqs_on_thunk;
#  define TRACE_IRQS_OFF        call trace_hardirqs_off_thunk;
#else
#  define TRACE_IRQS_ON
#  define TRACE_IRQS_OFF
#endif
#ifdef CONFIG_DEBUG_LOCK_ALLOC
#  ifdef CONFIG_X86_64
#    define LOCKDEP_SYS_EXIT                call lockdep_sys_exit_thunk
#    define LOCKDEP_SYS_EXIT_IRQ \
        TRACE_IRQS_ON; \
        sti; \
        call lockdep_sys_exit_thunk; \
        cli; \
        TRACE_IRQS_OFF;
#  else
#    define LOCKDEP_SYS_EXIT \
        pushl %eax;                                \
        pushl %ecx;                                \
        pushl %edx;                                \
        call lockdep_sys_exit;                        \
        popl %edx;                                \
        popl %ecx;                                \
        popl %eax;
#    define LOCKDEP_SYS_EXIT_IRQ
#  endif
#else
#  define LOCKDEP_SYS_EXIT
#  define LOCKDEP_SYS_EXIT_IRQ
#endif
#endif /* __ASSEMBLY__ */

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_JUMP_LABEL_H
#define _ASM_X86_JUMP_LABEL_H

#ifndef HAVE_JUMP_LABEL
/*
 * For better or for worse, if jump labels (the gcc extension) are missing,
 * then the entire static branch patching infrastructure is compiled out.
 * If that happens, the code in here will malfunction.  Raise a compiler
 * error instead.
 *
 * In theory, jump labels and the static branch patching infrastructure
 * could be decoupled to fix this.
 */
#error asm/jump_label.h included on a non-jump-label kernel
#endif

#define JUMP_LABEL_NOP_SIZE 5

#ifdef CONFIG_X86_64
# define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC
#else
# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
#endif

#include <asm/asm.h>
#include <asm/nops.h>

#ifndef __ASSEMBLY__

#include <linux/stringify.h>
#include <linux/types.h>

static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{
        asm_volatile_goto("1:" /*covered*/
                ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
                ".pushsection __jump_table,  \"aw\" \n\t"
                _ASM_ALIGN "\n\t"
                _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
                ".popsection \n\t"
                : :  "i" (key), "i" (branch) : : l_yes);

        return false;
l_yes:
        return true;
}

static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
        asm_volatile_goto("1:" /*covered*/
                ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
                "2:\n\t"
                ".pushsection __jump_table,  \"aw\" \n\t"
                _ASM_ALIGN "\n\t"
                _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
                ".popsection \n\t"
                : :  "i" (key), "i" (branch) : : l_yes);

        return false;
l_yes:
        return true;
}

#ifdef CONFIG_X86_64
typedef u64 jump_label_t;
#else
typedef u32 jump_label_t;
#endif

struct jump_entry {
        jump_label_t code;
        jump_label_t target;
        jump_label_t key;
};

#else        /* __ASSEMBLY__ */

.macro STATIC_JUMP_IF_TRUE target, key, def
.Lstatic_jump_\@:
        .if \def
        /* Equivalent to "jmp.d32 \target" */
        .byte                0xe9
        .long                \target - .Lstatic_jump_after_\@
.Lstatic_jump_after_\@:
        .else
        .byte                STATIC_KEY_INIT_NOP
        .endif
        .pushsection __jump_table, "aw"
        _ASM_ALIGN
        _ASM_PTR        .Lstatic_jump_\@, \target, \key
        .popsection
.endm

.macro STATIC_JUMP_IF_FALSE target, key, def
.Lstatic_jump_\@:
        .if \def
        .byte                STATIC_KEY_INIT_NOP
        .else
        /* Equivalent to "jmp.d32 \target" */
        .byte                0xe9
        .long                \target - .Lstatic_jump_after_\@
.Lstatic_jump_after_\@:
        .endif
        .pushsection __jump_table, "aw"
        _ASM_ALIGN
        _ASM_PTR        .Lstatic_jump_\@, \target, \key + 1
        .popsection
.endm

#endif        /* __ASSEMBLY__ */

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_LOCAL_H
#define _ASM_X86_LOCAL_H

#include <linux/percpu.h>

#include <linux/atomic.h>
#include <asm/asm.h>

typedef struct {
        atomic_long_t a;
} local_t;

#define LOCAL_INIT(i)        { ATOMIC_LONG_INIT(i) }

#define local_read(l)        atomic_long_read(&(l)->a)
#define local_set(l, i)        atomic_long_set(&(l)->a, (i))

static inline void local_inc(local_t *l)
{
        asm volatile(_ASM_INC "%0" /*covered*/
                     : "+m" (l->a.counter));
}

static inline void local_dec(local_t *l)
{
        asm volatile(_ASM_DEC "%0" /*covered*/
                     : "+m" (l->a.counter));
}

static inline void local_add(long i, local_t *l)
{
        asm volatile(_ASM_ADD "%1,%0"
                     : "+m" (l->a.counter)
                     : "ir" (i));
}

static inline void local_sub(long i, local_t *l)
{
        asm volatile(_ASM_SUB "%1,%0"
                     : "+m" (l->a.counter)
                     : "ir" (i));
}

/**
 * local_sub_and_test - subtract value from variable and test result
 * @i: integer value to subtract
 * @l: pointer to type local_t
 *
 * Atomically subtracts @i from @l and returns
 * true if the result is zero, or false for all
 * other cases.
 */
static inline bool local_sub_and_test(long i, local_t *l)
{
        GEN_BINARY_RMWcc(_ASM_SUB, l->a.counter, "er", i, "%0", e);
}

/**
 * local_dec_and_test - decrement and test
 * @l: pointer to type local_t
 *
 * Atomically decrements @l by 1 and
 * returns true if the result is 0, or false for all other
 * cases.
 */
static inline bool local_dec_and_test(local_t *l)
{
        GEN_UNARY_RMWcc(_ASM_DEC, l->a.counter, "%0", e);
}

/**
 * local_inc_and_test - increment and test
 * @l: pointer to type local_t
 *
 * Atomically increments @l by 1
 * and returns true if the result is zero, or false for all
 * other cases.
 */
static inline bool local_inc_and_test(local_t *l)
{
        GEN_UNARY_RMWcc(_ASM_INC, l->a.counter, "%0", e);
}

/**
 * local_add_negative - add and test if negative
 * @i: integer value to add
 * @l: pointer to type local_t
 *
 * Atomically adds @i to @l and returns true
 * if the result is negative, or false when
 * result is greater than or equal to zero.
 */
static inline bool local_add_negative(long i, local_t *l)
{
        GEN_BINARY_RMWcc(_ASM_ADD, l->a.counter, "er", i, "%0", s);
}

/**
 * local_add_return - add and return
 * @i: integer value to add
 * @l: pointer to type local_t
 *
 * Atomically adds @i to @l and returns @i + @l
 */
static inline long local_add_return(long i, local_t *l)
{
        long __i = i;
        asm volatile(_ASM_XADD "%0, %1;"
                     : "+r" (i), "+m" (l->a.counter)
                     : : "memory");
        return i + __i; /*covered*/
}

static inline long local_sub_return(long i, local_t *l)
{
        return local_add_return(-i, l);
}

#define local_inc_return(l)  (local_add_return(1, l))
#define local_dec_return(l)  (local_sub_return(1, l))

#define local_cmpxchg(l, o, n) \
        (cmpxchg_local(&((l)->a.counter), (o), (n)))
/* Always has a lock prefix */
#define local_xchg(l, n) (xchg(&((l)->a.counter), (n)))

/**
 * local_add_unless - add unless the number is a given value
 * @l: pointer of type local_t
 * @a: the amount to add to l...
 * @u: ...unless l is equal to u.
 *
 * Atomically adds @a to @l, so long as it was not @u.
 * Returns non-zero if @l was not @u, and zero otherwise.
 */
#define local_add_unless(l, a, u)                                \
({                                                                \
        long c, old;                                                \
        c = local_read((l));                                        \
        for (;;) {                                                \
                if (unlikely(c == (u)))                                \
                        break;                                        \
                old = local_cmpxchg((l), c, c + (a));                \
                if (likely(old == c))                                \
                        break;                                        \
                c = old;                                        \
        }                                                        \
        c != (u);                                                \
})
#define local_inc_not_zero(l) local_add_unless((l), 1, 0)

/* On x86_32, these are no better than the atomic variants.
 * On x86-64 these are better than the atomic variants on SMP kernels
 * because they dont use a lock prefix.
 */
#define __local_inc(l)                local_inc(l)
#define __local_dec(l)                local_dec(l)
#define __local_add(i, l)        local_add((i), (l))
#define __local_sub(i, l)        local_sub((i), (l))

#endif /* _ASM_X86_LOCAL_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_MMU_CONTEXT_H
#define _ASM_X86_MMU_CONTEXT_H

#include <asm/desc.h>
#include <linux/atomic.h>
#include <linux/mm_types.h>
#include <linux/pkeys.h>

#include <trace/events/tlb.h>

#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm/paravirt.h>
#include <asm/mpx.h>

extern atomic64_t last_mm_ctx_id;

#ifndef CONFIG_PARAVIRT
static inline void paravirt_activate_mm(struct mm_struct *prev,
                                        struct mm_struct *next)
{
}
#endif        /* !CONFIG_PARAVIRT */

#ifdef CONFIG_PERF_EVENTS

DECLARE_STATIC_KEY_FALSE(rdpmc_always_available_key);

static inline void load_mm_cr4(struct mm_struct *mm)
{
        if (static_branch_unlikely(&rdpmc_always_available_key) ||
            atomic_read(&mm->context.perf_rdpmc_allowed))
                cr4_set_bits(X86_CR4_PCE);
        else
                cr4_clear_bits(X86_CR4_PCE);
}
#else
static inline void load_mm_cr4(struct mm_struct *mm) {}
#endif

#ifdef CONFIG_MODIFY_LDT_SYSCALL
/*
 * ldt_structs can be allocated, used, and freed, but they are never
 * modified while live.
 */
struct ldt_struct {
        /*
         * Xen requires page-aligned LDTs with special permissions.  This is
         * needed to prevent us from installing evil descriptors such as
         * call gates.  On native, we could merge the ldt_struct and LDT
         * allocations, but it's not worth trying to optimize.
         */
        struct desc_struct        *entries;
        unsigned int                nr_entries;

        /*
         * If PTI is in use, then the entries array is not mapped while we're
         * in user mode.  The whole array will be aliased at the addressed
         * given by ldt_slot_va(slot).  We use two slots so that we can allocate
         * and map, and enable a new LDT without invalidating the mapping
         * of an older, still-in-use LDT.
         *
         * slot will be -1 if this LDT doesn't have an alias mapping.
         */
        int                        slot;
};

/* This is a multiple of PAGE_SIZE. */
#define LDT_SLOT_STRIDE (LDT_ENTRIES * LDT_ENTRY_SIZE)

static inline void *ldt_slot_va(int slot)
{
        return (void *)(LDT_BASE_ADDR + LDT_SLOT_STRIDE * slot);
}

/*
 * Used for LDT copy/destruction.
 */
static inline void init_new_context_ldt(struct mm_struct *mm)
{
        mm->context.ldt = NULL;
        init_rwsem(&mm->context.ldt_usr_sem);
}
int ldt_dup_context(struct mm_struct *oldmm, struct mm_struct *mm);
void destroy_context_ldt(struct mm_struct *mm);
void ldt_arch_exit_mmap(struct mm_struct *mm);
#else        /* CONFIG_MODIFY_LDT_SYSCALL */
static inline void init_new_context_ldt(struct mm_struct *mm) { }
static inline int ldt_dup_context(struct mm_struct *oldmm,
                                  struct mm_struct *mm)
{
        return 0;
}
static inline void destroy_context_ldt(struct mm_struct *mm) { }
static inline void ldt_arch_exit_mmap(struct mm_struct *mm) { }
#endif

static inline void load_mm_ldt(struct mm_struct *mm)
{
#ifdef CONFIG_MODIFY_LDT_SYSCALL
        struct ldt_struct *ldt;

        /* READ_ONCE synchronizes with smp_store_release */
        ldt = READ_ONCE(mm->context.ldt);

        /*
         * Any change to mm->context.ldt is followed by an IPI to all
         * CPUs with the mm active.  The LDT will not be freed until
         * after the IPI is handled by all such CPUs.  This means that,
         * if the ldt_struct changes before we return, the values we see
         * will be safe, and the new values will be loaded before we run
         * any user code.
         *
         * NB: don't try to convert this to use RCU without extreme care.
         * We would still need IRQs off, because we don't want to change
         * the local LDT after an IPI loaded a newer value than the one
         * that we can see.
         */

        if (unlikely(ldt)) {
                if (static_cpu_has(X86_FEATURE_PTI)) {
                        if (WARN_ON_ONCE((unsigned long)ldt->slot > 1)) {
                                /*
                                 * Whoops -- either the new LDT isn't mapped
                                 * (if slot == -1) or is mapped into a bogus
                                 * slot (if slot > 1).
                                 */
                                clear_LDT();
                                return;
                        }

                        /*
                         * If page table isolation is enabled, ldt->entries
                         * will not be mapped in the userspace pagetables.
                         * Tell the CPU to access the LDT through the alias
                         * at ldt_slot_va(ldt->slot).
                         */
                        set_ldt(ldt_slot_va(ldt->slot), ldt->nr_entries);
                } else {
                        set_ldt(ldt->entries, ldt->nr_entries);
                }
        } else {
                clear_LDT();
        }
#else
        clear_LDT();
#endif
}

static inline void switch_ldt(struct mm_struct *prev, struct mm_struct *next)
{
#ifdef CONFIG_MODIFY_LDT_SYSCALL
        /*
         * Load the LDT if either the old or new mm had an LDT.
         *
         * An mm will never go from having an LDT to not having an LDT.  Two
         * mms never share an LDT, so we don't gain anything by checking to
         * see whether the LDT changed.  There's also no guarantee that
         * prev->context.ldt actually matches LDTR, but, if LDTR is non-NULL,
         * then prev->context.ldt will also be non-NULL.
         *
         * If we really cared, we could optimize the case where prev == next
         * and we're exiting lazy mode.  Most of the time, if this happens,
         * we don't actually need to reload LDTR, but modify_ldt() is mostly
         * used by legacy code and emulators where we don't need this level of
         * performance.
         *
         * This uses | instead of || because it generates better code.
         */
        if (unlikely((unsigned long)prev->context.ldt |
                     (unsigned long)next->context.ldt))
                load_mm_ldt(next);
#endif

        DEBUG_LOCKS_WARN_ON(preemptible());
}

void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);

static inline int init_new_context(struct task_struct *tsk,
                                   struct mm_struct *mm)
{
        mutex_init(&mm->context.lock);

        mm->context.ctx_id = atomic64_inc_return(&last_mm_ctx_id);
        atomic64_set(&mm->context.tlb_gen, 0);

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
        if (cpu_feature_enabled(X86_FEATURE_OSPKE)) {
                /* pkey 0 is the default and allocated implicitly */
                mm->context.pkey_allocation_map = 0x1;
                /* -1 means unallocated or invalid */
                mm->context.execute_only_pkey = -1;
        }
#endif
        init_new_context_ldt(mm);
        return 0;
}
static inline void destroy_context(struct mm_struct *mm)
{
        destroy_context_ldt(mm);
}

extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
                      struct task_struct *tsk);

extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
                               struct task_struct *tsk);
#define switch_mm_irqs_off switch_mm_irqs_off

#define activate_mm(prev, next)                        \
do {                                                \
        paravirt_activate_mm((prev), (next));        \
        switch_mm((prev), (next), NULL);        \
} while (0);

#ifdef CONFIG_X86_32
#define deactivate_mm(tsk, mm)                        \
do {                                                \
        lazy_load_gs(0);                        \
} while (0)
#else
#define deactivate_mm(tsk, mm)                        \
do {                                                \
        load_gs_index(0);                        \
        loadsegment(fs, 0);                        \
} while (0)
#endif

static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
{
        paravirt_arch_dup_mmap(oldmm, mm);
        return ldt_dup_context(oldmm, mm);
}

static inline void arch_exit_mmap(struct mm_struct *mm)
{
        paravirt_arch_exit_mmap(mm);
        ldt_arch_exit_mmap(mm);
}

#ifdef CONFIG_X86_64
static inline bool is_64bit_mm(struct mm_struct *mm)
{
        return        !IS_ENABLED(CONFIG_IA32_EMULATION) ||
                !(mm->context.ia32_compat == TIF_IA32);
}
#else
static inline bool is_64bit_mm(struct mm_struct *mm)
{
        return false;
}
#endif

static inline void arch_bprm_mm_init(struct mm_struct *mm,
                struct vm_area_struct *vma)
{
        mpx_mm_init(mm);
}

static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
                              unsigned long start, unsigned long end)
{
        /*
         * mpx_notify_unmap() goes and reads a rarely-hot
         * cacheline in the mm_struct.  That can be expensive
         * enough to be seen in profiles.
         *
         * The mpx_notify_unmap() call and its contents have been
         * observed to affect munmap() performance on hardware
         * where MPX is not present.
         *
         * The unlikely() optimizes for the fast case: no MPX
         * in the CPU, or no MPX use in the process.  Even if
         * we get this wrong (in the unlikely event that MPX
         * is widely enabled on some system) the overhead of
         * MPX itself (reading bounds tables) is expected to
         * overwhelm the overhead of getting this unlikely()
         * consistently wrong.
         */
        if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX)))
                mpx_notify_unmap(mm, vma, start, end);
}

/*
 * We only want to enforce protection keys on the current process
 * because we effectively have no access to PKRU for other
 * processes or any way to tell *which * PKRU in a threaded
 * process we could use.
 *
 * So do not enforce things if the VMA is not from the current
 * mm, or if we are in a kernel thread.
 */
static inline bool vma_is_foreign(struct vm_area_struct *vma)
{
        if (!current->mm) /*covered*/
                return true;
        /*
         * Should PKRU be enforced on the access to this VMA?  If
         * the VMA is from another process, then PKRU has no
         * relevance and should not be enforced.
         */
        if (current->mm != vma->vm_mm) /*covered*/
                return true;

        return false;
}

static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
                bool write, bool execute, bool foreign)
{
        /* pkeys never affect instruction fetches */
        if (execute) /*covered*/
                return true;
        /* allow access if the VMA is not one from this process */
        if (foreign || vma_is_foreign(vma)) /*covered*/
                return true;
        return __pkru_allows_pkey(vma_pkey(vma), write); /*covered*/
}

/*
 * This can be used from process context to figure out what the value of
 * CR3 is without needing to do a (slow) __read_cr3().
 *
 * It's intended to be used for code like KVM that sneakily changes CR3
 * and needs to restore it.  It needs to be used very carefully.
 */
static inline unsigned long __get_current_cr3_fast(void)
{
        unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
                this_cpu_read(cpu_tlbstate.loaded_mm_asid));

        /* For now, be very restrictive about when this can be called. */
        VM_WARN_ON(in_nmi() || preemptible());

        VM_BUG_ON(cr3 != __read_cr3());
        return cr3;
}

#endif /* _ASM_X86_MMU_CONTEXT_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_MSR_H
#define _ASM_X86_MSR_H

#include "msr-index.h"

#ifndef __ASSEMBLY__

#include <asm/asm.h>
#include <asm/errno.h>
#include <asm/cpumask.h>
#include <uapi/asm/msr.h>

struct msr {
        union {
                struct {
                        u32 l;
                        u32 h;
                };
                u64 q;
        };
};

struct msr_info {
        u32 msr_no;
        struct msr reg;
        struct msr *msrs;
        int err;
};

struct msr_regs_info {
        u32 *regs;
        int err;
};

struct saved_msr {
        bool valid;
        struct msr_info info;
};

struct saved_msrs {
        unsigned int num;
        struct saved_msr *array;
};

/*
 * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A"
 * constraint has different meanings. For i386, "A" means exactly
 * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead,
 * it means rax *or* rdx.
 */
#ifdef CONFIG_X86_64
/* Using 64-bit values saves one instruction clearing the high half of low */
#define DECLARE_ARGS(val, low, high)        unsigned long low, high
#define EAX_EDX_VAL(val, low, high)        ((low) | (high) << 32)
#define EAX_EDX_RET(val, low, high)        "=a" (low), "=d" (high)
#else
#define DECLARE_ARGS(val, low, high)        unsigned long long val
#define EAX_EDX_VAL(val, low, high)        (val)
#define EAX_EDX_RET(val, low, high)        "=A" (val)
#endif

#ifdef CONFIG_TRACEPOINTS
/*
 * Be very careful with includes. This header is prone to include loops.
 */
#include <asm/atomic.h>
#include <linux/tracepoint-defs.h>

extern struct tracepoint __tracepoint_read_msr;
extern struct tracepoint __tracepoint_write_msr;
extern struct tracepoint __tracepoint_rdpmc;
#define msr_tracepoint_active(t) static_key_false(&(t).key)
extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
#else
#define msr_tracepoint_active(t) false
static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
#endif

/*
 * __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR
 * accessors and should not have any tracing or other functionality piggybacking
 * on them - those are *purely* for accessing MSRs and nothing more. So don't even
 * think of extending them - you will be slapped with a stinking trout or a frozen
 * shark will reach you, wherever you are! You've been warned.
 */
static inline unsigned long long notrace __rdmsr(unsigned int msr)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("1: rdmsr\n"
                     "2:\n"
                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_rdmsr_unsafe)
                     : EAX_EDX_RET(val, low, high) : "c" (msr));

        return EAX_EDX_VAL(val, low, high);
}

static inline void notrace __wrmsr(unsigned int msr, u32 low, u32 high)
{
        asm volatile("1: wrmsr\n"
                     "2:\n"
                     _ASM_EXTABLE_HANDLE(1b, 2b, ex_handler_wrmsr_unsafe)
                     : : "c" (msr), "a"(low), "d" (high) : "memory");
}

#define native_rdmsr(msr, val1, val2)                        \
do {                                                        \
        u64 __val = __rdmsr((msr));                        \
        (void)((val1) = (u32)__val);                        \
        (void)((val2) = (u32)(__val >> 32));                \
} while (0)

#define native_wrmsr(msr, low, high)                        \
        __wrmsr(msr, low, high)

#define native_wrmsrl(msr, val)                                \
        __wrmsr((msr), (u32)((u64)(val)),                \
                       (u32)((u64)(val) >> 32))

static inline unsigned long long native_read_msr(unsigned int msr)
{
        unsigned long long val;

        val = __rdmsr(msr);

        if (msr_tracepoint_active(__tracepoint_read_msr))
                do_trace_read_msr(msr, val, 0);

        return val;
}

static inline unsigned long long native_read_msr_safe(unsigned int msr,
                                                      int *err)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("2: rdmsr ; xor %[err],%[err]\n"
                     "1:\n\t"
                     ".section .fixup,\"ax\"\n\t"
                     "3: mov %[fault],%[err]\n\t"
                     "xorl %%eax, %%eax\n\t"
                     "xorl %%edx, %%edx\n\t"
                     "jmp 1b\n\t"
                     ".previous\n\t"
                     _ASM_EXTABLE(2b, 3b)
                     : [err] "=r" (*err), EAX_EDX_RET(val, low, high)
                     : "c" (msr), [fault] "i" (-EIO));
        if (msr_tracepoint_active(__tracepoint_read_msr))
                do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err);
        return EAX_EDX_VAL(val, low, high);
}

/* Can be uninlined because referenced by paravirt */
static inline void notrace
native_write_msr(unsigned int msr, u32 low, u32 high)
{
        __wrmsr(msr, low, high);

        if (msr_tracepoint_active(__tracepoint_write_msr))
                do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
}

/* Can be uninlined because referenced by paravirt */
static inline int notrace
native_write_msr_safe(unsigned int msr, u32 low, u32 high)
{
        int err;

        asm volatile("2: wrmsr ; xor %[err],%[err]\n"
                     "1:\n\t"
                     ".section .fixup,\"ax\"\n\t"
                     "3:  mov %[fault],%[err] ; jmp 1b\n\t"
                     ".previous\n\t"
                     _ASM_EXTABLE(2b, 3b)
                     : [err] "=a" (err)
                     : "c" (msr), "0" (low), "d" (high),
                       [fault] "i" (-EIO)
                     : "memory");
        if (msr_tracepoint_active(__tracepoint_write_msr))
                do_trace_write_msr(msr, ((u64)high << 32 | low), err);
        return err;
}

extern int rdmsr_safe_regs(u32 regs[8]);
extern int wrmsr_safe_regs(u32 regs[8]);

/**
 * rdtsc() - returns the current TSC without ordering constraints
 *
 * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
 * only ordering constraint it supplies is the ordering implied by
 * "asm volatile": it will put the RDTSC in the place you expect.  The
 * CPU can and will speculatively execute that RDTSC, though, so the
 * results can be non-monotonic if compared on different CPUs.
 */
static __always_inline unsigned long long rdtsc(void)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("rdtsc" : EAX_EDX_RET(val, low, high)); /*covered*/

        return EAX_EDX_VAL(val, low, high);
}

/**
 * rdtsc_ordered() - read the current TSC in program order
 *
 * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
 * It is ordered like a load to a global in-memory counter.  It should
 * be impossible to observe non-monotonic rdtsc_unordered() behavior
 * across multiple CPUs as long as the TSC is synced.
 */
static __always_inline unsigned long long rdtsc_ordered(void)
{
        /*
         * The RDTSC instruction is not ordered relative to memory
         * access.  The Intel SDM and the AMD APM are both vague on this
         * point, but empirically an RDTSC instruction can be
         * speculatively executed before prior loads.  An RDTSC
         * immediately after an appropriate barrier appears to be
         * ordered as a normal load, that is, it provides the same
         * ordering guarantees as reading from a global memory location
         * that some other imaginary CPU is updating continuously with a
         * time stamp.
         */
        barrier_nospec();
        return rdtsc();
}

static inline unsigned long long native_read_pmc(int counter)
{
        DECLARE_ARGS(val, low, high);

        asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
        if (msr_tracepoint_active(__tracepoint_rdpmc))
                do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0);
        return EAX_EDX_VAL(val, low, high);
}

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#include <linux/errno.h>
/*
 * Access to machine-specific registers (available on 586 and better only)
 * Note: the rd* operations modify the parameters directly (without using
 * pointer indirection), this allows gcc to optimize better
 */

#define rdmsr(msr, low, high)                                        \
do {                                                                \
        u64 __val = native_read_msr((msr));                        \
        (void)((low) = (u32)__val);                                \
        (void)((high) = (u32)(__val >> 32));                        \
} while (0)

static inline void wrmsr(unsigned int msr, u32 low, u32 high)
{
        native_write_msr(msr, low, high);
}

#define rdmsrl(msr, val)                        \
        ((val) = native_read_msr((msr)))

static inline void wrmsrl(unsigned int msr, u64 val)
{
        native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
}

/* wrmsr with exception handling */
static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
{
        return native_write_msr_safe(msr, low, high);
}

/* rdmsr with exception handling */
#define rdmsr_safe(msr, low, high)                                \
({                                                                \
        int __err;                                                \
        u64 __val = native_read_msr_safe((msr), &__err);        \
        (*low) = (u32)__val;                                        \
        (*high) = (u32)(__val >> 32);                                \
        __err;                                                        \
})

static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p)
{
        int err;

        *p = native_read_msr_safe(msr, &err);
        return err;
}

#define rdpmc(counter, low, high)                        \
do {                                                        \
        u64 _l = native_read_pmc((counter));                \
        (low)  = (u32)_l;                                \
        (high) = (u32)(_l >> 32);                        \
} while (0)

#define rdpmcl(counter, val) ((val) = native_read_pmc(counter))

#endif        /* !CONFIG_PARAVIRT */

/*
 * 64-bit version of wrmsr_safe():
 */
static inline int wrmsrl_safe(u32 msr, u64 val)
{
        return wrmsr_safe(msr, (u32)val,  (u32)(val >> 32));
}

#define write_tsc(low, high) wrmsr(MSR_IA32_TSC, (low), (high))

#define write_rdtscp_aux(val) wrmsr(MSR_TSC_AUX, (val), 0)

struct msr *msrs_alloc(void);
void msrs_free(struct msr *msrs);
int msr_set_bit(u32 msr, u8 bit);
int msr_clear_bit(u32 msr, u8 bit);

#ifdef CONFIG_SMP
int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
#else  /*  CONFIG_SMP  */
static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
{
        rdmsr(msr_no, *l, *h);
        return 0;
}
static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
{
        wrmsr(msr_no, l, h);
        return 0;
}
static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
{
        rdmsrl(msr_no, *q);
        return 0;
}
static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
{
        wrmsrl(msr_no, q);
        return 0;
}
static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no,
                                struct msr *msrs)
{
        rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h));
}
static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no,
                                struct msr *msrs)
{
        wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h);
}
static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no,
                                    u32 *l, u32 *h)
{
        return rdmsr_safe(msr_no, l, h);
}
static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
{
        return wrmsr_safe(msr_no, l, h);
}
static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
{
        return rdmsrl_safe(msr_no, q);
}
static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
{
        return wrmsrl_safe(msr_no, q);
}
static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
{
        return rdmsr_safe_regs(regs);
}
static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
{
        return wrmsr_safe_regs(regs);
}
#endif  /* CONFIG_SMP */
#endif /* __ASSEMBLY__ */
#endif /* _ASM_X86_MSR_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PARAVIRT_H
#define _ASM_X86_PARAVIRT_H
/* Various instructions on x86 need to be replaced for
 * para-virtualization: those hooks are defined here. */

#ifdef CONFIG_PARAVIRT
#include <asm/pgtable_types.h>
#include <asm/asm.h>
#include <asm/nospec-branch.h>

#include <asm/paravirt_types.h>

#ifndef __ASSEMBLY__
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/cpumask.h>
#include <asm/frame.h>

static inline void load_sp0(unsigned long sp0)
{
        PVOP_VCALL1(pv_cpu_ops.load_sp0, sp0);
}

/* The paravirtualized CPUID instruction. */
static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
                           unsigned int *ecx, unsigned int *edx)
{
        PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
}

/*
 * These special macros can be used to get or set a debugging register
 */
static inline unsigned long paravirt_get_debugreg(int reg)
{
        return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
}
#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
static inline void set_debugreg(unsigned long val, int reg)
{
        PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
}

static inline unsigned long read_cr0(void)
{
        return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
}

static inline void write_cr0(unsigned long x)
{
        PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
}

static inline unsigned long read_cr2(void)
{
        return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2); /*covered*/
}

static inline void write_cr2(unsigned long x)
{
        PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
}

static inline unsigned long __read_cr3(void)
{
        return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
}

static inline void write_cr3(unsigned long x)
{
        PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
}

static inline void __write_cr4(unsigned long x)
{
        PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
}

#ifdef CONFIG_X86_64
static inline unsigned long read_cr8(void)
{
        return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
}

static inline void write_cr8(unsigned long x)
{
        PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
}
#endif

static inline void arch_safe_halt(void)
{
        PVOP_VCALL0(pv_irq_ops.safe_halt);
}

static inline void halt(void)
{
        PVOP_VCALL0(pv_irq_ops.halt);
}

static inline void wbinvd(void)
{
        PVOP_VCALL0(pv_cpu_ops.wbinvd);
}

#define get_kernel_rpl()  (pv_info.kernel_rpl)

static inline u64 paravirt_read_msr(unsigned msr)
{
        return PVOP_CALL1(u64, pv_cpu_ops.read_msr, msr);
}

static inline void paravirt_write_msr(unsigned msr,
                                      unsigned low, unsigned high)
{
        PVOP_VCALL3(pv_cpu_ops.write_msr, msr, low, high);
}

static inline u64 paravirt_read_msr_safe(unsigned msr, int *err)
{
        return PVOP_CALL2(u64, pv_cpu_ops.read_msr_safe, msr, err);
}

static inline int paravirt_write_msr_safe(unsigned msr,
                                          unsigned low, unsigned high)
{
        return PVOP_CALL3(int, pv_cpu_ops.write_msr_safe, msr, low, high);
}

#define rdmsr(msr, val1, val2)                        \
do {                                                \
        u64 _l = paravirt_read_msr(msr);        \
        val1 = (u32)_l;                                \
        val2 = _l >> 32;                        \
} while (0)

#define wrmsr(msr, val1, val2)                        \
do {                                                \
        paravirt_write_msr(msr, val1, val2);        \
} while (0)

#define rdmsrl(msr, val)                        \
do {                                                \
        val = paravirt_read_msr(msr);                \
} while (0)

static inline void wrmsrl(unsigned msr, u64 val)
{
        wrmsr(msr, (u32)val, (u32)(val>>32));
}

#define wrmsr_safe(msr, a, b)        paravirt_write_msr_safe(msr, a, b)

/* rdmsr with exception handling */
#define rdmsr_safe(msr, a, b)                                \
({                                                        \
        int _err;                                        \
        u64 _l = paravirt_read_msr_safe(msr, &_err);        \
        (*a) = (u32)_l;                                        \
        (*b) = _l >> 32;                                \
        _err;                                                \
})

static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
{
        int err;

        *p = paravirt_read_msr_safe(msr, &err);
        return err;
}

static inline unsigned long long paravirt_sched_clock(void)
{
        return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
}

struct static_key;
extern struct static_key paravirt_steal_enabled;
extern struct static_key paravirt_steal_rq_enabled;

static inline u64 paravirt_steal_clock(int cpu)
{
        return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
}

static inline unsigned long long paravirt_read_pmc(int counter)
{
        return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
}

#define rdpmc(counter, low, high)                \
do {                                                \
        u64 _l = paravirt_read_pmc(counter);        \
        low = (u32)_l;                                \
        high = _l >> 32;                        \
} while (0)

#define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter))

static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
{
        PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
}

static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
{
        PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
}

static inline void load_TR_desc(void)
{
        PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
}
static inline void load_gdt(const struct desc_ptr *dtr)
{
        PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
}
static inline void load_idt(const struct desc_ptr *dtr)
{
        PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
}
static inline void set_ldt(const void *addr, unsigned entries)
{
        PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
}
static inline unsigned long paravirt_store_tr(void)
{
        return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
}
#define store_tr(tr)        ((tr) = paravirt_store_tr())
static inline void load_TLS(struct thread_struct *t, unsigned cpu)
{
        PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
}

#ifdef CONFIG_X86_64
static inline void load_gs_index(unsigned int gs)
{
        PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
}
#endif

static inline void write_ldt_entry(struct desc_struct *dt, int entry,
                                   const void *desc)
{
        PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
}

static inline void write_gdt_entry(struct desc_struct *dt, int entry,
                                   void *desc, int type)
{
        PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
}

static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
{
        PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
}
static inline void set_iopl_mask(unsigned mask)
{
        PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
}

/* The paravirtualized I/O functions */
static inline void slow_down_io(void)
{
        pv_cpu_ops.io_delay();
#ifdef REALLY_SLOW_IO
        pv_cpu_ops.io_delay();
        pv_cpu_ops.io_delay();
        pv_cpu_ops.io_delay();
#endif
}

static inline void paravirt_activate_mm(struct mm_struct *prev,
                                        struct mm_struct *next)
{
        PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
}

static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
                                          struct mm_struct *mm)
{
        PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
}

static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
{
        PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
}

static inline void __flush_tlb(void)
{
        PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
}
static inline void __flush_tlb_global(void)
{
        PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel); /*covered*/
}
static inline void __flush_tlb_one_user(unsigned long addr)
{
        PVOP_VCALL1(pv_mmu_ops.flush_tlb_one_user, addr); /*covered*/
}

static inline void flush_tlb_others(const struct cpumask *cpumask,
                                    const struct flush_tlb_info *info)
{
        PVOP_VCALL2(pv_mmu_ops.flush_tlb_others, cpumask, info);
}

static inline void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table)
{
        PVOP_VCALL2(pv_mmu_ops.tlb_remove_table, tlb, table); /*covered*/
}

static inline int paravirt_pgd_alloc(struct mm_struct *mm)
{
        return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
}

static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
        PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd); /*covered*/
}

static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn); /*covered*/
}
static inline void paravirt_release_pte(unsigned long pfn)
{
        PVOP_VCALL1(pv_mmu_ops.release_pte, pfn); /*covered*/
}

static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
}

static inline void paravirt_release_pmd(unsigned long pfn)
{
        PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
}

static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
}
static inline void paravirt_release_pud(unsigned long pfn)
{
        PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
}

static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn)
{
        PVOP_VCALL2(pv_mmu_ops.alloc_p4d, mm, pfn);
}

static inline void paravirt_release_p4d(unsigned long pfn)
{
        PVOP_VCALL1(pv_mmu_ops.release_p4d, pfn);
}

static inline pte_t __pte(pteval_t val)
{
        pteval_t ret;

        if (sizeof(pteval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pteval_t,
                                   pv_mmu_ops.make_pte,
                                   val, (u64)val >> 32);
        else
                ret = PVOP_CALLEE1(pteval_t, /*covered*/
                                   pv_mmu_ops.make_pte,
                                   val);

        return (pte_t) { .pte = ret };
}

static inline pteval_t pte_val(pte_t pte)
{
        pteval_t ret;

        if (sizeof(pteval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
                                   pte.pte, (u64)pte.pte >> 32);
        else
                ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val, /*covered*/
                                   pte.pte);

        return ret;
}

static inline pgd_t __pgd(pgdval_t val)
{
        pgdval_t ret;

        if (sizeof(pgdval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
                                   val, (u64)val >> 32);
        else
                ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
                                   val);

        return (pgd_t) { ret };
}

static inline pgdval_t pgd_val(pgd_t pgd)
{
        pgdval_t ret;

        if (sizeof(pgdval_t) > sizeof(long))
                ret =  PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
                                    pgd.pgd, (u64)pgd.pgd >> 32);
        else
                ret =  PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val, /*covered*/
                                    pgd.pgd);

        return ret;
}

#define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
                                           pte_t *ptep)
{
        pteval_t ret;

        ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
                         mm, addr, ptep);

        return (pte_t) { .pte = ret };
}

static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
                                           pte_t *ptep, pte_t pte)
{
        if (sizeof(pteval_t) > sizeof(long))
                /* 5 arg words */
                pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
        else
                PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
                            mm, addr, ptep, pte.pte);
}

static inline void set_pte(pte_t *ptep, pte_t pte)
{
        if (sizeof(pteval_t) > sizeof(long))
                PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
                            pte.pte, (u64)pte.pte >> 32);
        else
                PVOP_VCALL2(pv_mmu_ops.set_pte, ptep, /*covered*/
                            pte.pte);
}

static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
                              pte_t *ptep, pte_t pte)
{
        if (sizeof(pteval_t) > sizeof(long))
                /* 5 arg words */
                pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
        else
                PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte); /*covered*/
}

static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
{
        pmdval_t val = native_pmd_val(pmd);

        if (sizeof(pmdval_t) > sizeof(long))
                PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
        else
                PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val); /*covered*/
}

#if CONFIG_PGTABLE_LEVELS >= 3
static inline pmd_t __pmd(pmdval_t val)
{
        pmdval_t ret;

        if (sizeof(pmdval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
                                   val, (u64)val >> 32);
        else
                ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd, /*covered*/
                                   val);

        return (pmd_t) { ret };
}

static inline pmdval_t pmd_val(pmd_t pmd)
{
        pmdval_t ret;

        if (sizeof(pmdval_t) > sizeof(long))
                ret =  PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
                                    pmd.pmd, (u64)pmd.pmd >> 32);
        else
                ret =  PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val, /*covered*/
                                    pmd.pmd);

        return ret;
}

static inline void set_pud(pud_t *pudp, pud_t pud)
{
        pudval_t val = native_pud_val(pud);

        if (sizeof(pudval_t) > sizeof(long))
                PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
                            val, (u64)val >> 32);
        else
                PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
                            val);
}
#if CONFIG_PGTABLE_LEVELS >= 4
static inline pud_t __pud(pudval_t val)
{
        pudval_t ret;

        if (sizeof(pudval_t) > sizeof(long))
                ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
                                   val, (u64)val >> 32);
        else
                ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
                                   val);

        return (pud_t) { ret };
}

static inline pudval_t pud_val(pud_t pud)
{
        pudval_t ret;

        if (sizeof(pudval_t) > sizeof(long))
                ret =  PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
                                    pud.pud, (u64)pud.pud >> 32);
        else
                ret =  PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val, /*covered*/
                                    pud.pud);

        return ret;
}

static inline void pud_clear(pud_t *pudp)
{
        set_pud(pudp, __pud(0));
}

static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
{
        p4dval_t val = native_p4d_val(p4d);

        if (sizeof(p4dval_t) > sizeof(long))
                PVOP_VCALL3(pv_mmu_ops.set_p4d, p4dp,
                            val, (u64)val >> 32);
        else
                PVOP_VCALL2(pv_mmu_ops.set_p4d, p4dp,
                            val);
}

#if CONFIG_PGTABLE_LEVELS >= 5

static inline p4d_t __p4d(p4dval_t val)
{
        p4dval_t ret = PVOP_CALLEE1(p4dval_t, pv_mmu_ops.make_p4d, val);

        return (p4d_t) { ret };
}

static inline p4dval_t p4d_val(p4d_t p4d)
{
        return PVOP_CALLEE1(p4dval_t, pv_mmu_ops.p4d_val, p4d.p4d);
}

static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd)
{
        PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp, native_pgd_val(pgd));
}

#define set_pgd(pgdp, pgdval) do {                                        \
        if (pgtable_l5_enabled())                                                \
                __set_pgd(pgdp, pgdval);                                \
        else                                                                \
                set_p4d((p4d_t *)(pgdp), (p4d_t) { (pgdval).pgd });        \
} while (0)

#define pgd_clear(pgdp) do {                                                \
        if (pgtable_l5_enabled())                                                \
                set_pgd(pgdp, __pgd(0));                                \
} while (0)

#endif  /* CONFIG_PGTABLE_LEVELS == 5 */

static inline void p4d_clear(p4d_t *p4dp)
{
        set_p4d(p4dp, __p4d(0));
}

#endif        /* CONFIG_PGTABLE_LEVELS == 4 */

#endif        /* CONFIG_PGTABLE_LEVELS >= 3 */

#ifdef CONFIG_X86_PAE
/* Special-case pte-setting operations for PAE, which can't update a
   64-bit pte atomically */
static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
{
        PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
                    pte.pte, pte.pte >> 32);
}

static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
                             pte_t *ptep)
{
        PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
}

static inline void pmd_clear(pmd_t *pmdp)
{
        PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
}
#else  /* !CONFIG_X86_PAE */
static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
{
        set_pte(ptep, pte); /*covered*/
}

static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
                             pte_t *ptep)
{
        set_pte_at(mm, addr, ptep, __pte(0));
}

static inline void pmd_clear(pmd_t *pmdp)
{
        set_pmd(pmdp, __pmd(0)); /*covered*/
}
#endif        /* CONFIG_X86_PAE */

#define  __HAVE_ARCH_START_CONTEXT_SWITCH
static inline void arch_start_context_switch(struct task_struct *prev)
{
        PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
}

static inline void arch_end_context_switch(struct task_struct *next)
{
        PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
}

#define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
static inline void arch_enter_lazy_mmu_mode(void)
{
        PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter); /*covered*/
}

static inline void arch_leave_lazy_mmu_mode(void)
{
        PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave); /*covered*/
}

static inline void arch_flush_lazy_mmu_mode(void)
{
        PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush);
}

static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
                                phys_addr_t phys, pgprot_t flags)
{
        pv_mmu_ops.set_fixmap(idx, phys, flags);
}

#if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)

static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
                                                        u32 val)
{
        PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val);
}

static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
{
        PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock); /*covered*/
}

static __always_inline void pv_wait(u8 *ptr, u8 val)
{
        PVOP_VCALL2(pv_lock_ops.wait, ptr, val);
}

static __always_inline void pv_kick(int cpu)
{
        PVOP_VCALL1(pv_lock_ops.kick, cpu);
}

static __always_inline bool pv_vcpu_is_preempted(long cpu)
{
        return PVOP_CALLEE1(bool, pv_lock_ops.vcpu_is_preempted, cpu);
}

#endif /* SMP && PARAVIRT_SPINLOCKS */

#ifdef CONFIG_X86_32
#define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
#define PV_RESTORE_REGS "popl %edx; popl %ecx;"

/* save and restore all caller-save registers, except return value */
#define PV_SAVE_ALL_CALLER_REGS                "pushl %ecx;"
#define PV_RESTORE_ALL_CALLER_REGS        "popl  %ecx;"

#define PV_FLAGS_ARG "0"
#define PV_EXTRA_CLOBBERS
#define PV_VEXTRA_CLOBBERS
#else
/* save and restore all caller-save registers, except return value */
#define PV_SAVE_ALL_CALLER_REGS                                                \
        "push %rcx;"                                                        \
        "push %rdx;"                                                        \
        "push %rsi;"                                                        \
        "push %rdi;"                                                        \
        "push %r8;"                                                        \
        "push %r9;"                                                        \
        "push %r10;"                                                        \
        "push %r11;"
#define PV_RESTORE_ALL_CALLER_REGS                                        \
        "pop %r11;"                                                        \
        "pop %r10;"                                                        \
        "pop %r9;"                                                        \
        "pop %r8;"                                                        \
        "pop %rdi;"                                                        \
        "pop %rsi;"                                                        \
        "pop %rdx;"                                                        \
        "pop %rcx;"

/* We save some registers, but all of them, that's too much. We clobber all
 * caller saved registers but the argument parameter */
#define PV_SAVE_REGS "pushq %%rdi;"
#define PV_RESTORE_REGS "popq %%rdi;"
#define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
#define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
#define PV_FLAGS_ARG "D"
#endif

/*
 * Generate a thunk around a function which saves all caller-save
 * registers except for the return value.  This allows C functions to
 * be called from assembler code where fewer than normal registers are
 * available.  It may also help code generation around calls from C
 * code if the common case doesn't use many registers.
 *
 * When a callee is wrapped in a thunk, the caller can assume that all
 * arg regs and all scratch registers are preserved across the
 * call. The return value in rax/eax will not be saved, even for void
 * functions.
 */
#define PV_THUNK_NAME(func) "__raw_callee_save_" #func
#define PV_CALLEE_SAVE_REGS_THUNK(func)                                        \
        extern typeof(func) __raw_callee_save_##func;                        \
                                                                        \
        asm(".pushsection .text;"                                        \
            ".globl " PV_THUNK_NAME(func) ";"                                \
            ".type " PV_THUNK_NAME(func) ", @function;"                        \
            PV_THUNK_NAME(func) ":"                                        \
            FRAME_BEGIN                                                        \
            PV_SAVE_ALL_CALLER_REGS                                        \
            "call " #func ";"                                                \
            PV_RESTORE_ALL_CALLER_REGS                                        \
            FRAME_END                                                        \
            "ret;"                                                        \
            ".popsection")

/* Get a reference to a callee-save function */
#define PV_CALLEE_SAVE(func)                                                \
        ((struct paravirt_callee_save) { __raw_callee_save_##func })

/* Promise that "func" already uses the right calling convention */
#define __PV_IS_CALLEE_SAVE(func)                        \
        ((struct paravirt_callee_save) { func })

static inline notrace unsigned long arch_local_save_flags(void)
{
        return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl); /*covered*/
}

static inline notrace void arch_local_irq_restore(unsigned long f)
{
        PVOP_VCALLEE1(pv_irq_ops.restore_fl, f); /*covered*/
}

static inline notrace void arch_local_irq_disable(void)
{
        PVOP_VCALLEE0(pv_irq_ops.irq_disable); /*covered*/
}

static inline notrace void arch_local_irq_enable(void)
{
        PVOP_VCALLEE0(pv_irq_ops.irq_enable); /*covered*/
}

static inline notrace unsigned long arch_local_irq_save(void)
{
        unsigned long f;

        f = arch_local_save_flags(); /*covered*/
        arch_local_irq_disable(); /*covered*/
        return f;
}


/* Make sure as little as possible of this mess escapes. */
#undef PARAVIRT_CALL
#undef __PVOP_CALL
#undef __PVOP_VCALL
#undef PVOP_VCALL0
#undef PVOP_CALL0
#undef PVOP_VCALL1
#undef PVOP_CALL1
#undef PVOP_VCALL2
#undef PVOP_CALL2
#undef PVOP_VCALL3
#undef PVOP_CALL3
#undef PVOP_VCALL4
#undef PVOP_CALL4

extern void default_banner(void);

#else  /* __ASSEMBLY__ */

#define _PVSITE(ptype, clobbers, ops, word, algn)        \
771:;                                                \
        ops;                                        \
772:;                                                \
        .pushsection .parainstructions,"a";        \
         .align        algn;                                \
         word 771b;                                \
         .byte ptype;                                \
         .byte 772b-771b;                        \
         .short clobbers;                        \
        .popsection


#define COND_PUSH(set, mask, reg)                        \
        .if ((~(set)) & mask); push %reg; .endif
#define COND_POP(set, mask, reg)                        \
        .if ((~(set)) & mask); pop %reg; .endif

#ifdef CONFIG_X86_64

#define PV_SAVE_REGS(set)                        \
        COND_PUSH(set, CLBR_RAX, rax);                \
        COND_PUSH(set, CLBR_RCX, rcx);                \
        COND_PUSH(set, CLBR_RDX, rdx);                \
        COND_PUSH(set, CLBR_RSI, rsi);                \
        COND_PUSH(set, CLBR_RDI, rdi);                \
        COND_PUSH(set, CLBR_R8, r8);                \
        COND_PUSH(set, CLBR_R9, r9);                \
        COND_PUSH(set, CLBR_R10, r10);                \
        COND_PUSH(set, CLBR_R11, r11)
#define PV_RESTORE_REGS(set)                        \
        COND_POP(set, CLBR_R11, r11);                \
        COND_POP(set, CLBR_R10, r10);                \
        COND_POP(set, CLBR_R9, r9);                \
        COND_POP(set, CLBR_R8, r8);                \
        COND_POP(set, CLBR_RDI, rdi);                \
        COND_POP(set, CLBR_RSI, rsi);                \
        COND_POP(set, CLBR_RDX, rdx);                \
        COND_POP(set, CLBR_RCX, rcx);                \
        COND_POP(set, CLBR_RAX, rax)

#define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
#define PARA_INDIRECT(addr)        *addr(%rip)
#else
#define PV_SAVE_REGS(set)                        \
        COND_PUSH(set, CLBR_EAX, eax);                \
        COND_PUSH(set, CLBR_EDI, edi);                \
        COND_PUSH(set, CLBR_ECX, ecx);                \
        COND_PUSH(set, CLBR_EDX, edx)
#define PV_RESTORE_REGS(set)                        \
        COND_POP(set, CLBR_EDX, edx);                \
        COND_POP(set, CLBR_ECX, ecx);                \
        COND_POP(set, CLBR_EDI, edi);                \
        COND_POP(set, CLBR_EAX, eax)

#define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
#define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
#define PARA_INDIRECT(addr)        *%cs:addr
#endif

#define INTERRUPT_RETURN                                                \
        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,        \
                  ANNOTATE_RETPOLINE_SAFE;                                        \
                  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret);)

#define DISABLE_INTERRUPTS(clobbers)                                        \
        PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
                  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);                \
                  ANNOTATE_RETPOLINE_SAFE;                                        \
                  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);        \
                  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)

#define ENABLE_INTERRUPTS(clobbers)                                        \
        PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,        \
                  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);                \
                  ANNOTATE_RETPOLINE_SAFE;                                        \
                  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);        \
                  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)

#ifdef CONFIG_X86_32
#define GET_CR0_INTO_EAX                                \
        push %ecx; push %edx;                                \
        ANNOTATE_RETPOLINE_SAFE;                                \
        call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0);        \
        pop %edx; pop %ecx
#else        /* !CONFIG_X86_32 */

/*
 * If swapgs is used while the userspace stack is still current,
 * there's no way to call a pvop.  The PV replacement *must* be
 * inlined, or the swapgs instruction must be trapped and emulated.
 */
#define SWAPGS_UNSAFE_STACK                                                \
        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,        \
                  swapgs)

/*
 * Note: swapgs is very special, and in practise is either going to be
 * implemented with a single "swapgs" instruction or something very
 * special.  Either way, we don't need to save any registers for
 * it.
 */
#define SWAPGS                                                                \
        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,        \
                  ANNOTATE_RETPOLINE_SAFE;                                        \
                  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs);                \
                 )

#define GET_CR2_INTO_RAX                                \
        ANNOTATE_RETPOLINE_SAFE;                                \
        call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2);

#define USERGS_SYSRET64                                                        \
        PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),        \
                  CLBR_NONE,                                                \
                  ANNOTATE_RETPOLINE_SAFE;                                        \
                  jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64);)

#ifdef CONFIG_DEBUG_ENTRY
#define SAVE_FLAGS(clobbers)                                        \
        PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_save_fl), clobbers, \
                  PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);        \
                  ANNOTATE_RETPOLINE_SAFE;                                    \
                  call PARA_INDIRECT(pv_irq_ops+PV_IRQ_save_fl);    \
                  PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
#endif

#endif        /* CONFIG_X86_32 */

#endif /* __ASSEMBLY__ */
#else  /* CONFIG_PARAVIRT */
# define default_banner x86_init_noop
#ifndef __ASSEMBLY__
static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
                                          struct mm_struct *mm)
{
}

static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
{
}
#endif /* __ASSEMBLY__ */
#endif /* !CONFIG_PARAVIRT */
#endif /* _ASM_X86_PARAVIRT_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGALLOC_H
#define _ASM_X86_PGALLOC_H

#include <linux/threads.h>
#include <linux/mm.h>                /* for struct page */
#include <linux/pagemap.h>

static inline int  __paravirt_pgd_alloc(struct mm_struct *mm) { return 0; }

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else
#define paravirt_pgd_alloc(mm)        __paravirt_pgd_alloc(mm)
static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) {}
static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_alloc_pmd_clone(unsigned long pfn, unsigned long clonepfn,
                                            unsigned long start, unsigned long count) {}
static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_alloc_p4d(struct mm_struct *mm, unsigned long pfn)        {}
static inline void paravirt_release_pte(unsigned long pfn) {}
static inline void paravirt_release_pmd(unsigned long pfn) {}
static inline void paravirt_release_pud(unsigned long pfn) {}
static inline void paravirt_release_p4d(unsigned long pfn) {}
#endif

/*
 * Flags to use when allocating a user page table page.
 */
extern gfp_t __userpte_alloc_gfp;

#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
 * Instead of one PGD, we acquire two PGDs.  Being order-1, it is
 * both 8k in size and 8k-aligned.  That lets us just flip bit 12
 * in a pointer to swap between the two 4k halves.
 */
#define PGD_ALLOCATION_ORDER 1
#else
#define PGD_ALLOCATION_ORDER 0
#endif

/*
 * Allocate and free page tables.
 */
extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);

extern pte_t *pte_alloc_one_kernel(struct mm_struct *, unsigned long);
extern pgtable_t pte_alloc_one(struct mm_struct *, unsigned long);

/* Should really implement gc for free page table pages. This could be
   done with a reference count in struct page. */

static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
        BUG_ON((unsigned long)pte & (PAGE_SIZE-1));
        free_page((unsigned long)pte);
}

static inline void pte_free(struct mm_struct *mm, struct page *pte)
{
        pgtable_page_dtor(pte); /*covered*/
        __free_page(pte);
}

extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);

static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *pte,
                                  unsigned long address)
{
        ___pte_free_tlb(tlb, pte);
}

static inline void pmd_populate_kernel(struct mm_struct *mm,
                                       pmd_t *pmd, pte_t *pte)
{
        paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT); /*covered*/
        set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE)); /*covered*/
}

static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
                                struct page *pte)
{
        unsigned long pfn = page_to_pfn(pte); /*covered*/

        paravirt_alloc_pte(mm, pfn); /*covered*/
        set_pmd(pmd, __pmd(((pteval_t)pfn << PAGE_SHIFT) | _PAGE_TABLE)); /*covered*/
}

#define pmd_pgtable(pmd) pmd_page(pmd)

#if CONFIG_PGTABLE_LEVELS > 2
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
        struct page *page;
        gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO;

        if (mm == &init_mm)
                gfp &= ~__GFP_ACCOUNT;
        page = alloc_pages(gfp, 0);
        if (!page)
                return NULL;
        if (!pgtable_pmd_page_ctor(page)) {
                __free_pages(page, 0);
                return NULL;
        }
        return (pmd_t *)page_address(page);
}

static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
        BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
        pgtable_pmd_page_dtor(virt_to_page(pmd));
        free_page((unsigned long)pmd);
}

extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);

static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
                                  unsigned long address)
{
        ___pmd_free_tlb(tlb, pmd);
}

#ifdef CONFIG_X86_PAE
extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
#else        /* !CONFIG_X86_PAE */
static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
{
        paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
        set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
}
#endif        /* CONFIG_X86_PAE */

#if CONFIG_PGTABLE_LEVELS > 3
static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
{
        paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
        set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
}

static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{
        gfp_t gfp = GFP_KERNEL_ACCOUNT;

        if (mm == &init_mm)
                gfp &= ~__GFP_ACCOUNT;
        return (pud_t *)get_zeroed_page(gfp);
}

static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{
        BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
        free_page((unsigned long)pud);
}

extern void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud);

static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
                                  unsigned long address)
{
        ___pud_free_tlb(tlb, pud);
}

#if CONFIG_PGTABLE_LEVELS > 4
static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
{
        if (!pgtable_l5_enabled())
                return;
        paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
        set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
}

static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
{
        gfp_t gfp = GFP_KERNEL_ACCOUNT;

        if (mm == &init_mm)
                gfp &= ~__GFP_ACCOUNT;
        return (p4d_t *)get_zeroed_page(gfp);
}

static inline void p4d_free(struct mm_struct *mm, p4d_t *p4d)
{
        if (!pgtable_l5_enabled())
                return;

        BUG_ON((unsigned long)p4d & (PAGE_SIZE-1));
        free_page((unsigned long)p4d);
}

extern void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d);

static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
                                  unsigned long address)
{
        if (pgtable_l5_enabled())
                ___p4d_free_tlb(tlb, p4d);
}

#endif        /* CONFIG_PGTABLE_LEVELS > 4 */
#endif        /* CONFIG_PGTABLE_LEVELS > 3 */
#endif        /* CONFIG_PGTABLE_LEVELS > 2 */

#endif /* _ASM_X86_PGALLOC_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_PGTABLE_INVERT_H
#define _ASM_PGTABLE_INVERT_H 1

#ifndef __ASSEMBLY__

/*
 * A clear pte value is special, and doesn't get inverted.
 *
 * Note that even users that only pass a pgprot_t (rather
 * than a full pte) won't trigger the special zero case,
 * because even PAGE_NONE has _PAGE_PROTNONE | _PAGE_ACCESSED
 * set. So the all zero case really is limited to just the
 * cleared page table entry case.
 */
static inline bool __pte_needs_invert(u64 val)
{
        return val && !(val & _PAGE_PRESENT); /*covered*/
}

/* Get a mask to xor with the page table entry to get the correct pfn. */
static inline u64 protnone_mask(u64 val)
{
        return __pte_needs_invert(val) ?  ~0ull : 0; /*covered*/
}

static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask)
{
        /*
         * When a PTE transitions from NONE to !NONE or vice-versa
         * invert the PFN part to stop speculation.
         * pte_pfn undoes this when needed.
         */
        if (__pte_needs_invert(oldval) != __pte_needs_invert(val))
                val = (val & ~mask) | (~val & mask);
        return val;
}

#endif /* __ASSEMBLY__ */

#endif

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGTABLE_H
#define _ASM_X86_PGTABLE_H

#include <linux/mem_encrypt.h>
#include <asm/page.h>
#include <asm/pgtable_types.h>

/*
 * Macro to mark a page protection value as UC-
 */
#define pgprot_noncached(prot)                                                \
        ((boot_cpu_data.x86 > 3)                                        \
         ? (__pgprot(pgprot_val(prot) |                                        \
                     cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))        \
         : (prot))

/*
 * Macros to add or remove encryption attribute
 */
#define pgprot_encrypted(prot)        __pgprot(__sme_set(pgprot_val(prot)))
#define pgprot_decrypted(prot)        __pgprot(__sme_clr(pgprot_val(prot)))

#ifndef __ASSEMBLY__
#include <asm/x86_init.h>

extern pgd_t early_top_pgt[PTRS_PER_PGD];
int __init __early_make_pgtable(unsigned long address, pmdval_t pmd);

void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user);
void ptdump_walk_pgd_level_checkwx(void);
void ptdump_walk_user_pgd_level_checkwx(void);

#ifdef CONFIG_DEBUG_WX
#define debug_checkwx()                ptdump_walk_pgd_level_checkwx()
#define debug_checkwx_user()        ptdump_walk_user_pgd_level_checkwx()
#else
#define debug_checkwx()                do { } while (0)
#define debug_checkwx_user()        do { } while (0)
#endif

/*
 * ZERO_PAGE is a global shared page that is always zero: used
 * for zero-mapped memory areas etc..
 */
extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
        __visible;
#define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))

extern spinlock_t pgd_lock;
extern struct list_head pgd_list;

extern struct mm_struct *pgd_page_get_mm(struct page *page);

extern pmdval_t early_pmd_flags;

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else  /* !CONFIG_PARAVIRT */
#define set_pte(ptep, pte)                native_set_pte(ptep, pte)
#define set_pte_at(mm, addr, ptep, pte)        native_set_pte_at(mm, addr, ptep, pte)

#define set_pte_atomic(ptep, pte)                                        \
        native_set_pte_atomic(ptep, pte)

#define set_pmd(pmdp, pmd)                native_set_pmd(pmdp, pmd)

#ifndef __PAGETABLE_P4D_FOLDED
#define set_pgd(pgdp, pgd)                native_set_pgd(pgdp, pgd)
#define pgd_clear(pgd)                        (pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
#endif

#ifndef set_p4d
# define set_p4d(p4dp, p4d)                native_set_p4d(p4dp, p4d)
#endif

#ifndef __PAGETABLE_PUD_FOLDED
#define p4d_clear(p4d)                        native_p4d_clear(p4d)
#endif

#ifndef set_pud
# define set_pud(pudp, pud)                native_set_pud(pudp, pud)
#endif

#ifndef __PAGETABLE_PUD_FOLDED
#define pud_clear(pud)                        native_pud_clear(pud)
#endif

#define pte_clear(mm, addr, ptep)        native_pte_clear(mm, addr, ptep)
#define pmd_clear(pmd)                        native_pmd_clear(pmd)

#define pgd_val(x)        native_pgd_val(x)
#define __pgd(x)        native_make_pgd(x)

#ifndef __PAGETABLE_P4D_FOLDED
#define p4d_val(x)        native_p4d_val(x)
#define __p4d(x)        native_make_p4d(x)
#endif

#ifndef __PAGETABLE_PUD_FOLDED
#define pud_val(x)        native_pud_val(x)
#define __pud(x)        native_make_pud(x)
#endif

#ifndef __PAGETABLE_PMD_FOLDED
#define pmd_val(x)        native_pmd_val(x)
#define __pmd(x)        native_make_pmd(x)
#endif

#define pte_val(x)        native_pte_val(x)
#define __pte(x)        native_make_pte(x)

#define arch_end_context_switch(prev)        do {} while(0)

#endif        /* CONFIG_PARAVIRT */

/*
 * The following only work if pte_present() is true.
 * Undefined behaviour if not..
 */
static inline int pte_dirty(pte_t pte)
{
        return pte_flags(pte) & _PAGE_DIRTY; /*covered*/
}


static inline u32 read_pkru(void)
{
        if (boot_cpu_has(X86_FEATURE_OSPKE)) /*covered*/
                return __read_pkru();
        return 0;
}

static inline void write_pkru(u32 pkru)
{
        if (boot_cpu_has(X86_FEATURE_OSPKE))
                __write_pkru(pkru);
}

static inline int pte_young(pte_t pte)
{
        return pte_flags(pte) & _PAGE_ACCESSED; /*covered*/
}

static inline int pmd_dirty(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_DIRTY; /*covered*/
}

static inline int pmd_young(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_ACCESSED; /*covered*/
}

static inline int pud_dirty(pud_t pud)
{
        return pud_flags(pud) & _PAGE_DIRTY;
}

static inline int pud_young(pud_t pud)
{
        return pud_flags(pud) & _PAGE_ACCESSED;
}

static inline int pte_write(pte_t pte)
{
        return pte_flags(pte) & _PAGE_RW; /*covered*/
}

static inline int pte_huge(pte_t pte)
{
        return pte_flags(pte) & _PAGE_PSE;
}

static inline int pte_global(pte_t pte)
{
        return pte_flags(pte) & _PAGE_GLOBAL;
}

static inline int pte_exec(pte_t pte)
{
        return !(pte_flags(pte) & _PAGE_NX);
}

static inline int pte_special(pte_t pte)
{
        return pte_flags(pte) & _PAGE_SPECIAL; /*covered*/
}

/* Entries that were set to PROT_NONE are inverted */

static inline u64 protnone_mask(u64 val);

static inline unsigned long pte_pfn(pte_t pte)
{
        phys_addr_t pfn = pte_val(pte); /*covered*/
        pfn ^= protnone_mask(pfn); /*covered*/
        return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT; /*covered*/
}

static inline unsigned long pmd_pfn(pmd_t pmd)
{
        phys_addr_t pfn = pmd_val(pmd); /*covered*/
        pfn ^= protnone_mask(pfn); /*covered*/
        return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT; /*covered*/
}

static inline unsigned long pud_pfn(pud_t pud)
{
        phys_addr_t pfn = pud_val(pud); /*covered*/
        pfn ^= protnone_mask(pfn); /*covered*/
        return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT; /*covered*/
}

static inline unsigned long p4d_pfn(p4d_t p4d)
{
        return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
}

static inline unsigned long pgd_pfn(pgd_t pgd)
{
        return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
}

static inline int p4d_large(p4d_t p4d)
{
        /* No 512 GiB pages yet */
        return 0;
}

#define pte_page(pte)        pfn_to_page(pte_pfn(pte))

static inline int pmd_large(pmd_t pte)
{
        return pmd_flags(pte) & _PAGE_PSE; /*covered*/
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int pmd_trans_huge(pmd_t pmd)
{
        return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE; /*covered*/
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static inline int pud_trans_huge(pud_t pud)
{
        return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
}
#endif

#define has_transparent_hugepage has_transparent_hugepage
static inline int has_transparent_hugepage(void)
{
        return boot_cpu_has(X86_FEATURE_PSE);
}

#ifdef __HAVE_ARCH_PTE_DEVMAP
static inline int pmd_devmap(pmd_t pmd)
{
        return !!(pmd_val(pmd) & _PAGE_DEVMAP); /*covered*/
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
static inline int pud_devmap(pud_t pud)
{
        return !!(pud_val(pud) & _PAGE_DEVMAP); /*covered*/
}
#else
static inline int pud_devmap(pud_t pud)
{
        return 0;
}
#endif

static inline int pgd_devmap(pgd_t pgd)
{
        return 0;
}
#endif
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
{
        pteval_t v = native_pte_val(pte);

        return native_make_pte(v | set); /*covered*/
}

static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
{
        pteval_t v = native_pte_val(pte);

        return native_make_pte(v & ~clear);
}

static inline pte_t pte_mkclean(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_DIRTY);
}

static inline pte_t pte_mkold(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_ACCESSED);
}

static inline pte_t pte_wrprotect(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_RW);
}

static inline pte_t pte_mkexec(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_NX);
}

static inline pte_t pte_mkdirty(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); /*covered*/
}

static inline pte_t pte_mkyoung(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_ACCESSED); /*covered*/
}

static inline pte_t pte_mkwrite(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_RW); /*covered*/
}

static inline pte_t pte_mkhuge(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_PSE);
}

static inline pte_t pte_clrhuge(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_PSE);
}

static inline pte_t pte_mkglobal(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_GLOBAL);
}

static inline pte_t pte_clrglobal(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_GLOBAL);
}

static inline pte_t pte_mkspecial(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SPECIAL); /*covered*/
}

static inline pte_t pte_mkdevmap(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
}

static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
{
        pmdval_t v = native_pmd_val(pmd);

        return native_make_pmd(v | set); /*covered*/
}

static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
{
        pmdval_t v = native_pmd_val(pmd);

        return native_make_pmd(v & ~clear);
}

static inline pmd_t pmd_mkold(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_ACCESSED);
}

static inline pmd_t pmd_mkclean(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_DIRTY);
}

static inline pmd_t pmd_wrprotect(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_RW);
}

static inline pmd_t pmd_mkdirty(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); /*covered*/
}

static inline pmd_t pmd_mkdevmap(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_DEVMAP);
}

static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_PSE);
}

static inline pmd_t pmd_mkyoung(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_ACCESSED); /*covered*/
}

static inline pmd_t pmd_mkwrite(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_RW); /*covered*/
}

static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
{
        pudval_t v = native_pud_val(pud);

        return native_make_pud(v | set);
}

static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
{
        pudval_t v = native_pud_val(pud);

        return native_make_pud(v & ~clear);
}

static inline pud_t pud_mkold(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_ACCESSED);
}

static inline pud_t pud_mkclean(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_DIRTY);
}

static inline pud_t pud_wrprotect(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_RW);
}

static inline pud_t pud_mkdirty(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
}

static inline pud_t pud_mkdevmap(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_DEVMAP);
}

static inline pud_t pud_mkhuge(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_PSE);
}

static inline pud_t pud_mkyoung(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_ACCESSED);
}

static inline pud_t pud_mkwrite(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_RW);
}

#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
static inline int pte_soft_dirty(pte_t pte)
{
        return pte_flags(pte) & _PAGE_SOFT_DIRTY;
}

static inline int pmd_soft_dirty(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
}

static inline int pud_soft_dirty(pud_t pud)
{
        return pud_flags(pud) & _PAGE_SOFT_DIRTY;
}

static inline pte_t pte_mksoft_dirty(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SOFT_DIRTY); /*covered*/
}

static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
}

static inline pud_t pud_mksoft_dirty(pud_t pud)
{
        return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
}

static inline pte_t pte_clear_soft_dirty(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
}

static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
}

static inline pud_t pud_clear_soft_dirty(pud_t pud)
{
        return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
}

#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */

/*
 * Mask out unsupported bits in a present pgprot.  Non-present pgprots
 * can use those bits for other purposes, so leave them be.
 */
static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
{
        pgprotval_t protval = pgprot_val(pgprot);

        if (protval & _PAGE_PRESENT)
                protval &= __supported_pte_mask; /*covered*/

        return protval;
}

static inline pgprotval_t check_pgprot(pgprot_t pgprot)
{
        pgprotval_t massaged_val = massage_pgprot(pgprot); /*covered*/

        /* mmdebug.h can not be included here because of dependencies */
#ifdef CONFIG_DEBUG_VM
        WARN_ONCE(pgprot_val(pgprot) != massaged_val, /*covered*/
                  "attempted to set unsupported pgprot: %016llx "
                  "bits: %016llx supported: %016llx\n",
                  (u64)pgprot_val(pgprot),
                  (u64)pgprot_val(pgprot) ^ massaged_val,
                  (u64)__supported_pte_mask);
#endif

        return massaged_val;
}

static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
{
        phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT; /*covered*/
        pfn ^= protnone_mask(pgprot_val(pgprot)); /*covered*/
        pfn &= PTE_PFN_MASK; /*covered*/
        return __pte(pfn | check_pgprot(pgprot)); /*covered*/
}

static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
{
        phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
        pfn ^= protnone_mask(pgprot_val(pgprot)); /*covered*/
        pfn &= PHYSICAL_PMD_PAGE_MASK; /*covered*/
        return __pmd(pfn | check_pgprot(pgprot)); /*covered*/
}

static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
{
        phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
        pfn ^= protnone_mask(pgprot_val(pgprot));
        pfn &= PHYSICAL_PUD_PAGE_MASK;
        return __pud(pfn | check_pgprot(pgprot));
}

static inline pmd_t pmd_mknotpresent(pmd_t pmd)
{
        return pfn_pmd(pmd_pfn(pmd), /*covered*/
                      __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE))); /*covered*/
}

static inline pud_t pud_mknotpresent(pud_t pud)
{
        return pfn_pud(pud_pfn(pud),
              __pgprot(pud_flags(pud) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
}

static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);

static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
        pteval_t val = pte_val(pte), oldval = val;

        /*
         * Chop off the NX bit (if present), and add the NX portion of
         * the newprot (if present):
         */
        val &= _PAGE_CHG_MASK;
        val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
        val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
        return __pte(val);
}

static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
{
        pmdval_t val = pmd_val(pmd), oldval = val;

        val &= _HPAGE_CHG_MASK;
        val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
        val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
        return __pmd(val);
}

/* mprotect needs to preserve PAT bits when updating vm_page_prot */
#define pgprot_modify pgprot_modify
static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
{
        pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK; /*covered*/
        pgprotval_t addbits = pgprot_val(newprot);
        return __pgprot(preservebits | addbits);
}

#define pte_pgprot(x) __pgprot(pte_flags(x))
#define pmd_pgprot(x) __pgprot(pmd_flags(x))
#define pud_pgprot(x) __pgprot(pud_flags(x))
#define p4d_pgprot(x) __pgprot(p4d_flags(x))

#define canon_pgprot(p) __pgprot(massage_pgprot(p))

static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
{
        return canon_pgprot(prot); /*covered*/
}

static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
                                         enum page_cache_mode pcm,
                                         enum page_cache_mode new_pcm)
{
        /*
         * PAT type is always WB for untracked ranges, so no need to check.
         */
        if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
                return 1;

        /*
         * Certain new memtypes are not allowed with certain
         * requested memtype:
         * - request is uncached, return cannot be write-back
         * - request is write-combine, return cannot be write-back
         * - request is write-through, return cannot be write-back
         * - request is write-through, return cannot be write-combine
         */
        if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
             new_pcm == _PAGE_CACHE_MODE_WB) ||
            (pcm == _PAGE_CACHE_MODE_WC &&
             new_pcm == _PAGE_CACHE_MODE_WB) ||
            (pcm == _PAGE_CACHE_MODE_WT &&
             new_pcm == _PAGE_CACHE_MODE_WB) ||
            (pcm == _PAGE_CACHE_MODE_WT &&
             new_pcm == _PAGE_CACHE_MODE_WC)) {
                return 0;
        }

        return 1;
}

pmd_t *populate_extra_pmd(unsigned long vaddr);
pte_t *populate_extra_pte(unsigned long vaddr);

#ifdef CONFIG_PAGE_TABLE_ISOLATION
pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);

/*
 * Take a PGD location (pgdp) and a pgd value that needs to be set there.
 * Populates the user and returns the resulting PGD that must be set in
 * the kernel copy of the page tables.
 */
static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
{
        if (!static_cpu_has(X86_FEATURE_PTI))
                return pgd;
        return __pti_set_user_pgtbl(pgdp, pgd);
}
#else   /* CONFIG_PAGE_TABLE_ISOLATION */
static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
{
        return pgd;
}
#endif  /* CONFIG_PAGE_TABLE_ISOLATION */

#endif        /* __ASSEMBLY__ */


#ifdef CONFIG_X86_32
# include <asm/pgtable_32.h>
#else
# include <asm/pgtable_64.h>
#endif

#ifndef __ASSEMBLY__
#include <linux/mm_types.h>
#include <linux/mmdebug.h>
#include <linux/log2.h>
#include <asm/fixmap.h>

static inline int pte_none(pte_t pte)
{
        return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK)); /*covered*/
}

#define __HAVE_ARCH_PTE_SAME
static inline int pte_same(pte_t a, pte_t b)
{
        return a.pte == b.pte;
}

static inline int pte_present(pte_t a)
{
        return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE); /*covered*/
}

#ifdef __HAVE_ARCH_PTE_DEVMAP
static inline int pte_devmap(pte_t a)
{
        return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP; /*covered*/
}
#endif

#define pte_accessible pte_accessible
static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
{
        if (pte_flags(a) & _PAGE_PRESENT)
                return true;

        if ((pte_flags(a) & _PAGE_PROTNONE) &&
                        mm_tlb_flush_pending(mm))
                return true;

        return false;
}

static inline int pmd_present(pmd_t pmd)
{
        /*
         * Checking for _PAGE_PSE is needed too because
         * split_huge_page will temporarily clear the present bit (but
         * the _PAGE_PSE flag will remain set at all times while the
         * _PAGE_PRESENT bit is clear).
         */
        return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE); /*covered*/
}

#ifdef CONFIG_NUMA_BALANCING
/*
 * These work without NUMA balancing but the kernel does not care. See the
 * comment in include/asm-generic/pgtable.h
 */
static inline int pte_protnone(pte_t pte)
{
        return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT)) /*covered*/
                == _PAGE_PROTNONE;
}

static inline int pmd_protnone(pmd_t pmd)
{
        return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT)) /*covered*/
                == _PAGE_PROTNONE;
}
#endif /* CONFIG_NUMA_BALANCING */

static inline int pmd_none(pmd_t pmd)
{
        /* Only check low word on 32-bit platforms, since it might be
           out of sync with upper half. */
        unsigned long val = native_pmd_val(pmd);
        return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0; /*covered*/
}

static inline unsigned long pmd_page_vaddr(pmd_t pmd)
{
        return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd)); /*covered*/
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define pmd_page(pmd)        pfn_to_page(pmd_pfn(pmd))

/*
 * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
 *
 * this macro returns the index of the entry in the pmd page which would
 * control the given virtual address
 */
static inline unsigned long pmd_index(unsigned long address)
{
        return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); /*covered*/
}

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 *
 * (Currently stuck as a macro because of indirect forward reference
 * to linux/mm.h:page_to_nid())
 */
#define mk_pte(page, pgprot)   pfn_pte(page_to_pfn(page), (pgprot))

/*
 * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
 *
 * this function returns the index of the entry in the pte page which would
 * control the given virtual address
 */
static inline unsigned long pte_index(unsigned long address)
{
        return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); /*covered*/
}

static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
{
        return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); /*covered*/
}

static inline int pmd_bad(pmd_t pmd)
{
        return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; /*covered*/
}

static inline unsigned long pages_to_mb(unsigned long npg)
{
        return npg >> (20 - PAGE_SHIFT);
}

#if CONFIG_PGTABLE_LEVELS > 2
static inline int pud_none(pud_t pud)
{
        return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0; /*covered*/
}

static inline int pud_present(pud_t pud)
{
        return pud_flags(pud) & _PAGE_PRESENT; /*covered*/
}

static inline unsigned long pud_page_vaddr(pud_t pud)
{
        return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud)); /*covered*/
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define pud_page(pud)        pfn_to_page(pud_pfn(pud))

/* Find an entry in the second-level page table.. */
static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
{
        return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); /*covered*/
}

static inline int pud_large(pud_t pud)
{
        return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) == /*covered*/
                (_PAGE_PSE | _PAGE_PRESENT);
}

static inline int pud_bad(pud_t pud)
{
        return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0; /*covered*/
}
#else
static inline int pud_large(pud_t pud)
{
        return 0;
}
#endif        /* CONFIG_PGTABLE_LEVELS > 2 */

static inline unsigned long pud_index(unsigned long address)
{
        return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); /*covered*/
}

#if CONFIG_PGTABLE_LEVELS > 3
static inline int p4d_none(p4d_t p4d)
{
        return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
}

static inline int p4d_present(p4d_t p4d)
{
        return p4d_flags(p4d) & _PAGE_PRESENT;
}

static inline unsigned long p4d_page_vaddr(p4d_t p4d)
{
        return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d)); /*covered*/
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define p4d_page(p4d)        pfn_to_page(p4d_pfn(p4d))

/* Find an entry in the third-level page table.. */
static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
{
        return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address); /*covered*/
}

static inline int p4d_bad(p4d_t p4d)
{
        unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;

        if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
                ignore_flags |= _PAGE_NX;

        return (p4d_flags(p4d) & ~ignore_flags) != 0; /*covered*/
}
#endif  /* CONFIG_PGTABLE_LEVELS > 3 */

static inline unsigned long p4d_index(unsigned long address)
{
        return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
}

#if CONFIG_PGTABLE_LEVELS > 4
static inline int pgd_present(pgd_t pgd)
{
        if (!pgtable_l5_enabled())
                return 1;
        return pgd_flags(pgd) & _PAGE_PRESENT;
}

static inline unsigned long pgd_page_vaddr(pgd_t pgd)
{
        return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
}

/*
 * Currently stuck as a macro due to indirect forward reference to
 * linux/mmzone.h's __section_mem_map_addr() definition:
 */
#define pgd_page(pgd)        pfn_to_page(pgd_pfn(pgd))

/* to find an entry in a page-table-directory. */
static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
{
        if (!pgtable_l5_enabled())
                return (p4d_t *)pgd;
        return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
}

static inline int pgd_bad(pgd_t pgd)
{
        unsigned long ignore_flags = _PAGE_USER;

        if (!pgtable_l5_enabled())
                return 0;

        if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
                ignore_flags |= _PAGE_NX;

        return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
}

static inline int pgd_none(pgd_t pgd)
{
        if (!pgtable_l5_enabled())
                return 0;
        /*
         * There is no need to do a workaround for the KNL stray
         * A/D bit erratum here.  PGDs only point to page tables
         * except on 32-bit non-PAE which is not supported on
         * KNL.
         */
        return !native_pgd_val(pgd);
}
#endif        /* CONFIG_PGTABLE_LEVELS > 4 */

#endif        /* __ASSEMBLY__ */

/*
 * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
 *
 * this macro returns the index of the entry in the pgd page which would
 * control the given virtual address
 */
#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))

/*
 * pgd_offset() returns a (pgd_t *)
 * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
 */
#define pgd_offset_pgd(pgd, address) (pgd + pgd_index((address)))
/*
 * a shortcut to get a pgd_t in a given mm
 */
#define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
/*
 * a shortcut which implies the use of the kernel's pgd, instead
 * of a process's
 */
#define pgd_offset_k(address) pgd_offset(&init_mm, (address))


#define KERNEL_PGD_BOUNDARY        pgd_index(PAGE_OFFSET)
#define KERNEL_PGD_PTRS                (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)

#ifndef __ASSEMBLY__

extern int direct_gbpages;
void init_mem_mapping(void);
void early_alloc_pgt_buf(void);
extern void memblock_find_dma_reserve(void);

#ifdef CONFIG_X86_64
/* Realmode trampoline initialization. */
extern pgd_t trampoline_pgd_entry;
static inline void __meminit init_trampoline_default(void)
{
        /* Default trampoline pgd value */
        trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
}
# ifdef CONFIG_RANDOMIZE_MEMORY
void __meminit init_trampoline(void);
# else
#  define init_trampoline init_trampoline_default
# endif
#else
static inline void init_trampoline(void) { }
#endif

/* local pte updates need not use xchg for locking */
static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
{
        pte_t res = *ptep;

        /* Pure native function needs no input for mm, addr */
        native_pte_clear(NULL, 0, ptep);
        return res;
}

static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
{
        pmd_t res = *pmdp;

        native_pmd_clear(pmdp);
        return res;
}

static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
{
        pud_t res = *pudp;

        native_pud_clear(pudp);
        return res;
}

static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
                                     pte_t *ptep , pte_t pte)
{
        native_set_pte(ptep, pte);
}

static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
                              pmd_t *pmdp, pmd_t pmd)
{
        native_set_pmd(pmdp, pmd); /*covered*/
}

static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
                              pud_t *pudp, pud_t pud)
{
        native_set_pud(pudp, pud);
}

/*
 * We only update the dirty/accessed state if we set
 * the dirty bit by hand in the kernel, since the hardware
 * will do the accessed bit for us, and we don't want to
 * race with other CPU's that might be updating the dirty
 * bit at the same time.
 */
struct vm_area_struct;

#define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
extern int ptep_set_access_flags(struct vm_area_struct *vma,
                                 unsigned long address, pte_t *ptep,
                                 pte_t entry, int dirty);

#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
                                     unsigned long addr, pte_t *ptep);

#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
extern int ptep_clear_flush_young(struct vm_area_struct *vma,
                                  unsigned long address, pte_t *ptep);

#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
                                       pte_t *ptep)
{
        pte_t pte = native_ptep_get_and_clear(ptep); /*covered*/
        return pte;
}

#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
                                            unsigned long addr, pte_t *ptep,
                                            int full)
{
        pte_t pte;
        if (full) {
                /*
                 * Full address destruction in progress; paravirt does not
                 * care about updates and native needs no locking
                 */
                pte = native_local_ptep_get_and_clear(ptep);
        } else {
                pte = ptep_get_and_clear(mm, addr, ptep); /*covered*/
        }
        return pte;
}

#define __HAVE_ARCH_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(struct mm_struct *mm,
                                      unsigned long addr, pte_t *ptep)
{
        clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
}

#define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)

#define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))

#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
extern int pmdp_set_access_flags(struct vm_area_struct *vma,
                                 unsigned long address, pmd_t *pmdp,
                                 pmd_t entry, int dirty);
extern int pudp_set_access_flags(struct vm_area_struct *vma,
                                 unsigned long address, pud_t *pudp,
                                 pud_t entry, int dirty);

#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
                                     unsigned long addr, pmd_t *pmdp);
extern int pudp_test_and_clear_young(struct vm_area_struct *vma,
                                     unsigned long addr, pud_t *pudp);

#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
                                  unsigned long address, pmd_t *pmdp);


#define pmd_write pmd_write
static inline int pmd_write(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_RW; /*covered*/
}

#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
                                       pmd_t *pmdp)
{
        return native_pmdp_get_and_clear(pmdp); /*covered*/
}

#define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
                                        unsigned long addr, pud_t *pudp)
{
        return native_pudp_get_and_clear(pudp);
}

#define __HAVE_ARCH_PMDP_SET_WRPROTECT
static inline void pmdp_set_wrprotect(struct mm_struct *mm,
                                      unsigned long addr, pmd_t *pmdp)
{
        clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
}

#define pud_write pud_write
static inline int pud_write(pud_t pud)
{
        return pud_flags(pud) & _PAGE_RW;
}

#ifndef pmdp_establish
#define pmdp_establish pmdp_establish
static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
                unsigned long address, pmd_t *pmdp, pmd_t pmd)
{
        if (IS_ENABLED(CONFIG_SMP)) {
                return xchg(pmdp, pmd);
        } else {
                pmd_t old = *pmdp;
                WRITE_ONCE(*pmdp, pmd);
                return old;
        }
}
#endif
/*
 * Page table pages are page-aligned.  The lower half of the top
 * level is used for userspace and the top half for the kernel.
 *
 * Returns true for parts of the PGD that map userspace and
 * false for the parts that map the kernel.
 */
static inline bool pgdp_maps_userspace(void *__ptr)
{
        unsigned long ptr = (unsigned long)__ptr;

        return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
}

static inline int pgd_large(pgd_t pgd) { return 0; }

#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
 * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
 * (8k-aligned and 8k in size).  The kernel one is at the beginning 4k and
 * the user one is in the last 4k.  To switch between them, you
 * just need to flip the 12th bit in their addresses.
 */
#define PTI_PGTABLE_SWITCH_BIT        PAGE_SHIFT

/*
 * This generates better code than the inline assembly in
 * __set_bit().
 */
static inline void *ptr_set_bit(void *ptr, int bit)
{
        unsigned long __ptr = (unsigned long)ptr;

        __ptr |= BIT(bit);
        return (void *)__ptr;
}
static inline void *ptr_clear_bit(void *ptr, int bit)
{
        unsigned long __ptr = (unsigned long)ptr;

        __ptr &= ~BIT(bit);
        return (void *)__ptr;
}

static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
{
        return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
{
        return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
{
        return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}

static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
{
        return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}
#endif /* CONFIG_PAGE_TABLE_ISOLATION */

/*
 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
 *
 *  dst - pointer to pgd range anwhere on a pgd page
 *  src - ""
 *  count - the number of pgds to copy.
 *
 * dst and src can be on the same page, but the range must not overlap,
 * and must not cross a page boundary.
 */
static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
{
        memcpy(dst, src, count * sizeof(pgd_t));
#ifdef CONFIG_PAGE_TABLE_ISOLATION
        if (!static_cpu_has(X86_FEATURE_PTI))
                return;
        /* Clone the user space pgd as well */
        memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
               count * sizeof(pgd_t));
#endif
}

#define PTE_SHIFT ilog2(PTRS_PER_PTE)
static inline int page_level_shift(enum pg_level level)
{
        return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
}
static inline unsigned long page_level_size(enum pg_level level)
{
        return 1UL << page_level_shift(level);
}
static inline unsigned long page_level_mask(enum pg_level level)
{
        return ~(page_level_size(level) - 1);
}

/*
 * The x86 doesn't have any external MMU info: the kernel page
 * tables contain all the necessary information.
 */
static inline void update_mmu_cache(struct vm_area_struct *vma,
                unsigned long addr, pte_t *ptep)
{
}
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
                unsigned long addr, pmd_t *pmd)
{
}
static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
                unsigned long addr, pud_t *pud)
{
}

#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
{
        return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
}

static inline int pte_swp_soft_dirty(pte_t pte)
{
        return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
}

static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
{
        return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
}

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
{
        return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
}

static inline int pmd_swp_soft_dirty(pmd_t pmd)
{
        return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY; /*covered*/
}

static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
{
        return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
}
#endif
#endif

#define PKRU_AD_BIT 0x1
#define PKRU_WD_BIT 0x2
#define PKRU_BITS_PER_PKEY 2

static inline bool __pkru_allows_read(u32 pkru, u16 pkey)
{
        int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
        return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits));
}

static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
{
        int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
        /*
         * Access-disable disables writes too so we need to check
         * both bits here.
         */
        return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits));
}

static inline u16 pte_flags_pkey(unsigned long pte_flags)
{
#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
        /* ifdef to avoid doing 59-bit shift on 32-bit values */
        return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
#else
        return 0;
#endif
}

static inline bool __pkru_allows_pkey(u16 pkey, bool write)
{
        u32 pkru = read_pkru(); /*covered*/

        if (!__pkru_allows_read(pkru, pkey))
                return false;
        if (write && !__pkru_allows_write(pkru, pkey)) /*covered*/
                return false;

        return true;
}

/*
 * 'pteval' can come from a PTE, PMD or PUD.  We only check
 * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
 * same value on all 3 types.
 */
static inline bool __pte_access_permitted(unsigned long pteval, bool write)
{
        unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;

        if (write)
                need_pte_bits |= _PAGE_RW;

        if ((pteval & need_pte_bits) != need_pte_bits)
                return 0;

        return __pkru_allows_pkey(pte_flags_pkey(pteval), write); /*covered*/
}

#define pte_access_permitted pte_access_permitted
static inline bool pte_access_permitted(pte_t pte, bool write)
{
        return __pte_access_permitted(pte_val(pte), write); /*covered*/
}

#define pmd_access_permitted pmd_access_permitted
static inline bool pmd_access_permitted(pmd_t pmd, bool write)
{
        return __pte_access_permitted(pmd_val(pmd), write); /*covered*/
}

#define pud_access_permitted pud_access_permitted
static inline bool pud_access_permitted(pud_t pud, bool write)
{
        return __pte_access_permitted(pud_val(pud), write);
}

#define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1
extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot);

static inline bool arch_has_pfn_modify_check(void)
{
        return boot_cpu_has_bug(X86_BUG_L1TF);
}

#include <asm-generic/pgtable.h>
#endif        /* __ASSEMBLY__ */

#endif /* _ASM_X86_PGTABLE_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGTABLE_64_H
#define _ASM_X86_PGTABLE_64_H

#include <linux/const.h>
#include <asm/pgtable_64_types.h>

#ifndef __ASSEMBLY__

/*
 * This file contains the functions and defines necessary to modify and use
 * the x86-64 page table tree.
 */
#include <asm/processor.h>
#include <linux/bitops.h>
#include <linux/threads.h>
#include <asm/fixmap.h>

extern p4d_t level4_kernel_pgt[512];
extern p4d_t level4_ident_pgt[512];
extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
extern pmd_t level2_kernel_pgt[512];
extern pmd_t level2_fixmap_pgt[512];
extern pmd_t level2_ident_pgt[512];
extern pte_t level1_fixmap_pgt[512 * FIXMAP_PMD_NUM];
extern pgd_t init_top_pgt[];

#define swapper_pg_dir init_top_pgt

extern void paging_init(void);
static inline void sync_initial_page_table(void) { }

#define pte_ERROR(e)                                        \
        pr_err("%s:%d: bad pte %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pte_val(e))
#define pmd_ERROR(e)                                        \
        pr_err("%s:%d: bad pmd %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pmd_val(e))
#define pud_ERROR(e)                                        \
        pr_err("%s:%d: bad pud %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pud_val(e))

#if CONFIG_PGTABLE_LEVELS >= 5
#define p4d_ERROR(e)                                        \
        pr_err("%s:%d: bad p4d %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), p4d_val(e))
#endif

#define pgd_ERROR(e)                                        \
        pr_err("%s:%d: bad pgd %p(%016lx)\n",                \
               __FILE__, __LINE__, &(e), pgd_val(e))

struct mm_struct;

void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte);
void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);

static inline void native_set_pte(pte_t *ptep, pte_t pte)
{
        WRITE_ONCE(*ptep, pte);
}

static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
                                    pte_t *ptep)
{
        native_set_pte(ptep, native_make_pte(0));
}

static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
        native_set_pte(ptep, pte);
}

static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
        WRITE_ONCE(*pmdp, pmd); /*covered*/
}

static inline void native_pmd_clear(pmd_t *pmd)
{
        native_set_pmd(pmd, native_make_pmd(0));
}

static inline pte_t native_ptep_get_and_clear(pte_t *xp)
{
#ifdef CONFIG_SMP
        return native_make_pte(xchg(&xp->pte, 0)); /*covered*/
#else
        /* native_local_ptep_get_and_clear,
           but duplicated because of cyclic dependency */
        pte_t ret = *xp;
        native_pte_clear(NULL, 0, xp);
        return ret;
#endif
}

static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
{
#ifdef CONFIG_SMP
        return native_make_pmd(xchg(&xp->pmd, 0)); /*covered*/
#else
        /* native_local_pmdp_get_and_clear,
           but duplicated because of cyclic dependency */
        pmd_t ret = *xp;
        native_pmd_clear(xp);
        return ret;
#endif
}

static inline void native_set_pud(pud_t *pudp, pud_t pud)
{
        WRITE_ONCE(*pudp, pud);
}

static inline void native_pud_clear(pud_t *pud)
{
        native_set_pud(pud, native_make_pud(0));
}

static inline pud_t native_pudp_get_and_clear(pud_t *xp)
{
#ifdef CONFIG_SMP
        return native_make_pud(xchg(&xp->pud, 0));
#else
        /* native_local_pudp_get_and_clear,
         * but duplicated because of cyclic dependency
         */
        pud_t ret = *xp;

        native_pud_clear(xp);
        return ret;
#endif
}

static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
{
        pgd_t pgd;

        if (pgtable_l5_enabled() || !IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION)) {
                WRITE_ONCE(*p4dp, p4d);
                return;
        }

        pgd = native_make_pgd(native_p4d_val(p4d));
        pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd);
        WRITE_ONCE(*p4dp, native_make_p4d(native_pgd_val(pgd)));
}

static inline void native_p4d_clear(p4d_t *p4d)
{
        native_set_p4d(p4d, native_make_p4d(0));
}

static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
{
        WRITE_ONCE(*pgdp, pti_set_user_pgtbl(pgdp, pgd));
}

static inline void native_pgd_clear(pgd_t *pgd)
{
        native_set_pgd(pgd, native_make_pgd(0));
}

extern void sync_global_pgds(unsigned long start, unsigned long end);

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */

/*
 * Level 4 access.
 */
#define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE)

/* PUD - Level3 access */

/* PMD  - Level 2 access */

/* PTE - Level 1 access. */

/* x86-64 always has all page tables mapped. */
#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))
#define pte_unmap(pte) ((void)(pte))/* NOP */

/*
 * Encode and de-code a swap entry
 *
 * |     ...            | 11| 10|  9|8|7|6|5| 4| 3|2| 1|0| <- bit number
 * |     ...            |SW3|SW2|SW1|G|L|D|A|CD|WT|U| W|P| <- bit names
 * | TYPE (59-63) | ~OFFSET (9-58)  |0|0|X|X| X| X|X|SD|0| <- swp entry
 *
 * G (8) is aliased and used as a PROT_NONE indicator for
 * !present ptes.  We need to start storing swap entries above
 * there.  We also need to avoid using A and D because of an
 * erratum where they can be incorrectly set by hardware on
 * non-present PTEs.
 *
 * SD (1) in swp entry is used to store soft dirty bit, which helps us
 * remember soft dirty over page migration
 *
 * Bit 7 in swp entry should be 0 because pmd_present checks not only P,
 * but also L and G.
 *
 * The offset is inverted by a binary not operation to make the high
 * physical bits set.
 */
#define SWP_TYPE_BITS                5

#define SWP_OFFSET_FIRST_BIT        (_PAGE_BIT_PROTNONE + 1)

/* We always extract/encode the offset by shifting it all the way up, and then down again */
#define SWP_OFFSET_SHIFT        (SWP_OFFSET_FIRST_BIT+SWP_TYPE_BITS)

#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)

/* Extract the high bits for type */
#define __swp_type(x) ((x).val >> (64 - SWP_TYPE_BITS))

/* Shift up (to get rid of type), then down to get value */
#define __swp_offset(x) (~(x).val << SWP_TYPE_BITS >> SWP_OFFSET_SHIFT)

/*
 * Shift the offset up "too far" by TYPE bits, then down again
 * The offset is inverted by a binary not operation to make the high
 * physical bits set.
 */
#define __swp_entry(type, offset) ((swp_entry_t) { \
        (~(unsigned long)(offset) << SWP_OFFSET_SHIFT >> SWP_TYPE_BITS) \
        | ((unsigned long)(type) << (64-SWP_TYPE_BITS)) })

#define __pte_to_swp_entry(pte)                ((swp_entry_t) { pte_val((pte)) })
#define __pmd_to_swp_entry(pmd)                ((swp_entry_t) { pmd_val((pmd)) })
#define __swp_entry_to_pte(x)                ((pte_t) { .pte = (x).val })
#define __swp_entry_to_pmd(x)                ((pmd_t) { .pmd = (x).val })

extern int kern_addr_valid(unsigned long addr);
extern void cleanup_highmap(void);

#define HAVE_ARCH_UNMAPPED_AREA
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN

#define pgtable_cache_init()   do { } while (0)
#define check_pgt_cache()      do { } while (0)

#define PAGE_AGP    PAGE_KERNEL_NOCACHE
#define HAVE_PAGE_AGP 1

/* fs/proc/kcore.c */
#define        kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK)
#define        kc_offset_to_vaddr(o) ((o) | ~__VIRTUAL_MASK)

#define __HAVE_ARCH_PTE_SAME

#define vmemmap ((struct page *)VMEMMAP_START)

extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);

#define gup_fast_permitted gup_fast_permitted
static inline bool gup_fast_permitted(unsigned long start, int nr_pages,
                int write)
{
        unsigned long len, end;

        len = (unsigned long)nr_pages << PAGE_SHIFT;
        end = start + len;
        if (end < start) /*covered*/
                return false;
        if (end >> __VIRTUAL_MASK_SHIFT) /*covered*/
                return false;
        return true;
}

#include <asm/pgtable-invert.h>

#endif /* !__ASSEMBLY__ */
#endif /* _ASM_X86_PGTABLE_64_H */

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGTABLE_DEFS_H
#define _ASM_X86_PGTABLE_DEFS_H

#include <linux/const.h>
#include <linux/mem_encrypt.h>

#include <asm/page_types.h>

#define FIRST_USER_ADDRESS        0UL

#define _PAGE_BIT_PRESENT        0        /* is present */
#define _PAGE_BIT_RW                1        /* writeable */
#define _PAGE_BIT_USER                2        /* userspace addressable */
#define _PAGE_BIT_PWT                3        /* page write through */
#define _PAGE_BIT_PCD                4        /* page cache disabled */
#define _PAGE_BIT_ACCESSED        5        /* was accessed (raised by CPU) */
#define _PAGE_BIT_DIRTY                6        /* was written to (raised by CPU) */
#define _PAGE_BIT_PSE                7        /* 4 MB (or 2MB) page */
#define _PAGE_BIT_PAT                7        /* on 4KB pages */
#define _PAGE_BIT_GLOBAL        8        /* Global TLB entry PPro+ */
#define _PAGE_BIT_SOFTW1        9        /* available for programmer */
#define _PAGE_BIT_SOFTW2        10        /* " */
#define _PAGE_BIT_SOFTW3        11        /* " */
#define _PAGE_BIT_PAT_LARGE        12        /* On 2MB or 1GB pages */
#define _PAGE_BIT_SOFTW4        58        /* available for programmer */
#define _PAGE_BIT_PKEY_BIT0        59        /* Protection Keys, bit 1/4 */
#define _PAGE_BIT_PKEY_BIT1        60        /* Protection Keys, bit 2/4 */
#define _PAGE_BIT_PKEY_BIT2