arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/crypto.h>
  25 #include <linux/module.h>
  26 #include <linux/err.h>
  27 #include <crypto/algapi.h>
  28 #include <crypto/aes.h>
  29 #include <crypto/cryptd.h>
  30 #include <crypto/ctr.h>
  31 #include <asm/cpu_device_id.h>
  32 #include <asm/i387.h>
  33 #include <asm/aes.h>
  34 #include <crypto/scatterwalk.h>
  35 #include <crypto/internal/aead.h>
  36 #include <linux/workqueue.h>
  37 #include <linux/spinlock.h>
  38
  39 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
  40 #define HAS_CTR
  41 #endif
  42
  43 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
  44 #define HAS_LRW
  45 #endif
  46
  47 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
  48 #define HAS_PCBC
  49 #endif
  50
  51 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
  52 #define HAS_XTS
  53 #endif
  54
  55 struct async_aes_ctx {
  56         struct cryptd_ablkcipher *cryptd_tfm;
  57 };
  58
  59 /* This data is stored at the end of the crypto_tfm struct.
  60  * It's a type of per "session" data storage location.
  61  * This needs to be 16 byte aligned.
  62  */
  63 struct aesni_rfc4106_gcm_ctx {
  64         u8 hash_subkey[16];
  65         struct crypto_aes_ctx aes_key_expanded;
  66         u8 nonce[4];
  67         struct cryptd_aead *cryptd_tfm;
  68 };
  69
  70 struct aesni_gcm_set_hash_subkey_result {
  71         int err;
  72         struct completion completion;
  73 };
  74
  75 struct aesni_hash_subkey_req_data {
  76         u8 iv[16];
  77         struct aesni_gcm_set_hash_subkey_result result;
  78         struct scatterlist sg;
  79 };
  80
  81 #define AESNI_ALIGN     (16)
  82 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
  83 #define RFC4106_HASH_SUBKEY_SIZE 16
  84
  85 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  86                              unsigned int key_len);
  87 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  88                           const u8 *in);
  89 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  90                           const u8 *in);
  91 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  92                               const u8 *in, unsigned int len);
  93 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  94                               const u8 *in, unsigned int len);
  95 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  96                               const u8 *in, unsigned int len, u8 *iv);
  97 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  98                               const u8 *in, unsigned int len, u8 *iv);
  99
 100 int crypto_fpu_init(void);
 101 void crypto_fpu_exit(void);
 102
 103 #ifdef CONFIG_X86_64
 104 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
 105                               const u8 *in, unsigned int len, u8 *iv);
 106
 107 /* asmlinkage void aesni_gcm_enc()
 108  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 109  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 110  * const u8 *in, Plaintext input
 111  * unsigned long plaintext_len, Length of data in bytes for encryption.
 112  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 113  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 114  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 115  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 116  * const u8 *aad, Additional Authentication Data (AAD)
 117  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
 118  *          is going to be 8 or 12 bytes
 119  * u8 *auth_tag, Authenticated Tag output.
 120  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 121  *          Valid values are 16 (most likely), 12 or 8.
 122  */
 123 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 124                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 125                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 126                         u8 *auth_tag, unsigned long auth_tag_len);
 127
 128 /* asmlinkage void aesni_gcm_dec()
 129  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 130  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 131  * const u8 *in, Ciphertext input
 132  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 133  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 134  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 135  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 136  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 137  * const u8 *aad, Additional Authentication Data (AAD)
 138  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 139  * to be 8 or 12 bytes
 140  * u8 *auth_tag, Authenticated Tag output.
 141  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 142  * Valid values are 16 (most likely), 12 or 8.
 143  */
 144 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 145                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 146                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 147                         u8 *auth_tag, unsigned long auth_tag_len);
 148
 149 static inline struct
 150 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 151 {
 152         return
 153                 (struct aesni_rfc4106_gcm_ctx *)
 154                 PTR_ALIGN((u8 *)
 155                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
 156 }
 157 #endif
 158
 159 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 160 {
 161         unsigned long addr = (unsigned long)raw_ctx;
 162         unsigned long align = AESNI_ALIGN;
 163
 164         if (align <= crypto_tfm_ctx_alignment())
 165                 align = 1;
 166         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 167 }
 168
 169 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 170                               const u8 *in_key, unsigned int key_len)
 171 {
 172         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 173         u32 *flags = &tfm->crt_flags;
 174         int err;
 175
 176         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 177             key_len != AES_KEYSIZE_256) {
 178                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 179                 return -EINVAL;
 180         }
 181
 182         if (!irq_fpu_usable())
 183                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 184         else {
 185                 kernel_fpu_begin();
 186                 err = aesni_set_key(ctx, in_key, key_len);
 187                 kernel_fpu_end();
 188         }
 189
 190         return err;
 191 }
 192
 193 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 194                        unsigned int key_len)
 195 {
 196         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 197 }
 198
 199 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 200 {
 201         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 202
 203         if (!irq_fpu_usable())
 204                 crypto_aes_encrypt_x86(ctx, dst, src);
 205         else {
 206                 kernel_fpu_begin();
 207                 aesni_enc(ctx, dst, src);
 208                 kernel_fpu_end();
 209         }
 210 }
 211
 212 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 213 {
 214         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 215
 216         if (!irq_fpu_usable())
 217                 crypto_aes_decrypt_x86(ctx, dst, src);
 218         else {
 219                 kernel_fpu_begin();
 220                 aesni_dec(ctx, dst, src);
 221                 kernel_fpu_end();
 222         }
 223 }
 224
 225 static struct crypto_alg aesni_alg = {
 226         .cra_name               = "aes",
 227         .cra_driver_name        = "aes-aesni",
 228         .cra_priority           = 300,
 229         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 230         .cra_blocksize          = AES_BLOCK_SIZE,
 231         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 232         .cra_alignmask          = 0,
 233         .cra_module             = THIS_MODULE,
 234         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
 235         .cra_u  = {
 236                 .cipher = {
 237                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 238                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 239                         .cia_setkey             = aes_set_key,
 240                         .cia_encrypt            = aes_encrypt,
 241                         .cia_decrypt            = aes_decrypt
 242                 }
 243         }
 244 };
 245
 246 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 247 {
 248         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 249
 250         aesni_enc(ctx, dst, src);
 251 }
 252
 253 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 254 {
 255         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 256
 257         aesni_dec(ctx, dst, src);
 258 }
 259
 260 static struct crypto_alg __aesni_alg = {
 261         .cra_name               = "__aes-aesni",
 262         .cra_driver_name        = "__driver-aes-aesni",
 263         .cra_priority           = 0,
 264         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 265         .cra_blocksize          = AES_BLOCK_SIZE,
 266         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 267         .cra_alignmask          = 0,
 268         .cra_module             = THIS_MODULE,
 269         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
 270         .cra_u  = {
 271                 .cipher = {
 272                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 273                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 274                         .cia_setkey             = aes_set_key,
 275                         .cia_encrypt            = __aes_encrypt,
 276                         .cia_decrypt            = __aes_decrypt
 277                 }
 278         }
 279 };
 280
 281 static int ecb_encrypt(struct blkcipher_desc *desc,
 282                        struct scatterlist *dst, struct scatterlist *src,
 283                        unsigned int nbytes)
 284 {
 285         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 286         struct blkcipher_walk walk;
 287         int err;
 288
 289         blkcipher_walk_init(&walk, dst, src, nbytes);
 290         err = blkcipher_walk_virt(desc, &walk);
 291         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 292
 293         kernel_fpu_begin();
 294         while ((nbytes = walk.nbytes)) {
 295                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 296                               nbytes & AES_BLOCK_MASK);
 297                 nbytes &= AES_BLOCK_SIZE - 1;
 298                 err = blkcipher_walk_done(desc, &walk, nbytes);
 299         }
 300         kernel_fpu_end();
 301
 302         return err;
 303 }
 304
 305 static int ecb_decrypt(struct blkcipher_desc *desc,
 306                        struct scatterlist *dst, struct scatterlist *src,
 307                        unsigned int nbytes)
 308 {
 309         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 310         struct blkcipher_walk walk;
 311         int err;
 312
 313         blkcipher_walk_init(&walk, dst, src, nbytes);
 314         err = blkcipher_walk_virt(desc, &walk);
 315         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 316
 317         kernel_fpu_begin();
 318         while ((nbytes = walk.nbytes)) {
 319                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 320                               nbytes & AES_BLOCK_MASK);
 321                 nbytes &= AES_BLOCK_SIZE - 1;
 322                 err = blkcipher_walk_done(desc, &walk, nbytes);
 323         }
 324         kernel_fpu_end();
 325
 326         return err;
 327 }
 328
 329 static struct crypto_alg blk_ecb_alg = {
 330         .cra_name               = "__ecb-aes-aesni",
 331         .cra_driver_name        = "__driver-ecb-aes-aesni",
 332         .cra_priority           = 0,
 333         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 334         .cra_blocksize          = AES_BLOCK_SIZE,
 335         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 336         .cra_alignmask          = 0,
 337         .cra_type               = &crypto_blkcipher_type,
 338         .cra_module             = THIS_MODULE,
 339         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 340         .cra_u = {
 341                 .blkcipher = {
 342                         .min_keysize    = AES_MIN_KEY_SIZE,
 343                         .max_keysize    = AES_MAX_KEY_SIZE,
 344                         .setkey         = aes_set_key,
 345                         .encrypt        = ecb_encrypt,
 346                         .decrypt        = ecb_decrypt,
 347                 },
 348         },
 349 };
 350
 351 static int cbc_encrypt(struct blkcipher_desc *desc,
 352                        struct scatterlist *dst, struct scatterlist *src,
 353                        unsigned int nbytes)
 354 {
 355         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 356         struct blkcipher_walk walk;
 357         int err;
 358
 359         blkcipher_walk_init(&walk, dst, src, nbytes);
 360         err = blkcipher_walk_virt(desc, &walk);
 361         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 362
 363         kernel_fpu_begin();
 364         while ((nbytes = walk.nbytes)) {
 365                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 366                               nbytes & AES_BLOCK_MASK, walk.iv);
 367                 nbytes &= AES_BLOCK_SIZE - 1;
 368                 err = blkcipher_walk_done(desc, &walk, nbytes);
 369         }
 370         kernel_fpu_end();
 371
 372         return err;
 373 }
 374
 375 static int cbc_decrypt(struct blkcipher_desc *desc,
 376                        struct scatterlist *dst, struct scatterlist *src,
 377                        unsigned int nbytes)
 378 {
 379         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 380         struct blkcipher_walk walk;
 381         int err;
 382
 383         blkcipher_walk_init(&walk, dst, src, nbytes);
 384         err = blkcipher_walk_virt(desc, &walk);
 385         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 386
 387         kernel_fpu_begin();
 388         while ((nbytes = walk.nbytes)) {
 389                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 390                               nbytes & AES_BLOCK_MASK, walk.iv);
 391                 nbytes &= AES_BLOCK_SIZE - 1;
 392                 err = blkcipher_walk_done(desc, &walk, nbytes);
 393         }
 394         kernel_fpu_end();
 395
 396         return err;
 397 }
 398
 399 static struct crypto_alg blk_cbc_alg = {
 400         .cra_name               = "__cbc-aes-aesni",
 401         .cra_driver_name        = "__driver-cbc-aes-aesni",
 402         .cra_priority           = 0,
 403         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 404         .cra_blocksize          = AES_BLOCK_SIZE,
 405         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 406         .cra_alignmask          = 0,
 407         .cra_type               = &crypto_blkcipher_type,
 408         .cra_module             = THIS_MODULE,
 409         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 410         .cra_u = {
 411                 .blkcipher = {
 412                         .min_keysize    = AES_MIN_KEY_SIZE,
 413                         .max_keysize    = AES_MAX_KEY_SIZE,
 414                         .setkey         = aes_set_key,
 415                         .encrypt        = cbc_encrypt,
 416                         .decrypt        = cbc_decrypt,
 417                 },
 418         },
 419 };
 420
 421 #ifdef CONFIG_X86_64
 422 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 423                             struct blkcipher_walk *walk)
 424 {
 425         u8 *ctrblk = walk->iv;
 426         u8 keystream[AES_BLOCK_SIZE];
 427         u8 *src = walk->src.virt.addr;
 428         u8 *dst = walk->dst.virt.addr;
 429         unsigned int nbytes = walk->nbytes;
 430
 431         aesni_enc(ctx, keystream, ctrblk);
 432         crypto_xor(keystream, src, nbytes);
 433         memcpy(dst, keystream, nbytes);
 434         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 435 }
 436
 437 static int ctr_crypt(struct blkcipher_desc *desc,
 438                      struct scatterlist *dst, struct scatterlist *src,
 439                      unsigned int nbytes)
 440 {
 441         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 442         struct blkcipher_walk walk;
 443         int err;
 444
 445         blkcipher_walk_init(&walk, dst, src, nbytes);
 446         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 447         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 448
 449         kernel_fpu_begin();
 450         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 451                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 452                               nbytes & AES_BLOCK_MASK, walk.iv);
 453                 nbytes &= AES_BLOCK_SIZE - 1;
 454                 err = blkcipher_walk_done(desc, &walk, nbytes);
 455         }
 456         if (walk.nbytes) {
 457                 ctr_crypt_final(ctx, &walk);
 458                 err = blkcipher_walk_done(desc, &walk, 0);
 459         }
 460         kernel_fpu_end();
 461
 462         return err;
 463 }
 464
 465 static struct crypto_alg blk_ctr_alg = {
 466         .cra_name               = "__ctr-aes-aesni",
 467         .cra_driver_name        = "__driver-ctr-aes-aesni",
 468         .cra_priority           = 0,
 469         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 470         .cra_blocksize          = 1,
 471         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 472         .cra_alignmask          = 0,
 473         .cra_type               = &crypto_blkcipher_type,
 474         .cra_module             = THIS_MODULE,
 475         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 476         .cra_u = {
 477                 .blkcipher = {
 478                         .min_keysize    = AES_MIN_KEY_SIZE,
 479                         .max_keysize    = AES_MAX_KEY_SIZE,
 480                         .ivsize         = AES_BLOCK_SIZE,
 481                         .setkey         = aes_set_key,
 482                         .encrypt        = ctr_crypt,
 483                         .decrypt        = ctr_crypt,
 484                 },
 485         },
 486 };
 487 #endif
 488
 489 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
 490                         unsigned int key_len)
 491 {
 492         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 493         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
 494         int err;
 495
 496         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 497         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
 498                                     & CRYPTO_TFM_REQ_MASK);
 499         err = crypto_ablkcipher_setkey(child, key, key_len);
 500         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
 501                                     & CRYPTO_TFM_RES_MASK);
 502         return err;
 503 }
 504
 505 static int ablk_encrypt(struct ablkcipher_request *req)
 506 {
 507         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 508         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 509
 510         if (!irq_fpu_usable()) {
 511                 struct ablkcipher_request *cryptd_req =
 512                         ablkcipher_request_ctx(req);
 513                 memcpy(cryptd_req, req, sizeof(*req));
 514                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 515                 return crypto_ablkcipher_encrypt(cryptd_req);
 516         } else {
 517                 struct blkcipher_desc desc;
 518                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 519                 desc.info = req->info;
 520                 desc.flags = 0;
 521                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
 522                         &desc, req->dst, req->src, req->nbytes);
 523         }
 524 }
 525
 526 static int ablk_decrypt(struct ablkcipher_request *req)
 527 {
 528         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 529         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 530
 531         if (!irq_fpu_usable()) {
 532                 struct ablkcipher_request *cryptd_req =
 533                         ablkcipher_request_ctx(req);
 534                 memcpy(cryptd_req, req, sizeof(*req));
 535                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 536                 return crypto_ablkcipher_decrypt(cryptd_req);
 537         } else {
 538                 struct blkcipher_desc desc;
 539                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 540                 desc.info = req->info;
 541                 desc.flags = 0;
 542                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
 543                         &desc, req->dst, req->src, req->nbytes);
 544         }
 545 }
 546
 547 static void ablk_exit(struct crypto_tfm *tfm)
 548 {
 549         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 550
 551         cryptd_free_ablkcipher(ctx->cryptd_tfm);
 552 }
 553
 554 static void ablk_init_common(struct crypto_tfm *tfm,
 555                              struct cryptd_ablkcipher *cryptd_tfm)
 556 {
 557         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 558
 559         ctx->cryptd_tfm = cryptd_tfm;
 560         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
 561                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
 562 }
 563
 564 static int ablk_ecb_init(struct crypto_tfm *tfm)
 565 {
 566         struct cryptd_ablkcipher *cryptd_tfm;
 567
 568         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
 569         if (IS_ERR(cryptd_tfm))
 570                 return PTR_ERR(cryptd_tfm);
 571         ablk_init_common(tfm, cryptd_tfm);
 572         return 0;
 573 }
 574
 575 static struct crypto_alg ablk_ecb_alg = {
 576         .cra_name               = "ecb(aes)",
 577         .cra_driver_name        = "ecb-aes-aesni",
 578         .cra_priority           = 400,
 579         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 580         .cra_blocksize          = AES_BLOCK_SIZE,
 581         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 582         .cra_alignmask          = 0,
 583         .cra_type               = &crypto_ablkcipher_type,
 584         .cra_module             = THIS_MODULE,
 585         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
 586         .cra_init               = ablk_ecb_init,
 587         .cra_exit               = ablk_exit,
 588         .cra_u = {
 589                 .ablkcipher = {
 590                         .min_keysize    = AES_MIN_KEY_SIZE,
 591                         .max_keysize    = AES_MAX_KEY_SIZE,
 592                         .setkey         = ablk_set_key,
 593                         .encrypt        = ablk_encrypt,
 594                         .decrypt        = ablk_decrypt,
 595                 },
 596         },
 597 };
 598
 599 static int ablk_cbc_init(struct crypto_tfm *tfm)
 600 {
 601         struct cryptd_ablkcipher *cryptd_tfm;
 602
 603         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
 604         if (IS_ERR(cryptd_tfm))
 605                 return PTR_ERR(cryptd_tfm);
 606         ablk_init_common(tfm, cryptd_tfm);
 607         return 0;
 608 }
 609
 610 static struct crypto_alg ablk_cbc_alg = {
 611         .cra_name               = "cbc(aes)",
 612         .cra_driver_name        = "cbc-aes-aesni",
 613         .cra_priority           = 400,
 614         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 615         .cra_blocksize          = AES_BLOCK_SIZE,
 616         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 617         .cra_alignmask          = 0,
 618         .cra_type               = &crypto_ablkcipher_type,
 619         .cra_module             = THIS_MODULE,
 620         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
 621         .cra_init               = ablk_cbc_init,
 622         .cra_exit               = ablk_exit,
 623         .cra_u = {
 624                 .ablkcipher = {
 625                         .min_keysize    = AES_MIN_KEY_SIZE,
 626                         .max_keysize    = AES_MAX_KEY_SIZE,
 627                         .ivsize         = AES_BLOCK_SIZE,
 628                         .setkey         = ablk_set_key,
 629                         .encrypt        = ablk_encrypt,
 630                         .decrypt        = ablk_decrypt,
 631                 },
 632         },
 633 };
 634
 635 #ifdef CONFIG_X86_64
 636 static int ablk_ctr_init(struct crypto_tfm *tfm)
 637 {
 638         struct cryptd_ablkcipher *cryptd_tfm;
 639
 640         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
 641         if (IS_ERR(cryptd_tfm))
 642                 return PTR_ERR(cryptd_tfm);
 643         ablk_init_common(tfm, cryptd_tfm);
 644         return 0;
 645 }
 646
 647 static struct crypto_alg ablk_ctr_alg = {
 648         .cra_name               = "ctr(aes)",
 649         .cra_driver_name        = "ctr-aes-aesni",
 650         .cra_priority           = 400,
 651         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 652         .cra_blocksize          = 1,
 653         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 654         .cra_alignmask          = 0,
 655         .cra_type               = &crypto_ablkcipher_type,
 656         .cra_module             = THIS_MODULE,
 657         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
 658         .cra_init               = ablk_ctr_init,
 659         .cra_exit               = ablk_exit,
 660         .cra_u = {
 661                 .ablkcipher = {
 662                         .min_keysize    = AES_MIN_KEY_SIZE,
 663                         .max_keysize    = AES_MAX_KEY_SIZE,
 664                         .ivsize         = AES_BLOCK_SIZE,
 665                         .setkey         = ablk_set_key,
 666                         .encrypt        = ablk_encrypt,
 667                         .decrypt        = ablk_encrypt,
 668                         .geniv          = "chainiv",
 669                 },
 670         },
 671 };
 672
 673 #ifdef HAS_CTR
 674 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
 675 {
 676         struct cryptd_ablkcipher *cryptd_tfm;
 677
 678         cryptd_tfm = cryptd_alloc_ablkcipher(
 679                 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
 680         if (IS_ERR(cryptd_tfm))
 681                 return PTR_ERR(cryptd_tfm);
 682         ablk_init_common(tfm, cryptd_tfm);
 683         return 0;
 684 }
 685
 686 static struct crypto_alg ablk_rfc3686_ctr_alg = {
 687         .cra_name               = "rfc3686(ctr(aes))",
 688         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
 689         .cra_priority           = 400,
 690         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 691         .cra_blocksize          = 1,
 692         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 693         .cra_alignmask          = 0,
 694         .cra_type               = &crypto_ablkcipher_type,
 695         .cra_module             = THIS_MODULE,
 696         .cra_list               = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
 697         .cra_init               = ablk_rfc3686_ctr_init,
 698         .cra_exit               = ablk_exit,
 699         .cra_u = {
 700                 .ablkcipher = {
 701                         .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 702                         .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 703                         .ivsize      = CTR_RFC3686_IV_SIZE,
 704                         .setkey      = ablk_set_key,
 705                         .encrypt     = ablk_encrypt,
 706                         .decrypt     = ablk_decrypt,
 707                         .geniv       = "seqiv",
 708                 },
 709         },
 710 };
 711 #endif
 712 #endif
 713
 714 #ifdef HAS_LRW
 715 static int ablk_lrw_init(struct crypto_tfm *tfm)
 716 {
 717         struct cryptd_ablkcipher *cryptd_tfm;
 718
 719         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
 720                                              0, 0);
 721         if (IS_ERR(cryptd_tfm))
 722                 return PTR_ERR(cryptd_tfm);
 723         ablk_init_common(tfm, cryptd_tfm);
 724         return 0;
 725 }
 726
 727 static struct crypto_alg ablk_lrw_alg = {
 728         .cra_name               = "lrw(aes)",
 729         .cra_driver_name        = "lrw-aes-aesni",
 730         .cra_priority           = 400,
 731         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 732         .cra_blocksize          = AES_BLOCK_SIZE,
 733         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 734         .cra_alignmask          = 0,
 735         .cra_type               = &crypto_ablkcipher_type,
 736         .cra_module             = THIS_MODULE,
 737         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
 738         .cra_init               = ablk_lrw_init,
 739         .cra_exit               = ablk_exit,
 740         .cra_u = {
 741                 .ablkcipher = {
 742                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
 743                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
 744                         .ivsize         = AES_BLOCK_SIZE,
 745                         .setkey         = ablk_set_key,
 746                         .encrypt        = ablk_encrypt,
 747                         .decrypt        = ablk_decrypt,
 748                 },
 749         },
 750 };
 751 #endif
 752
 753 #ifdef HAS_PCBC
 754 static int ablk_pcbc_init(struct crypto_tfm *tfm)
 755 {
 756         struct cryptd_ablkcipher *cryptd_tfm;
 757
 758         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
 759                                              0, 0);
 760         if (IS_ERR(cryptd_tfm))
 761                 return PTR_ERR(cryptd_tfm);
 762         ablk_init_common(tfm, cryptd_tfm);
 763         return 0;
 764 }
 765
 766 static struct crypto_alg ablk_pcbc_alg = {
 767         .cra_name               = "pcbc(aes)",
 768         .cra_driver_name        = "pcbc-aes-aesni",
 769         .cra_priority           = 400,
 770         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 771         .cra_blocksize          = AES_BLOCK_SIZE,
 772         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 773         .cra_alignmask          = 0,
 774         .cra_type               = &crypto_ablkcipher_type,
 775         .cra_module             = THIS_MODULE,
 776         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
 777         .cra_init               = ablk_pcbc_init,
 778         .cra_exit               = ablk_exit,
 779         .cra_u = {
 780                 .ablkcipher = {
 781                         .min_keysize    = AES_MIN_KEY_SIZE,
 782                         .max_keysize    = AES_MAX_KEY_SIZE,
 783                         .ivsize         = AES_BLOCK_SIZE,
 784                         .setkey         = ablk_set_key,
 785                         .encrypt        = ablk_encrypt,
 786                         .decrypt        = ablk_decrypt,
 787                 },
 788         },
 789 };
 790 #endif
 791
 792 #ifdef HAS_XTS
 793 static int ablk_xts_init(struct crypto_tfm *tfm)
 794 {
 795         struct cryptd_ablkcipher *cryptd_tfm;
 796
 797         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
 798                                              0, 0);
 799         if (IS_ERR(cryptd_tfm))
 800                 return PTR_ERR(cryptd_tfm);
 801         ablk_init_common(tfm, cryptd_tfm);
 802         return 0;
 803 }
 804
 805 static struct crypto_alg ablk_xts_alg = {
 806         .cra_name               = "xts(aes)",
 807         .cra_driver_name        = "xts-aes-aesni",
 808         .cra_priority           = 400,
 809         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 810         .cra_blocksize          = AES_BLOCK_SIZE,
 811         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 812         .cra_alignmask          = 0,
 813         .cra_type               = &crypto_ablkcipher_type,
 814         .cra_module             = THIS_MODULE,
 815         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
 816         .cra_init               = ablk_xts_init,
 817         .cra_exit               = ablk_exit,
 818         .cra_u = {
 819                 .ablkcipher = {
 820                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 821                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 822                         .ivsize         = AES_BLOCK_SIZE,
 823                         .setkey         = ablk_set_key,
 824                         .encrypt        = ablk_encrypt,
 825                         .decrypt        = ablk_decrypt,
 826                 },
 827         },
 828 };
 829 #endif
 830
 831 #ifdef CONFIG_X86_64
 832 static int rfc4106_init(struct crypto_tfm *tfm)
 833 {
 834         struct cryptd_aead *cryptd_tfm;
 835         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
 836                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 837         struct crypto_aead *cryptd_child;
 838         struct aesni_rfc4106_gcm_ctx *child_ctx;
 839         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
 840         if (IS_ERR(cryptd_tfm))
 841                 return PTR_ERR(cryptd_tfm);
 842
 843         cryptd_child = cryptd_aead_child(cryptd_tfm);
 844         child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
 845         memcpy(child_ctx, ctx, sizeof(*ctx));
 846         ctx->cryptd_tfm = cryptd_tfm;
 847         tfm->crt_aead.reqsize = sizeof(struct aead_request)
 848                 + crypto_aead_reqsize(&cryptd_tfm->base);
 849         return 0;
 850 }
 851
 852 static void rfc4106_exit(struct crypto_tfm *tfm)
 853 {
 854         struct aesni_rfc4106_gcm_ctx *ctx =
 855                 (struct aesni_rfc4106_gcm_ctx *)
 856                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 857         if (!IS_ERR(ctx->cryptd_tfm))
 858                 cryptd_free_aead(ctx->cryptd_tfm);
 859         return;
 860 }
 861
 862 static void
 863 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
 864 {
 865         struct aesni_gcm_set_hash_subkey_result *result = req->data;
 866
 867         if (err == -EINPROGRESS)
 868                 return;
 869         result->err = err;
 870         complete(&result->completion);
 871 }
 872
 873 static int
 874 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 875 {
 876         struct crypto_ablkcipher *ctr_tfm;
 877         struct ablkcipher_request *req;
 878         int ret = -EINVAL;
 879         struct aesni_hash_subkey_req_data *req_data;
 880
 881         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
 882         if (IS_ERR(ctr_tfm))
 883                 return PTR_ERR(ctr_tfm);
 884
 885         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
 886
 887         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
 888         if (ret)
 889                 goto out_free_ablkcipher;
 890
 891         ret = -ENOMEM;
 892         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
 893         if (!req)
 894                 goto out_free_ablkcipher;
 895
 896         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
 897         if (!req_data)
 898                 goto out_free_request;
 899
 900         memset(req_data->iv, 0, sizeof(req_data->iv));
 901
 902         /* Clear the data in the hash sub key container to zero.*/
 903         /* We want to cipher all zeros to create the hash sub key. */
 904         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 905
 906         init_completion(&req_data->result.completion);
 907         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
 908         ablkcipher_request_set_tfm(req, ctr_tfm);
 909         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
 910                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
 911                                         rfc4106_set_hash_subkey_done,
 912                                         &req_data->result);
 913
 914         ablkcipher_request_set_crypt(req, &req_data->sg,
 915                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
 916
 917         ret = crypto_ablkcipher_encrypt(req);
 918         if (ret == -EINPROGRESS || ret == -EBUSY) {
 919                 ret = wait_for_completion_interruptible
 920                         (&req_data->result.completion);
 921                 if (!ret)
 922                         ret = req_data->result.err;
 923         }
 924         kfree(req_data);
 925 out_free_request:
 926         ablkcipher_request_free(req);
 927 out_free_ablkcipher:
 928         crypto_free_ablkcipher(ctr_tfm);
 929         return ret;
 930 }
 931
 932 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
 933                                                    unsigned int key_len)
 934 {
 935         int ret = 0;
 936         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
 937         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 938         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 939         struct aesni_rfc4106_gcm_ctx *child_ctx =
 940                                  aesni_rfc4106_gcm_ctx_get(cryptd_child);
 941         u8 *new_key_mem = NULL;
 942
 943         if (key_len < 4) {
 944                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 945                 return -EINVAL;
 946         }
 947         /*Account for 4 byte nonce at the end.*/
 948         key_len -= 4;
 949         if (key_len != AES_KEYSIZE_128) {
 950                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 951                 return -EINVAL;
 952         }
 953
 954         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 955         /*This must be on a 16 byte boundary!*/
 956         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
 957                 return -EINVAL;
 958
 959         if ((unsigned long)key % AESNI_ALIGN) {
 960                 /*key is not aligned: use an auxuliar aligned pointer*/
 961                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
 962                 if (!new_key_mem)
 963                         return -ENOMEM;
 964
 965                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
 966                 memcpy(new_key_mem, key, key_len);
 967                 key = new_key_mem;
 968         }
 969
 970         if (!irq_fpu_usable())
 971                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
 972                 key, key_len);
 973         else {
 974                 kernel_fpu_begin();
 975                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
 976                 kernel_fpu_end();
 977         }
 978         /*This must be on a 16 byte boundary!*/
 979         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
 980                 ret = -EINVAL;
 981                 goto exit;
 982         }
 983         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 984         memcpy(child_ctx, ctx, sizeof(*ctx));
 985 exit:
 986         kfree(new_key_mem);
 987         return ret;
 988 }
 989
 990 /* This is the Integrity Check Value (aka the authentication tag length and can
 991  * be 8, 12 or 16 bytes long. */
 992 static int rfc4106_set_authsize(struct crypto_aead *parent,
 993                                 unsigned int authsize)
 994 {
 995         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 996         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 997
 998         switch (authsize) {
 999         case 8:
1000         case 12:
1001         case 16:
1002                 break;
1003         default:
1004                 return -EINVAL;
1005         }
1006         crypto_aead_crt(parent)->authsize = authsize;
1007         crypto_aead_crt(cryptd_child)->authsize = authsize;
1008         return 0;
1009 }
1010
1011 static int rfc4106_encrypt(struct aead_request *req)
1012 {
1013         int ret;
1014         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1015         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1016
1017         if (!irq_fpu_usable()) {
1018                 struct aead_request *cryptd_req =
1019                         (struct aead_request *) aead_request_ctx(req);
1020                 memcpy(cryptd_req, req, sizeof(*req));
1021                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1022                 return crypto_aead_encrypt(cryptd_req);
1023         } else {
1024                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1025                 kernel_fpu_begin();
1026                 ret = cryptd_child->base.crt_aead.encrypt(req);
1027                 kernel_fpu_end();
1028                 return ret;
1029         }
1030 }
1031
1032 static int rfc4106_decrypt(struct aead_request *req)
1033 {
1034         int ret;
1035         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1036         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1037
1038         if (!irq_fpu_usable()) {
1039                 struct aead_request *cryptd_req =
1040                         (struct aead_request *) aead_request_ctx(req);
1041                 memcpy(cryptd_req, req, sizeof(*req));
1042                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1043                 return crypto_aead_decrypt(cryptd_req);
1044         } else {
1045                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1046                 kernel_fpu_begin();
1047                 ret = cryptd_child->base.crt_aead.decrypt(req);
1048                 kernel_fpu_end();
1049                 return ret;
1050         }
1051 }
1052
1053 static struct crypto_alg rfc4106_alg = {
1054         .cra_name = "rfc4106(gcm(aes))",
1055         .cra_driver_name = "rfc4106-gcm-aesni",
1056         .cra_priority = 400,
1057         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1058         .cra_blocksize = 1,
1059         .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1060         .cra_alignmask = 0,
1061         .cra_type = &crypto_nivaead_type,
1062         .cra_module = THIS_MODULE,
1063         .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1064         .cra_init = rfc4106_init,
1065         .cra_exit = rfc4106_exit,
1066         .cra_u = {
1067                 .aead = {
1068                         .setkey = rfc4106_set_key,
1069                         .setauthsize = rfc4106_set_authsize,
1070                         .encrypt = rfc4106_encrypt,
1071                         .decrypt = rfc4106_decrypt,
1072                         .geniv = "seqiv",
1073                         .ivsize = 8,
1074                         .maxauthsize = 16,
1075                 },
1076         },
1077 };
1078
1079 static int __driver_rfc4106_encrypt(struct aead_request *req)
1080 {
1081         u8 one_entry_in_sg = 0;
1082         u8 *src, *dst, *assoc;
1083         __be32 counter = cpu_to_be32(1);
1084         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1085         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1086         void *aes_ctx = &(ctx->aes_key_expanded);
1087         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1088         u8 iv_tab[16+AESNI_ALIGN];
1089         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1090         struct scatter_walk src_sg_walk;
1091         struct scatter_walk assoc_sg_walk;
1092         struct scatter_walk dst_sg_walk;
1093         unsigned int i;
1094
1095         /* Assuming we are supporting rfc4106 64-bit extended */
1096         /* sequence numbers We need to have the AAD length equal */
1097         /* to 8 or 12 bytes */
1098         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1099                 return -EINVAL;
1100         /* IV below built */
1101         for (i = 0; i < 4; i++)
1102                 *(iv+i) = ctx->nonce[i];
1103         for (i = 0; i < 8; i++)
1104                 *(iv+4+i) = req->iv[i];
1105         *((__be32 *)(iv+12)) = counter;
1106
1107         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1108                 one_entry_in_sg = 1;
1109                 scatterwalk_start(&src_sg_walk, req->src);
1110                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1111                 src = scatterwalk_map(&src_sg_walk);
1112                 assoc = scatterwalk_map(&assoc_sg_walk);
1113                 dst = src;
1114                 if (unlikely(req->src != req->dst)) {
1115                         scatterwalk_start(&dst_sg_walk, req->dst);
1116                         dst = scatterwalk_map(&dst_sg_walk);
1117                 }
1118
1119         } else {
1120                 /* Allocate memory for src, dst, assoc */
1121                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1122                         GFP_ATOMIC);
1123                 if (unlikely(!src))
1124                         return -ENOMEM;
1125                 assoc = (src + req->cryptlen + auth_tag_len);
1126                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1127                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1128                                         req->assoclen, 0);
1129                 dst = src;
1130         }
1131
1132         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1133                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1134                 + ((unsigned long)req->cryptlen), auth_tag_len);
1135
1136         /* The authTag (aka the Integrity Check Value) needs to be written
1137          * back to the packet. */
1138         if (one_entry_in_sg) {
1139                 if (unlikely(req->src != req->dst)) {
1140                         scatterwalk_unmap(dst);
1141                         scatterwalk_done(&dst_sg_walk, 0, 0);
1142                 }
1143                 scatterwalk_unmap(src);
1144                 scatterwalk_unmap(assoc);
1145                 scatterwalk_done(&src_sg_walk, 0, 0);
1146                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1147         } else {
1148                 scatterwalk_map_and_copy(dst, req->dst, 0,
1149                         req->cryptlen + auth_tag_len, 1);
1150                 kfree(src);
1151         }
1152         return 0;
1153 }
1154
1155 static int __driver_rfc4106_decrypt(struct aead_request *req)
1156 {
1157         u8 one_entry_in_sg = 0;
1158         u8 *src, *dst, *assoc;
1159         unsigned long tempCipherLen = 0;
1160         __be32 counter = cpu_to_be32(1);
1161         int retval = 0;
1162         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1163         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1164         void *aes_ctx = &(ctx->aes_key_expanded);
1165         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1166         u8 iv_and_authTag[32+AESNI_ALIGN];
1167         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1168         u8 *authTag = iv + 16;
1169         struct scatter_walk src_sg_walk;
1170         struct scatter_walk assoc_sg_walk;
1171         struct scatter_walk dst_sg_walk;
1172         unsigned int i;
1173
1174         if (unlikely((req->cryptlen < auth_tag_len) ||
1175                 (req->assoclen != 8 && req->assoclen != 12)))
1176                 return -EINVAL;
1177         /* Assuming we are supporting rfc4106 64-bit extended */
1178         /* sequence numbers We need to have the AAD length */
1179         /* equal to 8 or 12 bytes */
1180
1181         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1182         /* IV below built */
1183         for (i = 0; i < 4; i++)
1184                 *(iv+i) = ctx->nonce[i];
1185         for (i = 0; i < 8; i++)
1186                 *(iv+4+i) = req->iv[i];
1187         *((__be32 *)(iv+12)) = counter;
1188
1189         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1190                 one_entry_in_sg = 1;
1191                 scatterwalk_start(&src_sg_walk, req->src);
1192                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1193                 src = scatterwalk_map(&src_sg_walk);
1194                 assoc = scatterwalk_map(&assoc_sg_walk);
1195                 dst = src;
1196                 if (unlikely(req->src != req->dst)) {
1197                         scatterwalk_start(&dst_sg_walk, req->dst);
1198                         dst = scatterwalk_map(&dst_sg_walk);
1199                 }
1200
1201         } else {
1202                 /* Allocate memory for src, dst, assoc */
1203                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1204                 if (!src)
1205                         return -ENOMEM;
1206                 assoc = (src + req->cryptlen + auth_tag_len);
1207                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1208                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1209                         req->assoclen, 0);
1210                 dst = src;
1211         }
1212
1213         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1214                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1215                 authTag, auth_tag_len);
1216
1217         /* Compare generated tag with passed in tag. */
1218         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1219                 -EBADMSG : 0;
1220
1221         if (one_entry_in_sg) {
1222                 if (unlikely(req->src != req->dst)) {
1223                         scatterwalk_unmap(dst);
1224                         scatterwalk_done(&dst_sg_walk, 0, 0);
1225                 }
1226                 scatterwalk_unmap(src);
1227                 scatterwalk_unmap(assoc);
1228                 scatterwalk_done(&src_sg_walk, 0, 0);
1229                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1230         } else {
1231                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1232                 kfree(src);
1233         }
1234         return retval;
1235 }
1236
1237 static struct crypto_alg __rfc4106_alg = {
1238         .cra_name               = "__gcm-aes-aesni",
1239         .cra_driver_name        = "__driver-gcm-aes-aesni",
1240         .cra_priority           = 0,
1241         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1242         .cra_blocksize          = 1,
1243         .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1244         .cra_alignmask          = 0,
1245         .cra_type               = &crypto_aead_type,
1246         .cra_module             = THIS_MODULE,
1247         .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1248         .cra_u = {
1249                 .aead = {
1250                         .encrypt        = __driver_rfc4106_encrypt,
1251                         .decrypt        = __driver_rfc4106_decrypt,
1252                 },
1253         },
1254 };
1255 #endif
1256
1257
1258 static const struct x86_cpu_id aesni_cpu_id[] = {
1259         X86_FEATURE_MATCH(X86_FEATURE_AES),
1260         {}
1261 };
1262 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1263
1264 static int __init aesni_init(void)
1265 {
1266         int err;
1267
1268         if (!x86_match_cpu(aesni_cpu_id))
1269                 return -ENODEV;
1270
1271         if ((err = crypto_fpu_init()))
1272                 goto fpu_err;
1273         if ((err = crypto_register_alg(&aesni_alg)))
1274                 goto aes_err;
1275         if ((err = crypto_register_alg(&__aesni_alg)))
1276                 goto __aes_err;
1277         if ((err = crypto_register_alg(&blk_ecb_alg)))
1278                 goto blk_ecb_err;
1279         if ((err = crypto_register_alg(&blk_cbc_alg)))
1280                 goto blk_cbc_err;
1281         if ((err = crypto_register_alg(&ablk_ecb_alg)))
1282                 goto ablk_ecb_err;
1283         if ((err = crypto_register_alg(&ablk_cbc_alg)))
1284                 goto ablk_cbc_err;
1285 #ifdef CONFIG_X86_64
1286         if ((err = crypto_register_alg(&blk_ctr_alg)))
1287                 goto blk_ctr_err;
1288         if ((err = crypto_register_alg(&ablk_ctr_alg)))
1289                 goto ablk_ctr_err;
1290         if ((err = crypto_register_alg(&__rfc4106_alg)))
1291                 goto __aead_gcm_err;
1292         if ((err = crypto_register_alg(&rfc4106_alg)))
1293                 goto aead_gcm_err;
1294 #ifdef HAS_CTR
1295         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1296                 goto ablk_rfc3686_ctr_err;
1297 #endif
1298 #endif
1299 #ifdef HAS_LRW
1300         if ((err = crypto_register_alg(&ablk_lrw_alg)))
1301                 goto ablk_lrw_err;
1302 #endif
1303 #ifdef HAS_PCBC
1304         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1305                 goto ablk_pcbc_err;
1306 #endif
1307 #ifdef HAS_XTS
1308         if ((err = crypto_register_alg(&ablk_xts_alg)))
1309                 goto ablk_xts_err;
1310 #endif
1311         return err;
1312
1313 #ifdef HAS_XTS
1314 ablk_xts_err:
1315 #endif
1316 #ifdef HAS_PCBC
1317         crypto_unregister_alg(&ablk_pcbc_alg);
1318 ablk_pcbc_err:
1319 #endif
1320 #ifdef HAS_LRW
1321         crypto_unregister_alg(&ablk_lrw_alg);
1322 ablk_lrw_err:
1323 #endif
1324 #ifdef CONFIG_X86_64
1325 #ifdef HAS_CTR
1326         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1327 ablk_rfc3686_ctr_err:
1328 #endif
1329         crypto_unregister_alg(&rfc4106_alg);
1330 aead_gcm_err:
1331         crypto_unregister_alg(&__rfc4106_alg);
1332 __aead_gcm_err:
1333         crypto_unregister_alg(&ablk_ctr_alg);
1334 ablk_ctr_err:
1335         crypto_unregister_alg(&blk_ctr_alg);
1336 blk_ctr_err:
1337 #endif
1338         crypto_unregister_alg(&ablk_cbc_alg);
1339 ablk_cbc_err:
1340         crypto_unregister_alg(&ablk_ecb_alg);
1341 ablk_ecb_err:
1342         crypto_unregister_alg(&blk_cbc_alg);
1343 blk_cbc_err:
1344         crypto_unregister_alg(&blk_ecb_alg);
1345 blk_ecb_err:
1346         crypto_unregister_alg(&__aesni_alg);
1347 __aes_err:
1348         crypto_unregister_alg(&aesni_alg);
1349 aes_err:
1350 fpu_err:
1351         return err;
1352 }
1353
1354 static void __exit aesni_exit(void)
1355 {
1356 #ifdef HAS_XTS
1357         crypto_unregister_alg(&ablk_xts_alg);
1358 #endif
1359 #ifdef HAS_PCBC
1360         crypto_unregister_alg(&ablk_pcbc_alg);
1361 #endif
1362 #ifdef HAS_LRW
1363         crypto_unregister_alg(&ablk_lrw_alg);
1364 #endif
1365 #ifdef CONFIG_X86_64
1366 #ifdef HAS_CTR
1367         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1368 #endif
1369         crypto_unregister_alg(&rfc4106_alg);
1370         crypto_unregister_alg(&__rfc4106_alg);
1371         crypto_unregister_alg(&ablk_ctr_alg);
1372         crypto_unregister_alg(&blk_ctr_alg);
1373 #endif
1374         crypto_unregister_alg(&ablk_cbc_alg);
1375         crypto_unregister_alg(&ablk_ecb_alg);
1376         crypto_unregister_alg(&blk_cbc_alg);
1377         crypto_unregister_alg(&blk_ecb_alg);
1378         crypto_unregister_alg(&__aesni_alg);
1379         crypto_unregister_alg(&aesni_alg);
1380
1381         crypto_fpu_exit();
1382 }
1383
1384 module_init(aesni_init);
1385 module_exit(aesni_exit);
1386
1387 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1388 MODULE_LICENSE("GPL");
1389 MODULE_ALIAS("aes");