arch/x86/crypto/serpent_sse2_glue.c

   1 /*
   2  * Glue Code for SSE2 assembler versions of Serpent Cipher
   3  *
   4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * Glue code based on aesni-intel_glue.c by:
   7  *  Copyright (C) 2008, Intel Corp.
   8  *    Author: Huang Ying <ying.huang@intel.com>
   9  *
  10  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
  11  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  12  * CTR part based on code (crypto/ctr.c) by:
  13  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  14  *
  15  * This program is free software; you can redistribute it and/or modify
  16  * it under the terms of the GNU General Public License as published by
  17  * the Free Software Foundation; either version 2 of the License, or
  18  * (at your option) any later version.
  19  *
  20  * This program is distributed in the hope that it will be useful,
  21  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  22  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  23  * GNU General Public License for more details.
  24  *
  25  * You should have received a copy of the GNU General Public License
  26  * along with this program; if not, write to the Free Software
  27  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  28  * USA
  29  *
  30  */
  31
  32 #include <linux/module.h>
  33 #include <linux/hardirq.h>
  34 #include <linux/types.h>
  35 #include <linux/crypto.h>
  36 #include <linux/err.h>
  37 #include <crypto/algapi.h>
  38 #include <crypto/serpent.h>
  39 #include <crypto/cryptd.h>
  40 #include <crypto/b128ops.h>
  41 #include <crypto/ctr.h>
  42 #include <asm/i387.h>
  43 #include <asm/serpent.h>
  44 #include <crypto/scatterwalk.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/spinlock.h>
  47
  48 struct async_serpent_ctx {
  49         struct cryptd_ablkcipher *cryptd_tfm;
  50 };
  51
  52 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
  53 {
  54         if (fpu_enabled)
  55                 return true;
  56
  57         /* SSE2 is only used when chunk to be processed is large enough, so
  58          * do not enable FPU until it is necessary.
  59          */
  60         if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS)
  61                 return false;
  62
  63         kernel_fpu_begin();
  64         return true;
  65 }
  66
  67 static inline void serpent_fpu_end(bool fpu_enabled)
  68 {
  69         if (fpu_enabled)
  70                 kernel_fpu_end();
  71 }
  72
  73 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  74                      bool enc)
  75 {
  76         bool fpu_enabled = false;
  77         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  78         const unsigned int bsize = SERPENT_BLOCK_SIZE;
  79         unsigned int nbytes;
  80         int err;
  81
  82         err = blkcipher_walk_virt(desc, walk);
  83         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  84
  85         while ((nbytes = walk->nbytes)) {
  86                 u8 *wsrc = walk->src.virt.addr;
  87                 u8 *wdst = walk->dst.virt.addr;
  88
  89                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
  90
  91                 /* Process multi-block batch */
  92                 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
  93                         do {
  94                                 if (enc)
  95                                         serpent_enc_blk_xway(ctx, wdst, wsrc);
  96                                 else
  97                                         serpent_dec_blk_xway(ctx, wdst, wsrc);
  98
  99                                 wsrc += bsize * SERPENT_PARALLEL_BLOCKS;
 100                                 wdst += bsize * SERPENT_PARALLEL_BLOCKS;
 101                                 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 102                         } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 103
 104                         if (nbytes < bsize)
 105                                 goto done;
 106                 }
 107
 108                 /* Handle leftovers */
 109                 do {
 110                         if (enc)
 111                                 __serpent_encrypt(ctx, wdst, wsrc);
 112                         else
 113                                 __serpent_decrypt(ctx, wdst, wsrc);
 114
 115                         wsrc += bsize;
 116                         wdst += bsize;
 117                         nbytes -= bsize;
 118                 } while (nbytes >= bsize);
 119
 120 done:
 121                 err = blkcipher_walk_done(desc, walk, nbytes);
 122         }
 123
 124         serpent_fpu_end(fpu_enabled);
 125         return err;
 126 }
 127
 128 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 129                        struct scatterlist *src, unsigned int nbytes)
 130 {
 131         struct blkcipher_walk walk;
 132
 133         blkcipher_walk_init(&walk, dst, src, nbytes);
 134         return ecb_crypt(desc, &walk, true);
 135 }
 136
 137 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 138                        struct scatterlist *src, unsigned int nbytes)
 139 {
 140         struct blkcipher_walk walk;
 141
 142         blkcipher_walk_init(&walk, dst, src, nbytes);
 143         return ecb_crypt(desc, &walk, false);
 144 }
 145
 146 static struct crypto_alg blk_ecb_alg = {
 147         .cra_name               = "__ecb-serpent-sse2",
 148         .cra_driver_name        = "__driver-ecb-serpent-sse2",
 149         .cra_priority           = 0,
 150         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 151         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 152         .cra_ctxsize            = sizeof(struct serpent_ctx),
 153         .cra_alignmask          = 0,
 154         .cra_type               = &crypto_blkcipher_type,
 155         .cra_module             = THIS_MODULE,
 156         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 157         .cra_u = {
 158                 .blkcipher = {
 159                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 160                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 161                         .setkey         = serpent_setkey,
 162                         .encrypt        = ecb_encrypt,
 163                         .decrypt        = ecb_decrypt,
 164                 },
 165         },
 166 };
 167
 168 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
 169                                   struct blkcipher_walk *walk)
 170 {
 171         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 172         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 173         unsigned int nbytes = walk->nbytes;
 174         u128 *src = (u128 *)walk->src.virt.addr;
 175         u128 *dst = (u128 *)walk->dst.virt.addr;
 176         u128 *iv = (u128 *)walk->iv;
 177
 178         do {
 179                 u128_xor(dst, src, iv);
 180                 __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst);
 181                 iv = dst;
 182
 183                 src += 1;
 184                 dst += 1;
 185                 nbytes -= bsize;
 186         } while (nbytes >= bsize);
 187
 188         u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
 189         return nbytes;
 190 }
 191
 192 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 193                        struct scatterlist *src, unsigned int nbytes)
 194 {
 195         struct blkcipher_walk walk;
 196         int err;
 197
 198         blkcipher_walk_init(&walk, dst, src, nbytes);
 199         err = blkcipher_walk_virt(desc, &walk);
 200
 201         while ((nbytes = walk.nbytes)) {
 202                 nbytes = __cbc_encrypt(desc, &walk);
 203                 err = blkcipher_walk_done(desc, &walk, nbytes);
 204         }
 205
 206         return err;
 207 }
 208
 209 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
 210                                   struct blkcipher_walk *walk)
 211 {
 212         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 213         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 214         unsigned int nbytes = walk->nbytes;
 215         u128 *src = (u128 *)walk->src.virt.addr;
 216         u128 *dst = (u128 *)walk->dst.virt.addr;
 217         u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
 218         u128 last_iv;
 219         int i;
 220
 221         /* Start of the last block. */
 222         src += nbytes / bsize - 1;
 223         dst += nbytes / bsize - 1;
 224
 225         last_iv = *src;
 226
 227         /* Process multi-block batch */
 228         if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
 229                 do {
 230                         nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1);
 231                         src -= SERPENT_PARALLEL_BLOCKS - 1;
 232                         dst -= SERPENT_PARALLEL_BLOCKS - 1;
 233
 234                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
 235                                 ivs[i] = src[i];
 236
 237                         serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
 238
 239                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
 240                                 u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
 241
 242                         nbytes -= bsize;
 243                         if (nbytes < bsize)
 244                                 goto done;
 245
 246                         u128_xor(dst, dst, src - 1);
 247                         src -= 1;
 248                         dst -= 1;
 249                 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 250
 251                 if (nbytes < bsize)
 252                         goto done;
 253         }
 254
 255         /* Handle leftovers */
 256         for (;;) {
 257                 __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src);
 258
 259                 nbytes -= bsize;
 260                 if (nbytes < bsize)
 261                         break;
 262
 263                 u128_xor(dst, dst, src - 1);
 264                 src -= 1;
 265                 dst -= 1;
 266         }
 267
 268 done:
 269         u128_xor(dst, dst, (u128 *)walk->iv);
 270         *(u128 *)walk->iv = last_iv;
 271
 272         return nbytes;
 273 }
 274
 275 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 276                        struct scatterlist *src, unsigned int nbytes)
 277 {
 278         bool fpu_enabled = false;
 279         struct blkcipher_walk walk;
 280         int err;
 281
 282         blkcipher_walk_init(&walk, dst, src, nbytes);
 283         err = blkcipher_walk_virt(desc, &walk);
 284         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 285
 286         while ((nbytes = walk.nbytes)) {
 287                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
 288                 nbytes = __cbc_decrypt(desc, &walk);
 289                 err = blkcipher_walk_done(desc, &walk, nbytes);
 290         }
 291
 292         serpent_fpu_end(fpu_enabled);
 293         return err;
 294 }
 295
 296 static struct crypto_alg blk_cbc_alg = {
 297         .cra_name               = "__cbc-serpent-sse2",
 298         .cra_driver_name        = "__driver-cbc-serpent-sse2",
 299         .cra_priority           = 0,
 300         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 301         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 302         .cra_ctxsize            = sizeof(struct serpent_ctx),
 303         .cra_alignmask          = 0,
 304         .cra_type               = &crypto_blkcipher_type,
 305         .cra_module             = THIS_MODULE,
 306         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 307         .cra_u = {
 308                 .blkcipher = {
 309                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 310                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 311                         .setkey         = serpent_setkey,
 312                         .encrypt        = cbc_encrypt,
 313                         .decrypt        = cbc_decrypt,
 314                 },
 315         },
 316 };
 317
 318 static inline void u128_to_be128(be128 *dst, const u128 *src)
 319 {
 320         dst->a = cpu_to_be64(src->a);
 321         dst->b = cpu_to_be64(src->b);
 322 }
 323
 324 static inline void be128_to_u128(u128 *dst, const be128 *src)
 325 {
 326         dst->a = be64_to_cpu(src->a);
 327         dst->b = be64_to_cpu(src->b);
 328 }
 329
 330 static inline void u128_inc(u128 *i)
 331 {
 332         i->b++;
 333         if (!i->b)
 334                 i->a++;
 335 }
 336
 337 static void ctr_crypt_final(struct blkcipher_desc *desc,
 338                             struct blkcipher_walk *walk)
 339 {
 340         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 341         u8 *ctrblk = walk->iv;
 342         u8 keystream[SERPENT_BLOCK_SIZE];
 343         u8 *src = walk->src.virt.addr;
 344         u8 *dst = walk->dst.virt.addr;
 345         unsigned int nbytes = walk->nbytes;
 346
 347         __serpent_encrypt(ctx, keystream, ctrblk);
 348         crypto_xor(keystream, src, nbytes);
 349         memcpy(dst, keystream, nbytes);
 350
 351         crypto_inc(ctrblk, SERPENT_BLOCK_SIZE);
 352 }
 353
 354 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
 355                                 struct blkcipher_walk *walk)
 356 {
 357         struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 358         const unsigned int bsize = SERPENT_BLOCK_SIZE;
 359         unsigned int nbytes = walk->nbytes;
 360         u128 *src = (u128 *)walk->src.virt.addr;
 361         u128 *dst = (u128 *)walk->dst.virt.addr;
 362         u128 ctrblk;
 363         be128 ctrblocks[SERPENT_PARALLEL_BLOCKS];
 364         int i;
 365
 366         be128_to_u128(&ctrblk, (be128 *)walk->iv);
 367
 368         /* Process multi-block batch */
 369         if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
 370                 do {
 371                         /* create ctrblks for parallel encrypt */
 372                         for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
 373                                 if (dst != src)
 374                                         dst[i] = src[i];
 375
 376                                 u128_to_be128(&ctrblocks[i], &ctrblk);
 377                                 u128_inc(&ctrblk);
 378                         }
 379
 380                         serpent_enc_blk_xway_xor(ctx, (u8 *)dst,
 381                                                  (u8 *)ctrblocks);
 382
 383                         src += SERPENT_PARALLEL_BLOCKS;
 384                         dst += SERPENT_PARALLEL_BLOCKS;
 385                         nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
 386                 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
 387
 388                 if (nbytes < bsize)
 389                         goto done;
 390         }
 391
 392         /* Handle leftovers */
 393         do {
 394                 if (dst != src)
 395                         *dst = *src;
 396
 397                 u128_to_be128(&ctrblocks[0], &ctrblk);
 398                 u128_inc(&ctrblk);
 399
 400                 __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
 401                 u128_xor(dst, dst, (u128 *)ctrblocks);
 402
 403                 src += 1;
 404                 dst += 1;
 405                 nbytes -= bsize;
 406         } while (nbytes >= bsize);
 407
 408 done:
 409         u128_to_be128((be128 *)walk->iv, &ctrblk);
 410         return nbytes;
 411 }
 412
 413 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 414                      struct scatterlist *src, unsigned int nbytes)
 415 {
 416         bool fpu_enabled = false;
 417         struct blkcipher_walk walk;
 418         int err;
 419
 420         blkcipher_walk_init(&walk, dst, src, nbytes);
 421         err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE);
 422         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 423
 424         while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) {
 425                 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
 426                 nbytes = __ctr_crypt(desc, &walk);
 427                 err = blkcipher_walk_done(desc, &walk, nbytes);
 428         }
 429
 430         serpent_fpu_end(fpu_enabled);
 431
 432         if (walk.nbytes) {
 433                 ctr_crypt_final(desc, &walk);
 434                 err = blkcipher_walk_done(desc, &walk, 0);
 435         }
 436
 437         return err;
 438 }
 439
 440 static struct crypto_alg blk_ctr_alg = {
 441         .cra_name               = "__ctr-serpent-sse2",
 442         .cra_driver_name        = "__driver-ctr-serpent-sse2",
 443         .cra_priority           = 0,
 444         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 445         .cra_blocksize          = 1,
 446         .cra_ctxsize            = sizeof(struct serpent_ctx),
 447         .cra_alignmask          = 0,
 448         .cra_type               = &crypto_blkcipher_type,
 449         .cra_module             = THIS_MODULE,
 450         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 451         .cra_u = {
 452                 .blkcipher = {
 453                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 454                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 455                         .ivsize         = SERPENT_BLOCK_SIZE,
 456                         .setkey         = serpent_setkey,
 457                         .encrypt        = ctr_crypt,
 458                         .decrypt        = ctr_crypt,
 459                 },
 460         },
 461 };
 462
 463 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
 464                         unsigned int key_len)
 465 {
 466         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 467         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
 468         int err;
 469
 470         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 471         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
 472                                     & CRYPTO_TFM_REQ_MASK);
 473         err = crypto_ablkcipher_setkey(child, key, key_len);
 474         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
 475                                     & CRYPTO_TFM_RES_MASK);
 476         return err;
 477 }
 478
 479 static int __ablk_encrypt(struct ablkcipher_request *req)
 480 {
 481         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 482         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 483         struct blkcipher_desc desc;
 484
 485         desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 486         desc.info = req->info;
 487         desc.flags = 0;
 488
 489         return crypto_blkcipher_crt(desc.tfm)->encrypt(
 490                 &desc, req->dst, req->src, req->nbytes);
 491 }
 492
 493 static int ablk_encrypt(struct ablkcipher_request *req)
 494 {
 495         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 496         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 497
 498         if (!irq_fpu_usable()) {
 499                 struct ablkcipher_request *cryptd_req =
 500                         ablkcipher_request_ctx(req);
 501
 502                 memcpy(cryptd_req, req, sizeof(*req));
 503                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 504
 505                 return crypto_ablkcipher_encrypt(cryptd_req);
 506         } else {
 507                 return __ablk_encrypt(req);
 508         }
 509 }
 510
 511 static int ablk_decrypt(struct ablkcipher_request *req)
 512 {
 513         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 514         struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 515
 516         if (!irq_fpu_usable()) {
 517                 struct ablkcipher_request *cryptd_req =
 518                         ablkcipher_request_ctx(req);
 519
 520                 memcpy(cryptd_req, req, sizeof(*req));
 521                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 522
 523                 return crypto_ablkcipher_decrypt(cryptd_req);
 524         } else {
 525                 struct blkcipher_desc desc;
 526
 527                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 528                 desc.info = req->info;
 529                 desc.flags = 0;
 530
 531                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
 532                         &desc, req->dst, req->src, req->nbytes);
 533         }
 534 }
 535
 536 static void ablk_exit(struct crypto_tfm *tfm)
 537 {
 538         struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
 539
 540         cryptd_free_ablkcipher(ctx->cryptd_tfm);
 541 }
 542
 543 static void ablk_init_common(struct crypto_tfm *tfm,
 544                              struct cryptd_ablkcipher *cryptd_tfm)
 545 {
 546         struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
 547
 548         ctx->cryptd_tfm = cryptd_tfm;
 549         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
 550                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
 551 }
 552
 553 static int ablk_ecb_init(struct crypto_tfm *tfm)
 554 {
 555         struct cryptd_ablkcipher *cryptd_tfm;
 556
 557         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-serpent-sse2", 0, 0);
 558         if (IS_ERR(cryptd_tfm))
 559                 return PTR_ERR(cryptd_tfm);
 560         ablk_init_common(tfm, cryptd_tfm);
 561         return 0;
 562 }
 563
 564 static struct crypto_alg ablk_ecb_alg = {
 565         .cra_name               = "ecb(serpent)",
 566         .cra_driver_name        = "ecb-serpent-sse2",
 567         .cra_priority           = 400,
 568         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 569         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 570         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 571         .cra_alignmask          = 0,
 572         .cra_type               = &crypto_ablkcipher_type,
 573         .cra_module             = THIS_MODULE,
 574         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
 575         .cra_init               = ablk_ecb_init,
 576         .cra_exit               = ablk_exit,
 577         .cra_u = {
 578                 .ablkcipher = {
 579                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 580                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 581                         .setkey         = ablk_set_key,
 582                         .encrypt        = ablk_encrypt,
 583                         .decrypt        = ablk_decrypt,
 584                 },
 585         },
 586 };
 587
 588 static int ablk_cbc_init(struct crypto_tfm *tfm)
 589 {
 590         struct cryptd_ablkcipher *cryptd_tfm;
 591
 592         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-serpent-sse2", 0, 0);
 593         if (IS_ERR(cryptd_tfm))
 594                 return PTR_ERR(cryptd_tfm);
 595         ablk_init_common(tfm, cryptd_tfm);
 596         return 0;
 597 }
 598
 599 static struct crypto_alg ablk_cbc_alg = {
 600         .cra_name               = "cbc(serpent)",
 601         .cra_driver_name        = "cbc-serpent-sse2",
 602         .cra_priority           = 400,
 603         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 604         .cra_blocksize          = SERPENT_BLOCK_SIZE,
 605         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 606         .cra_alignmask          = 0,
 607         .cra_type               = &crypto_ablkcipher_type,
 608         .cra_module             = THIS_MODULE,
 609         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
 610         .cra_init               = ablk_cbc_init,
 611         .cra_exit               = ablk_exit,
 612         .cra_u = {
 613                 .ablkcipher = {
 614                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 615                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 616                         .ivsize         = SERPENT_BLOCK_SIZE,
 617                         .setkey         = ablk_set_key,
 618                         .encrypt        = __ablk_encrypt,
 619                         .decrypt        = ablk_decrypt,
 620                 },
 621         },
 622 };
 623
 624 static int ablk_ctr_init(struct crypto_tfm *tfm)
 625 {
 626         struct cryptd_ablkcipher *cryptd_tfm;
 627
 628         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-serpent-sse2", 0, 0);
 629         if (IS_ERR(cryptd_tfm))
 630                 return PTR_ERR(cryptd_tfm);
 631         ablk_init_common(tfm, cryptd_tfm);
 632         return 0;
 633 }
 634
 635 static struct crypto_alg ablk_ctr_alg = {
 636         .cra_name               = "ctr(serpent)",
 637         .cra_driver_name        = "ctr-serpent-sse2",
 638         .cra_priority           = 400,
 639         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 640         .cra_blocksize          = 1,
 641         .cra_ctxsize            = sizeof(struct async_serpent_ctx),
 642         .cra_alignmask          = 0,
 643         .cra_type               = &crypto_ablkcipher_type,
 644         .cra_module             = THIS_MODULE,
 645         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
 646         .cra_init               = ablk_ctr_init,
 647         .cra_exit               = ablk_exit,
 648         .cra_u = {
 649                 .ablkcipher = {
 650                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
 651                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
 652                         .ivsize         = SERPENT_BLOCK_SIZE,
 653                         .setkey         = ablk_set_key,
 654                         .encrypt        = ablk_encrypt,
 655                         .decrypt        = ablk_encrypt,
 656                         .geniv          = "chainiv",
 657                 },
 658         },
 659 };
 660
 661 static int __init serpent_sse2_init(void)
 662 {
 663         int err;
 664
 665         if (!cpu_has_xmm2) {
 666                 printk(KERN_INFO "SSE2 instructions are not detected.\n");
 667                 return -ENODEV;
 668         }
 669
 670         err = crypto_register_alg(&blk_ecb_alg);
 671         if (err)
 672                 goto blk_ecb_err;
 673         err = crypto_register_alg(&blk_cbc_alg);
 674         if (err)
 675                 goto blk_cbc_err;
 676         err = crypto_register_alg(&blk_ctr_alg);
 677         if (err)
 678                 goto blk_ctr_err;
 679         err = crypto_register_alg(&ablk_ecb_alg);
 680         if (err)
 681                 goto ablk_ecb_err;
 682         err = crypto_register_alg(&ablk_cbc_alg);
 683         if (err)
 684                 goto ablk_cbc_err;
 685         err = crypto_register_alg(&ablk_ctr_alg);
 686         if (err)
 687                 goto ablk_ctr_err;
 688         return err;
 689
 690 ablk_ctr_err:
 691         crypto_unregister_alg(&ablk_cbc_alg);
 692 ablk_cbc_err:
 693         crypto_unregister_alg(&ablk_ecb_alg);
 694 ablk_ecb_err:
 695         crypto_unregister_alg(&blk_ctr_alg);
 696 blk_ctr_err:
 697         crypto_unregister_alg(&blk_cbc_alg);
 698 blk_cbc_err:
 699         crypto_unregister_alg(&blk_ecb_alg);
 700 blk_ecb_err:
 701         return err;
 702 }
 703
 704 static void __exit serpent_sse2_exit(void)
 705 {
 706         crypto_unregister_alg(&ablk_ctr_alg);
 707         crypto_unregister_alg(&ablk_cbc_alg);
 708         crypto_unregister_alg(&ablk_ecb_alg);
 709         crypto_unregister_alg(&blk_ctr_alg);
 710         crypto_unregister_alg(&blk_cbc_alg);
 711         crypto_unregister_alg(&blk_ecb_alg);
 712 }
 713
 714 module_init(serpent_sse2_init);
 715 module_exit(serpent_sse2_exit);
 716
 717 MODULE_DESCRIPTION("Serpent Cipher Algorithm, SSE2 optimized");
 718 MODULE_LICENSE("GPL");
 719 MODULE_ALIAS("serpent");