drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/io.h>
  38 #include <linux/spinlock.h>
  39 #include <linux/rtnetlink.h>
  40 #include <linux/slab.h>
  41
  42 #include <crypto/algapi.h>
  43 #include <crypto/aes.h>
  44 #include <crypto/des.h>
  45 #include <crypto/sha.h>
  46 #include <crypto/md5.h>
  47 #include <crypto/aead.h>
  48 #include <crypto/authenc.h>
  49 #include <crypto/skcipher.h>
  50 #include <crypto/hash.h>
  51 #include <crypto/internal/hash.h>
  52 #include <crypto/scatterwalk.h>
  53
  54 #include "talitos.h"
  55
  56 #define TALITOS_TIMEOUT 100000
  57 #define TALITOS_MAX_DATA_LEN 65535
  58
  59 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
  60 #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
  61 #define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
  62
  63 /* descriptor pointer entry */
  64 struct talitos_ptr {
  65         __be16 len;     /* length */
  66         u8 j_extent;    /* jump to sg link table and/or extent */
  67         u8 eptr;        /* extended address */
  68         __be32 ptr;     /* address */
  69 };
  70
  71 static const struct talitos_ptr zero_entry = {
  72         .len = 0,
  73         .j_extent = 0,
  74         .eptr = 0,
  75         .ptr = 0
  76 };
  77
  78 /* descriptor */
  79 struct talitos_desc {
  80         __be32 hdr;                     /* header high bits */
  81         __be32 hdr_lo;                  /* header low bits */
  82         struct talitos_ptr ptr[7];      /* ptr/len pair array */
  83 };
  84
  85 /**
  86  * talitos_request - descriptor submission request
  87  * @desc: descriptor pointer (kernel virtual)
  88  * @dma_desc: descriptor's physical bus address
  89  * @callback: whom to call when descriptor processing is done
  90  * @context: caller context (optional)
  91  */
  92 struct talitos_request {
  93         struct talitos_desc *desc;
  94         dma_addr_t dma_desc;
  95         void (*callback) (struct device *dev, struct talitos_desc *desc,
  96                           void *context, int error);
  97         void *context;
  98 };
  99
 100 /* per-channel fifo management */
 101 struct talitos_channel {
 102         void __iomem *reg;
 103
 104         /* request fifo */
 105         struct talitos_request *fifo;
 106
 107         /* number of requests pending in channel h/w fifo */
 108         atomic_t submit_count ____cacheline_aligned;
 109
 110         /* request submission (head) lock */
 111         spinlock_t head_lock ____cacheline_aligned;
 112         /* index to next free descriptor request */
 113         int head;
 114
 115         /* request release (tail) lock */
 116         spinlock_t tail_lock ____cacheline_aligned;
 117         /* index to next in-progress/done descriptor request */
 118         int tail;
 119 };
 120
 121 struct talitos_private {
 122         struct device *dev;
 123         struct platform_device *ofdev;
 124         void __iomem *reg;
 125         int irq[2];
 126
 127         /* SEC version geometry (from device tree node) */
 128         unsigned int num_channels;
 129         unsigned int chfifo_len;
 130         unsigned int exec_units;
 131         unsigned int desc_types;
 132
 133         /* SEC Compatibility info */
 134         unsigned long features;
 135
 136         /*
 137          * length of the request fifo
 138          * fifo_len is chfifo_len rounded up to next power of 2
 139          * so we can use bitwise ops to wrap
 140          */
 141         unsigned int fifo_len;
 142
 143         struct talitos_channel *chan;
 144
 145         /* next channel to be assigned next incoming descriptor */
 146         atomic_t last_chan ____cacheline_aligned;
 147
 148         /* request callback tasklet */
 149         struct tasklet_struct done_task[2];
 150
 151         /* list of registered algorithms */
 152         struct list_head alg_list;
 153
 154         /* hwrng device */
 155         struct hwrng rng;
 156 };
 157
 158 /* .features flag */
 159 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
 160 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
 161 #define TALITOS_FTR_SHA224_HWINIT 0x00000004
 162 #define TALITOS_FTR_HMAC_OK 0x00000008
 163
 164 static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr)
 165 {
 166         talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
 167         talitos_ptr->eptr = upper_32_bits(dma_addr);
 168 }
 169
 170 /*
 171  * map virtual single (contiguous) pointer to h/w descriptor pointer
 172  */
 173 static void map_single_talitos_ptr(struct device *dev,
 174                                    struct talitos_ptr *talitos_ptr,
 175                                    unsigned short len, void *data,
 176                                    unsigned char extent,
 177                                    enum dma_data_direction dir)
 178 {
 179         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
 180
 181         talitos_ptr->len = cpu_to_be16(len);
 182         to_talitos_ptr(talitos_ptr, dma_addr);
 183         talitos_ptr->j_extent = extent;
 184 }
 185
 186 /*
 187  * unmap bus single (contiguous) h/w descriptor pointer
 188  */
 189 static void unmap_single_talitos_ptr(struct device *dev,
 190                                      struct talitos_ptr *talitos_ptr,
 191                                      enum dma_data_direction dir)
 192 {
 193         dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
 194                          be16_to_cpu(talitos_ptr->len), dir);
 195 }
 196
 197 static int reset_channel(struct device *dev, int ch)
 198 {
 199         struct talitos_private *priv = dev_get_drvdata(dev);
 200         unsigned int timeout = TALITOS_TIMEOUT;
 201
 202         setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET);
 203
 204         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET)
 205                && --timeout)
 206                 cpu_relax();
 207
 208         if (timeout == 0) {
 209                 dev_err(dev, "failed to reset channel %d\n", ch);
 210                 return -EIO;
 211         }
 212
 213         /* set 36-bit addressing, done writeback enable and done IRQ enable */
 214         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
 215                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
 216
 217         /* and ICCR writeback, if available */
 218         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 219                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
 220                           TALITOS_CCCR_LO_IWSE);
 221
 222         return 0;
 223 }
 224
 225 static int reset_device(struct device *dev)
 226 {
 227         struct talitos_private *priv = dev_get_drvdata(dev);
 228         unsigned int timeout = TALITOS_TIMEOUT;
 229         u32 mcr = TALITOS_MCR_SWR;
 230
 231         setbits32(priv->reg + TALITOS_MCR, mcr);
 232
 233         while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
 234                && --timeout)
 235                 cpu_relax();
 236
 237         if (priv->irq[1]) {
 238                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
 239                 setbits32(priv->reg + TALITOS_MCR, mcr);
 240         }
 241
 242         if (timeout == 0) {
 243                 dev_err(dev, "failed to reset device\n");
 244                 return -EIO;
 245         }
 246
 247         return 0;
 248 }
 249
 250 /*
 251  * Reset and initialize the device
 252  */
 253 static int init_device(struct device *dev)
 254 {
 255         struct talitos_private *priv = dev_get_drvdata(dev);
 256         int ch, err;
 257
 258         /*
 259          * Master reset
 260          * errata documentation: warning: certain SEC interrupts
 261          * are not fully cleared by writing the MCR:SWR bit,
 262          * set bit twice to completely reset
 263          */
 264         err = reset_device(dev);
 265         if (err)
 266                 return err;
 267
 268         err = reset_device(dev);
 269         if (err)
 270                 return err;
 271
 272         /* reset channels */
 273         for (ch = 0; ch < priv->num_channels; ch++) {
 274                 err = reset_channel(dev, ch);
 275                 if (err)
 276                         return err;
 277         }
 278
 279         /* enable channel done and error interrupts */
 280         setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
 281         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
 282
 283         /* disable integrity check error interrupts (use writeback instead) */
 284         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 285                 setbits32(priv->reg + TALITOS_MDEUICR_LO,
 286                           TALITOS_MDEUICR_LO_ICE);
 287
 288         return 0;
 289 }
 290
 291 /**
 292  * talitos_submit - submits a descriptor to the device for processing
 293  * @dev:        the SEC device to be used
 294  * @ch:         the SEC device channel to be used
 295  * @desc:       the descriptor to be processed by the device
 296  * @callback:   whom to call when processing is complete
 297  * @context:    a handle for use by caller (optional)
 298  *
 299  * desc must contain valid dma-mapped (bus physical) address pointers.
 300  * callback must check err and feedback in descriptor header
 301  * for device processing status.
 302  */
 303 static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 304                           void (*callback)(struct device *dev,
 305                                            struct talitos_desc *desc,
 306                                            void *context, int error),
 307                           void *context)
 308 {
 309         struct talitos_private *priv = dev_get_drvdata(dev);
 310         struct talitos_request *request;
 311         unsigned long flags;
 312         int head;
 313
 314         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 315
 316         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 317                 /* h/w fifo is full */
 318                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 319                 return -EAGAIN;
 320         }
 321
 322         head = priv->chan[ch].head;
 323         request = &priv->chan[ch].fifo[head];
 324
 325         /* map descriptor and save caller data */
 326         request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
 327                                            DMA_BIDIRECTIONAL);
 328         request->callback = callback;
 329         request->context = context;
 330
 331         /* increment fifo head */
 332         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 333
 334         smp_wmb();
 335         request->desc = desc;
 336
 337         /* GO! */
 338         wmb();
 339         out_be32(priv->chan[ch].reg + TALITOS_FF,
 340                  upper_32_bits(request->dma_desc));
 341         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
 342                  lower_32_bits(request->dma_desc));
 343
 344         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 345
 346         return -EINPROGRESS;
 347 }
 348
 349 /*
 350  * process what was done, notify callback of error if not
 351  */
 352 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 353 {
 354         struct talitos_private *priv = dev_get_drvdata(dev);
 355         struct talitos_request *request, saved_req;
 356         unsigned long flags;
 357         int tail, status;
 358
 359         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 360
 361         tail = priv->chan[ch].tail;
 362         while (priv->chan[ch].fifo[tail].desc) {
 363                 request = &priv->chan[ch].fifo[tail];
 364
 365                 /* descriptors with their done bits set don't get the error */
 366                 rmb();
 367                 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 368                         status = 0;
 369                 else
 370                         if (!error)
 371                                 break;
 372                         else
 373                                 status = error;
 374
 375                 dma_unmap_single(dev, request->dma_desc,
 376                                  sizeof(struct talitos_desc),
 377                                  DMA_BIDIRECTIONAL);
 378
 379                 /* copy entries so we can call callback outside lock */
 380                 saved_req.desc = request->desc;
 381                 saved_req.callback = request->callback;
 382                 saved_req.context = request->context;
 383
 384                 /* release request entry in fifo */
 385                 smp_wmb();
 386                 request->desc = NULL;
 387
 388                 /* increment fifo tail */
 389                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 390
 391                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 392
 393                 atomic_dec(&priv->chan[ch].submit_count);
 394
 395                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 396                                    status);
 397                 /* channel may resume processing in single desc error case */
 398                 if (error && !reset_ch && status == error)
 399                         return;
 400                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 401                 tail = priv->chan[ch].tail;
 402         }
 403
 404         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 405 }
 406
 407 /*
 408  * process completed requests for channels that have done status
 409  */
 410 #define DEF_TALITOS_DONE(name, ch_done_mask)                            \
 411 static void talitos_done_##name(unsigned long data)                     \
 412 {                                                                       \
 413         struct device *dev = (struct device *)data;                     \
 414         struct talitos_private *priv = dev_get_drvdata(dev);            \
 415                                                                         \
 416         if (ch_done_mask & 1)                                           \
 417                 flush_channel(dev, 0, 0, 0);                            \
 418         if (priv->num_channels == 1)                                    \
 419                 goto out;                                               \
 420         if (ch_done_mask & (1 << 2))                                    \
 421                 flush_channel(dev, 1, 0, 0);                            \
 422         if (ch_done_mask & (1 << 4))                                    \
 423                 flush_channel(dev, 2, 0, 0);                            \
 424         if (ch_done_mask & (1 << 6))                                    \
 425                 flush_channel(dev, 3, 0, 0);                            \
 426                                                                         \
 427 out:                                                                    \
 428         /* At this point, all completed channels have been processed */ \
 429         /* Unmask done interrupts for channels completed later on. */   \
 430         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
 431         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);     \
 432 }
 433 DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE)
 434 DEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE)
 435 DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE)
 436
 437 /*
 438  * locate current (offending) descriptor
 439  */
 440 static u32 current_desc_hdr(struct device *dev, int ch)
 441 {
 442         struct talitos_private *priv = dev_get_drvdata(dev);
 443         int tail = priv->chan[ch].tail;
 444         dma_addr_t cur_desc;
 445
 446         cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
 447
 448         while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) {
 449                 tail = (tail + 1) & (priv->fifo_len - 1);
 450                 if (tail == priv->chan[ch].tail) {
 451                         dev_err(dev, "couldn't locate current descriptor\n");
 452                         return 0;
 453                 }
 454         }
 455
 456         return priv->chan[ch].fifo[tail].desc->hdr;
 457 }
 458
 459 /*
 460  * user diagnostics; report root cause of error based on execution unit status
 461  */
 462 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
 463 {
 464         struct talitos_private *priv = dev_get_drvdata(dev);
 465         int i;
 466
 467         if (!desc_hdr)
 468                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
 469
 470         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
 471         case DESC_HDR_SEL0_AFEU:
 472                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 473                         in_be32(priv->reg + TALITOS_AFEUISR),
 474                         in_be32(priv->reg + TALITOS_AFEUISR_LO));
 475                 break;
 476         case DESC_HDR_SEL0_DEU:
 477                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 478                         in_be32(priv->reg + TALITOS_DEUISR),
 479                         in_be32(priv->reg + TALITOS_DEUISR_LO));
 480                 break;
 481         case DESC_HDR_SEL0_MDEUA:
 482         case DESC_HDR_SEL0_MDEUB:
 483                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 484                         in_be32(priv->reg + TALITOS_MDEUISR),
 485                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 486                 break;
 487         case DESC_HDR_SEL0_RNG:
 488                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 489                         in_be32(priv->reg + TALITOS_RNGUISR),
 490                         in_be32(priv->reg + TALITOS_RNGUISR_LO));
 491                 break;
 492         case DESC_HDR_SEL0_PKEU:
 493                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 494                         in_be32(priv->reg + TALITOS_PKEUISR),
 495                         in_be32(priv->reg + TALITOS_PKEUISR_LO));
 496                 break;
 497         case DESC_HDR_SEL0_AESU:
 498                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 499                         in_be32(priv->reg + TALITOS_AESUISR),
 500                         in_be32(priv->reg + TALITOS_AESUISR_LO));
 501                 break;
 502         case DESC_HDR_SEL0_CRCU:
 503                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 504                         in_be32(priv->reg + TALITOS_CRCUISR),
 505                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 506                 break;
 507         case DESC_HDR_SEL0_KEU:
 508                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 509                         in_be32(priv->reg + TALITOS_KEUISR),
 510                         in_be32(priv->reg + TALITOS_KEUISR_LO));
 511                 break;
 512         }
 513
 514         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
 515         case DESC_HDR_SEL1_MDEUA:
 516         case DESC_HDR_SEL1_MDEUB:
 517                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 518                         in_be32(priv->reg + TALITOS_MDEUISR),
 519                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 520                 break;
 521         case DESC_HDR_SEL1_CRCU:
 522                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 523                         in_be32(priv->reg + TALITOS_CRCUISR),
 524                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 525                 break;
 526         }
 527
 528         for (i = 0; i < 8; i++)
 529                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 530                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
 531                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
 532 }
 533
 534 /*
 535  * recover from error interrupts
 536  */
 537 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
 538 {
 539         struct talitos_private *priv = dev_get_drvdata(dev);
 540         unsigned int timeout = TALITOS_TIMEOUT;
 541         int ch, error, reset_dev = 0, reset_ch = 0;
 542         u32 v, v_lo;
 543
 544         for (ch = 0; ch < priv->num_channels; ch++) {
 545                 /* skip channels without errors */
 546                 if (!(isr & (1 << (ch * 2 + 1))))
 547                         continue;
 548
 549                 error = -EINVAL;
 550
 551                 v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR);
 552                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
 553
 554                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 555                         dev_err(dev, "double fetch fifo overflow error\n");
 556                         error = -EAGAIN;
 557                         reset_ch = 1;
 558                 }
 559                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 560                         /* h/w dropped descriptor */
 561                         dev_err(dev, "single fetch fifo overflow error\n");
 562                         error = -EAGAIN;
 563                 }
 564                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 565                         dev_err(dev, "master data transfer error\n");
 566                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 567                         dev_err(dev, "s/g data length zero error\n");
 568                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 569                         dev_err(dev, "fetch pointer zero error\n");
 570                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 571                         dev_err(dev, "illegal descriptor header error\n");
 572                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 573                         dev_err(dev, "invalid execution unit error\n");
 574                 if (v_lo & TALITOS_CCPSR_LO_EU)
 575                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
 576                 if (v_lo & TALITOS_CCPSR_LO_GB)
 577                         dev_err(dev, "gather boundary error\n");
 578                 if (v_lo & TALITOS_CCPSR_LO_GRL)
 579                         dev_err(dev, "gather return/length error\n");
 580                 if (v_lo & TALITOS_CCPSR_LO_SB)
 581                         dev_err(dev, "scatter boundary error\n");
 582                 if (v_lo & TALITOS_CCPSR_LO_SRL)
 583                         dev_err(dev, "scatter return/length error\n");
 584
 585                 flush_channel(dev, ch, error, reset_ch);
 586
 587                 if (reset_ch) {
 588                         reset_channel(dev, ch);
 589                 } else {
 590                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
 591                                   TALITOS_CCCR_CONT);
 592                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
 593                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
 594                                TALITOS_CCCR_CONT) && --timeout)
 595                                 cpu_relax();
 596                         if (timeout == 0) {
 597                                 dev_err(dev, "failed to restart channel %d\n",
 598                                         ch);
 599                                 reset_dev = 1;
 600                         }
 601                 }
 602         }
 603         if (reset_dev || isr & ~TALITOS_ISR_4CHERR || isr_lo) {
 604                 dev_err(dev, "done overflow, internal time out, or rngu error: "
 605                         "ISR 0x%08x_%08x\n", isr, isr_lo);
 606
 607                 /* purge request queues */
 608                 for (ch = 0; ch < priv->num_channels; ch++)
 609                         flush_channel(dev, ch, -EIO, 1);
 610
 611                 /* reset and reinitialize the device */
 612                 init_device(dev);
 613         }
 614 }
 615
 616 #define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)           \
 617 static irqreturn_t talitos_interrupt_##name(int irq, void *data)               \
 618 {                                                                              \
 619         struct device *dev = data;                                             \
 620         struct talitos_private *priv = dev_get_drvdata(dev);                   \
 621         u32 isr, isr_lo;                                                       \
 622                                                                                \
 623         isr = in_be32(priv->reg + TALITOS_ISR);                                \
 624         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
 625         /* Acknowledge interrupt */                                            \
 626         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
 627         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
 628                                                                                \
 629         if (unlikely((isr & ~TALITOS_ISR_4CHDONE) & ch_err_mask || isr_lo))    \
 630                 talitos_error(dev, isr, isr_lo);                               \
 631         else                                                                   \
 632                 if (likely(isr & ch_done_mask)) {                              \
 633                         /* mask further done interrupts. */                    \
 634                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
 635                         /* done_task will unmask done interrupts at exit */    \
 636                         tasklet_schedule(&priv->done_task[tlet]);              \
 637                 }                                                              \
 638                                                                                \
 639         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
 640                                                                 IRQ_NONE;      \
 641 }
 642 DEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0)
 643 DEF_TALITOS_INTERRUPT(ch0_2, TALITOS_ISR_CH_0_2_DONE, TALITOS_ISR_CH_0_2_ERR, 0)
 644 DEF_TALITOS_INTERRUPT(ch1_3, TALITOS_ISR_CH_1_3_DONE, TALITOS_ISR_CH_1_3_ERR, 1)
 645
 646 /*
 647  * hwrng
 648  */
 649 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 650 {
 651         struct device *dev = (struct device *)rng->priv;
 652         struct talitos_private *priv = dev_get_drvdata(dev);
 653         u32 ofl;
 654         int i;
 655
 656         for (i = 0; i < 20; i++) {
 657                 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
 658                       TALITOS_RNGUSR_LO_OFL;
 659                 if (ofl || !wait)
 660                         break;
 661                 udelay(10);
 662         }
 663
 664         return !!ofl;
 665 }
 666
 667 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 668 {
 669         struct device *dev = (struct device *)rng->priv;
 670         struct talitos_private *priv = dev_get_drvdata(dev);
 671
 672         /* rng fifo requires 64-bit accesses */
 673         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
 674         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
 675
 676         return sizeof(u32);
 677 }
 678
 679 static int talitos_rng_init(struct hwrng *rng)
 680 {
 681         struct device *dev = (struct device *)rng->priv;
 682         struct talitos_private *priv = dev_get_drvdata(dev);
 683         unsigned int timeout = TALITOS_TIMEOUT;
 684
 685         setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
 686         while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
 687                && --timeout)
 688                 cpu_relax();
 689         if (timeout == 0) {
 690                 dev_err(dev, "failed to reset rng hw\n");
 691                 return -ENODEV;
 692         }
 693
 694         /* start generating */
 695         setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
 696
 697         return 0;
 698 }
 699
 700 static int talitos_register_rng(struct device *dev)
 701 {
 702         struct talitos_private *priv = dev_get_drvdata(dev);
 703
 704         priv->rng.name          = dev_driver_string(dev),
 705         priv->rng.init          = talitos_rng_init,
 706         priv->rng.data_present  = talitos_rng_data_present,
 707         priv->rng.data_read     = talitos_rng_data_read,
 708         priv->rng.priv          = (unsigned long)dev;
 709
 710         return hwrng_register(&priv->rng);
 711 }
 712
 713 static void talitos_unregister_rng(struct device *dev)
 714 {
 715         struct talitos_private *priv = dev_get_drvdata(dev);
 716
 717         hwrng_unregister(&priv->rng);
 718 }
 719
 720 /*
 721  * crypto alg
 722  */
 723 #define TALITOS_CRA_PRIORITY            3000
 724 #define TALITOS_MAX_KEY_SIZE            64
 725 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 726
 727 #define MD5_BLOCK_SIZE    64
 728
 729 struct talitos_ctx {
 730         struct device *dev;
 731         int ch;
 732         __be32 desc_hdr_template;
 733         u8 key[TALITOS_MAX_KEY_SIZE];
 734         u8 iv[TALITOS_MAX_IV_LENGTH];
 735         unsigned int keylen;
 736         unsigned int enckeylen;
 737         unsigned int authkeylen;
 738         unsigned int authsize;
 739 };
 740
 741 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
 742 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
 743
 744 struct talitos_ahash_req_ctx {
 745         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 746         unsigned int hw_context_size;
 747         u8 buf[HASH_MAX_BLOCK_SIZE];
 748         u8 bufnext[HASH_MAX_BLOCK_SIZE];
 749         unsigned int swinit;
 750         unsigned int first;
 751         unsigned int last;
 752         unsigned int to_hash_later;
 753         u64 nbuf;
 754         struct scatterlist bufsl[2];
 755         struct scatterlist *psrc;
 756 };
 757
 758 static int aead_setauthsize(struct crypto_aead *authenc,
 759                             unsigned int authsize)
 760 {
 761         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 762
 763         ctx->authsize = authsize;
 764
 765         return 0;
 766 }
 767
 768 static int aead_setkey(struct crypto_aead *authenc,
 769                        const u8 *key, unsigned int keylen)
 770 {
 771         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 772         struct rtattr *rta = (void *)key;
 773         struct crypto_authenc_key_param *param;
 774         unsigned int authkeylen;
 775         unsigned int enckeylen;
 776
 777         if (!RTA_OK(rta, keylen))
 778                 goto badkey;
 779
 780         if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
 781                 goto badkey;
 782
 783         if (RTA_PAYLOAD(rta) < sizeof(*param))
 784                 goto badkey;
 785
 786         param = RTA_DATA(rta);
 787         enckeylen = be32_to_cpu(param->enckeylen);
 788
 789         key += RTA_ALIGN(rta->rta_len);
 790         keylen -= RTA_ALIGN(rta->rta_len);
 791
 792         if (keylen < enckeylen)
 793                 goto badkey;
 794
 795         authkeylen = keylen - enckeylen;
 796
 797         if (keylen > TALITOS_MAX_KEY_SIZE)
 798                 goto badkey;
 799
 800         memcpy(&ctx->key, key, keylen);
 801
 802         ctx->keylen = keylen;
 803         ctx->enckeylen = enckeylen;
 804         ctx->authkeylen = authkeylen;
 805
 806         return 0;
 807
 808 badkey:
 809         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 810         return -EINVAL;
 811 }
 812
 813 /*
 814  * talitos_edesc - s/w-extended descriptor
 815  * @src_nents: number of segments in input scatterlist
 816  * @dst_nents: number of segments in output scatterlist
 817  * @dma_len: length of dma mapped link_tbl space
 818  * @dma_link_tbl: bus physical address of link_tbl
 819  * @desc: h/w descriptor
 820  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
 821  *
 822  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 823  * is greater than 1, an integrity check value is concatenated to the end
 824  * of link_tbl data
 825  */
 826 struct talitos_edesc {
 827         int src_nents;
 828         int dst_nents;
 829         int src_is_chained;
 830         int dst_is_chained;
 831         int dma_len;
 832         dma_addr_t dma_link_tbl;
 833         struct talitos_desc desc;
 834         struct talitos_ptr link_tbl[0];
 835 };
 836
 837 static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
 838                           unsigned int nents, enum dma_data_direction dir,
 839                           int chained)
 840 {
 841         if (unlikely(chained))
 842                 while (sg) {
 843                         dma_map_sg(dev, sg, 1, dir);
 844                         sg = scatterwalk_sg_next(sg);
 845                 }
 846         else
 847                 dma_map_sg(dev, sg, nents, dir);
 848         return nents;
 849 }
 850
 851 static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
 852                                    enum dma_data_direction dir)
 853 {
 854         while (sg) {
 855                 dma_unmap_sg(dev, sg, 1, dir);
 856                 sg = scatterwalk_sg_next(sg);
 857         }
 858 }
 859
 860 static void talitos_sg_unmap(struct device *dev,
 861                              struct talitos_edesc *edesc,
 862                              struct scatterlist *src,
 863                              struct scatterlist *dst)
 864 {
 865         unsigned int src_nents = edesc->src_nents ? : 1;
 866         unsigned int dst_nents = edesc->dst_nents ? : 1;
 867
 868         if (src != dst) {
 869                 if (edesc->src_is_chained)
 870                         talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
 871                 else
 872                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 873
 874                 if (dst) {
 875                         if (edesc->dst_is_chained)
 876                                 talitos_unmap_sg_chain(dev, dst,
 877                                                        DMA_FROM_DEVICE);
 878                         else
 879                                 dma_unmap_sg(dev, dst, dst_nents,
 880                                              DMA_FROM_DEVICE);
 881                 }
 882         } else
 883                 if (edesc->src_is_chained)
 884                         talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
 885                 else
 886                         dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 887 }
 888
 889 static void ipsec_esp_unmap(struct device *dev,
 890                             struct talitos_edesc *edesc,
 891                             struct aead_request *areq)
 892 {
 893         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
 894         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 895         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 896         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 897
 898         dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
 899
 900         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
 901
 902         if (edesc->dma_len)
 903                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 904                                  DMA_BIDIRECTIONAL);
 905 }
 906
 907 /*
 908  * ipsec_esp descriptor callbacks
 909  */
 910 static void ipsec_esp_encrypt_done(struct device *dev,
 911                                    struct talitos_desc *desc, void *context,
 912                                    int err)
 913 {
 914         struct aead_request *areq = context;
 915         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 916         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 917         struct talitos_edesc *edesc;
 918         struct scatterlist *sg;
 919         void *icvdata;
 920
 921         edesc = container_of(desc, struct talitos_edesc, desc);
 922
 923         ipsec_esp_unmap(dev, edesc, areq);
 924
 925         /* copy the generated ICV to dst */
 926         if (edesc->dma_len) {
 927                 icvdata = &edesc->link_tbl[edesc->src_nents +
 928                                            edesc->dst_nents + 2];
 929                 sg = sg_last(areq->dst, edesc->dst_nents);
 930                 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
 931                        icvdata, ctx->authsize);
 932         }
 933
 934         kfree(edesc);
 935
 936         aead_request_complete(areq, err);
 937 }
 938
 939 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
 940                                           struct talitos_desc *desc,
 941                                           void *context, int err)
 942 {
 943         struct aead_request *req = context;
 944         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 945         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 946         struct talitos_edesc *edesc;
 947         struct scatterlist *sg;
 948         void *icvdata;
 949
 950         edesc = container_of(desc, struct talitos_edesc, desc);
 951
 952         ipsec_esp_unmap(dev, edesc, req);
 953
 954         if (!err) {
 955                 /* auth check */
 956                 if (edesc->dma_len)
 957                         icvdata = &edesc->link_tbl[edesc->src_nents +
 958                                                    edesc->dst_nents + 2];
 959                 else
 960                         icvdata = &edesc->link_tbl[0];
 961
 962                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
 963                 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
 964                              ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
 965         }
 966
 967         kfree(edesc);
 968
 969         aead_request_complete(req, err);
 970 }
 971
 972 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
 973                                           struct talitos_desc *desc,
 974                                           void *context, int err)
 975 {
 976         struct aead_request *req = context;
 977         struct talitos_edesc *edesc;
 978
 979         edesc = container_of(desc, struct talitos_edesc, desc);
 980
 981         ipsec_esp_unmap(dev, edesc, req);
 982
 983         /* check ICV auth status */
 984         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
 985                      DESC_HDR_LO_ICCR1_PASS))
 986                 err = -EBADMSG;
 987
 988         kfree(edesc);
 989
 990         aead_request_complete(req, err);
 991 }
 992
 993 /*
 994  * convert scatterlist to SEC h/w link table format
 995  * stop at cryptlen bytes
 996  */
 997 static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
 998                            int cryptlen, struct talitos_ptr *link_tbl_ptr)
 999 {
1000         int n_sg = sg_count;
1001
1002         while (n_sg--) {
1003                 to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg));
1004                 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
1005                 link_tbl_ptr->j_extent = 0;
1006                 link_tbl_ptr++;
1007                 cryptlen -= sg_dma_len(sg);
1008                 sg = scatterwalk_sg_next(sg);
1009         }
1010
1011         /* adjust (decrease) last one (or two) entry's len to cryptlen */
1012         link_tbl_ptr--;
1013         while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
1014                 /* Empty this entry, and move to previous one */
1015                 cryptlen += be16_to_cpu(link_tbl_ptr->len);
1016                 link_tbl_ptr->len = 0;
1017                 sg_count--;
1018                 link_tbl_ptr--;
1019         }
1020         link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
1021                                         + cryptlen);
1022
1023         /* tag end of link table */
1024         link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1025
1026         return sg_count;
1027 }
1028
1029 /*
1030  * fill in and submit ipsec_esp descriptor
1031  */
1032 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1033                      u8 *giv, u64 seq,
1034                      void (*callback) (struct device *dev,
1035                                        struct talitos_desc *desc,
1036                                        void *context, int error))
1037 {
1038         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1039         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1040         struct device *dev = ctx->dev;
1041         struct talitos_desc *desc = &edesc->desc;
1042         unsigned int cryptlen = areq->cryptlen;
1043         unsigned int authsize = ctx->authsize;
1044         unsigned int ivsize = crypto_aead_ivsize(aead);
1045         int sg_count, ret;
1046         int sg_link_tbl_len;
1047
1048         /* hmac key */
1049         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1050                                0, DMA_TO_DEVICE);
1051         /* hmac data */
1052         map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize,
1053                                sg_virt(areq->assoc), 0, DMA_TO_DEVICE);
1054         /* cipher iv */
1055         map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
1056                                DMA_TO_DEVICE);
1057
1058         /* cipher key */
1059         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1060                                (char *)&ctx->key + ctx->authkeylen, 0,
1061                                DMA_TO_DEVICE);
1062
1063         /*
1064          * cipher in
1065          * map and adjust cipher len to aead request cryptlen.
1066          * extent is bytes of HMAC postpended to ciphertext,
1067          * typically 12 for ipsec
1068          */
1069         desc->ptr[4].len = cpu_to_be16(cryptlen);
1070         desc->ptr[4].j_extent = authsize;
1071
1072         sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1073                                   (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1074                                                            : DMA_TO_DEVICE,
1075                                   edesc->src_is_chained);
1076
1077         if (sg_count == 1) {
1078                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src));
1079         } else {
1080                 sg_link_tbl_len = cryptlen;
1081
1082                 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1083                         sg_link_tbl_len = cryptlen + authsize;
1084
1085                 sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,
1086                                           &edesc->link_tbl[0]);
1087                 if (sg_count > 1) {
1088                         desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1089                         to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl);
1090                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1091                                                    edesc->dma_len,
1092                                                    DMA_BIDIRECTIONAL);
1093                 } else {
1094                         /* Only one segment now, so no link tbl needed */
1095                         to_talitos_ptr(&desc->ptr[4],
1096                                        sg_dma_address(areq->src));
1097                 }
1098         }
1099
1100         /* cipher out */
1101         desc->ptr[5].len = cpu_to_be16(cryptlen);
1102         desc->ptr[5].j_extent = authsize;
1103
1104         if (areq->src != areq->dst)
1105                 sg_count = talitos_map_sg(dev, areq->dst,
1106                                           edesc->dst_nents ? : 1,
1107                                           DMA_FROM_DEVICE,
1108                                           edesc->dst_is_chained);
1109
1110         if (sg_count == 1) {
1111                 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst));
1112         } else {
1113                 struct talitos_ptr *link_tbl_ptr =
1114                         &edesc->link_tbl[edesc->src_nents + 1];
1115
1116                 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1117                                (edesc->src_nents + 1) *
1118                                sizeof(struct talitos_ptr));
1119                 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1120                                           link_tbl_ptr);
1121
1122                 /* Add an entry to the link table for ICV data */
1123                 link_tbl_ptr += sg_count - 1;
1124                 link_tbl_ptr->j_extent = 0;
1125                 sg_count++;
1126                 link_tbl_ptr++;
1127                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1128                 link_tbl_ptr->len = cpu_to_be16(authsize);
1129
1130                 /* icv data follows link tables */
1131                 to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl +
1132                                (edesc->src_nents + edesc->dst_nents + 2) *
1133                                sizeof(struct talitos_ptr));
1134                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1135                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1136                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1137         }
1138
1139         /* iv out */
1140         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1141                                DMA_FROM_DEVICE);
1142
1143         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1144         if (ret != -EINPROGRESS) {
1145                 ipsec_esp_unmap(dev, edesc, areq);
1146                 kfree(edesc);
1147         }
1148         return ret;
1149 }
1150
1151 /*
1152  * derive number of elements in scatterlist
1153  */
1154 static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
1155 {
1156         struct scatterlist *sg = sg_list;
1157         int sg_nents = 0;
1158
1159         *chained = 0;
1160         while (nbytes > 0) {
1161                 sg_nents++;
1162                 nbytes -= sg->length;
1163                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
1164                         *chained = 1;
1165                 sg = scatterwalk_sg_next(sg);
1166         }
1167
1168         return sg_nents;
1169 }
1170
1171 /**
1172  * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer
1173  * @sgl:                 The SG list
1174  * @nents:               Number of SG entries
1175  * @buf:                 Where to copy to
1176  * @buflen:              The number of bytes to copy
1177  * @skip:                The number of bytes to skip before copying.
1178  *                       Note: skip + buflen should equal SG total size.
1179  *
1180  * Returns the number of copied bytes.
1181  *
1182  **/
1183 static size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents,
1184                                     void *buf, size_t buflen, unsigned int skip)
1185 {
1186         unsigned int offset = 0;
1187         unsigned int boffset = 0;
1188         struct sg_mapping_iter miter;
1189         unsigned long flags;
1190         unsigned int sg_flags = SG_MITER_ATOMIC;
1191         size_t total_buffer = buflen + skip;
1192
1193         sg_flags |= SG_MITER_FROM_SG;
1194
1195         sg_miter_start(&miter, sgl, nents, sg_flags);
1196
1197         local_irq_save(flags);
1198
1199         while (sg_miter_next(&miter) && offset < total_buffer) {
1200                 unsigned int len;
1201                 unsigned int ignore;
1202
1203                 if ((offset + miter.length) > skip) {
1204                         if (offset < skip) {
1205                                 /* Copy part of this segment */
1206                                 ignore = skip - offset;
1207                                 len = miter.length - ignore;
1208                                 if (boffset + len > buflen)
1209                                         len = buflen - boffset;
1210                                 memcpy(buf + boffset, miter.addr + ignore, len);
1211                         } else {
1212                                 /* Copy all of this segment (up to buflen) */
1213                                 len = miter.length;
1214                                 if (boffset + len > buflen)
1215                                         len = buflen - boffset;
1216                                 memcpy(buf + boffset, miter.addr, len);
1217                         }
1218                         boffset += len;
1219                 }
1220                 offset += miter.length;
1221         }
1222
1223         sg_miter_stop(&miter);
1224
1225         local_irq_restore(flags);
1226         return boffset;
1227 }
1228
1229 /*
1230  * allocate and map the extended descriptor
1231  */
1232 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1233                                                  struct scatterlist *src,
1234                                                  struct scatterlist *dst,
1235                                                  int hash_result,
1236                                                  unsigned int cryptlen,
1237                                                  unsigned int authsize,
1238                                                  int icv_stashing,
1239                                                  u32 cryptoflags)
1240 {
1241         struct talitos_edesc *edesc;
1242         int src_nents, dst_nents, alloc_len, dma_len;
1243         int src_chained, dst_chained = 0;
1244         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1245                       GFP_ATOMIC;
1246
1247         if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) {
1248                 dev_err(dev, "length exceeds h/w max limit\n");
1249                 return ERR_PTR(-EINVAL);
1250         }
1251
1252         src_nents = sg_count(src, cryptlen + authsize, &src_chained);
1253         src_nents = (src_nents == 1) ? 0 : src_nents;
1254
1255         if (hash_result) {
1256                 dst_nents = 0;
1257         } else {
1258                 if (dst == src) {
1259                         dst_nents = src_nents;
1260                 } else {
1261                         dst_nents = sg_count(dst, cryptlen + authsize,
1262                                              &dst_chained);
1263                         dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1264                 }
1265         }
1266
1267         /*
1268          * allocate space for base edesc plus the link tables,
1269          * allowing for two separate entries for ICV and generated ICV (+ 2),
1270          * and the ICV data itself
1271          */
1272         alloc_len = sizeof(struct talitos_edesc);
1273         if (src_nents || dst_nents) {
1274                 dma_len = (src_nents + dst_nents + 2) *
1275                                  sizeof(struct talitos_ptr) + authsize;
1276                 alloc_len += dma_len;
1277         } else {
1278                 dma_len = 0;
1279                 alloc_len += icv_stashing ? authsize : 0;
1280         }
1281
1282         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1283         if (!edesc) {
1284                 dev_err(dev, "could not allocate edescriptor\n");
1285                 return ERR_PTR(-ENOMEM);
1286         }
1287
1288         edesc->src_nents = src_nents;
1289         edesc->dst_nents = dst_nents;
1290         edesc->src_is_chained = src_chained;
1291         edesc->dst_is_chained = dst_chained;
1292         edesc->dma_len = dma_len;
1293         if (dma_len)
1294                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1295                                                      edesc->dma_len,
1296                                                      DMA_BIDIRECTIONAL);
1297
1298         return edesc;
1299 }
1300
1301 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq,
1302                                               int icv_stashing)
1303 {
1304         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1305         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1306
1307         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
1308                                    areq->cryptlen, ctx->authsize, icv_stashing,
1309                                    areq->base.flags);
1310 }
1311
1312 static int aead_encrypt(struct aead_request *req)
1313 {
1314         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1315         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1316         struct talitos_edesc *edesc;
1317
1318         /* allocate extended descriptor */
1319         edesc = aead_edesc_alloc(req, 0);
1320         if (IS_ERR(edesc))
1321                 return PTR_ERR(edesc);
1322
1323         /* set encrypt */
1324         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1325
1326         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1327 }
1328
1329 static int aead_decrypt(struct aead_request *req)
1330 {
1331         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1332         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1333         unsigned int authsize = ctx->authsize;
1334         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1335         struct talitos_edesc *edesc;
1336         struct scatterlist *sg;
1337         void *icvdata;
1338
1339         req->cryptlen -= authsize;
1340
1341         /* allocate extended descriptor */
1342         edesc = aead_edesc_alloc(req, 1);
1343         if (IS_ERR(edesc))
1344                 return PTR_ERR(edesc);
1345
1346         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1347             ((!edesc->src_nents && !edesc->dst_nents) ||
1348              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1349
1350                 /* decrypt and check the ICV */
1351                 edesc->desc.hdr = ctx->desc_hdr_template |
1352                                   DESC_HDR_DIR_INBOUND |
1353                                   DESC_HDR_MODE1_MDEU_CICV;
1354
1355                 /* reset integrity check result bits */
1356                 edesc->desc.hdr_lo = 0;
1357
1358                 return ipsec_esp(edesc, req, NULL, 0,
1359                                  ipsec_esp_decrypt_hwauth_done);
1360
1361         }
1362
1363         /* Have to check the ICV with software */
1364         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1365
1366         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1367         if (edesc->dma_len)
1368                 icvdata = &edesc->link_tbl[edesc->src_nents +
1369                                            edesc->dst_nents + 2];
1370         else
1371                 icvdata = &edesc->link_tbl[0];
1372
1373         sg = sg_last(req->src, edesc->src_nents ? : 1);
1374
1375         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1376                ctx->authsize);
1377
1378         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done);
1379 }
1380
1381 static int aead_givencrypt(struct aead_givcrypt_request *req)
1382 {
1383         struct aead_request *areq = &req->areq;
1384         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1385         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1386         struct talitos_edesc *edesc;
1387
1388         /* allocate extended descriptor */
1389         edesc = aead_edesc_alloc(areq, 0);
1390         if (IS_ERR(edesc))
1391                 return PTR_ERR(edesc);
1392
1393         /* set encrypt */
1394         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1395
1396         memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1397         /* avoid consecutive packets going out with same IV */
1398         *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
1399
1400         return ipsec_esp(edesc, areq, req->giv, req->seq,
1401                          ipsec_esp_encrypt_done);
1402 }
1403
1404 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1405                              const u8 *key, unsigned int keylen)
1406 {
1407         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1408
1409         memcpy(&ctx->key, key, keylen);
1410         ctx->keylen = keylen;
1411
1412         return 0;
1413 }
1414
1415 static void common_nonsnoop_unmap(struct device *dev,
1416                                   struct talitos_edesc *edesc,
1417                                   struct ablkcipher_request *areq)
1418 {
1419         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1420         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1421         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1422
1423         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
1424
1425         if (edesc->dma_len)
1426                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1427                                  DMA_BIDIRECTIONAL);
1428 }
1429
1430 static void ablkcipher_done(struct device *dev,
1431                             struct talitos_desc *desc, void *context,
1432                             int err)
1433 {
1434         struct ablkcipher_request *areq = context;
1435         struct talitos_edesc *edesc;
1436
1437         edesc = container_of(desc, struct talitos_edesc, desc);
1438
1439         common_nonsnoop_unmap(dev, edesc, areq);
1440
1441         kfree(edesc);
1442
1443         areq->base.complete(&areq->base, err);
1444 }
1445
1446 static int common_nonsnoop(struct talitos_edesc *edesc,
1447                            struct ablkcipher_request *areq,
1448                            void (*callback) (struct device *dev,
1449                                              struct talitos_desc *desc,
1450                                              void *context, int error))
1451 {
1452         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1453         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1454         struct device *dev = ctx->dev;
1455         struct talitos_desc *desc = &edesc->desc;
1456         unsigned int cryptlen = areq->nbytes;
1457         unsigned int ivsize;
1458         int sg_count, ret;
1459
1460         /* first DWORD empty */
1461         desc->ptr[0].len = 0;
1462         to_talitos_ptr(&desc->ptr[0], 0);
1463         desc->ptr[0].j_extent = 0;
1464
1465         /* cipher iv */
1466         ivsize = crypto_ablkcipher_ivsize(cipher);
1467         map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0,
1468                                DMA_TO_DEVICE);
1469
1470         /* cipher key */
1471         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1472                                (char *)&ctx->key, 0, DMA_TO_DEVICE);
1473
1474         /*
1475          * cipher in
1476          */
1477         desc->ptr[3].len = cpu_to_be16(cryptlen);
1478         desc->ptr[3].j_extent = 0;
1479
1480         sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1481                                   (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1482                                                            : DMA_TO_DEVICE,
1483                                   edesc->src_is_chained);
1484
1485         if (sg_count == 1) {
1486                 to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src));
1487         } else {
1488                 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
1489                                           &edesc->link_tbl[0]);
1490                 if (sg_count > 1) {
1491                         to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
1492                         desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
1493                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1494                                                    edesc->dma_len,
1495                                                    DMA_BIDIRECTIONAL);
1496                 } else {
1497                         /* Only one segment now, so no link tbl needed */
1498                         to_talitos_ptr(&desc->ptr[3],
1499                                        sg_dma_address(areq->src));
1500                 }
1501         }
1502
1503         /* cipher out */
1504         desc->ptr[4].len = cpu_to_be16(cryptlen);
1505         desc->ptr[4].j_extent = 0;
1506
1507         if (areq->src != areq->dst)
1508                 sg_count = talitos_map_sg(dev, areq->dst,
1509                                           edesc->dst_nents ? : 1,
1510                                           DMA_FROM_DEVICE,
1511                                           edesc->dst_is_chained);
1512
1513         if (sg_count == 1) {
1514                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst));
1515         } else {
1516                 struct talitos_ptr *link_tbl_ptr =
1517                         &edesc->link_tbl[edesc->src_nents + 1];
1518
1519                 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1520                                               (edesc->src_nents + 1) *
1521                                               sizeof(struct talitos_ptr));
1522                 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1523                 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1524                                           link_tbl_ptr);
1525                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1526                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1527         }
1528
1529         /* iv out */
1530         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 0,
1531                                DMA_FROM_DEVICE);
1532
1533         /* last DWORD empty */
1534         desc->ptr[6].len = 0;
1535         to_talitos_ptr(&desc->ptr[6], 0);
1536         desc->ptr[6].j_extent = 0;
1537
1538         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1539         if (ret != -EINPROGRESS) {
1540                 common_nonsnoop_unmap(dev, edesc, areq);
1541                 kfree(edesc);
1542         }
1543         return ret;
1544 }
1545
1546 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1547                                                     areq)
1548 {
1549         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1550         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1551
1552         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
1553                                    areq->nbytes, 0, 0, areq->base.flags);
1554 }
1555
1556 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1557 {
1558         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1559         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1560         struct talitos_edesc *edesc;
1561
1562         /* allocate extended descriptor */
1563         edesc = ablkcipher_edesc_alloc(areq);
1564         if (IS_ERR(edesc))
1565                 return PTR_ERR(edesc);
1566
1567         /* set encrypt */
1568         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1569
1570         return common_nonsnoop(edesc, areq, ablkcipher_done);
1571 }
1572
1573 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1574 {
1575         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1576         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1577         struct talitos_edesc *edesc;
1578
1579         /* allocate extended descriptor */
1580         edesc = ablkcipher_edesc_alloc(areq);
1581         if (IS_ERR(edesc))
1582                 return PTR_ERR(edesc);
1583
1584         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1585
1586         return common_nonsnoop(edesc, areq, ablkcipher_done);
1587 }
1588
1589 static void common_nonsnoop_hash_unmap(struct device *dev,
1590                                        struct talitos_edesc *edesc,
1591                                        struct ahash_request *areq)
1592 {
1593         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1594
1595         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1596
1597         /* When using hashctx-in, must unmap it. */
1598         if (edesc->desc.ptr[1].len)
1599                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1600                                          DMA_TO_DEVICE);
1601
1602         if (edesc->desc.ptr[2].len)
1603                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1604                                          DMA_TO_DEVICE);
1605
1606         talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL);
1607
1608         if (edesc->dma_len)
1609                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1610                                  DMA_BIDIRECTIONAL);
1611
1612 }
1613
1614 static void ahash_done(struct device *dev,
1615                        struct talitos_desc *desc, void *context,
1616                        int err)
1617 {
1618         struct ahash_request *areq = context;
1619         struct talitos_edesc *edesc =
1620                  container_of(desc, struct talitos_edesc, desc);
1621         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1622
1623         if (!req_ctx->last && req_ctx->to_hash_later) {
1624                 /* Position any partial block for next update/final/finup */
1625                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1626                 req_ctx->nbuf = req_ctx->to_hash_later;
1627         }
1628         common_nonsnoop_hash_unmap(dev, edesc, areq);
1629
1630         kfree(edesc);
1631
1632         areq->base.complete(&areq->base, err);
1633 }
1634
1635 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1636                                 struct ahash_request *areq, unsigned int length,
1637                                 void (*callback) (struct device *dev,
1638                                                   struct talitos_desc *desc,
1639                                                   void *context, int error))
1640 {
1641         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1642         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1643         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1644         struct device *dev = ctx->dev;
1645         struct talitos_desc *desc = &edesc->desc;
1646         int sg_count, ret;
1647
1648         /* first DWORD empty */
1649         desc->ptr[0] = zero_entry;
1650
1651         /* hash context in */
1652         if (!req_ctx->first || req_ctx->swinit) {
1653                 map_single_talitos_ptr(dev, &desc->ptr[1],
1654                                        req_ctx->hw_context_size,
1655                                        (char *)req_ctx->hw_context, 0,
1656                                        DMA_TO_DEVICE);
1657                 req_ctx->swinit = 0;
1658         } else {
1659                 desc->ptr[1] = zero_entry;
1660                 /* Indicate next op is not the first. */
1661                 req_ctx->first = 0;
1662         }
1663
1664         /* HMAC key */
1665         if (ctx->keylen)
1666                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1667                                        (char *)&ctx->key, 0, DMA_TO_DEVICE);
1668         else
1669                 desc->ptr[2] = zero_entry;
1670
1671         /*
1672          * data in
1673          */
1674         desc->ptr[3].len = cpu_to_be16(length);
1675         desc->ptr[3].j_extent = 0;
1676
1677         sg_count = talitos_map_sg(dev, req_ctx->psrc,
1678                                   edesc->src_nents ? : 1,
1679                                   DMA_TO_DEVICE,
1680                                   edesc->src_is_chained);
1681
1682         if (sg_count == 1) {
1683                 to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc));
1684         } else {
1685                 sg_count = sg_to_link_tbl(req_ctx->psrc, sg_count, length,
1686                                           &edesc->link_tbl[0]);
1687                 if (sg_count > 1) {
1688                         desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
1689                         to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
1690                         dma_sync_single_for_device(ctx->dev,
1691                                                    edesc->dma_link_tbl,
1692                                                    edesc->dma_len,
1693                                                    DMA_BIDIRECTIONAL);
1694                 } else {
1695                         /* Only one segment now, so no link tbl needed */
1696                         to_talitos_ptr(&desc->ptr[3],
1697                                        sg_dma_address(req_ctx->psrc));
1698                 }
1699         }
1700
1701         /* fifth DWORD empty */
1702         desc->ptr[4] = zero_entry;
1703
1704         /* hash/HMAC out -or- hash context out */
1705         if (req_ctx->last)
1706                 map_single_talitos_ptr(dev, &desc->ptr[5],
1707                                        crypto_ahash_digestsize(tfm),
1708                                        areq->result, 0, DMA_FROM_DEVICE);
1709         else
1710                 map_single_talitos_ptr(dev, &desc->ptr[5],
1711                                        req_ctx->hw_context_size,
1712                                        req_ctx->hw_context, 0, DMA_FROM_DEVICE);
1713
1714         /* last DWORD empty */
1715         desc->ptr[6] = zero_entry;
1716
1717         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1718         if (ret != -EINPROGRESS) {
1719                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1720                 kfree(edesc);
1721         }
1722         return ret;
1723 }
1724
1725 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1726                                                unsigned int nbytes)
1727 {
1728         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1729         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1730         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1731
1732         return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1,
1733                                    nbytes, 0, 0, areq->base.flags);
1734 }
1735
1736 static int ahash_init(struct ahash_request *areq)
1737 {
1738         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1739         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1740
1741         /* Initialize the context */
1742         req_ctx->nbuf = 0;
1743         req_ctx->first = 1; /* first indicates h/w must init its context */
1744         req_ctx->swinit = 0; /* assume h/w init of context */
1745         req_ctx->hw_context_size =
1746                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1747                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1748                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1749
1750         return 0;
1751 }
1752
1753 /*
1754  * on h/w without explicit sha224 support, we initialize h/w context
1755  * manually with sha224 constants, and tell it to run sha256.
1756  */
1757 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1758 {
1759         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1760
1761         ahash_init(areq);
1762         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1763
1764         req_ctx->hw_context[0] = SHA224_H0;
1765         req_ctx->hw_context[1] = SHA224_H1;
1766         req_ctx->hw_context[2] = SHA224_H2;
1767         req_ctx->hw_context[3] = SHA224_H3;
1768         req_ctx->hw_context[4] = SHA224_H4;
1769         req_ctx->hw_context[5] = SHA224_H5;
1770         req_ctx->hw_context[6] = SHA224_H6;
1771         req_ctx->hw_context[7] = SHA224_H7;
1772
1773         /* init 64-bit count */
1774         req_ctx->hw_context[8] = 0;
1775         req_ctx->hw_context[9] = 0;
1776
1777         return 0;
1778 }
1779
1780 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1781 {
1782         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1783         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1784         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1785         struct talitos_edesc *edesc;
1786         unsigned int blocksize =
1787                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1788         unsigned int nbytes_to_hash;
1789         unsigned int to_hash_later;
1790         unsigned int nsg;
1791         int chained;
1792
1793         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1794                 /* Buffer up to one whole block */
1795                 sg_copy_to_buffer(areq->src,
1796                                   sg_count(areq->src, nbytes, &chained),
1797                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1798                 req_ctx->nbuf += nbytes;
1799                 return 0;
1800         }
1801
1802         /* At least (blocksize + 1) bytes are available to hash */
1803         nbytes_to_hash = nbytes + req_ctx->nbuf;
1804         to_hash_later = nbytes_to_hash & (blocksize - 1);
1805
1806         if (req_ctx->last)
1807                 to_hash_later = 0;
1808         else if (to_hash_later)
1809                 /* There is a partial block. Hash the full block(s) now */
1810                 nbytes_to_hash -= to_hash_later;
1811         else {
1812                 /* Keep one block buffered */
1813                 nbytes_to_hash -= blocksize;
1814                 to_hash_later = blocksize;
1815         }
1816
1817         /* Chain in any previously buffered data */
1818         if (req_ctx->nbuf) {
1819                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1820                 sg_init_table(req_ctx->bufsl, nsg);
1821                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1822                 if (nsg > 1)
1823                         scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src);
1824                 req_ctx->psrc = req_ctx->bufsl;
1825         } else
1826                 req_ctx->psrc = areq->src;
1827
1828         if (to_hash_later) {
1829                 int nents = sg_count(areq->src, nbytes, &chained);
1830                 sg_copy_end_to_buffer(areq->src, nents,
1831                                       req_ctx->bufnext,
1832                                       to_hash_later,
1833                                       nbytes - to_hash_later);
1834         }
1835         req_ctx->to_hash_later = to_hash_later;
1836
1837         /* Allocate extended descriptor */
1838         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1839         if (IS_ERR(edesc))
1840                 return PTR_ERR(edesc);
1841
1842         edesc->desc.hdr = ctx->desc_hdr_template;
1843
1844         /* On last one, request SEC to pad; otherwise continue */
1845         if (req_ctx->last)
1846                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1847         else
1848                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1849
1850         /* request SEC to INIT hash. */
1851         if (req_ctx->first && !req_ctx->swinit)
1852                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1853
1854         /* When the tfm context has a keylen, it's an HMAC.
1855          * A first or last (ie. not middle) descriptor must request HMAC.
1856          */
1857         if (ctx->keylen && (req_ctx->first || req_ctx->last))
1858                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1859
1860         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1861                                     ahash_done);
1862 }
1863
1864 static int ahash_update(struct ahash_request *areq)
1865 {
1866         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1867
1868         req_ctx->last = 0;
1869
1870         return ahash_process_req(areq, areq->nbytes);
1871 }
1872
1873 static int ahash_final(struct ahash_request *areq)
1874 {
1875         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1876
1877         req_ctx->last = 1;
1878
1879         return ahash_process_req(areq, 0);
1880 }
1881
1882 static int ahash_finup(struct ahash_request *areq)
1883 {
1884         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1885
1886         req_ctx->last = 1;
1887
1888         return ahash_process_req(areq, areq->nbytes);
1889 }
1890
1891 static int ahash_digest(struct ahash_request *areq)
1892 {
1893         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1894         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1895
1896         ahash->init(areq);
1897         req_ctx->last = 1;
1898
1899         return ahash_process_req(areq, areq->nbytes);
1900 }
1901
1902 struct keyhash_result {
1903         struct completion completion;
1904         int err;
1905 };
1906
1907 static void keyhash_complete(struct crypto_async_request *req, int err)
1908 {
1909         struct keyhash_result *res = req->data;
1910
1911         if (err == -EINPROGRESS)
1912                 return;
1913
1914         res->err = err;
1915         complete(&res->completion);
1916 }
1917
1918 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
1919                    u8 *hash)
1920 {
1921         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1922
1923         struct scatterlist sg[1];
1924         struct ahash_request *req;
1925         struct keyhash_result hresult;
1926         int ret;
1927
1928         init_completion(&hresult.completion);
1929
1930         req = ahash_request_alloc(tfm, GFP_KERNEL);
1931         if (!req)
1932                 return -ENOMEM;
1933
1934         /* Keep tfm keylen == 0 during hash of the long key */
1935         ctx->keylen = 0;
1936         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1937                                    keyhash_complete, &hresult);
1938
1939         sg_init_one(&sg[0], key, keylen);
1940
1941         ahash_request_set_crypt(req, sg, hash, keylen);
1942         ret = crypto_ahash_digest(req);
1943         switch (ret) {
1944         case 0:
1945                 break;
1946         case -EINPROGRESS:
1947         case -EBUSY:
1948                 ret = wait_for_completion_interruptible(
1949                         &hresult.completion);
1950                 if (!ret)
1951                         ret = hresult.err;
1952                 break;
1953         default:
1954                 break;
1955         }
1956         ahash_request_free(req);
1957
1958         return ret;
1959 }
1960
1961 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1962                         unsigned int keylen)
1963 {
1964         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1965         unsigned int blocksize =
1966                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1967         unsigned int digestsize = crypto_ahash_digestsize(tfm);
1968         unsigned int keysize = keylen;
1969         u8 hash[SHA512_DIGEST_SIZE];
1970         int ret;
1971
1972         if (keylen <= blocksize)
1973                 memcpy(ctx->key, key, keysize);
1974         else {
1975                 /* Must get the hash of the long key */
1976                 ret = keyhash(tfm, key, keylen, hash);
1977
1978                 if (ret) {
1979                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1980                         return -EINVAL;
1981                 }
1982
1983                 keysize = digestsize;
1984                 memcpy(ctx->key, hash, digestsize);
1985         }
1986
1987         ctx->keylen = keysize;
1988
1989         return 0;
1990 }
1991
1992
1993 struct talitos_alg_template {
1994         u32 type;
1995         union {
1996                 struct crypto_alg crypto;
1997                 struct ahash_alg hash;
1998         } alg;
1999         __be32 desc_hdr_template;
2000 };
2001
2002 static struct talitos_alg_template driver_algs[] = {
2003         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2004         {       .type = CRYPTO_ALG_TYPE_AEAD,
2005                 .alg.crypto = {
2006                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
2007                         .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
2008                         .cra_blocksize = AES_BLOCK_SIZE,
2009                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2010                         .cra_type = &crypto_aead_type,
2011                         .cra_aead = {
2012                                 .setkey = aead_setkey,
2013                                 .setauthsize = aead_setauthsize,
2014                                 .encrypt = aead_encrypt,
2015                                 .decrypt = aead_decrypt,
2016                                 .givencrypt = aead_givencrypt,
2017                                 .geniv = "<built-in>",
2018                                 .ivsize = AES_BLOCK_SIZE,
2019                                 .maxauthsize = SHA1_DIGEST_SIZE,
2020                         }
2021                 },
2022                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2023                                      DESC_HDR_SEL0_AESU |
2024                                      DESC_HDR_MODE0_AESU_CBC |
2025                                      DESC_HDR_SEL1_MDEUA |
2026                                      DESC_HDR_MODE1_MDEU_INIT |
2027                                      DESC_HDR_MODE1_MDEU_PAD |
2028                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2029         },
2030         {       .type = CRYPTO_ALG_TYPE_AEAD,
2031                 .alg.crypto = {
2032                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
2033                         .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
2034                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2035                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2036                         .cra_type = &crypto_aead_type,
2037                         .cra_aead = {
2038                                 .setkey = aead_setkey,
2039                                 .setauthsize = aead_setauthsize,
2040                                 .encrypt = aead_encrypt,
2041                                 .decrypt = aead_decrypt,
2042                                 .givencrypt = aead_givencrypt,
2043                                 .geniv = "<built-in>",
2044                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2045                                 .maxauthsize = SHA1_DIGEST_SIZE,
2046                         }
2047                 },
2048                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2049                                      DESC_HDR_SEL0_DEU |
2050                                      DESC_HDR_MODE0_DEU_CBC |
2051                                      DESC_HDR_MODE0_DEU_3DES |
2052                                      DESC_HDR_SEL1_MDEUA |
2053                                      DESC_HDR_MODE1_MDEU_INIT |
2054                                      DESC_HDR_MODE1_MDEU_PAD |
2055                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2056         },
2057         {       .type = CRYPTO_ALG_TYPE_AEAD,
2058                 .alg.crypto = {
2059                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
2060                         .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
2061                         .cra_blocksize = AES_BLOCK_SIZE,
2062                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2063                         .cra_type = &crypto_aead_type,
2064                         .cra_aead = {
2065                                 .setkey = aead_setkey,
2066                                 .setauthsize = aead_setauthsize,
2067                                 .encrypt = aead_encrypt,
2068                                 .decrypt = aead_decrypt,
2069                                 .givencrypt = aead_givencrypt,
2070                                 .geniv = "<built-in>",
2071                                 .ivsize = AES_BLOCK_SIZE,
2072                                 .maxauthsize = SHA256_DIGEST_SIZE,
2073                         }
2074                 },
2075                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2076                                      DESC_HDR_SEL0_AESU |
2077                                      DESC_HDR_MODE0_AESU_CBC |
2078                                      DESC_HDR_SEL1_MDEUA |
2079                                      DESC_HDR_MODE1_MDEU_INIT |
2080                                      DESC_HDR_MODE1_MDEU_PAD |
2081                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2082         },
2083         {       .type = CRYPTO_ALG_TYPE_AEAD,
2084                 .alg.crypto = {
2085                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
2086                         .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
2087                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2088                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2089                         .cra_type = &crypto_aead_type,
2090                         .cra_aead = {
2091                                 .setkey = aead_setkey,
2092                                 .setauthsize = aead_setauthsize,
2093                                 .encrypt = aead_encrypt,
2094                                 .decrypt = aead_decrypt,
2095                                 .givencrypt = aead_givencrypt,
2096                                 .geniv = "<built-in>",
2097                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2098                                 .maxauthsize = SHA256_DIGEST_SIZE,
2099                         }
2100                 },
2101                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2102                                      DESC_HDR_SEL0_DEU |
2103                                      DESC_HDR_MODE0_DEU_CBC |
2104                                      DESC_HDR_MODE0_DEU_3DES |
2105                                      DESC_HDR_SEL1_MDEUA |
2106                                      DESC_HDR_MODE1_MDEU_INIT |
2107                                      DESC_HDR_MODE1_MDEU_PAD |
2108                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2109         },
2110         {       .type = CRYPTO_ALG_TYPE_AEAD,
2111                 .alg.crypto = {
2112                         .cra_name = "authenc(hmac(md5),cbc(aes))",
2113                         .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
2114                         .cra_blocksize = AES_BLOCK_SIZE,
2115                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2116                         .cra_type = &crypto_aead_type,
2117                         .cra_aead = {
2118                                 .setkey = aead_setkey,
2119                                 .setauthsize = aead_setauthsize,
2120                                 .encrypt = aead_encrypt,
2121                                 .decrypt = aead_decrypt,
2122                                 .givencrypt = aead_givencrypt,
2123                                 .geniv = "<built-in>",
2124                                 .ivsize = AES_BLOCK_SIZE,
2125                                 .maxauthsize = MD5_DIGEST_SIZE,
2126                         }
2127                 },
2128                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2129                                      DESC_HDR_SEL0_AESU |
2130                                      DESC_HDR_MODE0_AESU_CBC |
2131                                      DESC_HDR_SEL1_MDEUA |
2132                                      DESC_HDR_MODE1_MDEU_INIT |
2133                                      DESC_HDR_MODE1_MDEU_PAD |
2134                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2135         },
2136         {       .type = CRYPTO_ALG_TYPE_AEAD,
2137                 .alg.crypto = {
2138                         .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2139                         .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
2140                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2141                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
2142                         .cra_type = &crypto_aead_type,
2143                         .cra_aead = {
2144                                 .setkey = aead_setkey,
2145                                 .setauthsize = aead_setauthsize,
2146                                 .encrypt = aead_encrypt,
2147                                 .decrypt = aead_decrypt,
2148                                 .givencrypt = aead_givencrypt,
2149                                 .geniv = "<built-in>",
2150                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2151                                 .maxauthsize = MD5_DIGEST_SIZE,
2152                         }
2153                 },
2154                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2155                                      DESC_HDR_SEL0_DEU |
2156                                      DESC_HDR_MODE0_DEU_CBC |
2157                                      DESC_HDR_MODE0_DEU_3DES |
2158                                      DESC_HDR_SEL1_MDEUA |
2159                                      DESC_HDR_MODE1_MDEU_INIT |
2160                                      DESC_HDR_MODE1_MDEU_PAD |
2161                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2162         },
2163         /* ABLKCIPHER algorithms. */
2164         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2165                 .alg.crypto = {
2166                         .cra_name = "cbc(aes)",
2167                         .cra_driver_name = "cbc-aes-talitos",
2168                         .cra_blocksize = AES_BLOCK_SIZE,
2169                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2170                                      CRYPTO_ALG_ASYNC,
2171                         .cra_type = &crypto_ablkcipher_type,
2172                         .cra_ablkcipher = {
2173                                 .setkey = ablkcipher_setkey,
2174                                 .encrypt = ablkcipher_encrypt,
2175                                 .decrypt = ablkcipher_decrypt,
2176                                 .geniv = "eseqiv",
2177                                 .min_keysize = AES_MIN_KEY_SIZE,
2178                                 .max_keysize = AES_MAX_KEY_SIZE,
2179                                 .ivsize = AES_BLOCK_SIZE,
2180                         }
2181                 },
2182                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2183                                      DESC_HDR_SEL0_AESU |
2184                                      DESC_HDR_MODE0_AESU_CBC,
2185         },
2186         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2187                 .alg.crypto = {
2188                         .cra_name = "cbc(des3_ede)",
2189                         .cra_driver_name = "cbc-3des-talitos",
2190                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2191                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2192                                      CRYPTO_ALG_ASYNC,
2193                         .cra_type = &crypto_ablkcipher_type,
2194                         .cra_ablkcipher = {
2195                                 .setkey = ablkcipher_setkey,
2196                                 .encrypt = ablkcipher_encrypt,
2197                                 .decrypt = ablkcipher_decrypt,
2198                                 .geniv = "eseqiv",
2199                                 .min_keysize = DES3_EDE_KEY_SIZE,
2200                                 .max_keysize = DES3_EDE_KEY_SIZE,
2201                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2202                         }
2203                 },
2204                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2205                                      DESC_HDR_SEL0_DEU |
2206                                      DESC_HDR_MODE0_DEU_CBC |
2207                                      DESC_HDR_MODE0_DEU_3DES,
2208         },
2209         /* AHASH algorithms. */
2210         {       .type = CRYPTO_ALG_TYPE_AHASH,
2211                 .alg.hash = {
2212                         .init = ahash_init,
2213                         .update = ahash_update,
2214                         .final = ahash_final,
2215                         .finup = ahash_finup,
2216                         .digest = ahash_digest,
2217                         .halg.digestsize = MD5_DIGEST_SIZE,
2218                         .halg.base = {
2219                                 .cra_name = "md5",
2220                                 .cra_driver_name = "md5-talitos",
2221                                 .cra_blocksize = MD5_BLOCK_SIZE,
2222                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2223                                              CRYPTO_ALG_ASYNC,
2224                                 .cra_type = &crypto_ahash_type
2225                         }
2226                 },
2227                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2228                                      DESC_HDR_SEL0_MDEUA |
2229                                      DESC_HDR_MODE0_MDEU_MD5,
2230         },
2231         {       .type = CRYPTO_ALG_TYPE_AHASH,
2232                 .alg.hash = {
2233                         .init = ahash_init,
2234                         .update = ahash_update,
2235                         .final = ahash_final,
2236                         .finup = ahash_finup,
2237                         .digest = ahash_digest,
2238                         .halg.digestsize = SHA1_DIGEST_SIZE,
2239                         .halg.base = {
2240                                 .cra_name = "sha1",
2241                                 .cra_driver_name = "sha1-talitos",
2242                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2243                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2244                                              CRYPTO_ALG_ASYNC,
2245                                 .cra_type = &crypto_ahash_type
2246                         }
2247                 },
2248                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2249                                      DESC_HDR_SEL0_MDEUA |
2250                                      DESC_HDR_MODE0_MDEU_SHA1,
2251         },
2252         {       .type = CRYPTO_ALG_TYPE_AHASH,
2253                 .alg.hash = {
2254                         .init = ahash_init,
2255                         .update = ahash_update,
2256                         .final = ahash_final,
2257                         .finup = ahash_finup,
2258                         .digest = ahash_digest,
2259                         .halg.digestsize = SHA224_DIGEST_SIZE,
2260                         .halg.base = {
2261                                 .cra_name = "sha224",
2262                                 .cra_driver_name = "sha224-talitos",
2263                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2264                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2265                                              CRYPTO_ALG_ASYNC,
2266                                 .cra_type = &crypto_ahash_type
2267                         }
2268                 },
2269                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2270                                      DESC_HDR_SEL0_MDEUA |
2271                                      DESC_HDR_MODE0_MDEU_SHA224,
2272         },
2273         {       .type = CRYPTO_ALG_TYPE_AHASH,
2274                 .alg.hash = {
2275                         .init = ahash_init,
2276                         .update = ahash_update,
2277                         .final = ahash_final,
2278                         .finup = ahash_finup,
2279                         .digest = ahash_digest,
2280                         .halg.digestsize = SHA256_DIGEST_SIZE,
2281                         .halg.base = {
2282                                 .cra_name = "sha256",
2283                                 .cra_driver_name = "sha256-talitos",
2284                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2285                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2286                                              CRYPTO_ALG_ASYNC,
2287                                 .cra_type = &crypto_ahash_type
2288                         }
2289                 },
2290                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2291                                      DESC_HDR_SEL0_MDEUA |
2292                                      DESC_HDR_MODE0_MDEU_SHA256,
2293         },
2294         {       .type = CRYPTO_ALG_TYPE_AHASH,
2295                 .alg.hash = {
2296                         .init = ahash_init,
2297                         .update = ahash_update,
2298                         .final = ahash_final,
2299                         .finup = ahash_finup,
2300                         .digest = ahash_digest,
2301                         .halg.digestsize = SHA384_DIGEST_SIZE,
2302                         .halg.base = {
2303                                 .cra_name = "sha384",
2304                                 .cra_driver_name = "sha384-talitos",
2305                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2306                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2307                                              CRYPTO_ALG_ASYNC,
2308                                 .cra_type = &crypto_ahash_type
2309                         }
2310                 },
2311                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2312                                      DESC_HDR_SEL0_MDEUB |
2313                                      DESC_HDR_MODE0_MDEUB_SHA384,
2314         },
2315         {       .type = CRYPTO_ALG_TYPE_AHASH,
2316                 .alg.hash = {
2317                         .init = ahash_init,
2318                         .update = ahash_update,
2319                         .final = ahash_final,
2320                         .finup = ahash_finup,
2321                         .digest = ahash_digest,
2322                         .halg.digestsize = SHA512_DIGEST_SIZE,
2323                         .halg.base = {
2324                                 .cra_name = "sha512",
2325                                 .cra_driver_name = "sha512-talitos",
2326                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2327                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2328                                              CRYPTO_ALG_ASYNC,
2329                                 .cra_type = &crypto_ahash_type
2330                         }
2331                 },
2332                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2333                                      DESC_HDR_SEL0_MDEUB |
2334                                      DESC_HDR_MODE0_MDEUB_SHA512,
2335         },
2336         {       .type = CRYPTO_ALG_TYPE_AHASH,
2337                 .alg.hash = {
2338                         .init = ahash_init,
2339                         .update = ahash_update,
2340                         .final = ahash_final,
2341                         .finup = ahash_finup,
2342                         .digest = ahash_digest,
2343                         .setkey = ahash_setkey,
2344                         .halg.digestsize = MD5_DIGEST_SIZE,
2345                         .halg.base = {
2346                                 .cra_name = "hmac(md5)",
2347                                 .cra_driver_name = "hmac-md5-talitos",
2348                                 .cra_blocksize = MD5_BLOCK_SIZE,
2349                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2350                                              CRYPTO_ALG_ASYNC,
2351                                 .cra_type = &crypto_ahash_type
2352                         }
2353                 },
2354                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2355                                      DESC_HDR_SEL0_MDEUA |
2356                                      DESC_HDR_MODE0_MDEU_MD5,
2357         },
2358         {       .type = CRYPTO_ALG_TYPE_AHASH,
2359                 .alg.hash = {
2360                         .init = ahash_init,
2361                         .update = ahash_update,
2362                         .final = ahash_final,
2363                         .finup = ahash_finup,
2364                         .digest = ahash_digest,
2365                         .setkey = ahash_setkey,
2366                         .halg.digestsize = SHA1_DIGEST_SIZE,
2367                         .halg.base = {
2368                                 .cra_name = "hmac(sha1)",
2369                                 .cra_driver_name = "hmac-sha1-talitos",
2370                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2371                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2372                                              CRYPTO_ALG_ASYNC,
2373                                 .cra_type = &crypto_ahash_type
2374                         }
2375                 },
2376                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2377                                      DESC_HDR_SEL0_MDEUA |
2378                                      DESC_HDR_MODE0_MDEU_SHA1,
2379         },
2380         {       .type = CRYPTO_ALG_TYPE_AHASH,
2381                 .alg.hash = {
2382                         .init = ahash_init,
2383                         .update = ahash_update,
2384                         .final = ahash_final,
2385                         .finup = ahash_finup,
2386                         .digest = ahash_digest,
2387                         .setkey = ahash_setkey,
2388                         .halg.digestsize = SHA224_DIGEST_SIZE,
2389                         .halg.base = {
2390                                 .cra_name = "hmac(sha224)",
2391                                 .cra_driver_name = "hmac-sha224-talitos",
2392                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2393                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2394                                              CRYPTO_ALG_ASYNC,
2395                                 .cra_type = &crypto_ahash_type
2396                         }
2397                 },
2398                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2399                                      DESC_HDR_SEL0_MDEUA |
2400                                      DESC_HDR_MODE0_MDEU_SHA224,
2401         },
2402         {       .type = CRYPTO_ALG_TYPE_AHASH,
2403                 .alg.hash = {
2404                         .init = ahash_init,
2405                         .update = ahash_update,
2406                         .final = ahash_final,
2407                         .finup = ahash_finup,
2408                         .digest = ahash_digest,
2409                         .setkey = ahash_setkey,
2410                         .halg.digestsize = SHA256_DIGEST_SIZE,
2411                         .halg.base = {
2412                                 .cra_name = "hmac(sha256)",
2413                                 .cra_driver_name = "hmac-sha256-talitos",
2414                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2415                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2416                                              CRYPTO_ALG_ASYNC,
2417                                 .cra_type = &crypto_ahash_type
2418                         }
2419                 },
2420                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2421                                      DESC_HDR_SEL0_MDEUA |
2422                                      DESC_HDR_MODE0_MDEU_SHA256,
2423         },
2424         {       .type = CRYPTO_ALG_TYPE_AHASH,
2425                 .alg.hash = {
2426                         .init = ahash_init,
2427                         .update = ahash_update,
2428                         .final = ahash_final,
2429                         .finup = ahash_finup,
2430                         .digest = ahash_digest,
2431                         .setkey = ahash_setkey,
2432                         .halg.digestsize = SHA384_DIGEST_SIZE,
2433                         .halg.base = {
2434                                 .cra_name = "hmac(sha384)",
2435                                 .cra_driver_name = "hmac-sha384-talitos",
2436                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2437                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2438                                              CRYPTO_ALG_ASYNC,
2439                                 .cra_type = &crypto_ahash_type
2440                         }
2441                 },
2442                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2443                                      DESC_HDR_SEL0_MDEUB |
2444                                      DESC_HDR_MODE0_MDEUB_SHA384,
2445         },
2446         {       .type = CRYPTO_ALG_TYPE_AHASH,
2447                 .alg.hash = {
2448                         .init = ahash_init,
2449                         .update = ahash_update,
2450                         .final = ahash_final,
2451                         .finup = ahash_finup,
2452                         .digest = ahash_digest,
2453                         .setkey = ahash_setkey,
2454                         .halg.digestsize = SHA512_DIGEST_SIZE,
2455                         .halg.base = {
2456                                 .cra_name = "hmac(sha512)",
2457                                 .cra_driver_name = "hmac-sha512-talitos",
2458                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2459                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2460                                              CRYPTO_ALG_ASYNC,
2461                                 .cra_type = &crypto_ahash_type
2462                         }
2463                 },
2464                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2465                                      DESC_HDR_SEL0_MDEUB |
2466                                      DESC_HDR_MODE0_MDEUB_SHA512,
2467         }
2468 };
2469
2470 struct talitos_crypto_alg {
2471         struct list_head entry;
2472         struct device *dev;
2473         struct talitos_alg_template algt;
2474 };
2475
2476 static int talitos_cra_init(struct crypto_tfm *tfm)
2477 {
2478         struct crypto_alg *alg = tfm->__crt_alg;
2479         struct talitos_crypto_alg *talitos_alg;
2480         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2481         struct talitos_private *priv;
2482
2483         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2484                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2485                                            struct talitos_crypto_alg,
2486                                            algt.alg.hash);
2487         else
2488                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2489                                            algt.alg.crypto);
2490
2491         /* update context with ptr to dev */
2492         ctx->dev = talitos_alg->dev;
2493
2494         /* assign SEC channel to tfm in round-robin fashion */
2495         priv = dev_get_drvdata(ctx->dev);
2496         ctx->ch = atomic_inc_return(&priv->last_chan) &
2497                   (priv->num_channels - 1);
2498
2499         /* copy descriptor header template value */
2500         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2501
2502         /* select done notification */
2503         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2504
2505         return 0;
2506 }
2507
2508 static int talitos_cra_init_aead(struct crypto_tfm *tfm)
2509 {
2510         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2511
2512         talitos_cra_init(tfm);
2513
2514         /* random first IV */
2515         get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
2516
2517         return 0;
2518 }
2519
2520 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2521 {
2522         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2523
2524         talitos_cra_init(tfm);
2525
2526         ctx->keylen = 0;
2527         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2528                                  sizeof(struct talitos_ahash_req_ctx));
2529
2530         return 0;
2531 }
2532
2533 /*
2534  * given the alg's descriptor header template, determine whether descriptor
2535  * type and primary/secondary execution units required match the hw
2536  * capabilities description provided in the device tree node.
2537  */
2538 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2539 {
2540         struct talitos_private *priv = dev_get_drvdata(dev);
2541         int ret;
2542
2543         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2544               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2545
2546         if (SECONDARY_EU(desc_hdr_template))
2547                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2548                               & priv->exec_units);
2549
2550         return ret;
2551 }
2552
2553 static int talitos_remove(struct platform_device *ofdev)
2554 {
2555         struct device *dev = &ofdev->dev;
2556         struct talitos_private *priv = dev_get_drvdata(dev);
2557         struct talitos_crypto_alg *t_alg, *n;
2558         int i;
2559
2560         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2561                 switch (t_alg->algt.type) {
2562                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2563                 case CRYPTO_ALG_TYPE_AEAD:
2564                         crypto_unregister_alg(&t_alg->algt.alg.crypto);
2565                         break;
2566                 case CRYPTO_ALG_TYPE_AHASH:
2567                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2568                         break;
2569                 }
2570                 list_del(&t_alg->entry);
2571                 kfree(t_alg);
2572         }
2573
2574         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2575                 talitos_unregister_rng(dev);
2576
2577         for (i = 0; i < priv->num_channels; i++)
2578                 kfree(priv->chan[i].fifo);
2579
2580         kfree(priv->chan);
2581
2582         for (i = 0; i < 2; i++)
2583                 if (priv->irq[i]) {
2584                         free_irq(priv->irq[i], dev);
2585                         irq_dispose_mapping(priv->irq[i]);
2586                 }
2587
2588         tasklet_kill(&priv->done_task[0]);
2589         if (priv->irq[1])
2590                 tasklet_kill(&priv->done_task[1]);
2591
2592         iounmap(priv->reg);
2593
2594         dev_set_drvdata(dev, NULL);
2595
2596         kfree(priv);
2597
2598         return 0;
2599 }
2600
2601 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2602                                                     struct talitos_alg_template
2603                                                            *template)
2604 {
2605         struct talitos_private *priv = dev_get_drvdata(dev);
2606         struct talitos_crypto_alg *t_alg;
2607         struct crypto_alg *alg;
2608
2609         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2610         if (!t_alg)
2611                 return ERR_PTR(-ENOMEM);
2612
2613         t_alg->algt = *template;
2614
2615         switch (t_alg->algt.type) {
2616         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2617                 alg = &t_alg->algt.alg.crypto;
2618                 alg->cra_init = talitos_cra_init;
2619                 break;
2620         case CRYPTO_ALG_TYPE_AEAD:
2621                 alg = &t_alg->algt.alg.crypto;
2622                 alg->cra_init = talitos_cra_init_aead;
2623                 break;
2624         case CRYPTO_ALG_TYPE_AHASH:
2625                 alg = &t_alg->algt.alg.hash.halg.base;
2626                 alg->cra_init = talitos_cra_init_ahash;
2627                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2628                     !strncmp(alg->cra_name, "hmac", 4)) {
2629                         kfree(t_alg);
2630                         return ERR_PTR(-ENOTSUPP);
2631                 }
2632                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2633                     (!strcmp(alg->cra_name, "sha224") ||
2634                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
2635                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2636                         t_alg->algt.desc_hdr_template =
2637                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2638                                         DESC_HDR_SEL0_MDEUA |
2639                                         DESC_HDR_MODE0_MDEU_SHA256;
2640                 }
2641                 break;
2642         default:
2643                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2644                 return ERR_PTR(-EINVAL);
2645         }
2646
2647         alg->cra_module = THIS_MODULE;
2648         alg->cra_priority = TALITOS_CRA_PRIORITY;
2649         alg->cra_alignmask = 0;
2650         alg->cra_ctxsize = sizeof(struct talitos_ctx);
2651         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2652
2653         t_alg->dev = dev;
2654
2655         return t_alg;
2656 }
2657
2658 static int talitos_probe_irq(struct platform_device *ofdev)
2659 {
2660         struct device *dev = &ofdev->dev;
2661         struct device_node *np = ofdev->dev.of_node;
2662         struct talitos_private *priv = dev_get_drvdata(dev);
2663         int err;
2664
2665         priv->irq[0] = irq_of_parse_and_map(np, 0);
2666         if (!priv->irq[0]) {
2667                 dev_err(dev, "failed to map irq\n");
2668                 return -EINVAL;
2669         }
2670
2671         priv->irq[1] = irq_of_parse_and_map(np, 1);
2672
2673         /* get the primary irq line */
2674         if (!priv->irq[1]) {
2675                 err = request_irq(priv->irq[0], talitos_interrupt_4ch, 0,
2676                                   dev_driver_string(dev), dev);
2677                 goto primary_out;
2678         }
2679
2680         err = request_irq(priv->irq[0], talitos_interrupt_ch0_2, 0,
2681                           dev_driver_string(dev), dev);
2682         if (err)
2683                 goto primary_out;
2684
2685         /* get the secondary irq line */
2686         err = request_irq(priv->irq[1], talitos_interrupt_ch1_3, 0,
2687                           dev_driver_string(dev), dev);
2688         if (err) {
2689                 dev_err(dev, "failed to request secondary irq\n");
2690                 irq_dispose_mapping(priv->irq[1]);
2691                 priv->irq[1] = 0;
2692         }
2693
2694         return err;
2695
2696 primary_out:
2697         if (err) {
2698                 dev_err(dev, "failed to request primary irq\n");
2699                 irq_dispose_mapping(priv->irq[0]);
2700                 priv->irq[0] = 0;
2701         }
2702
2703         return err;
2704 }
2705
2706 static int talitos_probe(struct platform_device *ofdev)
2707 {
2708         struct device *dev = &ofdev->dev;
2709         struct device_node *np = ofdev->dev.of_node;
2710         struct talitos_private *priv;
2711         const unsigned int *prop;
2712         int i, err;
2713
2714         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2715         if (!priv)
2716                 return -ENOMEM;
2717
2718         dev_set_drvdata(dev, priv);
2719
2720         priv->ofdev = ofdev;
2721
2722         err = talitos_probe_irq(ofdev);
2723         if (err)
2724                 goto err_out;
2725
2726         if (!priv->irq[1]) {
2727                 tasklet_init(&priv->done_task[0], talitos_done_4ch,
2728                              (unsigned long)dev);
2729         } else {
2730                 tasklet_init(&priv->done_task[0], talitos_done_ch0_2,
2731                              (unsigned long)dev);
2732                 tasklet_init(&priv->done_task[1], talitos_done_ch1_3,
2733                              (unsigned long)dev);
2734         }
2735
2736         INIT_LIST_HEAD(&priv->alg_list);
2737
2738         priv->reg = of_iomap(np, 0);
2739         if (!priv->reg) {
2740                 dev_err(dev, "failed to of_iomap\n");
2741                 err = -ENOMEM;
2742                 goto err_out;
2743         }
2744
2745         /* get SEC version capabilities from device tree */
2746         prop = of_get_property(np, "fsl,num-channels", NULL);
2747         if (prop)
2748                 priv->num_channels = *prop;
2749
2750         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2751         if (prop)
2752                 priv->chfifo_len = *prop;
2753
2754         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2755         if (prop)
2756                 priv->exec_units = *prop;
2757
2758         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2759         if (prop)
2760                 priv->desc_types = *prop;
2761
2762         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2763             !priv->exec_units || !priv->desc_types) {
2764                 dev_err(dev, "invalid property data in device tree node\n");
2765                 err = -EINVAL;
2766                 goto err_out;
2767         }
2768
2769         if (of_device_is_compatible(np, "fsl,sec3.0"))
2770                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2771
2772         if (of_device_is_compatible(np, "fsl,sec2.1"))
2773                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2774                                   TALITOS_FTR_SHA224_HWINIT |
2775                                   TALITOS_FTR_HMAC_OK;
2776
2777         priv->chan = kzalloc(sizeof(struct talitos_channel) *
2778                              priv->num_channels, GFP_KERNEL);
2779         if (!priv->chan) {
2780                 dev_err(dev, "failed to allocate channel management space\n");
2781                 err = -ENOMEM;
2782                 goto err_out;
2783         }
2784
2785         for (i = 0; i < priv->num_channels; i++) {
2786                 priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1);
2787                 if (!priv->irq[1] || !(i & 1))
2788                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
2789         }
2790
2791         for (i = 0; i < priv->num_channels; i++) {
2792                 spin_lock_init(&priv->chan[i].head_lock);
2793                 spin_lock_init(&priv->chan[i].tail_lock);
2794         }
2795
2796         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
2797
2798         for (i = 0; i < priv->num_channels; i++) {
2799                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
2800                                              priv->fifo_len, GFP_KERNEL);
2801                 if (!priv->chan[i].fifo) {
2802                         dev_err(dev, "failed to allocate request fifo %d\n", i);
2803                         err = -ENOMEM;
2804                         goto err_out;
2805                 }
2806         }
2807
2808         for (i = 0; i < priv->num_channels; i++)
2809                 atomic_set(&priv->chan[i].submit_count,
2810                            -(priv->chfifo_len - 1));
2811
2812         dma_set_mask(dev, DMA_BIT_MASK(36));
2813
2814         /* reset and initialize the h/w */
2815         err = init_device(dev);
2816         if (err) {
2817                 dev_err(dev, "failed to initialize device\n");
2818                 goto err_out;
2819         }
2820
2821         /* register the RNG, if available */
2822         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
2823                 err = talitos_register_rng(dev);
2824                 if (err) {
2825                         dev_err(dev, "failed to register hwrng: %d\n", err);
2826                         goto err_out;
2827                 } else
2828                         dev_info(dev, "hwrng\n");
2829         }
2830
2831         /* register crypto algorithms the device supports */
2832         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2833                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
2834                         struct talitos_crypto_alg *t_alg;
2835                         char *name = NULL;
2836
2837                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
2838                         if (IS_ERR(t_alg)) {
2839                                 err = PTR_ERR(t_alg);
2840                                 if (err == -ENOTSUPP)
2841                                         continue;
2842                                 goto err_out;
2843                         }
2844
2845                         switch (t_alg->algt.type) {
2846                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2847                         case CRYPTO_ALG_TYPE_AEAD:
2848                                 err = crypto_register_alg(
2849                                                 &t_alg->algt.alg.crypto);
2850                                 name = t_alg->algt.alg.crypto.cra_driver_name;
2851                                 break;
2852                         case CRYPTO_ALG_TYPE_AHASH:
2853                                 err = crypto_register_ahash(
2854                                                 &t_alg->algt.alg.hash);
2855                                 name =
2856                                  t_alg->algt.alg.hash.halg.base.cra_driver_name;
2857                                 break;
2858                         }
2859                         if (err) {
2860                                 dev_err(dev, "%s alg registration failed\n",
2861                                         name);
2862                                 kfree(t_alg);
2863                         } else
2864                                 list_add_tail(&t_alg->entry, &priv->alg_list);
2865                 }
2866         }
2867         if (!list_empty(&priv->alg_list))
2868                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
2869                          (char *)of_get_property(np, "compatible", NULL));
2870
2871         return 0;
2872
2873 err_out:
2874         talitos_remove(ofdev);
2875
2876         return err;
2877 }
2878
2879 static const struct of_device_id talitos_match[] = {
2880         {
2881                 .compatible = "fsl,sec2.0",
2882         },
2883         {},
2884 };
2885 MODULE_DEVICE_TABLE(of, talitos_match);
2886
2887 static struct platform_driver talitos_driver = {
2888         .driver = {
2889                 .name = "talitos",
2890                 .owner = THIS_MODULE,
2891                 .of_match_table = talitos_match,
2892         },
2893         .probe = talitos_probe,
2894         .remove = talitos_remove,
2895 };
2896
2897 module_platform_driver(talitos_driver);
2898
2899 MODULE_LICENSE("GPL");
2900 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
2901 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");