drivers/dma/ioat/dma.c

   1 /*
   2  * Intel I/OAT DMA Linux driver
   3  * Copyright(c) 2004 - 2009 Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  */
  22
  23 /*
  24  * This driver supports an Intel I/OAT DMA engine, which does asynchronous
  25  * copy operations.
  26  */
  27
  28 #include <linux/init.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/dmaengine.h>
  33 #include <linux/delay.h>
  34 #include <linux/dma-mapping.h>
  35 #include <linux/workqueue.h>
  36 #include <linux/i7300_idle.h>
  37 #include "dma.h"
  38 #include "registers.h"
  39 #include "hw.h"
  40
  41 static int ioat_pending_level = 4;
  42 module_param(ioat_pending_level, int, 0644);
  43 MODULE_PARM_DESC(ioat_pending_level,
  44                  "high-water mark for pushing ioat descriptors (default: 4)");
  45
  46 static void ioat_dma_chan_reset_part2(struct work_struct *work);
  47 static void ioat_dma_chan_watchdog(struct work_struct *work);
  48
  49 /* internal functions */
  50 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan);
  51 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
  52
  53 static struct ioat_desc_sw *
  54 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
  55 static struct ioat_desc_sw *
  56 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan);
  57
  58 static inline struct ioat_dma_chan *
  59 ioat_chan_by_index(struct ioatdma_device *device, int index)
  60 {
  61         return device->idx[index];
  62 }
  63
  64 /**
  65  * ioat_dma_do_interrupt - handler used for single vector interrupt mode
  66  * @irq: interrupt id
  67  * @data: interrupt data
  68  */
  69 static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
  70 {
  71         struct ioatdma_device *instance = data;
  72         struct ioat_dma_chan *ioat_chan;
  73         unsigned long attnstatus;
  74         int bit;
  75         u8 intrctrl;
  76
  77         intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
  78
  79         if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
  80                 return IRQ_NONE;
  81
  82         if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
  83                 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  84                 return IRQ_NONE;
  85         }
  86
  87         attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
  88         for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
  89                 ioat_chan = ioat_chan_by_index(instance, bit);
  90                 tasklet_schedule(&ioat_chan->cleanup_task);
  91         }
  92
  93         writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
  94         return IRQ_HANDLED;
  95 }
  96
  97 /**
  98  * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
  99  * @irq: interrupt id
 100  * @data: interrupt data
 101  */
 102 static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
 103 {
 104         struct ioat_dma_chan *ioat_chan = data;
 105
 106         tasklet_schedule(&ioat_chan->cleanup_task);
 107
 108         return IRQ_HANDLED;
 109 }
 110
 111 static void ioat_dma_cleanup_tasklet(unsigned long data);
 112
 113 /**
 114  * ioat_dma_enumerate_channels - find and initialize the device's channels
 115  * @device: the device to be enumerated
 116  */
 117 static int ioat_dma_enumerate_channels(struct ioatdma_device *device)
 118 {
 119         u8 xfercap_scale;
 120         u32 xfercap;
 121         int i;
 122         struct ioat_dma_chan *ioat_chan;
 123         struct device *dev = &device->pdev->dev;
 124
 125         /*
 126          * IOAT ver.3 workarounds
 127          */
 128         if (device->version == IOAT_VER_3_0) {
 129                 u32 chan_err_mask;
 130                 u16 dev_id;
 131                 u32 dmauncerrsts;
 132
 133                 /*
 134                  * Write CHANERRMSK_INT with 3E07h to mask out the errors
 135                  * that can cause stability issues for IOAT ver.3
 136                  */
 137                 chan_err_mask = 0x3E07;
 138                 pci_write_config_dword(device->pdev,
 139                         IOAT_PCI_CHANERRMASK_INT_OFFSET,
 140                         chan_err_mask);
 141
 142                 /*
 143                  * Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit
 144                  * (workaround for spurious config parity error after restart)
 145                  */
 146                 pci_read_config_word(device->pdev,
 147                         IOAT_PCI_DEVICE_ID_OFFSET,
 148                         &dev_id);
 149                 if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) {
 150                         dmauncerrsts = 0x10;
 151                         pci_write_config_dword(device->pdev,
 152                                 IOAT_PCI_DMAUNCERRSTS_OFFSET,
 153                                 dmauncerrsts);
 154                 }
 155         }
 156
 157         device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
 158         xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
 159         xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
 160
 161 #ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
 162         if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) {
 163                 device->common.chancnt--;
 164         }
 165 #endif
 166         for (i = 0; i < device->common.chancnt; i++) {
 167                 ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL);
 168                 if (!ioat_chan) {
 169                         device->common.chancnt = i;
 170                         break;
 171                 }
 172
 173                 ioat_chan->device = device;
 174                 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
 175                 ioat_chan->xfercap = xfercap;
 176                 ioat_chan->desccount = 0;
 177                 INIT_DELAYED_WORK(&ioat_chan->work, ioat_dma_chan_reset_part2);
 178                 if (ioat_chan->device->version == IOAT_VER_2_0)
 179                         writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE |
 180                                IOAT_DMA_DCA_ANY_CPU,
 181                                ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
 182                 else if (ioat_chan->device->version == IOAT_VER_3_0)
 183                         writel(IOAT_DMA_DCA_ANY_CPU,
 184                                ioat_chan->reg_base + IOAT_DCACTRL_OFFSET);
 185                 spin_lock_init(&ioat_chan->cleanup_lock);
 186                 spin_lock_init(&ioat_chan->desc_lock);
 187                 INIT_LIST_HEAD(&ioat_chan->free_desc);
 188                 INIT_LIST_HEAD(&ioat_chan->used_desc);
 189                 /* This should be made common somewhere in dmaengine.c */
 190                 ioat_chan->common.device = &device->common;
 191                 list_add_tail(&ioat_chan->common.device_node,
 192                               &device->common.channels);
 193                 device->idx[i] = ioat_chan;
 194                 tasklet_init(&ioat_chan->cleanup_task,
 195                              ioat_dma_cleanup_tasklet,
 196                              (unsigned long) ioat_chan);
 197                 tasklet_disable(&ioat_chan->cleanup_task);
 198         }
 199         return device->common.chancnt;
 200 }
 201
 202 /**
 203  * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
 204  *                                 descriptors to hw
 205  * @chan: DMA channel handle
 206  */
 207 static inline void
 208 __ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat_chan)
 209 {
 210         ioat_chan->pending = 0;
 211         writeb(IOAT_CHANCMD_APPEND, ioat_chan->reg_base + IOAT1_CHANCMD_OFFSET);
 212 }
 213
 214 static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
 215 {
 216         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 217
 218         if (ioat_chan->pending > 0) {
 219                 spin_lock_bh(&ioat_chan->desc_lock);
 220                 __ioat1_dma_memcpy_issue_pending(ioat_chan);
 221                 spin_unlock_bh(&ioat_chan->desc_lock);
 222         }
 223 }
 224
 225 static inline void
 226 __ioat2_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat_chan)
 227 {
 228         ioat_chan->pending = 0;
 229         writew(ioat_chan->dmacount,
 230                ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
 231 }
 232
 233 static void ioat2_dma_memcpy_issue_pending(struct dma_chan *chan)
 234 {
 235         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 236
 237         if (ioat_chan->pending > 0) {
 238                 spin_lock_bh(&ioat_chan->desc_lock);
 239                 __ioat2_dma_memcpy_issue_pending(ioat_chan);
 240                 spin_unlock_bh(&ioat_chan->desc_lock);
 241         }
 242 }
 243
 244
 245 /**
 246  * ioat_dma_chan_reset_part2 - reinit the channel after a reset
 247  */
 248 static void ioat_dma_chan_reset_part2(struct work_struct *work)
 249 {
 250         struct ioat_dma_chan *ioat_chan =
 251                 container_of(work, struct ioat_dma_chan, work.work);
 252         struct ioat_desc_sw *desc;
 253
 254         spin_lock_bh(&ioat_chan->cleanup_lock);
 255         spin_lock_bh(&ioat_chan->desc_lock);
 256
 257         ioat_chan->completion_virt->low = 0;
 258         ioat_chan->completion_virt->high = 0;
 259         ioat_chan->pending = 0;
 260
 261         /*
 262          * count the descriptors waiting, and be sure to do it
 263          * right for both the CB1 line and the CB2 ring
 264          */
 265         ioat_chan->dmacount = 0;
 266         if (ioat_chan->used_desc.prev) {
 267                 desc = to_ioat_desc(ioat_chan->used_desc.prev);
 268                 do {
 269                         ioat_chan->dmacount++;
 270                         desc = to_ioat_desc(desc->node.next);
 271                 } while (&desc->node != ioat_chan->used_desc.next);
 272         }
 273
 274         /*
 275          * write the new starting descriptor address
 276          * this puts channel engine into ARMED state
 277          */
 278         desc = to_ioat_desc(ioat_chan->used_desc.prev);
 279         switch (ioat_chan->device->version) {
 280         case IOAT_VER_1_2:
 281                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 282                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
 283                 writel(((u64) desc->txd.phys) >> 32,
 284                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
 285
 286                 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
 287                         + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
 288                 break;
 289         case IOAT_VER_2_0:
 290                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
 291                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
 292                 writel(((u64) desc->txd.phys) >> 32,
 293                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
 294
 295                 /* tell the engine to go with what's left to be done */
 296                 writew(ioat_chan->dmacount,
 297                        ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
 298
 299                 break;
 300         }
 301         dev_err(to_dev(ioat_chan),
 302                 "chan%d reset - %d descs waiting, %d total desc\n",
 303                 chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
 304
 305         spin_unlock_bh(&ioat_chan->desc_lock);
 306         spin_unlock_bh(&ioat_chan->cleanup_lock);
 307 }
 308
 309 /**
 310  * ioat_dma_reset_channel - restart a channel
 311  * @ioat_chan: IOAT DMA channel handle
 312  */
 313 static void ioat_dma_reset_channel(struct ioat_dma_chan *ioat_chan)
 314 {
 315         u32 chansts, chanerr;
 316
 317         if (!ioat_chan->used_desc.prev)
 318                 return;
 319
 320         chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 321         chansts = (ioat_chan->completion_virt->low
 322                                         & IOAT_CHANSTS_DMA_TRANSFER_STATUS);
 323         if (chanerr) {
 324                 dev_err(to_dev(ioat_chan),
 325                         "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
 326                         chan_num(ioat_chan), chansts, chanerr);
 327                 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 328         }
 329
 330         /*
 331          * whack it upside the head with a reset
 332          * and wait for things to settle out.
 333          * force the pending count to a really big negative
 334          * to make sure no one forces an issue_pending
 335          * while we're waiting.
 336          */
 337
 338         spin_lock_bh(&ioat_chan->desc_lock);
 339         ioat_chan->pending = INT_MIN;
 340         writeb(IOAT_CHANCMD_RESET,
 341                ioat_chan->reg_base
 342                + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
 343         spin_unlock_bh(&ioat_chan->desc_lock);
 344
 345         /* schedule the 2nd half instead of sleeping a long time */
 346         schedule_delayed_work(&ioat_chan->work, RESET_DELAY);
 347 }
 348
 349 /**
 350  * ioat_dma_chan_watchdog - watch for stuck channels
 351  */
 352 static void ioat_dma_chan_watchdog(struct work_struct *work)
 353 {
 354         struct ioatdma_device *device =
 355                 container_of(work, struct ioatdma_device, work.work);
 356         struct ioat_dma_chan *ioat_chan;
 357         int i;
 358
 359         union {
 360                 u64 full;
 361                 struct {
 362                         u32 low;
 363                         u32 high;
 364                 };
 365         } completion_hw;
 366         unsigned long compl_desc_addr_hw;
 367
 368         for (i = 0; i < device->common.chancnt; i++) {
 369                 ioat_chan = ioat_chan_by_index(device, i);
 370
 371                 if (ioat_chan->device->version == IOAT_VER_1_2
 372                         /* have we started processing anything yet */
 373                     && ioat_chan->last_completion
 374                         /* have we completed any since last watchdog cycle? */
 375                     && (ioat_chan->last_completion ==
 376                                 ioat_chan->watchdog_completion)
 377                         /* has TCP stuck on one cookie since last watchdog? */
 378                     && (ioat_chan->watchdog_tcp_cookie ==
 379                                 ioat_chan->watchdog_last_tcp_cookie)
 380                     && (ioat_chan->watchdog_tcp_cookie !=
 381                                 ioat_chan->completed_cookie)
 382                         /* is there something in the chain to be processed? */
 383                         /* CB1 chain always has at least the last one processed */
 384                     && (ioat_chan->used_desc.prev != ioat_chan->used_desc.next)
 385                     && ioat_chan->pending == 0) {
 386
 387                         /*
 388                          * check CHANSTS register for completed
 389                          * descriptor address.
 390                          * if it is different than completion writeback,
 391                          * it is not zero
 392                          * and it has changed since the last watchdog
 393                          *     we can assume that channel
 394                          *     is still working correctly
 395                          *     and the problem is in completion writeback.
 396                          *     update completion writeback
 397                          *     with actual CHANSTS value
 398                          * else
 399                          *     try resetting the channel
 400                          */
 401
 402                         completion_hw.low = readl(ioat_chan->reg_base +
 403                                 IOAT_CHANSTS_OFFSET_LOW(ioat_chan->device->version));
 404                         completion_hw.high = readl(ioat_chan->reg_base +
 405                                 IOAT_CHANSTS_OFFSET_HIGH(ioat_chan->device->version));
 406 #if (BITS_PER_LONG == 64)
 407                         compl_desc_addr_hw =
 408                                 completion_hw.full
 409                                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
 410 #else
 411                         compl_desc_addr_hw =
 412                                 completion_hw.low & IOAT_LOW_COMPLETION_MASK;
 413 #endif
 414
 415                         if ((compl_desc_addr_hw != 0)
 416                            && (compl_desc_addr_hw != ioat_chan->watchdog_completion)
 417                            && (compl_desc_addr_hw != ioat_chan->last_compl_desc_addr_hw)) {
 418                                 ioat_chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
 419                                 ioat_chan->completion_virt->low = completion_hw.low;
 420                                 ioat_chan->completion_virt->high = completion_hw.high;
 421                         } else {
 422                                 ioat_dma_reset_channel(ioat_chan);
 423                                 ioat_chan->watchdog_completion = 0;
 424                                 ioat_chan->last_compl_desc_addr_hw = 0;
 425                         }
 426
 427                 /*
 428                  * for version 2.0 if there are descriptors yet to be processed
 429                  * and the last completed hasn't changed since the last watchdog
 430                  *      if they haven't hit the pending level
 431                  *          issue the pending to push them through
 432                  *      else
 433                  *          try resetting the channel
 434                  */
 435                 } else if (ioat_chan->device->version == IOAT_VER_2_0
 436                     && ioat_chan->used_desc.prev
 437                     && ioat_chan->last_completion
 438                     && ioat_chan->last_completion == ioat_chan->watchdog_completion) {
 439
 440                         if (ioat_chan->pending < ioat_pending_level)
 441                                 ioat2_dma_memcpy_issue_pending(&ioat_chan->common);
 442                         else {
 443                                 ioat_dma_reset_channel(ioat_chan);
 444                                 ioat_chan->watchdog_completion = 0;
 445                         }
 446                 } else {
 447                         ioat_chan->last_compl_desc_addr_hw = 0;
 448                         ioat_chan->watchdog_completion
 449                                         = ioat_chan->last_completion;
 450                 }
 451
 452                 ioat_chan->watchdog_last_tcp_cookie =
 453                         ioat_chan->watchdog_tcp_cookie;
 454         }
 455
 456         schedule_delayed_work(&device->work, WATCHDOG_DELAY);
 457 }
 458
 459 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
 460 {
 461         struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
 462         struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
 463         struct ioat_desc_sw *prev, *new;
 464         struct ioat_dma_descriptor *hw;
 465         dma_cookie_t cookie;
 466         LIST_HEAD(new_chain);
 467         u32 copy;
 468         size_t len;
 469         dma_addr_t src, dst;
 470         unsigned long orig_flags;
 471         unsigned int desc_count = 0;
 472
 473         /* src and dest and len are stored in the initial descriptor */
 474         len = first->len;
 475         src = first->src;
 476         dst = first->dst;
 477         orig_flags = first->txd.flags;
 478         new = first;
 479
 480         spin_lock_bh(&ioat_chan->desc_lock);
 481         prev = to_ioat_desc(ioat_chan->used_desc.prev);
 482         prefetch(prev->hw);
 483         do {
 484                 copy = min_t(size_t, len, ioat_chan->xfercap);
 485
 486                 async_tx_ack(&new->txd);
 487
 488                 hw = new->hw;
 489                 hw->size = copy;
 490                 hw->ctl = 0;
 491                 hw->src_addr = src;
 492                 hw->dst_addr = dst;
 493                 hw->next = 0;
 494
 495                 /* chain together the physical address list for the HW */
 496                 wmb();
 497                 prev->hw->next = (u64) new->txd.phys;
 498
 499                 len -= copy;
 500                 dst += copy;
 501                 src += copy;
 502
 503                 list_add_tail(&new->node, &new_chain);
 504                 desc_count++;
 505                 prev = new;
 506         } while (len && (new = ioat1_dma_get_next_descriptor(ioat_chan)));
 507
 508         if (!new) {
 509                 dev_err(to_dev(ioat_chan), "tx submit failed\n");
 510                 spin_unlock_bh(&ioat_chan->desc_lock);
 511                 return -ENOMEM;
 512         }
 513
 514         hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
 515         if (first->txd.callback) {
 516                 hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
 517                 if (first != new) {
 518                         /* move callback into to last desc */
 519                         new->txd.callback = first->txd.callback;
 520                         new->txd.callback_param
 521                                         = first->txd.callback_param;
 522                         first->txd.callback = NULL;
 523                         first->txd.callback_param = NULL;
 524                 }
 525         }
 526
 527         new->tx_cnt = desc_count;
 528         new->txd.flags = orig_flags; /* client is in control of this ack */
 529
 530         /* store the original values for use in later cleanup */
 531         if (new != first) {
 532                 new->src = first->src;
 533                 new->dst = first->dst;
 534                 new->len = first->len;
 535         }
 536
 537         /* cookie incr and addition to used_list must be atomic */
 538         cookie = ioat_chan->common.cookie;
 539         cookie++;
 540         if (cookie < 0)
 541                 cookie = 1;
 542         ioat_chan->common.cookie = new->txd.cookie = cookie;
 543
 544         /* write address into NextDescriptor field of last desc in chain */
 545         to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
 546                                                         first->txd.phys;
 547         list_splice_tail(&new_chain, &ioat_chan->used_desc);
 548
 549         ioat_chan->dmacount += desc_count;
 550         ioat_chan->pending += desc_count;
 551         if (ioat_chan->pending >= ioat_pending_level)
 552                 __ioat1_dma_memcpy_issue_pending(ioat_chan);
 553         spin_unlock_bh(&ioat_chan->desc_lock);
 554
 555         return cookie;
 556 }
 557
 558 static dma_cookie_t ioat2_tx_submit(struct dma_async_tx_descriptor *tx)
 559 {
 560         struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
 561         struct ioat_desc_sw *first = tx_to_ioat_desc(tx);
 562         struct ioat_desc_sw *new;
 563         struct ioat_dma_descriptor *hw;
 564         dma_cookie_t cookie;
 565         u32 copy;
 566         size_t len;
 567         dma_addr_t src, dst;
 568         unsigned long orig_flags;
 569         unsigned int desc_count = 0;
 570
 571         /* src and dest and len are stored in the initial descriptor */
 572         len = first->len;
 573         src = first->src;
 574         dst = first->dst;
 575         orig_flags = first->txd.flags;
 576         new = first;
 577
 578         /*
 579          * ioat_chan->desc_lock is still in force in version 2 path
 580          * it gets unlocked at end of this function
 581          */
 582         do {
 583                 copy = min_t(size_t, len, ioat_chan->xfercap);
 584
 585                 async_tx_ack(&new->txd);
 586
 587                 hw = new->hw;
 588                 hw->size = copy;
 589                 hw->ctl = 0;
 590                 hw->src_addr = src;
 591                 hw->dst_addr = dst;
 592
 593                 len -= copy;
 594                 dst += copy;
 595                 src += copy;
 596                 desc_count++;
 597         } while (len && (new = ioat2_dma_get_next_descriptor(ioat_chan)));
 598
 599         if (!new) {
 600                 dev_err(to_dev(ioat_chan), "tx submit failed\n");
 601                 spin_unlock_bh(&ioat_chan->desc_lock);
 602                 return -ENOMEM;
 603         }
 604
 605         hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
 606         if (first->txd.callback) {
 607                 hw->ctl |= IOAT_DMA_DESCRIPTOR_CTL_INT_GN;
 608                 if (first != new) {
 609                         /* move callback into to last desc */
 610                         new->txd.callback = first->txd.callback;
 611                         new->txd.callback_param
 612                                         = first->txd.callback_param;
 613                         first->txd.callback = NULL;
 614                         first->txd.callback_param = NULL;
 615                 }
 616         }
 617
 618         new->tx_cnt = desc_count;
 619         new->txd.flags = orig_flags; /* client is in control of this ack */
 620
 621         /* store the original values for use in later cleanup */
 622         if (new != first) {
 623                 new->src = first->src;
 624                 new->dst = first->dst;
 625                 new->len = first->len;
 626         }
 627
 628         /* cookie incr and addition to used_list must be atomic */
 629         cookie = ioat_chan->common.cookie;
 630         cookie++;
 631         if (cookie < 0)
 632                 cookie = 1;
 633         ioat_chan->common.cookie = new->txd.cookie = cookie;
 634
 635         ioat_chan->dmacount += desc_count;
 636         ioat_chan->pending += desc_count;
 637         if (ioat_chan->pending >= ioat_pending_level)
 638                 __ioat2_dma_memcpy_issue_pending(ioat_chan);
 639         spin_unlock_bh(&ioat_chan->desc_lock);
 640
 641         return cookie;
 642 }
 643
 644 /**
 645  * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
 646  * @ioat_chan: the channel supplying the memory pool for the descriptors
 647  * @flags: allocation flags
 648  */
 649 static struct ioat_desc_sw *
 650 ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat_chan, gfp_t flags)
 651 {
 652         struct ioat_dma_descriptor *desc;
 653         struct ioat_desc_sw *desc_sw;
 654         struct ioatdma_device *ioatdma_device;
 655         dma_addr_t phys;
 656
 657         ioatdma_device = to_ioatdma_device(ioat_chan->common.device);
 658         desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
 659         if (unlikely(!desc))
 660                 return NULL;
 661
 662         desc_sw = kzalloc(sizeof(*desc_sw), flags);
 663         if (unlikely(!desc_sw)) {
 664                 pci_pool_free(ioatdma_device->dma_pool, desc, phys);
 665                 return NULL;
 666         }
 667
 668         memset(desc, 0, sizeof(*desc));
 669         dma_async_tx_descriptor_init(&desc_sw->txd, &ioat_chan->common);
 670         switch (ioat_chan->device->version) {
 671         case IOAT_VER_1_2:
 672                 desc_sw->txd.tx_submit = ioat1_tx_submit;
 673                 break;
 674         case IOAT_VER_2_0:
 675         case IOAT_VER_3_0:
 676                 desc_sw->txd.tx_submit = ioat2_tx_submit;
 677                 break;
 678         }
 679
 680         desc_sw->hw = desc;
 681         desc_sw->txd.phys = phys;
 682
 683         return desc_sw;
 684 }
 685
 686 static int ioat_initial_desc_count = 256;
 687 module_param(ioat_initial_desc_count, int, 0644);
 688 MODULE_PARM_DESC(ioat_initial_desc_count,
 689                  "initial descriptors per channel (default: 256)");
 690
 691 /**
 692  * ioat2_dma_massage_chan_desc - link the descriptors into a circle
 693  * @ioat_chan: the channel to be massaged
 694  */
 695 static void ioat2_dma_massage_chan_desc(struct ioat_dma_chan *ioat_chan)
 696 {
 697         struct ioat_desc_sw *desc, *_desc;
 698
 699         /* setup used_desc */
 700         ioat_chan->used_desc.next = ioat_chan->free_desc.next;
 701         ioat_chan->used_desc.prev = NULL;
 702
 703         /* pull free_desc out of the circle so that every node is a hw
 704          * descriptor, but leave it pointing to the list
 705          */
 706         ioat_chan->free_desc.prev->next = ioat_chan->free_desc.next;
 707         ioat_chan->free_desc.next->prev = ioat_chan->free_desc.prev;
 708
 709         /* circle link the hw descriptors */
 710         desc = to_ioat_desc(ioat_chan->free_desc.next);
 711         desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
 712         list_for_each_entry_safe(desc, _desc, ioat_chan->free_desc.next, node) {
 713                 desc->hw->next = to_ioat_desc(desc->node.next)->txd.phys;
 714         }
 715 }
 716
 717 /**
 718  * ioat_dma_alloc_chan_resources - returns the number of allocated descriptors
 719  * @chan: the channel to be filled out
 720  */
 721 static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
 722 {
 723         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 724         struct ioat_desc_sw *desc;
 725         u16 chanctrl;
 726         u32 chanerr;
 727         int i;
 728         LIST_HEAD(tmp_list);
 729
 730         /* have we already been set up? */
 731         if (!list_empty(&ioat_chan->free_desc))
 732                 return ioat_chan->desccount;
 733
 734         /* Setup register to interrupt and write completion status on error */
 735         chanctrl = IOAT_CHANCTRL_ERR_INT_EN |
 736                 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
 737                 IOAT_CHANCTRL_ERR_COMPLETION_EN;
 738         writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
 739
 740         chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 741         if (chanerr) {
 742                 dev_err(to_dev(ioat_chan), "CHANERR = %x, clearing\n", chanerr);
 743                 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
 744         }
 745
 746         /* Allocate descriptors */
 747         for (i = 0; i < ioat_initial_desc_count; i++) {
 748                 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
 749                 if (!desc) {
 750                         dev_err(to_dev(ioat_chan),
 751                                 "Only %d initial descriptors\n", i);
 752                         break;
 753                 }
 754                 list_add_tail(&desc->node, &tmp_list);
 755         }
 756         spin_lock_bh(&ioat_chan->desc_lock);
 757         ioat_chan->desccount = i;
 758         list_splice(&tmp_list, &ioat_chan->free_desc);
 759         if (ioat_chan->device->version != IOAT_VER_1_2)
 760                 ioat2_dma_massage_chan_desc(ioat_chan);
 761         spin_unlock_bh(&ioat_chan->desc_lock);
 762
 763         /* allocate a completion writeback area */
 764         /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
 765         ioat_chan->completion_virt =
 766                 pci_pool_alloc(ioat_chan->device->completion_pool,
 767                                GFP_KERNEL,
 768                                &ioat_chan->completion_addr);
 769         memset(ioat_chan->completion_virt, 0,
 770                sizeof(*ioat_chan->completion_virt));
 771         writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
 772                ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
 773         writel(((u64) ioat_chan->completion_addr) >> 32,
 774                ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
 775
 776         tasklet_enable(&ioat_chan->cleanup_task);
 777         ioat_dma_start_null_desc(ioat_chan);  /* give chain to dma device */
 778         return ioat_chan->desccount;
 779 }
 780
 781 /**
 782  * ioat_dma_free_chan_resources - release all the descriptors
 783  * @chan: the channel to be cleaned
 784  */
 785 static void ioat_dma_free_chan_resources(struct dma_chan *chan)
 786 {
 787         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 788         struct ioatdma_device *ioatdma_device = to_ioatdma_device(chan->device);
 789         struct ioat_desc_sw *desc, *_desc;
 790         int in_use_descs = 0;
 791
 792         /* Before freeing channel resources first check
 793          * if they have been previously allocated for this channel.
 794          */
 795         if (ioat_chan->desccount == 0)
 796                 return;
 797
 798         tasklet_disable(&ioat_chan->cleanup_task);
 799         ioat_dma_memcpy_cleanup(ioat_chan);
 800
 801         /* Delay 100ms after reset to allow internal DMA logic to quiesce
 802          * before removing DMA descriptor resources.
 803          */
 804         writeb(IOAT_CHANCMD_RESET,
 805                ioat_chan->reg_base
 806                         + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
 807         mdelay(100);
 808
 809         spin_lock_bh(&ioat_chan->desc_lock);
 810         switch (ioat_chan->device->version) {
 811         case IOAT_VER_1_2:
 812                 list_for_each_entry_safe(desc, _desc,
 813                                          &ioat_chan->used_desc, node) {
 814                         in_use_descs++;
 815                         list_del(&desc->node);
 816                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 817                                       desc->txd.phys);
 818                         kfree(desc);
 819                 }
 820                 list_for_each_entry_safe(desc, _desc,
 821                                          &ioat_chan->free_desc, node) {
 822                         list_del(&desc->node);
 823                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 824                                       desc->txd.phys);
 825                         kfree(desc);
 826                 }
 827                 break;
 828         case IOAT_VER_2_0:
 829         case IOAT_VER_3_0:
 830                 list_for_each_entry_safe(desc, _desc,
 831                                          ioat_chan->free_desc.next, node) {
 832                         list_del(&desc->node);
 833                         pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 834                                       desc->txd.phys);
 835                         kfree(desc);
 836                 }
 837                 desc = to_ioat_desc(ioat_chan->free_desc.next);
 838                 pci_pool_free(ioatdma_device->dma_pool, desc->hw,
 839                               desc->txd.phys);
 840                 kfree(desc);
 841                 INIT_LIST_HEAD(&ioat_chan->free_desc);
 842                 INIT_LIST_HEAD(&ioat_chan->used_desc);
 843                 break;
 844         }
 845         spin_unlock_bh(&ioat_chan->desc_lock);
 846
 847         pci_pool_free(ioatdma_device->completion_pool,
 848                       ioat_chan->completion_virt,
 849                       ioat_chan->completion_addr);
 850
 851         /* one is ok since we left it on there on purpose */
 852         if (in_use_descs > 1)
 853                 dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n",
 854                         in_use_descs - 1);
 855
 856         ioat_chan->last_completion = ioat_chan->completion_addr = 0;
 857         ioat_chan->pending = 0;
 858         ioat_chan->dmacount = 0;
 859         ioat_chan->desccount = 0;
 860         ioat_chan->watchdog_completion = 0;
 861         ioat_chan->last_compl_desc_addr_hw = 0;
 862         ioat_chan->watchdog_tcp_cookie =
 863                 ioat_chan->watchdog_last_tcp_cookie = 0;
 864 }
 865
 866 /**
 867  * ioat_dma_get_next_descriptor - return the next available descriptor
 868  * @ioat_chan: IOAT DMA channel handle
 869  *
 870  * Gets the next descriptor from the chain, and must be called with the
 871  * channel's desc_lock held.  Allocates more descriptors if the channel
 872  * has run out.
 873  */
 874 static struct ioat_desc_sw *
 875 ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
 876 {
 877         struct ioat_desc_sw *new;
 878
 879         if (!list_empty(&ioat_chan->free_desc)) {
 880                 new = to_ioat_desc(ioat_chan->free_desc.next);
 881                 list_del(&new->node);
 882         } else {
 883                 /* try to get another desc */
 884                 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
 885                 if (!new) {
 886                         dev_err(to_dev(ioat_chan), "alloc failed\n");
 887                         return NULL;
 888                 }
 889         }
 890
 891         prefetch(new->hw);
 892         return new;
 893 }
 894
 895 static struct ioat_desc_sw *
 896 ioat2_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
 897 {
 898         struct ioat_desc_sw *new;
 899
 900         /*
 901          * used.prev points to where to start processing
 902          * used.next points to next free descriptor
 903          * if used.prev == NULL, there are none waiting to be processed
 904          * if used.next == used.prev.prev, there is only one free descriptor,
 905          *      and we need to use it to as a noop descriptor before
 906          *      linking in a new set of descriptors, since the device
 907          *      has probably already read the pointer to it
 908          */
 909         if (ioat_chan->used_desc.prev &&
 910             ioat_chan->used_desc.next == ioat_chan->used_desc.prev->prev) {
 911
 912                 struct ioat_desc_sw *desc;
 913                 struct ioat_desc_sw *noop_desc;
 914                 int i;
 915
 916                 /* set up the noop descriptor */
 917                 noop_desc = to_ioat_desc(ioat_chan->used_desc.next);
 918                 /* set size to non-zero value (channel returns error when size is 0) */
 919                 noop_desc->hw->size = NULL_DESC_BUFFER_SIZE;
 920                 noop_desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
 921                 noop_desc->hw->src_addr = 0;
 922                 noop_desc->hw->dst_addr = 0;
 923
 924                 ioat_chan->used_desc.next = ioat_chan->used_desc.next->next;
 925                 ioat_chan->pending++;
 926                 ioat_chan->dmacount++;
 927
 928                 /* try to get a few more descriptors */
 929                 for (i = 16; i; i--) {
 930                         desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
 931                         if (!desc) {
 932                                 dev_err(to_dev(ioat_chan), "alloc failed\n");
 933                                 break;
 934                         }
 935                         list_add_tail(&desc->node, ioat_chan->used_desc.next);
 936
 937                         desc->hw->next
 938                                 = to_ioat_desc(desc->node.next)->txd.phys;
 939                         to_ioat_desc(desc->node.prev)->hw->next
 940                                 = desc->txd.phys;
 941                         ioat_chan->desccount++;
 942                 }
 943
 944                 ioat_chan->used_desc.next = noop_desc->node.next;
 945         }
 946         new = to_ioat_desc(ioat_chan->used_desc.next);
 947         prefetch(new);
 948         ioat_chan->used_desc.next = new->node.next;
 949
 950         if (ioat_chan->used_desc.prev == NULL)
 951                 ioat_chan->used_desc.prev = &new->node;
 952
 953         prefetch(new->hw);
 954         return new;
 955 }
 956
 957 static struct ioat_desc_sw *
 958 ioat_dma_get_next_descriptor(struct ioat_dma_chan *ioat_chan)
 959 {
 960         if (!ioat_chan)
 961                 return NULL;
 962
 963         switch (ioat_chan->device->version) {
 964         case IOAT_VER_1_2:
 965                 return ioat1_dma_get_next_descriptor(ioat_chan);
 966         case IOAT_VER_2_0:
 967         case IOAT_VER_3_0:
 968                 return ioat2_dma_get_next_descriptor(ioat_chan);
 969         }
 970         return NULL;
 971 }
 972
 973 static struct dma_async_tx_descriptor *
 974 ioat1_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
 975                       dma_addr_t dma_src, size_t len, unsigned long flags)
 976 {
 977         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
 978         struct ioat_desc_sw *new;
 979
 980         spin_lock_bh(&ioat_chan->desc_lock);
 981         new = ioat_dma_get_next_descriptor(ioat_chan);
 982         spin_unlock_bh(&ioat_chan->desc_lock);
 983
 984         if (new) {
 985                 new->len = len;
 986                 new->dst = dma_dest;
 987                 new->src = dma_src;
 988                 new->txd.flags = flags;
 989                 return &new->txd;
 990         } else {
 991                 dev_err(to_dev(ioat_chan),
 992                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
 993                         chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
 994                 return NULL;
 995         }
 996 }
 997
 998 static struct dma_async_tx_descriptor *
 999 ioat2_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
1000                       dma_addr_t dma_src, size_t len, unsigned long flags)
1001 {
1002         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
1003         struct ioat_desc_sw *new;
1004
1005         spin_lock_bh(&ioat_chan->desc_lock);
1006         new = ioat2_dma_get_next_descriptor(ioat_chan);
1007
1008         /*
1009          * leave ioat_chan->desc_lock set in ioat 2 path
1010          * it will get unlocked at end of tx_submit
1011          */
1012
1013         if (new) {
1014                 new->len = len;
1015                 new->dst = dma_dest;
1016                 new->src = dma_src;
1017                 new->txd.flags = flags;
1018                 return &new->txd;
1019         } else {
1020                 spin_unlock_bh(&ioat_chan->desc_lock);
1021                 dev_err(to_dev(ioat_chan),
1022                         "chan%d - get_next_desc failed: %d descs waiting, %d total desc\n",
1023                         chan_num(ioat_chan), ioat_chan->dmacount, ioat_chan->desccount);
1024                 return NULL;
1025         }
1026 }
1027
1028 static void ioat_dma_cleanup_tasklet(unsigned long data)
1029 {
1030         struct ioat_dma_chan *chan = (void *)data;
1031         ioat_dma_memcpy_cleanup(chan);
1032         writew(IOAT_CHANCTRL_INT_DISABLE,
1033                chan->reg_base + IOAT_CHANCTRL_OFFSET);
1034 }
1035
1036 static void
1037 ioat_dma_unmap(struct ioat_dma_chan *ioat_chan, struct ioat_desc_sw *desc)
1038 {
1039         if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
1040                 if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
1041                         pci_unmap_single(ioat_chan->device->pdev,
1042                                          pci_unmap_addr(desc, dst),
1043                                          pci_unmap_len(desc, len),
1044                                          PCI_DMA_FROMDEVICE);
1045                 else
1046                         pci_unmap_page(ioat_chan->device->pdev,
1047                                        pci_unmap_addr(desc, dst),
1048                                        pci_unmap_len(desc, len),
1049                                        PCI_DMA_FROMDEVICE);
1050         }
1051
1052         if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
1053                 if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
1054                         pci_unmap_single(ioat_chan->device->pdev,
1055                                          pci_unmap_addr(desc, src),
1056                                          pci_unmap_len(desc, len),
1057                                          PCI_DMA_TODEVICE);
1058                 else
1059                         pci_unmap_page(ioat_chan->device->pdev,
1060                                        pci_unmap_addr(desc, src),
1061                                        pci_unmap_len(desc, len),
1062                                        PCI_DMA_TODEVICE);
1063         }
1064 }
1065
1066 /**
1067  * ioat_dma_memcpy_cleanup - cleanup up finished descriptors
1068  * @chan: ioat channel to be cleaned up
1069  */
1070 static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan)
1071 {
1072         unsigned long phys_complete;
1073         struct ioat_desc_sw *desc, *_desc;
1074         dma_cookie_t cookie = 0;
1075         unsigned long desc_phys;
1076         struct ioat_desc_sw *latest_desc;
1077         struct dma_async_tx_descriptor *tx;
1078
1079         prefetch(ioat_chan->completion_virt);
1080
1081         if (!spin_trylock_bh(&ioat_chan->cleanup_lock))
1082                 return;
1083
1084         /* The completion writeback can happen at any time,
1085            so reads by the driver need to be atomic operations
1086            The descriptor physical addresses are limited to 32-bits
1087            when the CPU can only do a 32-bit mov */
1088
1089 #if (BITS_PER_LONG == 64)
1090         phys_complete =
1091                 ioat_chan->completion_virt->full
1092                 & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1093 #else
1094         phys_complete =
1095                 ioat_chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
1096 #endif
1097
1098         if ((ioat_chan->completion_virt->full
1099                 & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
1100                                 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
1101                 dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n",
1102                         readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET));
1103
1104                 /* TODO do something to salvage the situation */
1105         }
1106
1107         if (phys_complete == ioat_chan->last_completion) {
1108                 spin_unlock_bh(&ioat_chan->cleanup_lock);
1109                 /*
1110                  * perhaps we're stuck so hard that the watchdog can't go off?
1111                  * try to catch it after 2 seconds
1112                  */
1113                 if (ioat_chan->device->version != IOAT_VER_3_0) {
1114                         if (time_after(jiffies,
1115                                        ioat_chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
1116                                 ioat_dma_chan_watchdog(&(ioat_chan->device->work.work));
1117                                 ioat_chan->last_completion_time = jiffies;
1118                         }
1119                 }
1120                 return;
1121         }
1122         ioat_chan->last_completion_time = jiffies;
1123
1124         cookie = 0;
1125         if (!spin_trylock_bh(&ioat_chan->desc_lock)) {
1126                 spin_unlock_bh(&ioat_chan->cleanup_lock);
1127                 return;
1128         }
1129
1130         switch (ioat_chan->device->version) {
1131         case IOAT_VER_1_2:
1132                 list_for_each_entry_safe(desc, _desc,
1133                                          &ioat_chan->used_desc, node) {
1134                         tx = &desc->txd;
1135                         /*
1136                          * Incoming DMA requests may use multiple descriptors,
1137                          * due to exceeding xfercap, perhaps. If so, only the
1138                          * last one will have a cookie, and require unmapping.
1139                          */
1140                         if (tx->cookie) {
1141                                 cookie = tx->cookie;
1142                                 ioat_dma_unmap(ioat_chan, desc);
1143                                 if (tx->callback) {
1144                                         tx->callback(tx->callback_param);
1145                                         tx->callback = NULL;
1146                                 }
1147                         }
1148
1149                         if (tx->phys != phys_complete) {
1150                                 /*
1151                                  * a completed entry, but not the last, so clean
1152                                  * up if the client is done with the descriptor
1153                                  */
1154                                 if (async_tx_test_ack(tx)) {
1155                                         list_move_tail(&desc->node,
1156                                                        &ioat_chan->free_desc);
1157                                 } else
1158                                         tx->cookie = 0;
1159                         } else {
1160                                 /*
1161                                  * last used desc. Do not remove, so we can
1162                                  * append from it, but don't look at it next
1163                                  * time, either
1164                                  */
1165                                 tx->cookie = 0;
1166
1167                                 /* TODO check status bits? */
1168                                 break;
1169                         }
1170                 }
1171                 break;
1172         case IOAT_VER_2_0:
1173         case IOAT_VER_3_0:
1174                 /* has some other thread has already cleaned up? */
1175                 if (ioat_chan->used_desc.prev == NULL)
1176                         break;
1177
1178                 /* work backwards to find latest finished desc */
1179                 desc = to_ioat_desc(ioat_chan->used_desc.next);
1180                 tx = &desc->txd;
1181                 latest_desc = NULL;
1182                 do {
1183                         desc = to_ioat_desc(desc->node.prev);
1184                         desc_phys = (unsigned long)tx->phys
1185                                        & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
1186                         if (desc_phys == phys_complete) {
1187                                 latest_desc = desc;
1188                                 break;
1189                         }
1190                 } while (&desc->node != ioat_chan->used_desc.prev);
1191
1192                 if (latest_desc != NULL) {
1193                         /* work forwards to clear finished descriptors */
1194                         for (desc = to_ioat_desc(ioat_chan->used_desc.prev);
1195                              &desc->node != latest_desc->node.next &&
1196                              &desc->node != ioat_chan->used_desc.next;
1197                              desc = to_ioat_desc(desc->node.next)) {
1198                                 if (tx->cookie) {
1199                                         cookie = tx->cookie;
1200                                         tx->cookie = 0;
1201                                         ioat_dma_unmap(ioat_chan, desc);
1202                                         if (tx->callback) {
1203                                                 tx->callback(tx->callback_param);
1204                                                 tx->callback = NULL;
1205                                         }
1206                                 }
1207                         }
1208
1209                         /* move used.prev up beyond those that are finished */
1210                         if (&desc->node == ioat_chan->used_desc.next)
1211                                 ioat_chan->used_desc.prev = NULL;
1212                         else
1213                                 ioat_chan->used_desc.prev = &desc->node;
1214                 }
1215                 break;
1216         }
1217
1218         spin_unlock_bh(&ioat_chan->desc_lock);
1219
1220         ioat_chan->last_completion = phys_complete;
1221         if (cookie != 0)
1222                 ioat_chan->completed_cookie = cookie;
1223
1224         spin_unlock_bh(&ioat_chan->cleanup_lock);
1225 }
1226
1227 /**
1228  * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
1229  * @chan: IOAT DMA channel handle
1230  * @cookie: DMA transaction identifier
1231  * @done: if not %NULL, updated with last completed transaction
1232  * @used: if not %NULL, updated with last used transaction
1233  */
1234 static enum dma_status
1235 ioat_dma_is_complete(struct dma_chan *chan, dma_cookie_t cookie,
1236                      dma_cookie_t *done, dma_cookie_t *used)
1237 {
1238         struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
1239         dma_cookie_t last_used;
1240         dma_cookie_t last_complete;
1241         enum dma_status ret;
1242
1243         last_used = chan->cookie;
1244         last_complete = ioat_chan->completed_cookie;
1245         ioat_chan->watchdog_tcp_cookie = cookie;
1246
1247         if (done)
1248                 *done = last_complete;
1249         if (used)
1250                 *used = last_used;
1251
1252         ret = dma_async_is_complete(cookie, last_complete, last_used);
1253         if (ret == DMA_SUCCESS)
1254                 return ret;
1255
1256         ioat_dma_memcpy_cleanup(ioat_chan);
1257
1258         last_used = chan->cookie;
1259         last_complete = ioat_chan->completed_cookie;
1260
1261         if (done)
1262                 *done = last_complete;
1263         if (used)
1264                 *used = last_used;
1265
1266         return dma_async_is_complete(cookie, last_complete, last_used);
1267 }
1268
1269 static void ioat_dma_start_null_desc(struct ioat_dma_chan *ioat_chan)
1270 {
1271         struct ioat_desc_sw *desc;
1272
1273         spin_lock_bh(&ioat_chan->desc_lock);
1274
1275         desc = ioat_dma_get_next_descriptor(ioat_chan);
1276
1277         if (!desc) {
1278                 dev_err(to_dev(ioat_chan),
1279                         "Unable to start null desc - get next desc failed\n");
1280                 spin_unlock_bh(&ioat_chan->desc_lock);
1281                 return;
1282         }
1283
1284         desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL
1285                                 | IOAT_DMA_DESCRIPTOR_CTL_INT_GN
1286                                 | IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
1287         /* set size to non-zero value (channel returns error when size is 0) */
1288         desc->hw->size = NULL_DESC_BUFFER_SIZE;
1289         desc->hw->src_addr = 0;
1290         desc->hw->dst_addr = 0;
1291         async_tx_ack(&desc->txd);
1292         switch (ioat_chan->device->version) {
1293         case IOAT_VER_1_2:
1294                 desc->hw->next = 0;
1295                 list_add_tail(&desc->node, &ioat_chan->used_desc);
1296
1297                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1298                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
1299                 writel(((u64) desc->txd.phys) >> 32,
1300                        ioat_chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
1301
1302                 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base
1303                         + IOAT_CHANCMD_OFFSET(ioat_chan->device->version));
1304                 break;
1305         case IOAT_VER_2_0:
1306         case IOAT_VER_3_0:
1307                 writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
1308                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
1309                 writel(((u64) desc->txd.phys) >> 32,
1310                        ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
1311
1312                 ioat_chan->dmacount++;
1313                 __ioat2_dma_memcpy_issue_pending(ioat_chan);
1314                 break;
1315         }
1316         spin_unlock_bh(&ioat_chan->desc_lock);
1317 }
1318
1319 /*
1320  * Perform a IOAT transaction to verify the HW works.
1321  */
1322 #define IOAT_TEST_SIZE 2000
1323
1324 static void ioat_dma_test_callback(void *dma_async_param)
1325 {
1326         struct completion *cmp = dma_async_param;
1327
1328         complete(cmp);
1329 }
1330
1331 /**
1332  * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
1333  * @device: device to be tested
1334  */
1335 static int ioat_dma_self_test(struct ioatdma_device *device)
1336 {
1337         int i;
1338         u8 *src;
1339         u8 *dest;
1340         struct dma_device *dma = &device->common;
1341         struct device *dev = &device->pdev->dev;
1342         struct dma_chan *dma_chan;
1343         struct dma_async_tx_descriptor *tx;
1344         dma_addr_t dma_dest, dma_src;
1345         dma_cookie_t cookie;
1346         int err = 0;
1347         struct completion cmp;
1348         unsigned long tmo;
1349         unsigned long flags;
1350
1351         src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1352         if (!src)
1353                 return -ENOMEM;
1354         dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
1355         if (!dest) {
1356                 kfree(src);
1357                 return -ENOMEM;
1358         }
1359
1360         /* Fill in src buffer */
1361         for (i = 0; i < IOAT_TEST_SIZE; i++)
1362                 src[i] = (u8)i;
1363
1364         /* Start copy, using first DMA channel */
1365         dma_chan = container_of(dma->channels.next, struct dma_chan,
1366                                 device_node);
1367         if (dma->device_alloc_chan_resources(dma_chan) < 1) {
1368                 dev_err(dev, "selftest cannot allocate chan resource\n");
1369                 err = -ENODEV;
1370                 goto out;
1371         }
1372
1373         dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
1374         dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
1375         flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE;
1376         tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
1377                                                    IOAT_TEST_SIZE, flags);
1378         if (!tx) {
1379                 dev_err(dev, "Self-test prep failed, disabling\n");
1380                 err = -ENODEV;
1381                 goto free_resources;
1382         }
1383
1384         async_tx_ack(tx);
1385         init_completion(&cmp);
1386         tx->callback = ioat_dma_test_callback;
1387         tx->callback_param = &cmp;
1388         cookie = tx->tx_submit(tx);
1389         if (cookie < 0) {
1390                 dev_err(dev, "Self-test setup failed, disabling\n");
1391                 err = -ENODEV;
1392                 goto free_resources;
1393         }
1394         dma->device_issue_pending(dma_chan);
1395
1396         tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
1397
1398         if (tmo == 0 ||
1399             dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
1400                                         != DMA_SUCCESS) {
1401                 dev_err(dev, "Self-test copy timed out, disabling\n");
1402                 err = -ENODEV;
1403                 goto free_resources;
1404         }
1405         if (memcmp(src, dest, IOAT_TEST_SIZE)) {
1406                 dev_err(dev, "Self-test copy failed compare, disabling\n");
1407                 err = -ENODEV;
1408                 goto free_resources;
1409         }
1410
1411 free_resources:
1412         dma->device_free_chan_resources(dma_chan);
1413 out:
1414         kfree(src);
1415         kfree(dest);
1416         return err;
1417 }
1418
1419 static char ioat_interrupt_style[32] = "msix";
1420 module_param_string(ioat_interrupt_style, ioat_interrupt_style,
1421                     sizeof(ioat_interrupt_style), 0644);
1422 MODULE_PARM_DESC(ioat_interrupt_style,
1423                  "set ioat interrupt style: msix (default), "
1424                  "msix-single-vector, msi, intx)");
1425
1426 /**
1427  * ioat_dma_setup_interrupts - setup interrupt handler
1428  * @device: ioat device
1429  */
1430 static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
1431 {
1432         struct ioat_dma_chan *ioat_chan;
1433         struct pci_dev *pdev = device->pdev;
1434         struct device *dev = &pdev->dev;
1435         struct msix_entry *msix;
1436         int i, j, msixcnt;
1437         int err = -EINVAL;
1438         u8 intrctrl = 0;
1439
1440         if (!strcmp(ioat_interrupt_style, "msix"))
1441                 goto msix;
1442         if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
1443                 goto msix_single_vector;
1444         if (!strcmp(ioat_interrupt_style, "msi"))
1445                 goto msi;
1446         if (!strcmp(ioat_interrupt_style, "intx"))
1447                 goto intx;
1448         dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
1449         goto err_no_irq;
1450
1451 msix:
1452         /* The number of MSI-X vectors should equal the number of channels */
1453         msixcnt = device->common.chancnt;
1454         for (i = 0; i < msixcnt; i++)
1455                 device->msix_entries[i].entry = i;
1456
1457         err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
1458         if (err < 0)
1459                 goto msi;
1460         if (err > 0)
1461                 goto msix_single_vector;
1462
1463         for (i = 0; i < msixcnt; i++) {
1464                 msix = &device->msix_entries[i];
1465                 ioat_chan = ioat_chan_by_index(device, i);
1466                 err = devm_request_irq(dev, msix->vector,
1467                                        ioat_dma_do_interrupt_msix, 0,
1468                                        "ioat-msix", ioat_chan);
1469                 if (err) {
1470                         for (j = 0; j < i; j++) {
1471                                 msix = &device->msix_entries[j];
1472                                 ioat_chan = ioat_chan_by_index(device, j);
1473                                 devm_free_irq(dev, msix->vector, ioat_chan);
1474                         }
1475                         goto msix_single_vector;
1476                 }
1477         }
1478         intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
1479         goto done;
1480
1481 msix_single_vector:
1482         msix = &device->msix_entries[0];
1483         msix->entry = 0;
1484         err = pci_enable_msix(pdev, device->msix_entries, 1);
1485         if (err)
1486                 goto msi;
1487
1488         err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
1489                                "ioat-msix", device);
1490         if (err) {
1491                 pci_disable_msix(pdev);
1492                 goto msi;
1493         }
1494         goto done;
1495
1496 msi:
1497         err = pci_enable_msi(pdev);
1498         if (err)
1499                 goto intx;
1500
1501         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
1502                                "ioat-msi", device);
1503         if (err) {
1504                 pci_disable_msi(pdev);
1505                 goto intx;
1506         }
1507         /*
1508          * CB 1.2 devices need a bit set in configuration space to enable MSI
1509          */
1510         if (device->version == IOAT_VER_1_2) {
1511                 u32 dmactrl;
1512                 pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
1513                 dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
1514                 pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
1515         }
1516         goto done;
1517
1518 intx:
1519         err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
1520                                IRQF_SHARED, "ioat-intx", device);
1521         if (err)
1522                 goto err_no_irq;
1523
1524 done:
1525         intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
1526         writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
1527         return 0;
1528
1529 err_no_irq:
1530         /* Disable all interrupt generation */
1531         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1532         dev_err(dev, "no usable interrupts\n");
1533         return err;
1534 }
1535
1536 static void ioat_disable_interrupts(struct ioatdma_device *device)
1537 {
1538         /* Disable all interrupt generation */
1539         writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
1540 }
1541
1542 struct ioatdma_device *
1543 ioat_dma_probe(struct pci_dev *pdev, void __iomem *iobase)
1544 {
1545         int err;
1546         struct device *dev = &pdev->dev;
1547         struct ioatdma_device *device;
1548         struct dma_device *dma;
1549
1550         device = devm_kzalloc(dev, sizeof(*device), GFP_KERNEL);
1551         if (!device)
1552                 err = -ENOMEM;
1553         device->pdev = pdev;
1554         device->reg_base = iobase;
1555         device->version = readb(device->reg_base + IOAT_VER_OFFSET);
1556         dma = &device->common;
1557
1558         /* DMA coherent memory pool for DMA descriptor allocations */
1559         device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
1560                                            sizeof(struct ioat_dma_descriptor),
1561                                            64, 0);
1562         if (!device->dma_pool) {
1563                 err = -ENOMEM;
1564                 goto err_dma_pool;
1565         }
1566
1567         device->completion_pool = pci_pool_create("completion_pool", pdev,
1568                                                   sizeof(u64), SMP_CACHE_BYTES,
1569                                                   SMP_CACHE_BYTES);
1570         if (!device->completion_pool) {
1571                 err = -ENOMEM;
1572                 goto err_completion_pool;
1573         }
1574
1575         INIT_LIST_HEAD(&dma->channels);
1576         ioat_dma_enumerate_channels(device);
1577
1578         dma->device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
1579         dma->device_free_chan_resources = ioat_dma_free_chan_resources;
1580         dma->dev = &pdev->dev;
1581
1582         dma_cap_set(DMA_MEMCPY, dma->cap_mask);
1583         dma->device_is_tx_complete = ioat_dma_is_complete;
1584         switch (device->version) {
1585         case IOAT_VER_1_2:
1586                 dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
1587                 dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
1588                 break;
1589         case IOAT_VER_2_0:
1590         case IOAT_VER_3_0:
1591                 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy;
1592                 dma->device_issue_pending = ioat2_dma_memcpy_issue_pending;
1593                 break;
1594         }
1595
1596         dev_err(dev, "Intel(R) I/OAT DMA Engine found,"
1597                 " %d channels, device version 0x%02x, driver version %s\n",
1598                 dma->chancnt, device->version, IOAT_DMA_VERSION);
1599
1600         if (!dma->chancnt) {
1601                 dev_err(dev, "Intel(R) I/OAT DMA Engine problem found: "
1602                         "zero channels detected\n");
1603                 goto err_setup_interrupts;
1604         }
1605
1606         err = ioat_dma_setup_interrupts(device);
1607         if (err)
1608                 goto err_setup_interrupts;
1609
1610         err = ioat_dma_self_test(device);
1611         if (err)
1612                 goto err_self_test;
1613
1614         err = dma_async_device_register(dma);
1615         if (err)
1616                 goto err_self_test;
1617
1618         ioat_set_tcp_copy_break(device);
1619
1620         if (device->version != IOAT_VER_3_0) {
1621                 INIT_DELAYED_WORK(&device->work, ioat_dma_chan_watchdog);
1622                 schedule_delayed_work(&device->work,
1623                                       WATCHDOG_DELAY);
1624         }
1625
1626         return device;
1627
1628 err_self_test:
1629         ioat_disable_interrupts(device);
1630 err_setup_interrupts:
1631         pci_pool_destroy(device->completion_pool);
1632 err_completion_pool:
1633         pci_pool_destroy(device->dma_pool);
1634 err_dma_pool:
1635         return NULL;
1636 }
1637
1638 void ioat_dma_remove(struct ioatdma_device *device)
1639 {
1640         struct dma_chan *chan, *_chan;
1641         struct ioat_dma_chan *ioat_chan;
1642         struct dma_device *dma = &device->common;
1643
1644         if (device->version != IOAT_VER_3_0)
1645                 cancel_delayed_work(&device->work);
1646
1647         ioat_disable_interrupts(device);
1648
1649         dma_async_device_unregister(dma);
1650
1651         pci_pool_destroy(device->dma_pool);
1652         pci_pool_destroy(device->completion_pool);
1653
1654         list_for_each_entry_safe(chan, _chan, &dma->channels, device_node) {
1655                 ioat_chan = to_ioat_chan(chan);
1656                 list_del(&chan->device_node);
1657         }
1658 }
1659