drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 static u32 i915_gem_get_seqno(struct drm_device *dev)
  38 {
  39         drm_i915_private_t *dev_priv = dev->dev_private;
  40         u32 seqno;
  41
  42         seqno = dev_priv->next_seqno;
  43
  44         /* reserve 0 for non-seqno */
  45         if (++dev_priv->next_seqno == 0)
  46                 dev_priv->next_seqno = 1;
  47
  48         return seqno;
  49 }
  50
  51 static void
  52 render_ring_flush(struct intel_ring_buffer *ring,
  53                   u32   invalidate_domains,
  54                   u32   flush_domains)
  55 {
  56         struct drm_device *dev = ring->dev;
  57         drm_i915_private_t *dev_priv = dev->dev_private;
  58         u32 cmd;
  59
  60 #if WATCH_EXEC
  61         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  62                   invalidate_domains, flush_domains);
  63 #endif
  64
  65         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
  66                                      invalidate_domains, flush_domains);
  67
  68         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  69                 /*
  70                  * read/write caches:
  71                  *
  72                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  73                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  74                  * also flushed at 2d versus 3d pipeline switches.
  75                  *
  76                  * read-only caches:
  77                  *
  78                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  79                  * MI_READ_FLUSH is set, and is always flushed on 965.
  80                  *
  81                  * I915_GEM_DOMAIN_COMMAND may not exist?
  82                  *
  83                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  84                  * invalidated when MI_EXE_FLUSH is set.
  85                  *
  86                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  87                  * invalidated with every MI_FLUSH.
  88                  *
  89                  * TLBs:
  90                  *
  91                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  92                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  93                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  94                  * are flushed at any MI_FLUSH.
  95                  */
  96
  97                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  98                 if ((invalidate_domains|flush_domains) &
  99                     I915_GEM_DOMAIN_RENDER)
 100                         cmd &= ~MI_NO_WRITE_FLUSH;
 101                 if (INTEL_INFO(dev)->gen < 4) {
 102                         /*
 103                          * On the 965, the sampler cache always gets flushed
 104                          * and this bit is reserved.
 105                          */
 106                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 107                                 cmd |= MI_READ_FLUSH;
 108                 }
 109                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 110                         cmd |= MI_EXE_FLUSH;
 111
 112                 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 113                     (IS_G4X(dev) || IS_GEN5(dev)))
 114                         cmd |= MI_INVALIDATE_ISP;
 115
 116 #if WATCH_EXEC
 117                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 118 #endif
 119                 if (intel_ring_begin(ring, 2) == 0) {
 120                         intel_ring_emit(ring, cmd);
 121                         intel_ring_emit(ring, MI_NOOP);
 122                         intel_ring_advance(ring);
 123                 }
 124         }
 125 }
 126
 127 static void ring_write_tail(struct intel_ring_buffer *ring,
 128                             u32 value)
 129 {
 130         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 131         I915_WRITE_TAIL(ring, value);
 132 }
 133
 134 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 135 {
 136         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 137         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 138                         RING_ACTHD(ring->mmio_base) : ACTHD;
 139
 140         return I915_READ(acthd_reg);
 141 }
 142
 143 static int init_ring_common(struct intel_ring_buffer *ring)
 144 {
 145         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 146         struct drm_i915_gem_object *obj = ring->obj;
 147         u32 head;
 148
 149         /* Stop the ring if it's running. */
 150         I915_WRITE_CTL(ring, 0);
 151         I915_WRITE_HEAD(ring, 0);
 152         ring->write_tail(ring, 0);
 153
 154         /* Initialize the ring. */
 155         I915_WRITE_START(ring, obj->gtt_offset);
 156         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 157
 158         /* G45 ring initialization fails to reset head to zero */
 159         if (head != 0) {
 160                 DRM_ERROR("%s head not reset to zero "
 161                                 "ctl %08x head %08x tail %08x start %08x\n",
 162                                 ring->name,
 163                                 I915_READ_CTL(ring),
 164                                 I915_READ_HEAD(ring),
 165                                 I915_READ_TAIL(ring),
 166                                 I915_READ_START(ring));
 167
 168                 I915_WRITE_HEAD(ring, 0);
 169
 170                 DRM_ERROR("%s head forced to zero "
 171                                 "ctl %08x head %08x tail %08x start %08x\n",
 172                                 ring->name,
 173                                 I915_READ_CTL(ring),
 174                                 I915_READ_HEAD(ring),
 175                                 I915_READ_TAIL(ring),
 176                                 I915_READ_START(ring));
 177         }
 178
 179         I915_WRITE_CTL(ring,
 180                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 181                         | RING_REPORT_64K | RING_VALID);
 182
 183         /* If the head is still not zero, the ring is dead */
 184         if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
 185             I915_READ_START(ring) != obj->gtt_offset ||
 186             (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
 187                 DRM_ERROR("%s initialization failed "
 188                                 "ctl %08x head %08x tail %08x start %08x\n",
 189                                 ring->name,
 190                                 I915_READ_CTL(ring),
 191                                 I915_READ_HEAD(ring),
 192                                 I915_READ_TAIL(ring),
 193                                 I915_READ_START(ring));
 194                 return -EIO;
 195         }
 196
 197         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 198                 i915_kernel_lost_context(ring->dev);
 199         else {
 200                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 201                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 202                 ring->space = ring->head - (ring->tail + 8);
 203                 if (ring->space < 0)
 204                         ring->space += ring->size;
 205         }
 206         return 0;
 207 }
 208
 209 /*
 210  * 965+ support PIPE_CONTROL commands, which provide finer grained control
 211  * over cache flushing.
 212  */
 213 struct pipe_control {
 214         struct drm_i915_gem_object *obj;
 215         volatile u32 *cpu_page;
 216         u32 gtt_offset;
 217 };
 218
 219 static int
 220 init_pipe_control(struct intel_ring_buffer *ring)
 221 {
 222         struct pipe_control *pc;
 223         struct drm_i915_gem_object *obj;
 224         int ret;
 225
 226         if (ring->private)
 227                 return 0;
 228
 229         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
 230         if (!pc)
 231                 return -ENOMEM;
 232
 233         obj = i915_gem_alloc_object(ring->dev, 4096);
 234         if (obj == NULL) {
 235                 DRM_ERROR("Failed to allocate seqno page\n");
 236                 ret = -ENOMEM;
 237                 goto err;
 238         }
 239         obj->agp_type = AGP_USER_CACHED_MEMORY;
 240
 241         ret = i915_gem_object_pin(obj, 4096, true);
 242         if (ret)
 243                 goto err_unref;
 244
 245         pc->gtt_offset = obj->gtt_offset;
 246         pc->cpu_page =  kmap(obj->pages[0]);
 247         if (pc->cpu_page == NULL)
 248                 goto err_unpin;
 249
 250         pc->obj = obj;
 251         ring->private = pc;
 252         return 0;
 253
 254 err_unpin:
 255         i915_gem_object_unpin(obj);
 256 err_unref:
 257         drm_gem_object_unreference(&obj->base);
 258 err:
 259         kfree(pc);
 260         return ret;
 261 }
 262
 263 static void
 264 cleanup_pipe_control(struct intel_ring_buffer *ring)
 265 {
 266         struct pipe_control *pc = ring->private;
 267         struct drm_i915_gem_object *obj;
 268
 269         if (!ring->private)
 270                 return;
 271
 272         obj = pc->obj;
 273         kunmap(obj->pages[0]);
 274         i915_gem_object_unpin(obj);
 275         drm_gem_object_unreference(&obj->base);
 276
 277         kfree(pc);
 278         ring->private = NULL;
 279 }
 280
 281 static int init_render_ring(struct intel_ring_buffer *ring)
 282 {
 283         struct drm_device *dev = ring->dev;
 284         int ret = init_ring_common(ring);
 285
 286         if (INTEL_INFO(dev)->gen > 3) {
 287                 drm_i915_private_t *dev_priv = dev->dev_private;
 288                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
 289                 if (IS_GEN6(dev))
 290                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
 291                 I915_WRITE(MI_MODE, mode);
 292         }
 293
 294         if (HAS_PIPE_CONTROL(dev)) {
 295                 ret = init_pipe_control(ring);
 296                 if (ret)
 297                         return ret;
 298         }
 299
 300         return ret;
 301 }
 302
 303 static void render_ring_cleanup(struct intel_ring_buffer *ring)
 304 {
 305         if (!ring->private)
 306                 return;
 307
 308         cleanup_pipe_control(ring);
 309 }
 310
 311 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 312 do {                                                                    \
 313         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
 314                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
 315         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 316         intel_ring_emit(ring__, 0);                                                     \
 317         intel_ring_emit(ring__, 0);                                                     \
 318 } while (0)
 319
 320 /**
 321  * Creates a new sequence number, emitting a write of it to the status page
 322  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 323  *
 324  * Must be called with struct_lock held.
 325  *
 326  * Returned sequence numbers are nonzero on success.
 327  */
 328 static int
 329 render_ring_add_request(struct intel_ring_buffer *ring,
 330                         u32 *result)
 331 {
 332         struct drm_device *dev = ring->dev;
 333         u32 seqno = i915_gem_get_seqno(dev);
 334         struct pipe_control *pc = ring->private;
 335         int ret;
 336
 337         if (IS_GEN6(dev)) {
 338                 ret = intel_ring_begin(ring, 6);
 339                 if (ret)
 340                     return ret;
 341
 342                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | 3);
 343                 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE |
 344                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
 345                                 PIPE_CONTROL_NOTIFY);
 346                 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 347                 intel_ring_emit(ring, seqno);
 348                 intel_ring_emit(ring, 0);
 349                 intel_ring_emit(ring, 0);
 350         } else if (HAS_PIPE_CONTROL(dev)) {
 351                 u32 scratch_addr = pc->gtt_offset + 128;
 352
 353                 /*
 354                  * Workaround qword write incoherence by flushing the
 355                  * PIPE_NOTIFY buffers out to memory before requesting
 356                  * an interrupt.
 357                  */
 358                 ret = intel_ring_begin(ring, 32);
 359                 if (ret)
 360                         return ret;
 361
 362                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 363                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 364                 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 365                 intel_ring_emit(ring, seqno);
 366                 intel_ring_emit(ring, 0);
 367                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 368                 scratch_addr += 128; /* write to separate cachelines */
 369                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 370                 scratch_addr += 128;
 371                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 372                 scratch_addr += 128;
 373                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 374                 scratch_addr += 128;
 375                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 376                 scratch_addr += 128;
 377                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 378                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 379                                 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 380                                 PIPE_CONTROL_NOTIFY);
 381                 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 382                 intel_ring_emit(ring, seqno);
 383                 intel_ring_emit(ring, 0);
 384         } else {
 385                 ret = intel_ring_begin(ring, 4);
 386                 if (ret)
 387                     return ret;
 388
 389                 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 390                 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 391                 intel_ring_emit(ring, seqno);
 392
 393                 intel_ring_emit(ring, MI_USER_INTERRUPT);
 394         }
 395
 396         intel_ring_advance(ring);
 397         *result = seqno;
 398         return 0;
 399 }
 400
 401 static u32
 402 render_ring_get_seqno(struct intel_ring_buffer *ring)
 403 {
 404         struct drm_device *dev = ring->dev;
 405         if (HAS_PIPE_CONTROL(dev)) {
 406                 struct pipe_control *pc = ring->private;
 407                 return pc->cpu_page[0];
 408         } else
 409                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 410 }
 411
 412 static void
 413 render_ring_get_user_irq(struct intel_ring_buffer *ring)
 414 {
 415         struct drm_device *dev = ring->dev;
 416         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 417         unsigned long irqflags;
 418
 419         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 420         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
 421                 if (HAS_PCH_SPLIT(dev))
 422                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 423                 else
 424                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 425         }
 426         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 427 }
 428
 429 static void
 430 render_ring_put_user_irq(struct intel_ring_buffer *ring)
 431 {
 432         struct drm_device *dev = ring->dev;
 433         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 434         unsigned long irqflags;
 435
 436         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 437         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
 438         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
 439                 if (HAS_PCH_SPLIT(dev))
 440                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 441                 else
 442                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 443         }
 444         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 445 }
 446
 447 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 448 {
 449         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 450         u32 mmio = IS_GEN6(ring->dev) ?
 451                 RING_HWS_PGA_GEN6(ring->mmio_base) :
 452                 RING_HWS_PGA(ring->mmio_base);
 453         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 454         POSTING_READ(mmio);
 455 }
 456
 457 static void
 458 bsd_ring_flush(struct intel_ring_buffer *ring,
 459                u32     invalidate_domains,
 460                u32     flush_domains)
 461 {
 462         if (intel_ring_begin(ring, 2) == 0) {
 463                 intel_ring_emit(ring, MI_FLUSH);
 464                 intel_ring_emit(ring, MI_NOOP);
 465                 intel_ring_advance(ring);
 466         }
 467 }
 468
 469 static int
 470 ring_add_request(struct intel_ring_buffer *ring,
 471                  u32 *result)
 472 {
 473         u32 seqno;
 474         int ret;
 475
 476         ret = intel_ring_begin(ring, 4);
 477         if (ret)
 478                 return ret;
 479
 480         seqno = i915_gem_get_seqno(ring->dev);
 481
 482         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 483         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 484         intel_ring_emit(ring, seqno);
 485         intel_ring_emit(ring, MI_USER_INTERRUPT);
 486         intel_ring_advance(ring);
 487
 488         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
 489         *result = seqno;
 490         return 0;
 491 }
 492
 493 static void
 494 bsd_ring_get_user_irq(struct intel_ring_buffer *ring)
 495 {
 496         /* do nothing */
 497 }
 498 static void
 499 bsd_ring_put_user_irq(struct intel_ring_buffer *ring)
 500 {
 501         /* do nothing */
 502 }
 503
 504 static u32
 505 ring_status_page_get_seqno(struct intel_ring_buffer *ring)
 506 {
 507         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 508 }
 509
 510 static int
 511 ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 512 {
 513         int ret;
 514
 515         ret = intel_ring_begin(ring, 2);
 516         if (ret)
 517                 return ret;
 518
 519         intel_ring_emit(ring,
 520                         MI_BATCH_BUFFER_START | (2 << 6) |
 521                         MI_BATCH_NON_SECURE_I965);
 522         intel_ring_emit(ring, offset);
 523         intel_ring_advance(ring);
 524
 525         return 0;
 526 }
 527
 528 static int
 529 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 530                                 u32 offset, u32 len)
 531 {
 532         struct drm_device *dev = ring->dev;
 533         drm_i915_private_t *dev_priv = dev->dev_private;
 534         int ret;
 535
 536         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
 537
 538         if (IS_I830(dev) || IS_845G(dev)) {
 539                 ret = intel_ring_begin(ring, 4);
 540                 if (ret)
 541                         return ret;
 542
 543                 intel_ring_emit(ring, MI_BATCH_BUFFER);
 544                 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 545                 intel_ring_emit(ring, offset + len - 8);
 546                 intel_ring_emit(ring, 0);
 547         } else {
 548                 ret = intel_ring_begin(ring, 2);
 549                 if (ret)
 550                         return ret;
 551
 552                 if (INTEL_INFO(dev)->gen >= 4) {
 553                         intel_ring_emit(ring,
 554                                         MI_BATCH_BUFFER_START | (2 << 6) |
 555                                         MI_BATCH_NON_SECURE_I965);
 556                         intel_ring_emit(ring, offset);
 557                 } else {
 558                         intel_ring_emit(ring,
 559                                         MI_BATCH_BUFFER_START | (2 << 6));
 560                         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 561                 }
 562         }
 563         intel_ring_advance(ring);
 564
 565         return 0;
 566 }
 567
 568 static void cleanup_status_page(struct intel_ring_buffer *ring)
 569 {
 570         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 571         struct drm_i915_gem_object *obj;
 572
 573         obj = ring->status_page.obj;
 574         if (obj == NULL)
 575                 return;
 576
 577         kunmap(obj->pages[0]);
 578         i915_gem_object_unpin(obj);
 579         drm_gem_object_unreference(&obj->base);
 580         ring->status_page.obj = NULL;
 581
 582         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 583 }
 584
 585 static int init_status_page(struct intel_ring_buffer *ring)
 586 {
 587         struct drm_device *dev = ring->dev;
 588         drm_i915_private_t *dev_priv = dev->dev_private;
 589         struct drm_i915_gem_object *obj;
 590         int ret;
 591
 592         obj = i915_gem_alloc_object(dev, 4096);
 593         if (obj == NULL) {
 594                 DRM_ERROR("Failed to allocate status page\n");
 595                 ret = -ENOMEM;
 596                 goto err;
 597         }
 598         obj->agp_type = AGP_USER_CACHED_MEMORY;
 599
 600         ret = i915_gem_object_pin(obj, 4096, true);
 601         if (ret != 0) {
 602                 goto err_unref;
 603         }
 604
 605         ring->status_page.gfx_addr = obj->gtt_offset;
 606         ring->status_page.page_addr = kmap(obj->pages[0]);
 607         if (ring->status_page.page_addr == NULL) {
 608                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 609                 goto err_unpin;
 610         }
 611         ring->status_page.obj = obj;
 612         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 613
 614         intel_ring_setup_status_page(ring);
 615         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 616                         ring->name, ring->status_page.gfx_addr);
 617
 618         return 0;
 619
 620 err_unpin:
 621         i915_gem_object_unpin(obj);
 622 err_unref:
 623         drm_gem_object_unreference(&obj->base);
 624 err:
 625         return ret;
 626 }
 627
 628 int intel_init_ring_buffer(struct drm_device *dev,
 629                            struct intel_ring_buffer *ring)
 630 {
 631         struct drm_i915_gem_object *obj;
 632         int ret;
 633
 634         ring->dev = dev;
 635         INIT_LIST_HEAD(&ring->active_list);
 636         INIT_LIST_HEAD(&ring->request_list);
 637         INIT_LIST_HEAD(&ring->gpu_write_list);
 638
 639         if (I915_NEED_GFX_HWS(dev)) {
 640                 ret = init_status_page(ring);
 641                 if (ret)
 642                         return ret;
 643         }
 644
 645         obj = i915_gem_alloc_object(dev, ring->size);
 646         if (obj == NULL) {
 647                 DRM_ERROR("Failed to allocate ringbuffer\n");
 648                 ret = -ENOMEM;
 649                 goto err_hws;
 650         }
 651
 652         ring->obj = obj;
 653
 654         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
 655         if (ret)
 656                 goto err_unref;
 657
 658         ring->map.size = ring->size;
 659         ring->map.offset = dev->agp->base + obj->gtt_offset;
 660         ring->map.type = 0;
 661         ring->map.flags = 0;
 662         ring->map.mtrr = 0;
 663
 664         drm_core_ioremap_wc(&ring->map, dev);
 665         if (ring->map.handle == NULL) {
 666                 DRM_ERROR("Failed to map ringbuffer.\n");
 667                 ret = -EINVAL;
 668                 goto err_unpin;
 669         }
 670
 671         ring->virtual_start = ring->map.handle;
 672         ret = ring->init(ring);
 673         if (ret)
 674                 goto err_unmap;
 675
 676         return 0;
 677
 678 err_unmap:
 679         drm_core_ioremapfree(&ring->map, dev);
 680 err_unpin:
 681         i915_gem_object_unpin(obj);
 682 err_unref:
 683         drm_gem_object_unreference(&obj->base);
 684         ring->obj = NULL;
 685 err_hws:
 686         cleanup_status_page(ring);
 687         return ret;
 688 }
 689
 690 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 691 {
 692         struct drm_i915_private *dev_priv;
 693         int ret;
 694
 695         if (ring->obj == NULL)
 696                 return;
 697
 698         /* Disable the ring buffer. The ring must be idle at this point */
 699         dev_priv = ring->dev->dev_private;
 700         ret = intel_wait_ring_buffer(ring, ring->size - 8);
 701         I915_WRITE_CTL(ring, 0);
 702
 703         drm_core_ioremapfree(&ring->map, ring->dev);
 704
 705         i915_gem_object_unpin(ring->obj);
 706         drm_gem_object_unreference(&ring->obj->base);
 707         ring->obj = NULL;
 708
 709         if (ring->cleanup)
 710                 ring->cleanup(ring);
 711
 712         cleanup_status_page(ring);
 713 }
 714
 715 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 716 {
 717         unsigned int *virt;
 718         int rem;
 719         rem = ring->size - ring->tail;
 720
 721         if (ring->space < rem) {
 722                 int ret = intel_wait_ring_buffer(ring, rem);
 723                 if (ret)
 724                         return ret;
 725         }
 726
 727         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 728         rem /= 8;
 729         while (rem--) {
 730                 *virt++ = MI_NOOP;
 731                 *virt++ = MI_NOOP;
 732         }
 733
 734         ring->tail = 0;
 735         ring->space = ring->head - 8;
 736
 737         return 0;
 738 }
 739
 740 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
 741 {
 742         struct drm_device *dev = ring->dev;
 743         struct drm_i915_private *dev_priv = dev->dev_private;
 744         unsigned long end;
 745         u32 head;
 746
 747         head = intel_read_status_page(ring, 4);
 748         if (head) {
 749                 ring->head = head & HEAD_ADDR;
 750                 ring->space = ring->head - (ring->tail + 8);
 751                 if (ring->space < 0)
 752                         ring->space += ring->size;
 753                 if (ring->space >= n)
 754                         return 0;
 755         }
 756
 757         trace_i915_ring_wait_begin (dev);
 758         end = jiffies + 3 * HZ;
 759         do {
 760                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 761                 ring->space = ring->head - (ring->tail + 8);
 762                 if (ring->space < 0)
 763                         ring->space += ring->size;
 764                 if (ring->space >= n) {
 765                         trace_i915_ring_wait_end(dev);
 766                         return 0;
 767                 }
 768
 769                 if (dev->primary->master) {
 770                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 771                         if (master_priv->sarea_priv)
 772                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 773                 }
 774
 775                 msleep(1);
 776                 if (atomic_read(&dev_priv->mm.wedged))
 777                         return -EAGAIN;
 778         } while (!time_after(jiffies, end));
 779         trace_i915_ring_wait_end (dev);
 780         return -EBUSY;
 781 }
 782
 783 int intel_ring_begin(struct intel_ring_buffer *ring,
 784                      int num_dwords)
 785 {
 786         int n = 4*num_dwords;
 787         int ret;
 788
 789         if (unlikely(ring->tail + n > ring->size)) {
 790                 ret = intel_wrap_ring_buffer(ring);
 791                 if (unlikely(ret))
 792                         return ret;
 793         }
 794
 795         if (unlikely(ring->space < n)) {
 796                 ret = intel_wait_ring_buffer(ring, n);
 797                 if (unlikely(ret))
 798                         return ret;
 799         }
 800
 801         ring->space -= n;
 802         return 0;
 803 }
 804
 805 void intel_ring_advance(struct intel_ring_buffer *ring)
 806 {
 807         ring->tail &= ring->size - 1;
 808         ring->write_tail(ring, ring->tail);
 809 }
 810
 811 static const struct intel_ring_buffer render_ring = {
 812         .name                   = "render ring",
 813         .id                     = RING_RENDER,
 814         .mmio_base              = RENDER_RING_BASE,
 815         .size                   = 32 * PAGE_SIZE,
 816         .init                   = init_render_ring,
 817         .write_tail             = ring_write_tail,
 818         .flush                  = render_ring_flush,
 819         .add_request            = render_ring_add_request,
 820         .get_seqno              = render_ring_get_seqno,
 821         .user_irq_get           = render_ring_get_user_irq,
 822         .user_irq_put           = render_ring_put_user_irq,
 823         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
 824        .cleanup                 = render_ring_cleanup,
 825 };
 826
 827 /* ring buffer for bit-stream decoder */
 828
 829 static const struct intel_ring_buffer bsd_ring = {
 830         .name                   = "bsd ring",
 831         .id                     = RING_BSD,
 832         .mmio_base              = BSD_RING_BASE,
 833         .size                   = 32 * PAGE_SIZE,
 834         .init                   = init_ring_common,
 835         .write_tail             = ring_write_tail,
 836         .flush                  = bsd_ring_flush,
 837         .add_request            = ring_add_request,
 838         .get_seqno              = ring_status_page_get_seqno,
 839         .user_irq_get           = bsd_ring_get_user_irq,
 840         .user_irq_put           = bsd_ring_put_user_irq,
 841         .dispatch_execbuffer    = ring_dispatch_execbuffer,
 842 };
 843
 844
 845 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
 846                                      u32 value)
 847 {
 848        drm_i915_private_t *dev_priv = ring->dev->dev_private;
 849
 850        /* Every tail move must follow the sequence below */
 851        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
 852                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
 853                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
 854        I915_WRITE(GEN6_BSD_RNCID, 0x0);
 855
 856        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
 857                                GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
 858                        50))
 859                DRM_ERROR("timed out waiting for IDLE Indicator\n");
 860
 861        I915_WRITE_TAIL(ring, value);
 862        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
 863                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
 864                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
 865 }
 866
 867 static void gen6_ring_flush(struct intel_ring_buffer *ring,
 868                             u32 invalidate_domains,
 869                             u32 flush_domains)
 870 {
 871         if (intel_ring_begin(ring, 4) == 0) {
 872                 intel_ring_emit(ring, MI_FLUSH_DW);
 873                 intel_ring_emit(ring, 0);
 874                 intel_ring_emit(ring, 0);
 875                 intel_ring_emit(ring, 0);
 876                 intel_ring_advance(ring);
 877         }
 878 }
 879
 880 static int
 881 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 882                               u32 offset, u32 len)
 883 {
 884        int ret;
 885
 886        ret = intel_ring_begin(ring, 2);
 887        if (ret)
 888                return ret;
 889
 890        intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
 891        /* bit0-7 is the length on GEN6+ */
 892        intel_ring_emit(ring, offset);
 893        intel_ring_advance(ring);
 894
 895        return 0;
 896 }
 897
 898 /* ring buffer for Video Codec for Gen6+ */
 899 static const struct intel_ring_buffer gen6_bsd_ring = {
 900        .name                    = "gen6 bsd ring",
 901        .id                      = RING_BSD,
 902        .mmio_base               = GEN6_BSD_RING_BASE,
 903        .size                    = 32 * PAGE_SIZE,
 904        .init                    = init_ring_common,
 905        .write_tail              = gen6_bsd_ring_write_tail,
 906        .flush                   = gen6_ring_flush,
 907        .add_request             = ring_add_request,
 908        .get_seqno               = ring_status_page_get_seqno,
 909        .user_irq_get            = bsd_ring_get_user_irq,
 910        .user_irq_put            = bsd_ring_put_user_irq,
 911        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
 912 };
 913
 914 /* Blitter support (SandyBridge+) */
 915
 916 static void
 917 blt_ring_get_user_irq(struct intel_ring_buffer *ring)
 918 {
 919         /* do nothing */
 920 }
 921 static void
 922 blt_ring_put_user_irq(struct intel_ring_buffer *ring)
 923 {
 924         /* do nothing */
 925 }
 926
 927
 928 /* Workaround for some stepping of SNB,
 929  * each time when BLT engine ring tail moved,
 930  * the first command in the ring to be parsed
 931  * should be MI_BATCH_BUFFER_START
 932  */
 933 #define NEED_BLT_WORKAROUND(dev) \
 934         (IS_GEN6(dev) && (dev->pdev->revision < 8))
 935
 936 static inline struct drm_i915_gem_object *
 937 to_blt_workaround(struct intel_ring_buffer *ring)
 938 {
 939         return ring->private;
 940 }
 941
 942 static int blt_ring_init(struct intel_ring_buffer *ring)
 943 {
 944         if (NEED_BLT_WORKAROUND(ring->dev)) {
 945                 struct drm_i915_gem_object *obj;
 946                 u32 *ptr;
 947                 int ret;
 948
 949                 obj = i915_gem_alloc_object(ring->dev, 4096);
 950                 if (obj == NULL)
 951                         return -ENOMEM;
 952
 953                 ret = i915_gem_object_pin(obj, 4096, true);
 954                 if (ret) {
 955                         drm_gem_object_unreference(&obj->base);
 956                         return ret;
 957                 }
 958
 959                 ptr = kmap(obj->pages[0]);
 960                 *ptr++ = MI_BATCH_BUFFER_END;
 961                 *ptr++ = MI_NOOP;
 962                 kunmap(obj->pages[0]);
 963
 964                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 965                 if (ret) {
 966                         i915_gem_object_unpin(obj);
 967                         drm_gem_object_unreference(&obj->base);
 968                         return ret;
 969                 }
 970
 971                 ring->private = obj;
 972         }
 973
 974         return init_ring_common(ring);
 975 }
 976
 977 static int blt_ring_begin(struct intel_ring_buffer *ring,
 978                           int num_dwords)
 979 {
 980         if (ring->private) {
 981                 int ret = intel_ring_begin(ring, num_dwords+2);
 982                 if (ret)
 983                         return ret;
 984
 985                 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
 986                 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
 987
 988                 return 0;
 989         } else
 990                 return intel_ring_begin(ring, 4);
 991 }
 992
 993 static void blt_ring_flush(struct intel_ring_buffer *ring,
 994                            u32 invalidate_domains,
 995                            u32 flush_domains)
 996 {
 997         if (blt_ring_begin(ring, 4) == 0) {
 998                 intel_ring_emit(ring, MI_FLUSH_DW);
 999                 intel_ring_emit(ring, 0);
1000                 intel_ring_emit(ring, 0);
1001                 intel_ring_emit(ring, 0);
1002                 intel_ring_advance(ring);
1003         }
1004 }
1005
1006 static int
1007 blt_ring_add_request(struct intel_ring_buffer *ring,
1008                      u32 *result)
1009 {
1010         u32 seqno;
1011         int ret;
1012
1013         ret = blt_ring_begin(ring, 4);
1014         if (ret)
1015                 return ret;
1016
1017         seqno = i915_gem_get_seqno(ring->dev);
1018
1019         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1020         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1021         intel_ring_emit(ring, seqno);
1022         intel_ring_emit(ring, MI_USER_INTERRUPT);
1023         intel_ring_advance(ring);
1024
1025         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
1026         *result = seqno;
1027         return 0;
1028 }
1029
1030 static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1031 {
1032         if (!ring->private)
1033                 return;
1034
1035         i915_gem_object_unpin(ring->private);
1036         drm_gem_object_unreference(ring->private);
1037         ring->private = NULL;
1038 }
1039
1040 static const struct intel_ring_buffer gen6_blt_ring = {
1041        .name                    = "blt ring",
1042        .id                      = RING_BLT,
1043        .mmio_base               = BLT_RING_BASE,
1044        .size                    = 32 * PAGE_SIZE,
1045        .init                    = blt_ring_init,
1046        .write_tail              = ring_write_tail,
1047        .flush                   = blt_ring_flush,
1048        .add_request             = blt_ring_add_request,
1049        .get_seqno               = ring_status_page_get_seqno,
1050        .user_irq_get            = blt_ring_get_user_irq,
1051        .user_irq_put            = blt_ring_put_user_irq,
1052        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
1053        .cleanup                 = blt_ring_cleanup,
1054 };
1055
1056 int intel_init_render_ring_buffer(struct drm_device *dev)
1057 {
1058         drm_i915_private_t *dev_priv = dev->dev_private;
1059
1060         dev_priv->render_ring = render_ring;
1061
1062         if (!I915_NEED_GFX_HWS(dev)) {
1063                 dev_priv->render_ring.status_page.page_addr
1064                         = dev_priv->status_page_dmah->vaddr;
1065                 memset(dev_priv->render_ring.status_page.page_addr,
1066                                 0, PAGE_SIZE);
1067         }
1068
1069         return intel_init_ring_buffer(dev, &dev_priv->render_ring);
1070 }
1071
1072 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1073 {
1074         drm_i915_private_t *dev_priv = dev->dev_private;
1075
1076         if (IS_GEN6(dev))
1077                 dev_priv->bsd_ring = gen6_bsd_ring;
1078         else
1079                 dev_priv->bsd_ring = bsd_ring;
1080
1081         return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
1082 }
1083
1084 int intel_init_blt_ring_buffer(struct drm_device *dev)
1085 {
1086         drm_i915_private_t *dev_priv = dev->dev_private;
1087
1088         dev_priv->blt_ring = gen6_blt_ring;
1089
1090         return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
1091 }