drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 static u32 i915_gem_get_seqno(struct drm_device *dev)
  38 {
  39         drm_i915_private_t *dev_priv = dev->dev_private;
  40         u32 seqno;
  41
  42         seqno = dev_priv->next_seqno;
  43
  44         /* reserve 0 for non-seqno */
  45         if (++dev_priv->next_seqno == 0)
  46                 dev_priv->next_seqno = 1;
  47
  48         return seqno;
  49 }
  50
  51 static void
  52 render_ring_flush(struct intel_ring_buffer *ring,
  53                   u32   invalidate_domains,
  54                   u32   flush_domains)
  55 {
  56         struct drm_device *dev = ring->dev;
  57         drm_i915_private_t *dev_priv = dev->dev_private;
  58         u32 cmd;
  59
  60 #if WATCH_EXEC
  61         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  62                   invalidate_domains, flush_domains);
  63 #endif
  64
  65         trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
  66                                      invalidate_domains, flush_domains);
  67
  68         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  69                 /*
  70                  * read/write caches:
  71                  *
  72                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  73                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  74                  * also flushed at 2d versus 3d pipeline switches.
  75                  *
  76                  * read-only caches:
  77                  *
  78                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  79                  * MI_READ_FLUSH is set, and is always flushed on 965.
  80                  *
  81                  * I915_GEM_DOMAIN_COMMAND may not exist?
  82                  *
  83                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  84                  * invalidated when MI_EXE_FLUSH is set.
  85                  *
  86                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  87                  * invalidated with every MI_FLUSH.
  88                  *
  89                  * TLBs:
  90                  *
  91                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  92                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  93                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  94                  * are flushed at any MI_FLUSH.
  95                  */
  96
  97                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  98                 if ((invalidate_domains|flush_domains) &
  99                     I915_GEM_DOMAIN_RENDER)
 100                         cmd &= ~MI_NO_WRITE_FLUSH;
 101                 if (INTEL_INFO(dev)->gen < 4) {
 102                         /*
 103                          * On the 965, the sampler cache always gets flushed
 104                          * and this bit is reserved.
 105                          */
 106                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
 107                                 cmd |= MI_READ_FLUSH;
 108                 }
 109                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 110                         cmd |= MI_EXE_FLUSH;
 111
 112 #if WATCH_EXEC
 113                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
 114 #endif
 115                 intel_ring_begin(ring, 2);
 116                 intel_ring_emit(ring, cmd);
 117                 intel_ring_emit(ring, MI_NOOP);
 118                 intel_ring_advance(ring);
 119         }
 120 }
 121
 122 static void ring_write_tail(struct intel_ring_buffer *ring,
 123                             u32 value)
 124 {
 125         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 126         I915_WRITE_TAIL(ring, value);
 127 }
 128
 129 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 130 {
 131         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 132         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 133                         RING_ACTHD(ring->mmio_base) : ACTHD;
 134
 135         return I915_READ(acthd_reg);
 136 }
 137
 138 static int init_ring_common(struct intel_ring_buffer *ring)
 139 {
 140         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 141         struct drm_i915_gem_object *obj_priv = to_intel_bo(ring->gem_object);
 142         u32 head;
 143
 144         /* Stop the ring if it's running. */
 145         I915_WRITE_CTL(ring, 0);
 146         I915_WRITE_HEAD(ring, 0);
 147         ring->write_tail(ring, 0);
 148
 149         /* Initialize the ring. */
 150         I915_WRITE_START(ring, obj_priv->gtt_offset);
 151         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 152
 153         /* G45 ring initialization fails to reset head to zero */
 154         if (head != 0) {
 155                 DRM_ERROR("%s head not reset to zero "
 156                                 "ctl %08x head %08x tail %08x start %08x\n",
 157                                 ring->name,
 158                                 I915_READ_CTL(ring),
 159                                 I915_READ_HEAD(ring),
 160                                 I915_READ_TAIL(ring),
 161                                 I915_READ_START(ring));
 162
 163                 I915_WRITE_HEAD(ring, 0);
 164
 165                 DRM_ERROR("%s head forced to zero "
 166                                 "ctl %08x head %08x tail %08x start %08x\n",
 167                                 ring->name,
 168                                 I915_READ_CTL(ring),
 169                                 I915_READ_HEAD(ring),
 170                                 I915_READ_TAIL(ring),
 171                                 I915_READ_START(ring));
 172         }
 173
 174         I915_WRITE_CTL(ring,
 175                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
 176                         | RING_NO_REPORT | RING_VALID);
 177
 178         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 179         /* If the head is still not zero, the ring is dead */
 180         if (head != 0) {
 181                 DRM_ERROR("%s initialization failed "
 182                                 "ctl %08x head %08x tail %08x start %08x\n",
 183                                 ring->name,
 184                                 I915_READ_CTL(ring),
 185                                 I915_READ_HEAD(ring),
 186                                 I915_READ_TAIL(ring),
 187                                 I915_READ_START(ring));
 188                 return -EIO;
 189         }
 190
 191         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 192                 i915_kernel_lost_context(ring->dev);
 193         else {
 194                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 195                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 196                 ring->space = ring->head - (ring->tail + 8);
 197                 if (ring->space < 0)
 198                         ring->space += ring->size;
 199         }
 200         return 0;
 201 }
 202
 203 static int init_render_ring(struct intel_ring_buffer *ring)
 204 {
 205         struct drm_device *dev = ring->dev;
 206         int ret = init_ring_common(ring);
 207
 208         if (INTEL_INFO(dev)->gen > 3) {
 209                 drm_i915_private_t *dev_priv = dev->dev_private;
 210                 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
 211                 if (IS_GEN6(dev))
 212                         mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
 213                 I915_WRITE(MI_MODE, mode);
 214         }
 215
 216         return ret;
 217 }
 218
 219 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 220 do {                                                                    \
 221         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |           \
 222                  PIPE_CONTROL_DEPTH_STALL | 2);                         \
 223         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 224         intel_ring_emit(ring__, 0);                                                     \
 225         intel_ring_emit(ring__, 0);                                                     \
 226 } while (0)
 227
 228 /**
 229  * Creates a new sequence number, emitting a write of it to the status page
 230  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 231  *
 232  * Must be called with struct_lock held.
 233  *
 234  * Returned sequence numbers are nonzero on success.
 235  */
 236 static u32
 237 render_ring_add_request(struct intel_ring_buffer *ring,
 238                         u32 flush_domains)
 239 {
 240         struct drm_device *dev = ring->dev;
 241         drm_i915_private_t *dev_priv = dev->dev_private;
 242         u32 seqno;
 243
 244         seqno = i915_gem_get_seqno(dev);
 245
 246         if (IS_GEN6(dev)) {
 247                 intel_ring_begin(ring, 6);
 248                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | 3);
 249                 intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE |
 250                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
 251                          PIPE_CONTROL_NOTIFY);
 252                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 253                 intel_ring_emit(ring, seqno);
 254                 intel_ring_emit(ring, 0);
 255                 intel_ring_emit(ring, 0);
 256                 intel_ring_advance(ring);
 257         } else if (HAS_PIPE_CONTROL(dev)) {
 258                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
 259
 260                 /*
 261                  * Workaround qword write incoherence by flushing the
 262                  * PIPE_NOTIFY buffers out to memory before requesting
 263                  * an interrupt.
 264                  */
 265                 intel_ring_begin(ring, 32);
 266                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 267                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 268                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 269                 intel_ring_emit(ring, seqno);
 270                 intel_ring_emit(ring, 0);
 271                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 272                 scratch_addr += 128; /* write to separate cachelines */
 273                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 274                 scratch_addr += 128;
 275                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 276                 scratch_addr += 128;
 277                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 278                 scratch_addr += 128;
 279                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 280                 scratch_addr += 128;
 281                 PIPE_CONTROL_FLUSH(ring, scratch_addr);
 282                 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 283                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 284                          PIPE_CONTROL_NOTIFY);
 285                 intel_ring_emit(ring, dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 286                 intel_ring_emit(ring, seqno);
 287                 intel_ring_emit(ring, 0);
 288                 intel_ring_advance(ring);
 289         } else {
 290                 intel_ring_begin(ring, 4);
 291                 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 292                 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 293                 intel_ring_emit(ring, seqno);
 294
 295                 intel_ring_emit(ring, MI_USER_INTERRUPT);
 296                 intel_ring_advance(ring);
 297         }
 298         return seqno;
 299 }
 300
 301 static u32
 302 render_ring_get_seqno(struct intel_ring_buffer *ring)
 303 {
 304         struct drm_device *dev = ring->dev;
 305         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 306         if (HAS_PIPE_CONTROL(dev))
 307                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
 308         else
 309                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 310 }
 311
 312 static void
 313 render_ring_get_user_irq(struct intel_ring_buffer *ring)
 314 {
 315         struct drm_device *dev = ring->dev;
 316         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 317         unsigned long irqflags;
 318
 319         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 320         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
 321                 if (HAS_PCH_SPLIT(dev))
 322                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 323                 else
 324                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 325         }
 326         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 327 }
 328
 329 static void
 330 render_ring_put_user_irq(struct intel_ring_buffer *ring)
 331 {
 332         struct drm_device *dev = ring->dev;
 333         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 334         unsigned long irqflags;
 335
 336         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 337         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
 338         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
 339                 if (HAS_PCH_SPLIT(dev))
 340                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 341                 else
 342                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 343         }
 344         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 345 }
 346
 347 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 348 {
 349         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 350         u32 mmio = IS_GEN6(ring->dev) ?
 351                 RING_HWS_PGA_GEN6(ring->mmio_base) :
 352                 RING_HWS_PGA(ring->mmio_base);
 353         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 354         POSTING_READ(mmio);
 355 }
 356
 357 static void
 358 bsd_ring_flush(struct intel_ring_buffer *ring,
 359                u32     invalidate_domains,
 360                u32     flush_domains)
 361 {
 362         intel_ring_begin(ring, 2);
 363         intel_ring_emit(ring, MI_FLUSH);
 364         intel_ring_emit(ring, MI_NOOP);
 365         intel_ring_advance(ring);
 366 }
 367
 368 static u32
 369 ring_add_request(struct intel_ring_buffer *ring,
 370                  u32 flush_domains)
 371 {
 372         u32 seqno;
 373
 374         seqno = i915_gem_get_seqno(ring->dev);
 375
 376         intel_ring_begin(ring, 4);
 377         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 378         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 379         intel_ring_emit(ring, seqno);
 380         intel_ring_emit(ring, MI_USER_INTERRUPT);
 381         intel_ring_advance(ring);
 382
 383         DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
 384
 385         return seqno;
 386 }
 387
 388 static void
 389 bsd_ring_get_user_irq(struct intel_ring_buffer *ring)
 390 {
 391         /* do nothing */
 392 }
 393 static void
 394 bsd_ring_put_user_irq(struct intel_ring_buffer *ring)
 395 {
 396         /* do nothing */
 397 }
 398
 399 static u32
 400 ring_status_page_get_seqno(struct intel_ring_buffer *ring)
 401 {
 402         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 403 }
 404
 405 static int
 406 ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 407                          struct drm_i915_gem_execbuffer2 *exec,
 408                          struct drm_clip_rect *cliprects,
 409                          uint64_t exec_offset)
 410 {
 411         uint32_t exec_start;
 412
 413         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 414
 415         intel_ring_begin(ring, 2);
 416         intel_ring_emit(ring,
 417                         MI_BATCH_BUFFER_START |
 418                         (2 << 6) |
 419                         MI_BATCH_NON_SECURE_I965);
 420         intel_ring_emit(ring, exec_start);
 421         intel_ring_advance(ring);
 422
 423         return 0;
 424 }
 425
 426 static int
 427 render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 428                                 struct drm_i915_gem_execbuffer2 *exec,
 429                                 struct drm_clip_rect *cliprects,
 430                                 uint64_t exec_offset)
 431 {
 432         struct drm_device *dev = ring->dev;
 433         drm_i915_private_t *dev_priv = dev->dev_private;
 434         int nbox = exec->num_cliprects;
 435         int i = 0, count;
 436         uint32_t exec_start, exec_len;
 437
 438         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 439         exec_len = (uint32_t) exec->batch_len;
 440
 441         trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
 442
 443         count = nbox ? nbox : 1;
 444
 445         for (i = 0; i < count; i++) {
 446                 if (i < nbox) {
 447                         int ret = i915_emit_box(dev, cliprects, i,
 448                                                 exec->DR1, exec->DR4);
 449                         if (ret)
 450                                 return ret;
 451                 }
 452
 453                 if (IS_I830(dev) || IS_845G(dev)) {
 454                         intel_ring_begin(ring, 4);
 455                         intel_ring_emit(ring, MI_BATCH_BUFFER);
 456                         intel_ring_emit(ring, exec_start | MI_BATCH_NON_SECURE);
 457                         intel_ring_emit(ring, exec_start + exec_len - 4);
 458                         intel_ring_emit(ring, 0);
 459                 } else {
 460                         intel_ring_begin(ring, 2);
 461                         if (INTEL_INFO(dev)->gen >= 4) {
 462                                 intel_ring_emit(ring,
 463                                                 MI_BATCH_BUFFER_START | (2 << 6)
 464                                                 | MI_BATCH_NON_SECURE_I965);
 465                                 intel_ring_emit(ring, exec_start);
 466                         } else {
 467                                 intel_ring_emit(ring, MI_BATCH_BUFFER_START
 468                                                 | (2 << 6));
 469                                 intel_ring_emit(ring, exec_start |
 470                                                 MI_BATCH_NON_SECURE);
 471                         }
 472                 }
 473                 intel_ring_advance(ring);
 474         }
 475
 476         if (IS_G4X(dev) || IS_GEN5(dev)) {
 477                 intel_ring_begin(ring, 2);
 478                 intel_ring_emit(ring, MI_FLUSH |
 479                                 MI_NO_WRITE_FLUSH |
 480                                 MI_INVALIDATE_ISP );
 481                 intel_ring_emit(ring, MI_NOOP);
 482                 intel_ring_advance(ring);
 483         }
 484         /* XXX breadcrumb */
 485
 486         return 0;
 487 }
 488
 489 static void cleanup_status_page(struct intel_ring_buffer *ring)
 490 {
 491         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 492         struct drm_gem_object *obj;
 493         struct drm_i915_gem_object *obj_priv;
 494
 495         obj = ring->status_page.obj;
 496         if (obj == NULL)
 497                 return;
 498         obj_priv = to_intel_bo(obj);
 499
 500         kunmap(obj_priv->pages[0]);
 501         i915_gem_object_unpin(obj);
 502         drm_gem_object_unreference(obj);
 503         ring->status_page.obj = NULL;
 504
 505         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 506 }
 507
 508 static int init_status_page(struct intel_ring_buffer *ring)
 509 {
 510         struct drm_device *dev = ring->dev;
 511         drm_i915_private_t *dev_priv = dev->dev_private;
 512         struct drm_gem_object *obj;
 513         struct drm_i915_gem_object *obj_priv;
 514         int ret;
 515
 516         obj = i915_gem_alloc_object(dev, 4096);
 517         if (obj == NULL) {
 518                 DRM_ERROR("Failed to allocate status page\n");
 519                 ret = -ENOMEM;
 520                 goto err;
 521         }
 522         obj_priv = to_intel_bo(obj);
 523         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
 524
 525         ret = i915_gem_object_pin(obj, 4096);
 526         if (ret != 0) {
 527                 goto err_unref;
 528         }
 529
 530         ring->status_page.gfx_addr = obj_priv->gtt_offset;
 531         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
 532         if (ring->status_page.page_addr == NULL) {
 533                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 534                 goto err_unpin;
 535         }
 536         ring->status_page.obj = obj;
 537         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 538
 539         intel_ring_setup_status_page(ring);
 540         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 541                         ring->name, ring->status_page.gfx_addr);
 542
 543         return 0;
 544
 545 err_unpin:
 546         i915_gem_object_unpin(obj);
 547 err_unref:
 548         drm_gem_object_unreference(obj);
 549 err:
 550         return ret;
 551 }
 552
 553 int intel_init_ring_buffer(struct drm_device *dev,
 554                            struct intel_ring_buffer *ring)
 555 {
 556         struct drm_i915_private *dev_priv = dev->dev_private;
 557         struct drm_i915_gem_object *obj_priv;
 558         struct drm_gem_object *obj;
 559         int ret;
 560
 561         ring->dev = dev;
 562         INIT_LIST_HEAD(&ring->active_list);
 563         INIT_LIST_HEAD(&ring->request_list);
 564         INIT_LIST_HEAD(&ring->gpu_write_list);
 565
 566         if (I915_NEED_GFX_HWS(dev)) {
 567                 ret = init_status_page(ring);
 568                 if (ret)
 569                         return ret;
 570         }
 571
 572         obj = i915_gem_alloc_object(dev, ring->size);
 573         if (obj == NULL) {
 574                 DRM_ERROR("Failed to allocate ringbuffer\n");
 575                 ret = -ENOMEM;
 576                 goto err_hws;
 577         }
 578
 579         ring->gem_object = obj;
 580
 581         ret = i915_gem_object_pin(obj, PAGE_SIZE);
 582         if (ret)
 583                 goto err_unref;
 584
 585         obj_priv = to_intel_bo(obj);
 586         ring->map.size = ring->size;
 587         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
 588         ring->map.type = 0;
 589         ring->map.flags = 0;
 590         ring->map.mtrr = 0;
 591
 592         drm_core_ioremap_wc(&ring->map, dev);
 593         if (ring->map.handle == NULL) {
 594                 DRM_ERROR("Failed to map ringbuffer.\n");
 595                 ret = -EINVAL;
 596                 goto err_unpin;
 597         }
 598
 599         ring->virtual_start = ring->map.handle;
 600         ret = ring->init(ring);
 601         if (ret)
 602                 goto err_unmap;
 603
 604         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 605                 i915_kernel_lost_context(dev);
 606         else {
 607                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 608                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 609                 ring->space = ring->head - (ring->tail + 8);
 610                 if (ring->space < 0)
 611                         ring->space += ring->size;
 612         }
 613         return ret;
 614
 615 err_unmap:
 616         drm_core_ioremapfree(&ring->map, dev);
 617 err_unpin:
 618         i915_gem_object_unpin(obj);
 619 err_unref:
 620         drm_gem_object_unreference(obj);
 621         ring->gem_object = NULL;
 622 err_hws:
 623         cleanup_status_page(ring);
 624         return ret;
 625 }
 626
 627 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 628 {
 629         if (ring->gem_object == NULL)
 630                 return;
 631
 632         drm_core_ioremapfree(&ring->map, ring->dev);
 633
 634         i915_gem_object_unpin(ring->gem_object);
 635         drm_gem_object_unreference(ring->gem_object);
 636         ring->gem_object = NULL;
 637
 638         cleanup_status_page(ring);
 639 }
 640
 641 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 642 {
 643         unsigned int *virt;
 644         int rem;
 645         rem = ring->size - ring->tail;
 646
 647         if (ring->space < rem) {
 648                 int ret = intel_wait_ring_buffer(ring, rem);
 649                 if (ret)
 650                         return ret;
 651         }
 652
 653         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 654         rem /= 8;
 655         while (rem--) {
 656                 *virt++ = MI_NOOP;
 657                 *virt++ = MI_NOOP;
 658         }
 659
 660         ring->tail = 0;
 661         ring->space = ring->head - 8;
 662
 663         return 0;
 664 }
 665
 666 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
 667 {
 668         struct drm_device *dev = ring->dev;
 669         drm_i915_private_t *dev_priv = dev->dev_private;
 670         unsigned long end;
 671
 672         trace_i915_ring_wait_begin (dev);
 673         end = jiffies + 3 * HZ;
 674         do {
 675                 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 676                 ring->space = ring->head - (ring->tail + 8);
 677                 if (ring->space < 0)
 678                         ring->space += ring->size;
 679                 if (ring->space >= n) {
 680                         trace_i915_ring_wait_end(dev);
 681                         return 0;
 682                 }
 683
 684                 if (dev->primary->master) {
 685                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 686                         if (master_priv->sarea_priv)
 687                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 688                 }
 689
 690                 msleep(1);
 691         } while (!time_after(jiffies, end));
 692         trace_i915_ring_wait_end (dev);
 693         return -EBUSY;
 694 }
 695
 696 void intel_ring_begin(struct intel_ring_buffer *ring,
 697                       int num_dwords)
 698 {
 699         int n = 4*num_dwords;
 700
 701         if (unlikely(ring->tail + n > ring->size))
 702                 intel_wrap_ring_buffer(ring);
 703
 704         if (unlikely(ring->space < n))
 705                 intel_wait_ring_buffer(ring, n);
 706
 707         ring->space -= n;
 708 }
 709
 710 void intel_ring_advance(struct intel_ring_buffer *ring)
 711 {
 712         ring->tail &= ring->size - 1;
 713         ring->write_tail(ring, ring->tail);
 714 }
 715
 716 static const struct intel_ring_buffer render_ring = {
 717         .name                   = "render ring",
 718         .id                     = RING_RENDER,
 719         .mmio_base              = RENDER_RING_BASE,
 720         .size                   = 32 * PAGE_SIZE,
 721         .init                   = init_render_ring,
 722         .write_tail             = ring_write_tail,
 723         .flush                  = render_ring_flush,
 724         .add_request            = render_ring_add_request,
 725         .get_seqno              = render_ring_get_seqno,
 726         .user_irq_get           = render_ring_get_user_irq,
 727         .user_irq_put           = render_ring_put_user_irq,
 728         .dispatch_execbuffer    = render_ring_dispatch_execbuffer,
 729 };
 730
 731 /* ring buffer for bit-stream decoder */
 732
 733 static const struct intel_ring_buffer bsd_ring = {
 734         .name                   = "bsd ring",
 735         .id                     = RING_BSD,
 736         .mmio_base              = BSD_RING_BASE,
 737         .size                   = 32 * PAGE_SIZE,
 738         .init                   = init_ring_common,
 739         .write_tail             = ring_write_tail,
 740         .flush                  = bsd_ring_flush,
 741         .add_request            = ring_add_request,
 742         .get_seqno              = ring_status_page_get_seqno,
 743         .user_irq_get           = bsd_ring_get_user_irq,
 744         .user_irq_put           = bsd_ring_put_user_irq,
 745         .dispatch_execbuffer    = ring_dispatch_execbuffer,
 746 };
 747
 748
 749 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
 750                                      u32 value)
 751 {
 752        drm_i915_private_t *dev_priv = ring->dev->dev_private;
 753
 754        /* Every tail move must follow the sequence below */
 755        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
 756                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
 757                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
 758        I915_WRITE(GEN6_BSD_RNCID, 0x0);
 759
 760        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
 761                                GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
 762                        50))
 763                DRM_ERROR("timed out waiting for IDLE Indicator\n");
 764
 765        I915_WRITE_TAIL(ring, value);
 766        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
 767                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
 768                GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
 769 }
 770
 771 static void gen6_ring_flush(struct intel_ring_buffer *ring,
 772                             u32 invalidate_domains,
 773                             u32 flush_domains)
 774 {
 775        intel_ring_begin(ring, 4);
 776        intel_ring_emit(ring, MI_FLUSH_DW);
 777        intel_ring_emit(ring, 0);
 778        intel_ring_emit(ring, 0);
 779        intel_ring_emit(ring, 0);
 780        intel_ring_advance(ring);
 781 }
 782
 783 static int
 784 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 785                               struct drm_i915_gem_execbuffer2 *exec,
 786                               struct drm_clip_rect *cliprects,
 787                               uint64_t exec_offset)
 788 {
 789        uint32_t exec_start;
 790
 791        exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 792
 793        intel_ring_begin(ring, 2);
 794        intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
 795        /* bit0-7 is the length on GEN6+ */
 796        intel_ring_emit(ring, exec_start);
 797        intel_ring_advance(ring);
 798
 799        return 0;
 800 }
 801
 802 /* ring buffer for Video Codec for Gen6+ */
 803 static const struct intel_ring_buffer gen6_bsd_ring = {
 804        .name                    = "gen6 bsd ring",
 805        .id                      = RING_BSD,
 806        .mmio_base               = GEN6_BSD_RING_BASE,
 807        .size                    = 32 * PAGE_SIZE,
 808        .init                    = init_ring_common,
 809        .write_tail              = gen6_bsd_ring_write_tail,
 810        .flush                   = gen6_ring_flush,
 811        .add_request             = ring_add_request,
 812        .get_seqno               = ring_status_page_get_seqno,
 813        .user_irq_get            = bsd_ring_get_user_irq,
 814        .user_irq_put            = bsd_ring_put_user_irq,
 815        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
 816 };
 817
 818 /* Blitter support (SandyBridge+) */
 819
 820 static void
 821 blt_ring_get_user_irq(struct intel_ring_buffer *ring)
 822 {
 823         /* do nothing */
 824 }
 825 static void
 826 blt_ring_put_user_irq(struct intel_ring_buffer *ring)
 827 {
 828         /* do nothing */
 829 }
 830
 831 static const struct intel_ring_buffer gen6_blt_ring = {
 832        .name                    = "blt ring",
 833        .id                      = RING_BLT,
 834        .mmio_base               = BLT_RING_BASE,
 835        .size                    = 32 * PAGE_SIZE,
 836        .init                    = init_ring_common,
 837        .write_tail              = ring_write_tail,
 838        .flush                   = gen6_ring_flush,
 839        .add_request             = ring_add_request,
 840        .get_seqno               = ring_status_page_get_seqno,
 841        .user_irq_get            = blt_ring_get_user_irq,
 842        .user_irq_put            = blt_ring_put_user_irq,
 843        .dispatch_execbuffer     = gen6_ring_dispatch_execbuffer,
 844 };
 845
 846 int intel_init_render_ring_buffer(struct drm_device *dev)
 847 {
 848         drm_i915_private_t *dev_priv = dev->dev_private;
 849
 850         dev_priv->render_ring = render_ring;
 851
 852         if (!I915_NEED_GFX_HWS(dev)) {
 853                 dev_priv->render_ring.status_page.page_addr
 854                         = dev_priv->status_page_dmah->vaddr;
 855                 memset(dev_priv->render_ring.status_page.page_addr,
 856                                 0, PAGE_SIZE);
 857         }
 858
 859         return intel_init_ring_buffer(dev, &dev_priv->render_ring);
 860 }
 861
 862 int intel_init_bsd_ring_buffer(struct drm_device *dev)
 863 {
 864         drm_i915_private_t *dev_priv = dev->dev_private;
 865
 866         if (IS_GEN6(dev))
 867                 dev_priv->bsd_ring = gen6_bsd_ring;
 868         else
 869                 dev_priv->bsd_ring = bsd_ring;
 870
 871         return intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
 872 }
 873
 874 int intel_init_blt_ring_buffer(struct drm_device *dev)
 875 {
 876         drm_i915_private_t *dev_priv = dev->dev_private;
 877
 878         dev_priv->blt_ring = gen6_blt_ring;
 879
 880         return intel_init_ring_buffer(dev, &dev_priv->blt_ring);
 881 }