drivers/video/tegra/host/host1x/host1x_cdma.c

   1 /*
   2  * drivers/video/tegra/host/host1x/host1x_cdma.c
   3  *
   4  * Tegra Graphics Host Command DMA
   5  *
   6  * Copyright (c) 2010-2012, NVIDIA Corporation.
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms and conditions of the GNU General Public License,
  10  * version 2, as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope it will be useful, but WITHOUT
  13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15  * more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19  */
  20
  21 #include <linux/slab.h>
  22 #include "nvhost_acm.h"
  23 #include "nvhost_cdma.h"
  24 #include "nvhost_channel.h"
  25 #include "dev.h"
  26 #include "chip_support.h"
  27 #include "nvhost_memmgr.h"
  28
  29 #include "host1x_hardware.h"
  30 #include "host1x_syncpt.h"
  31 #include "host1x_cdma.h"
  32 #include "host1x_hwctx.h"
  33
  34 static inline u32 host1x_channel_dmactrl(int stop, int get_rst, int init_get)
  35 {
  36         return host1x_channel_dmactrl_dmastop_f(stop)
  37                 | host1x_channel_dmactrl_dmagetrst_f(get_rst)
  38                 | host1x_channel_dmactrl_dmainitget_f(init_get);
  39 }
  40
  41 static void cdma_timeout_handler(struct work_struct *work);
  42
  43 /*
  44  * push_buffer
  45  *
  46  * The push buffer is a circular array of words to be fetched by command DMA.
  47  * Note that it works slightly differently to the sync queue; fence == cur
  48  * means that the push buffer is full, not empty.
  49  */
  50
  51
  52 /**
  53  * Reset to empty push buffer
  54  */
  55 static void push_buffer_reset(struct push_buffer *pb)
  56 {
  57         pb->fence = PUSH_BUFFER_SIZE - 8;
  58         pb->cur = 0;
  59 }
  60
  61 /**
  62  * Init push buffer resources
  63  */
  64 static int push_buffer_init(struct push_buffer *pb)
  65 {
  66         struct nvhost_cdma *cdma = pb_to_cdma(pb);
  67         struct mem_mgr *mgr = cdma_to_memmgr(cdma);
  68         pb->mem = NULL;
  69         pb->mapped = NULL;
  70         pb->phys = 0;
  71         pb->client_handle = NULL;
  72
  73         BUG_ON(!cdma_pb_op().reset);
  74         cdma_pb_op().reset(pb);
  75
  76         /* allocate and map pushbuffer memory */
  77         pb->mem = mem_op().alloc(mgr, PUSH_BUFFER_SIZE + 4, 32,
  78                               mem_mgr_flag_write_combine);
  79         if (IS_ERR_OR_NULL(pb->mem)) {
  80                 pb->mem = NULL;
  81                 goto fail;
  82         }
  83         pb->mapped = mem_op().mmap(pb->mem);
  84         if (pb->mapped == NULL)
  85                 goto fail;
  86
  87         /* pin pushbuffer and get physical address */
  88         pb->phys = mem_op().pin(mgr, pb->mem);
  89         if (pb->phys >= 0xfffff000) {
  90                 pb->phys = 0;
  91                 goto fail;
  92         }
  93
  94         /* memory for storing nvmap client and handles for each opcode pair */
  95         pb->client_handle = kzalloc(NVHOST_GATHER_QUEUE_SIZE *
  96                                 sizeof(struct mem_mgr_handle),
  97                         GFP_KERNEL);
  98         if (!pb->client_handle)
  99                 goto fail;
 100
 101         /* put the restart at the end of pushbuffer memory */
 102         *(pb->mapped + (PUSH_BUFFER_SIZE >> 2)) =
 103                 nvhost_opcode_restart(pb->phys);
 104
 105         return 0;
 106
 107 fail:
 108         cdma_pb_op().destroy(pb);
 109         return -ENOMEM;
 110 }
 111
 112 /**
 113  * Clean up push buffer resources
 114  */
 115 static void push_buffer_destroy(struct push_buffer *pb)
 116 {
 117         struct nvhost_cdma *cdma = pb_to_cdma(pb);
 118         struct mem_mgr *mgr = cdma_to_memmgr(cdma);
 119         if (pb->mapped)
 120                 mem_op().munmap(pb->mem, pb->mapped);
 121
 122         if (pb->phys != 0)
 123                 mem_op().unpin(mgr, pb->mem);
 124
 125         if (pb->mem)
 126                 mem_op().put(mgr, pb->mem);
 127
 128         kfree(pb->client_handle);
 129
 130         pb->mem = NULL;
 131         pb->mapped = NULL;
 132         pb->phys = 0;
 133         pb->client_handle = 0;
 134 }
 135
 136 /**
 137  * Push two words to the push buffer
 138  * Caller must ensure push buffer is not full
 139  */
 140 static void push_buffer_push_to(struct push_buffer *pb,
 141                 struct mem_mgr *client, struct mem_handle *handle,
 142                 u32 op1, u32 op2)
 143 {
 144         u32 cur = pb->cur;
 145         u32 *p = (u32 *)((u32)pb->mapped + cur);
 146         u32 cur_nvmap = (cur/8) & (NVHOST_GATHER_QUEUE_SIZE - 1);
 147         BUG_ON(cur == pb->fence);
 148         *(p++) = op1;
 149         *(p++) = op2;
 150         pb->client_handle[cur_nvmap].client = client;
 151         pb->client_handle[cur_nvmap].handle = handle;
 152         pb->cur = (cur + 8) & (PUSH_BUFFER_SIZE - 1);
 153 }
 154
 155 /**
 156  * Pop a number of two word slots from the push buffer
 157  * Caller must ensure push buffer is not empty
 158  */
 159 static void push_buffer_pop_from(struct push_buffer *pb,
 160                 unsigned int slots)
 161 {
 162         /* Clear the nvmap references for old items from pb */
 163         unsigned int i;
 164         u32 fence_nvmap = pb->fence/8;
 165         for (i = 0; i < slots; i++) {
 166                 int cur_fence_nvmap = (fence_nvmap+i)
 167                                 & (NVHOST_GATHER_QUEUE_SIZE - 1);
 168                 struct mem_mgr_handle *h = &pb->client_handle[cur_fence_nvmap];
 169                 h->client = NULL;
 170                 h->handle = NULL;
 171         }
 172         /* Advance the next write position */
 173         pb->fence = (pb->fence + slots * 8) & (PUSH_BUFFER_SIZE - 1);
 174 }
 175
 176 /**
 177  * Return the number of two word slots free in the push buffer
 178  */
 179 static u32 push_buffer_space(struct push_buffer *pb)
 180 {
 181         return ((pb->fence - pb->cur) & (PUSH_BUFFER_SIZE - 1)) / 8;
 182 }
 183
 184 static u32 push_buffer_putptr(struct push_buffer *pb)
 185 {
 186         return pb->phys + pb->cur;
 187 }
 188
 189 /*
 190  * The syncpt incr buffer is filled with methods to increment syncpts, which
 191  * is later GATHER-ed into the mainline PB. It's used when a timed out context
 192  * is interleaved with other work, so needs to inline the syncpt increments
 193  * to maintain the count (but otherwise does no work).
 194  */
 195
 196 /**
 197  * Init timeout resources
 198  */
 199 static int cdma_timeout_init(struct nvhost_cdma *cdma,
 200                                  u32 syncpt_id)
 201 {
 202         if (syncpt_id == NVSYNCPT_INVALID)
 203                 return -EINVAL;
 204
 205         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 206         cdma->timeout.initialized = true;
 207
 208         return 0;
 209 }
 210
 211 /**
 212  * Clean up timeout resources
 213  */
 214 static void cdma_timeout_destroy(struct nvhost_cdma *cdma)
 215 {
 216         if (cdma->timeout.initialized)
 217                 cancel_delayed_work(&cdma->timeout.wq);
 218         cdma->timeout.initialized = false;
 219 }
 220
 221 /**
 222  * Increment timedout buffer's syncpt via CPU.
 223  */
 224 static void cdma_timeout_cpu_incr(struct nvhost_cdma *cdma, u32 getptr,
 225                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots,
 226                                 u32 waitbases)
 227 {
 228         struct nvhost_master *dev = cdma_to_dev(cdma);
 229         struct push_buffer *pb = &cdma->push_buffer;
 230         u32 i, getidx;
 231
 232         for (i = 0; i < syncpt_incrs; i++)
 233                 nvhost_syncpt_cpu_incr(&dev->syncpt, cdma->timeout.syncpt_id);
 234
 235         /* after CPU incr, ensure shadow is up to date */
 236         nvhost_syncpt_update_min(&dev->syncpt, cdma->timeout.syncpt_id);
 237
 238         /* update WAITBASE_3D by same number of incrs */
 239         if (waitbases) {
 240                 void __iomem *p;
 241                 p = dev->sync_aperture + host1x_sync_syncpt_base_0_r() +
 242                                 (ffs(waitbases) * sizeof(u32));
 243                 writel(syncval, p);
 244         }
 245
 246         /* NOP all the PB slots */
 247         getidx = getptr - pb->phys;
 248         while (nr_slots--) {
 249                 u32 *p = (u32 *)((u32)pb->mapped + getidx);
 250                 *(p++) = NVHOST_OPCODE_NOOP;
 251                 *(p++) = NVHOST_OPCODE_NOOP;
 252                 dev_dbg(&dev->dev->dev, "%s: NOP at 0x%x\n",
 253                         __func__, pb->phys + getidx);
 254                 getidx = (getidx + 8) & (PUSH_BUFFER_SIZE - 1);
 255         }
 256         wmb();
 257 }
 258
 259 /**
 260  * Start channel DMA
 261  */
 262 static void cdma_start(struct nvhost_cdma *cdma)
 263 {
 264         void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
 265
 266         if (cdma->running)
 267                 return;
 268
 269         BUG_ON(!cdma_pb_op().putptr);
 270         cdma->last_put = cdma_pb_op().putptr(&cdma->push_buffer);
 271
 272         writel(host1x_channel_dmactrl(true, false, false),
 273                 chan_regs + host1x_channel_dmactrl_r());
 274
 275         /* set base, put, end pointer (all of memory) */
 276         writel(0, chan_regs + host1x_channel_dmastart_r());
 277         writel(cdma->last_put, chan_regs + host1x_channel_dmaput_r());
 278         writel(0xFFFFFFFF, chan_regs + host1x_channel_dmaend_r());
 279
 280         /* reset GET */
 281         writel(host1x_channel_dmactrl(true, true, true),
 282                 chan_regs + host1x_channel_dmactrl_r());
 283
 284         /* start the command DMA */
 285         writel(host1x_channel_dmactrl(false, false, false),
 286                 chan_regs + host1x_channel_dmactrl_r());
 287
 288         cdma->running = true;
 289 }
 290
 291 /**
 292  * Similar to cdma_start(), but rather than starting from an idle
 293  * state (where DMA GET is set to DMA PUT), on a timeout we restore
 294  * DMA GET from an explicit value (so DMA may again be pending).
 295  */
 296 static void cdma_timeout_restart(struct nvhost_cdma *cdma, u32 getptr)
 297 {
 298         struct nvhost_master *dev = cdma_to_dev(cdma);
 299         void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
 300
 301         if (cdma->running)
 302                 return;
 303
 304         BUG_ON(!cdma_pb_op().putptr);
 305         cdma->last_put = cdma_pb_op().putptr(&cdma->push_buffer);
 306
 307         writel(host1x_channel_dmactrl(true, false, false),
 308                 chan_regs + host1x_channel_dmactrl_r());
 309
 310         /* set base, end pointer (all of memory) */
 311         writel(0, chan_regs + host1x_channel_dmastart_r());
 312         writel(0xFFFFFFFF, chan_regs + host1x_channel_dmaend_r());
 313
 314         /* set GET, by loading the value in PUT (then reset GET) */
 315         writel(getptr, chan_regs + host1x_channel_dmaput_r());
 316         writel(host1x_channel_dmactrl(true, true, true),
 317                 chan_regs + host1x_channel_dmactrl_r());
 318
 319         dev_dbg(&dev->dev->dev,
 320                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 321                 __func__,
 322                 readl(chan_regs + host1x_channel_dmaget_r()),
 323                 readl(chan_regs + host1x_channel_dmaput_r()),
 324                 cdma->last_put);
 325
 326         /* deassert GET reset and set PUT */
 327         writel(host1x_channel_dmactrl(true, false, false),
 328                 chan_regs + host1x_channel_dmactrl_r());
 329         writel(cdma->last_put, chan_regs + host1x_channel_dmaput_r());
 330
 331         /* start the command DMA */
 332         writel(host1x_channel_dmactrl(false, false, false),
 333                 chan_regs + host1x_channel_dmactrl_r());
 334
 335         cdma->running = true;
 336 }
 337
 338 /**
 339  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 340  */
 341 static void cdma_kick(struct nvhost_cdma *cdma)
 342 {
 343         u32 put;
 344         BUG_ON(!cdma_pb_op().putptr);
 345
 346         put = cdma_pb_op().putptr(&cdma->push_buffer);
 347
 348         if (put != cdma->last_put) {
 349                 void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
 350                 wmb();
 351                 writel(put, chan_regs + host1x_channel_dmaput_r());
 352                 cdma->last_put = put;
 353         }
 354 }
 355
 356 static void cdma_stop(struct nvhost_cdma *cdma)
 357 {
 358         void __iomem *chan_regs = cdma_to_channel(cdma)->aperture;
 359
 360         mutex_lock(&cdma->lock);
 361         if (cdma->running) {
 362                 nvhost_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 363                 writel(host1x_channel_dmactrl(true, false, false),
 364                         chan_regs + host1x_channel_dmactrl_r());
 365                 cdma->running = false;
 366         }
 367         mutex_unlock(&cdma->lock);
 368 }
 369
 370 /**
 371  * Stops both channel's command processor and CDMA immediately.
 372  * Also, tears down the channel and resets corresponding module.
 373  */
 374 void cdma_timeout_teardown_begin(struct nvhost_cdma *cdma)
 375 {
 376         struct nvhost_master *dev = cdma_to_dev(cdma);
 377         struct nvhost_channel *ch = cdma_to_channel(cdma);
 378         u32 cmdproc_stop;
 379
 380         BUG_ON(cdma->torndown);
 381
 382         dev_dbg(&dev->dev->dev,
 383                 "begin channel teardown (channel id %d)\n", ch->chid);
 384
 385         cmdproc_stop = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
 386         cmdproc_stop |= BIT(ch->chid);
 387         writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
 388
 389         dev_dbg(&dev->dev->dev,
 390                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 391                 __func__,
 392                 readl(ch->aperture + host1x_channel_dmaget_r()),
 393                 readl(ch->aperture + host1x_channel_dmaput_r()),
 394                 cdma->last_put);
 395
 396         writel(host1x_channel_dmactrl(true, false, false),
 397                 ch->aperture + host1x_channel_dmactrl_r());
 398
 399         writel(BIT(ch->chid), dev->sync_aperture + host1x_sync_ch_teardown_r());
 400         nvhost_module_reset(ch->dev);
 401
 402         cdma->running = false;
 403         cdma->torndown = true;
 404 }
 405
 406 void cdma_timeout_teardown_end(struct nvhost_cdma *cdma, u32 getptr)
 407 {
 408         struct nvhost_master *dev = cdma_to_dev(cdma);
 409         struct nvhost_channel *ch = cdma_to_channel(cdma);
 410         u32 cmdproc_stop;
 411
 412         BUG_ON(!cdma->torndown || cdma->running);
 413
 414         dev_dbg(&dev->dev->dev,
 415                 "end channel teardown (id %d, DMAGET restart = 0x%x)\n",
 416                 ch->chid, getptr);
 417
 418         cmdproc_stop = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
 419         cmdproc_stop &= ~(BIT(ch->chid));
 420         writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
 421
 422         cdma->torndown = false;
 423         cdma_timeout_restart(cdma, getptr);
 424 }
 425
 426 /**
 427  * If this timeout fires, it indicates the current sync_queue entry has
 428  * exceeded its TTL and the userctx should be timed out and remaining
 429  * submits already issued cleaned up (future submits return an error).
 430  */
 431 static void cdma_timeout_handler(struct work_struct *work)
 432 {
 433         struct nvhost_cdma *cdma;
 434         struct nvhost_master *dev;
 435         struct nvhost_syncpt *sp;
 436         struct nvhost_channel *ch;
 437
 438         u32 syncpt_val;
 439
 440         u32 prev_cmdproc, cmdproc_stop;
 441
 442         cdma = container_of(to_delayed_work(work), struct nvhost_cdma,
 443                             timeout.wq);
 444         dev = cdma_to_dev(cdma);
 445         sp = &dev->syncpt;
 446         ch = cdma_to_channel(cdma);
 447
 448         mutex_lock(&cdma->lock);
 449
 450         if (!cdma->timeout.clientid) {
 451                 dev_dbg(&dev->dev->dev,
 452                          "cdma_timeout: expired, but has no clientid\n");
 453                 mutex_unlock(&cdma->lock);
 454                 return;
 455         }
 456
 457         /* stop processing to get a clean snapshot */
 458         prev_cmdproc = readl(dev->sync_aperture + host1x_sync_cmdproc_stop_r());
 459         cmdproc_stop = prev_cmdproc | BIT(ch->chid);
 460         writel(cmdproc_stop, dev->sync_aperture + host1x_sync_cmdproc_stop_r());
 461
 462         dev_dbg(&dev->dev->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
 463                 prev_cmdproc, cmdproc_stop);
 464
 465         syncpt_val = nvhost_syncpt_update_min(&dev->syncpt,
 466                         cdma->timeout.syncpt_id);
 467
 468         /* has buffer actually completed? */
 469         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 470                 dev_dbg(&dev->dev->dev,
 471                          "cdma_timeout: expired, but buffer had completed\n");
 472                 /* restore */
 473                 cmdproc_stop = prev_cmdproc & ~(BIT(ch->chid));
 474                 writel(cmdproc_stop,
 475                         dev->sync_aperture + host1x_sync_cmdproc_stop_r());
 476                 mutex_unlock(&cdma->lock);
 477                 return;
 478         }
 479
 480         dev_warn(&dev->dev->dev,
 481                 "%s: timeout: %d (%s) ctx 0x%p, HW thresh %d, done %d\n",
 482                 __func__,
 483                 cdma->timeout.syncpt_id,
 484                 syncpt_op().name(sp, cdma->timeout.syncpt_id),
 485                 cdma->timeout.ctx,
 486                 syncpt_val, cdma->timeout.syncpt_val);
 487
 488         /* stop HW, resetting channel/module */
 489         cdma_op().timeout_teardown_begin(cdma);
 490
 491         nvhost_cdma_update_sync_queue(cdma, sp, dev->dev);
 492         mutex_unlock(&cdma->lock);
 493 }
 494
 495 int host1x_init_cdma_support(struct nvhost_chip_support *op)
 496 {
 497         op->cdma.start = cdma_start;
 498         op->cdma.stop = cdma_stop;
 499         op->cdma.kick = cdma_kick;
 500
 501         op->cdma.timeout_init = cdma_timeout_init;
 502         op->cdma.timeout_destroy = cdma_timeout_destroy;
 503         op->cdma.timeout_teardown_begin = cdma_timeout_teardown_begin;
 504         op->cdma.timeout_teardown_end = cdma_timeout_teardown_end;
 505         op->cdma.timeout_cpu_incr = cdma_timeout_cpu_incr;
 506
 507         op->push_buffer.reset = push_buffer_reset;
 508         op->push_buffer.init = push_buffer_init;
 509         op->push_buffer.destroy = push_buffer_destroy;
 510         op->push_buffer.push_to = push_buffer_push_to;
 511         op->push_buffer.pop_from = push_buffer_pop_from;
 512         op->push_buffer.space = push_buffer_space;
 513         op->push_buffer.putptr = push_buffer_putptr;
 514
 515         return 0;
 516 }