blob: df53a46285a3fe3430c90eb5496f945f4d7a304e [file] [log] [blame]
Thomas Gleixnerd2912cb2019-06-04 10:11:33 +02001// SPDX-License-Identifier: GPL-2.0-only
Arto Merilainende2ba662013-03-22 16:34:08 +02002/*
3 * NVIDIA Tegra DRM GEM helper functions
4 *
5 * Copyright (C) 2012 Sascha Hauer, Pengutronix
Arto Merilainen7ecada32016-11-08 19:51:34 +02006 * Copyright (C) 2013-2015 NVIDIA CORPORATION, All rights reserved.
Arto Merilainende2ba662013-03-22 16:34:08 +02007 *
8 * Based on the GEM/CMA helpers
9 *
10 * Copyright (c) 2011 Samsung Electronics Co., Ltd.
Arto Merilainende2ba662013-03-22 16:34:08 +020011 */
12
Thierry Reding38003912013-12-12 10:00:43 +010013#include <linux/dma-buf.h>
Thierry Redingdf06b752014-06-26 21:41:53 +020014#include <linux/iommu.h>
Thierry Reding773af772013-10-04 22:34:01 +020015#include <drm/tegra_drm.h>
16
Thierry Redingd1f3e1e2014-07-11 08:29:14 +020017#include "drm.h"
Arto Merilainende2ba662013-03-22 16:34:08 +020018#include "gem.h"
19
Arto Merilainende2ba662013-03-22 16:34:08 +020020static void tegra_bo_put(struct host1x_bo *bo)
21{
Thierry Reding3be82742013-09-24 16:34:05 +020022 struct tegra_bo *obj = host1x_to_tegra_bo(bo);
Arto Merilainende2ba662013-03-22 16:34:08 +020023
Cihangir Akturk7664b2f2017-08-11 15:33:07 +030024 drm_gem_object_put_unlocked(&obj->gem);
Arto Merilainende2ba662013-03-22 16:34:08 +020025}
26
27static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt)
28{
Thierry Reding3be82742013-09-24 16:34:05 +020029 struct tegra_bo *obj = host1x_to_tegra_bo(bo);
Arto Merilainende2ba662013-03-22 16:34:08 +020030
Mikko Perttunen585ee0f2016-11-08 19:51:35 +020031 *sgt = obj->sgt;
32
Arto Merilainende2ba662013-03-22 16:34:08 +020033 return obj->paddr;
34}
35
36static void tegra_bo_unpin(struct host1x_bo *bo, struct sg_table *sgt)
37{
38}
39
40static void *tegra_bo_mmap(struct host1x_bo *bo)
41{
Thierry Reding3be82742013-09-24 16:34:05 +020042 struct tegra_bo *obj = host1x_to_tegra_bo(bo);
Arto Merilainende2ba662013-03-22 16:34:08 +020043
Arto Merilainen7ecada32016-11-08 19:51:34 +020044 if (obj->vaddr)
45 return obj->vaddr;
46 else if (obj->gem.import_attach)
47 return dma_buf_vmap(obj->gem.import_attach->dmabuf);
48 else
49 return vmap(obj->pages, obj->num_pages, VM_MAP,
50 pgprot_writecombine(PAGE_KERNEL));
Arto Merilainende2ba662013-03-22 16:34:08 +020051}
52
53static void tegra_bo_munmap(struct host1x_bo *bo, void *addr)
54{
Arto Merilainen7ecada32016-11-08 19:51:34 +020055 struct tegra_bo *obj = host1x_to_tegra_bo(bo);
56
57 if (obj->vaddr)
58 return;
59 else if (obj->gem.import_attach)
60 dma_buf_vunmap(obj->gem.import_attach->dmabuf, addr);
61 else
62 vunmap(addr);
Arto Merilainende2ba662013-03-22 16:34:08 +020063}
64
65static void *tegra_bo_kmap(struct host1x_bo *bo, unsigned int page)
66{
Thierry Reding3be82742013-09-24 16:34:05 +020067 struct tegra_bo *obj = host1x_to_tegra_bo(bo);
Arto Merilainende2ba662013-03-22 16:34:08 +020068
Arto Merilainen7ecada32016-11-08 19:51:34 +020069 if (obj->vaddr)
70 return obj->vaddr + page * PAGE_SIZE;
71 else if (obj->gem.import_attach)
72 return dma_buf_kmap(obj->gem.import_attach->dmabuf, page);
73 else
74 return vmap(obj->pages + page, 1, VM_MAP,
75 pgprot_writecombine(PAGE_KERNEL));
Arto Merilainende2ba662013-03-22 16:34:08 +020076}
77
78static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page,
79 void *addr)
80{
Arto Merilainen7ecada32016-11-08 19:51:34 +020081 struct tegra_bo *obj = host1x_to_tegra_bo(bo);
82
83 if (obj->vaddr)
84 return;
85 else if (obj->gem.import_attach)
86 dma_buf_kunmap(obj->gem.import_attach->dmabuf, page, addr);
87 else
88 vunmap(addr);
Arto Merilainende2ba662013-03-22 16:34:08 +020089}
90
91static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo)
92{
Thierry Reding3be82742013-09-24 16:34:05 +020093 struct tegra_bo *obj = host1x_to_tegra_bo(bo);
Arto Merilainende2ba662013-03-22 16:34:08 +020094
Cihangir Akturk7664b2f2017-08-11 15:33:07 +030095 drm_gem_object_get(&obj->gem);
Arto Merilainende2ba662013-03-22 16:34:08 +020096
97 return bo;
98}
99
Thierry Reding425c0fd2013-12-12 10:10:46 +0100100static const struct host1x_bo_ops tegra_bo_ops = {
Arto Merilainende2ba662013-03-22 16:34:08 +0200101 .get = tegra_bo_get,
102 .put = tegra_bo_put,
103 .pin = tegra_bo_pin,
104 .unpin = tegra_bo_unpin,
105 .mmap = tegra_bo_mmap,
106 .munmap = tegra_bo_munmap,
107 .kmap = tegra_bo_kmap,
108 .kunmap = tegra_bo_kunmap,
109};
110
Thierry Redingdf06b752014-06-26 21:41:53 +0200111static int tegra_bo_iommu_map(struct tegra_drm *tegra, struct tegra_bo *bo)
112{
113 int prot = IOMMU_READ | IOMMU_WRITE;
Dmitry Osipenko04184b12017-12-20 18:46:13 +0300114 int err;
Thierry Redingdf06b752014-06-26 21:41:53 +0200115
116 if (bo->mm)
117 return -EBUSY;
118
119 bo->mm = kzalloc(sizeof(*bo->mm), GFP_KERNEL);
120 if (!bo->mm)
121 return -ENOMEM;
122
Thierry Reding347ad49d2017-03-09 20:04:56 +0100123 mutex_lock(&tegra->mm_lock);
124
Chris Wilson4e64e552017-02-02 21:04:38 +0000125 err = drm_mm_insert_node_generic(&tegra->mm,
126 bo->mm, bo->gem.size, PAGE_SIZE, 0, 0);
Thierry Redingdf06b752014-06-26 21:41:53 +0200127 if (err < 0) {
Dmitry Osipenko04184b12017-12-20 18:46:13 +0300128 dev_err(tegra->drm->dev, "out of I/O virtual memory: %d\n",
Thierry Redingdf06b752014-06-26 21:41:53 +0200129 err);
Thierry Reding347ad49d2017-03-09 20:04:56 +0100130 goto unlock;
Thierry Redingdf06b752014-06-26 21:41:53 +0200131 }
132
133 bo->paddr = bo->mm->start;
134
Dmitry Osipenko04184b12017-12-20 18:46:13 +0300135 bo->size = iommu_map_sg(tegra->domain, bo->paddr, bo->sgt->sgl,
136 bo->sgt->nents, prot);
137 if (!bo->size) {
138 dev_err(tegra->drm->dev, "failed to map buffer\n");
139 err = -ENOMEM;
Thierry Redingdf06b752014-06-26 21:41:53 +0200140 goto remove;
141 }
142
Thierry Reding347ad49d2017-03-09 20:04:56 +0100143 mutex_unlock(&tegra->mm_lock);
144
Thierry Redingdf06b752014-06-26 21:41:53 +0200145 return 0;
146
147remove:
148 drm_mm_remove_node(bo->mm);
Thierry Reding347ad49d2017-03-09 20:04:56 +0100149unlock:
150 mutex_unlock(&tegra->mm_lock);
Thierry Redingdf06b752014-06-26 21:41:53 +0200151 kfree(bo->mm);
152 return err;
153}
154
155static int tegra_bo_iommu_unmap(struct tegra_drm *tegra, struct tegra_bo *bo)
156{
157 if (!bo->mm)
158 return 0;
159
Thierry Reding347ad49d2017-03-09 20:04:56 +0100160 mutex_lock(&tegra->mm_lock);
Thierry Redingdf06b752014-06-26 21:41:53 +0200161 iommu_unmap(tegra->domain, bo->paddr, bo->size);
162 drm_mm_remove_node(bo->mm);
Thierry Reding347ad49d2017-03-09 20:04:56 +0100163 mutex_unlock(&tegra->mm_lock);
164
Thierry Redingdf06b752014-06-26 21:41:53 +0200165 kfree(bo->mm);
166
167 return 0;
168}
169
Thierry Redingc28d4a32014-10-16 14:18:50 +0200170static struct tegra_bo *tegra_bo_alloc_object(struct drm_device *drm,
171 size_t size)
Arto Merilainende2ba662013-03-22 16:34:08 +0200172{
173 struct tegra_bo *bo;
174 int err;
175
176 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
177 if (!bo)
178 return ERR_PTR(-ENOMEM);
179
180 host1x_bo_init(&bo->base, &tegra_bo_ops);
181 size = round_up(size, PAGE_SIZE);
182
Thierry Redingc28d4a32014-10-16 14:18:50 +0200183 err = drm_gem_object_init(drm, &bo->gem, size);
184 if (err < 0)
185 goto free;
186
187 err = drm_gem_create_mmap_offset(&bo->gem);
188 if (err < 0)
189 goto release;
190
191 return bo;
192
193release:
194 drm_gem_object_release(&bo->gem);
195free:
196 kfree(bo);
197 return ERR_PTR(err);
198}
199
Thierry Redingdf06b752014-06-26 21:41:53 +0200200static void tegra_bo_free(struct drm_device *drm, struct tegra_bo *bo)
Thierry Redingc28d4a32014-10-16 14:18:50 +0200201{
Thierry Redingdf06b752014-06-26 21:41:53 +0200202 if (bo->pages) {
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100203 dma_unmap_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents,
Dmitry Osipenko61b51fb2019-03-07 01:55:19 +0300204 DMA_FROM_DEVICE);
Thierry Redingdf06b752014-06-26 21:41:53 +0200205 drm_gem_put_pages(&bo->gem, bo->pages, true, true);
206 sg_free_table(bo->sgt);
207 kfree(bo->sgt);
Thierry Reding7e0180e2014-11-06 14:41:31 +0100208 } else if (bo->vaddr) {
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800209 dma_free_wc(drm->dev, bo->gem.size, bo->vaddr, bo->paddr);
Thierry Redingdf06b752014-06-26 21:41:53 +0200210 }
211}
212
Thierry Reding73c42c72014-12-16 16:41:47 +0100213static int tegra_bo_get_pages(struct drm_device *drm, struct tegra_bo *bo)
Thierry Redingdf06b752014-06-26 21:41:53 +0200214{
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100215 int err;
Thierry Redinga04251f2014-12-16 16:35:26 +0100216
Thierry Redingdf06b752014-06-26 21:41:53 +0200217 bo->pages = drm_gem_get_pages(&bo->gem);
218 if (IS_ERR(bo->pages))
219 return PTR_ERR(bo->pages);
220
Thierry Reding73c42c72014-12-16 16:41:47 +0100221 bo->num_pages = bo->gem.size >> PAGE_SHIFT;
Thierry Redingdf06b752014-06-26 21:41:53 +0200222
Thierry Redingfd73caa2015-04-14 12:52:36 +0200223 bo->sgt = drm_prime_pages_to_sg(bo->pages, bo->num_pages);
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100224 if (IS_ERR(bo->sgt)) {
225 err = PTR_ERR(bo->sgt);
Thierry Redinga04251f2014-12-16 16:35:26 +0100226 goto put_pages;
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100227 }
Thierry Redinga04251f2014-12-16 16:35:26 +0100228
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100229 err = dma_map_sg(drm->dev, bo->sgt->sgl, bo->sgt->nents,
Dmitry Osipenko61b51fb2019-03-07 01:55:19 +0300230 DMA_FROM_DEVICE);
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100231 if (err == 0) {
232 err = -EFAULT;
233 goto free_sgt;
234 }
Thierry Redinga04251f2014-12-16 16:35:26 +0100235
Thierry Redingdf06b752014-06-26 21:41:53 +0200236 return 0;
Thierry Redinga04251f2014-12-16 16:35:26 +0100237
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100238free_sgt:
239 sg_free_table(bo->sgt);
240 kfree(bo->sgt);
Thierry Redinga04251f2014-12-16 16:35:26 +0100241put_pages:
242 drm_gem_put_pages(&bo->gem, bo->pages, false, false);
Thierry Redingbd43c9f2017-12-13 12:22:48 +0100243 return err;
Thierry Redingdf06b752014-06-26 21:41:53 +0200244}
245
Thierry Reding73c42c72014-12-16 16:41:47 +0100246static int tegra_bo_alloc(struct drm_device *drm, struct tegra_bo *bo)
Thierry Redingdf06b752014-06-26 21:41:53 +0200247{
248 struct tegra_drm *tegra = drm->dev_private;
249 int err;
250
251 if (tegra->domain) {
Thierry Reding73c42c72014-12-16 16:41:47 +0100252 err = tegra_bo_get_pages(drm, bo);
Thierry Redingdf06b752014-06-26 21:41:53 +0200253 if (err < 0)
254 return err;
255
256 err = tegra_bo_iommu_map(tegra, bo);
257 if (err < 0) {
258 tegra_bo_free(drm, bo);
259 return err;
260 }
261 } else {
Thierry Reding73c42c72014-12-16 16:41:47 +0100262 size_t size = bo->gem.size;
263
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800264 bo->vaddr = dma_alloc_wc(drm->dev, size, &bo->paddr,
265 GFP_KERNEL | __GFP_NOWARN);
Thierry Redingdf06b752014-06-26 21:41:53 +0200266 if (!bo->vaddr) {
267 dev_err(drm->dev,
268 "failed to allocate buffer of size %zu\n",
269 size);
270 return -ENOMEM;
271 }
272 }
273
274 return 0;
Thierry Redingc28d4a32014-10-16 14:18:50 +0200275}
276
Thierry Reding71c38622014-11-03 13:23:02 +0100277struct tegra_bo *tegra_bo_create(struct drm_device *drm, size_t size,
Thierry Redingc28d4a32014-10-16 14:18:50 +0200278 unsigned long flags)
279{
280 struct tegra_bo *bo;
281 int err;
282
283 bo = tegra_bo_alloc_object(drm, size);
284 if (IS_ERR(bo))
285 return bo;
286
Thierry Reding73c42c72014-12-16 16:41:47 +0100287 err = tegra_bo_alloc(drm, bo);
Thierry Redingdf06b752014-06-26 21:41:53 +0200288 if (err < 0)
289 goto release;
Arto Merilainende2ba662013-03-22 16:34:08 +0200290
Thierry Reding773af772013-10-04 22:34:01 +0200291 if (flags & DRM_TEGRA_GEM_CREATE_TILED)
Thierry Redingc134f012014-06-03 14:48:12 +0200292 bo->tiling.mode = TEGRA_BO_TILING_MODE_TILED;
Thierry Reding773af772013-10-04 22:34:01 +0200293
Thierry Redingdb7fbdf2013-10-07 09:47:58 +0200294 if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP)
295 bo->flags |= TEGRA_BO_BOTTOM_UP;
296
Arto Merilainende2ba662013-03-22 16:34:08 +0200297 return bo;
298
Thierry Redingdf06b752014-06-26 21:41:53 +0200299release:
300 drm_gem_object_release(&bo->gem);
Arto Merilainende2ba662013-03-22 16:34:08 +0200301 kfree(bo);
Arto Merilainende2ba662013-03-22 16:34:08 +0200302 return ERR_PTR(err);
Arto Merilainende2ba662013-03-22 16:34:08 +0200303}
304
305struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,
Thierry Reding3be82742013-09-24 16:34:05 +0200306 struct drm_device *drm,
Thierry Reding71c38622014-11-03 13:23:02 +0100307 size_t size,
Thierry Reding773af772013-10-04 22:34:01 +0200308 unsigned long flags,
Thierry Reding71c38622014-11-03 13:23:02 +0100309 u32 *handle)
Arto Merilainende2ba662013-03-22 16:34:08 +0200310{
311 struct tegra_bo *bo;
Thierry Redinga8b48df2014-10-16 14:22:50 +0200312 int err;
Arto Merilainende2ba662013-03-22 16:34:08 +0200313
Thierry Reding773af772013-10-04 22:34:01 +0200314 bo = tegra_bo_create(drm, size, flags);
Arto Merilainende2ba662013-03-22 16:34:08 +0200315 if (IS_ERR(bo))
316 return bo;
317
Thierry Redinga8b48df2014-10-16 14:22:50 +0200318 err = drm_gem_handle_create(file, &bo->gem, handle);
319 if (err) {
320 tegra_bo_free_object(&bo->gem);
321 return ERR_PTR(err);
322 }
Arto Merilainende2ba662013-03-22 16:34:08 +0200323
Cihangir Akturk7664b2f2017-08-11 15:33:07 +0300324 drm_gem_object_put_unlocked(&bo->gem);
Arto Merilainende2ba662013-03-22 16:34:08 +0200325
326 return bo;
Arto Merilainende2ba662013-03-22 16:34:08 +0200327}
328
Thierry Reding540457c2014-05-13 16:46:11 +0200329static struct tegra_bo *tegra_bo_import(struct drm_device *drm,
330 struct dma_buf *buf)
Thierry Reding38003912013-12-12 10:00:43 +0100331{
Thierry Redingdf06b752014-06-26 21:41:53 +0200332 struct tegra_drm *tegra = drm->dev_private;
Thierry Reding38003912013-12-12 10:00:43 +0100333 struct dma_buf_attachment *attach;
334 struct tegra_bo *bo;
Thierry Reding38003912013-12-12 10:00:43 +0100335 int err;
336
Thierry Redingc28d4a32014-10-16 14:18:50 +0200337 bo = tegra_bo_alloc_object(drm, buf->size);
338 if (IS_ERR(bo))
339 return bo;
Thierry Reding38003912013-12-12 10:00:43 +0100340
341 attach = dma_buf_attach(buf, drm->dev);
342 if (IS_ERR(attach)) {
343 err = PTR_ERR(attach);
Thierry Redingc28d4a32014-10-16 14:18:50 +0200344 goto free;
Thierry Reding38003912013-12-12 10:00:43 +0100345 }
346
347 get_dma_buf(buf);
348
349 bo->sgt = dma_buf_map_attachment(attach, DMA_TO_DEVICE);
Thierry Reding38003912013-12-12 10:00:43 +0100350 if (IS_ERR(bo->sgt)) {
351 err = PTR_ERR(bo->sgt);
352 goto detach;
353 }
354
Thierry Redingdf06b752014-06-26 21:41:53 +0200355 if (tegra->domain) {
356 err = tegra_bo_iommu_map(tegra, bo);
357 if (err < 0)
358 goto detach;
359 } else {
360 if (bo->sgt->nents > 1) {
361 err = -EINVAL;
362 goto detach;
363 }
364
365 bo->paddr = sg_dma_address(bo->sgt->sgl);
Thierry Reding38003912013-12-12 10:00:43 +0100366 }
367
Thierry Reding38003912013-12-12 10:00:43 +0100368 bo->gem.import_attach = attach;
369
370 return bo;
371
372detach:
373 if (!IS_ERR_OR_NULL(bo->sgt))
374 dma_buf_unmap_attachment(attach, bo->sgt, DMA_TO_DEVICE);
375
376 dma_buf_detach(buf, attach);
377 dma_buf_put(buf);
Thierry Reding38003912013-12-12 10:00:43 +0100378free:
Thierry Redingc28d4a32014-10-16 14:18:50 +0200379 drm_gem_object_release(&bo->gem);
Thierry Reding38003912013-12-12 10:00:43 +0100380 kfree(bo);
Thierry Reding38003912013-12-12 10:00:43 +0100381 return ERR_PTR(err);
382}
383
Arto Merilainende2ba662013-03-22 16:34:08 +0200384void tegra_bo_free_object(struct drm_gem_object *gem)
385{
Thierry Redingdf06b752014-06-26 21:41:53 +0200386 struct tegra_drm *tegra = gem->dev->dev_private;
Arto Merilainende2ba662013-03-22 16:34:08 +0200387 struct tegra_bo *bo = to_tegra_bo(gem);
388
Thierry Redingdf06b752014-06-26 21:41:53 +0200389 if (tegra->domain)
390 tegra_bo_iommu_unmap(tegra, bo);
391
Thierry Reding38003912013-12-12 10:00:43 +0100392 if (gem->import_attach) {
393 dma_buf_unmap_attachment(gem->import_attach, bo->sgt,
394 DMA_TO_DEVICE);
395 drm_prime_gem_destroy(gem, NULL);
396 } else {
Thierry Redingdf06b752014-06-26 21:41:53 +0200397 tegra_bo_free(gem->dev, bo);
Thierry Reding38003912013-12-12 10:00:43 +0100398 }
399
Arto Merilainende2ba662013-03-22 16:34:08 +0200400 drm_gem_object_release(gem);
Arto Merilainende2ba662013-03-22 16:34:08 +0200401 kfree(bo);
402}
403
404int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
405 struct drm_mode_create_dumb *args)
406{
Thierry Redingdc6057e2014-10-30 15:32:56 +0100407 unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
Thierry Redingd1f3e1e2014-07-11 08:29:14 +0200408 struct tegra_drm *tegra = drm->dev_private;
Arto Merilainende2ba662013-03-22 16:34:08 +0200409 struct tegra_bo *bo;
410
Thierry Redingdc6057e2014-10-30 15:32:56 +0100411 args->pitch = round_up(min_pitch, tegra->pitch_align);
412 args->size = args->pitch * args->height;
Arto Merilainende2ba662013-03-22 16:34:08 +0200413
Thierry Reding773af772013-10-04 22:34:01 +0200414 bo = tegra_bo_create_with_handle(file, drm, args->size, 0,
Thierry Reding3be82742013-09-24 16:34:05 +0200415 &args->handle);
Arto Merilainende2ba662013-03-22 16:34:08 +0200416 if (IS_ERR(bo))
417 return PTR_ERR(bo);
418
419 return 0;
420}
421
Souptick Joardercc7add72018-04-17 19:17:55 +0530422static vm_fault_t tegra_bo_fault(struct vm_fault *vmf)
Thierry Redingdf06b752014-06-26 21:41:53 +0200423{
Dave Jiang11bac802017-02-24 14:56:41 -0800424 struct vm_area_struct *vma = vmf->vma;
Thierry Redingdf06b752014-06-26 21:41:53 +0200425 struct drm_gem_object *gem = vma->vm_private_data;
426 struct tegra_bo *bo = to_tegra_bo(gem);
427 struct page *page;
428 pgoff_t offset;
Thierry Redingdf06b752014-06-26 21:41:53 +0200429
430 if (!bo->pages)
431 return VM_FAULT_SIGBUS;
432
Jan Kara1a29d852016-12-14 15:07:01 -0800433 offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
Thierry Redingdf06b752014-06-26 21:41:53 +0200434 page = bo->pages[offset];
435
Souptick Joardercc7add72018-04-17 19:17:55 +0530436 return vmf_insert_page(vma, vmf->address, page);
Thierry Redingdf06b752014-06-26 21:41:53 +0200437}
438
Arto Merilainende2ba662013-03-22 16:34:08 +0200439const struct vm_operations_struct tegra_bo_vm_ops = {
Thierry Redingdf06b752014-06-26 21:41:53 +0200440 .fault = tegra_bo_fault,
Arto Merilainende2ba662013-03-22 16:34:08 +0200441 .open = drm_gem_vm_open,
442 .close = drm_gem_vm_close,
443};
444
Thierry Reding04c07462018-02-07 18:45:55 +0100445int __tegra_gem_mmap(struct drm_gem_object *gem, struct vm_area_struct *vma)
Arto Merilainende2ba662013-03-22 16:34:08 +0200446{
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200447 struct tegra_bo *bo = to_tegra_bo(gem);
Arto Merilainende2ba662013-03-22 16:34:08 +0200448
Thierry Redingdf06b752014-06-26 21:41:53 +0200449 if (!bo->pages) {
450 unsigned long vm_pgoff = vma->vm_pgoff;
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200451 int err;
Arto Merilainende2ba662013-03-22 16:34:08 +0200452
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200453 /*
454 * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(),
455 * and set the vm_pgoff (used as a fake buffer offset by DRM)
456 * to 0 as we want to map the whole buffer.
457 */
Thierry Redingdf06b752014-06-26 21:41:53 +0200458 vma->vm_flags &= ~VM_PFNMAP;
459 vma->vm_pgoff = 0;
460
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200461 err = dma_mmap_wc(gem->dev->dev, vma, bo->vaddr, bo->paddr,
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800462 gem->size);
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200463 if (err < 0) {
Thierry Redingdf06b752014-06-26 21:41:53 +0200464 drm_gem_vm_close(vma);
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200465 return err;
Thierry Redingdf06b752014-06-26 21:41:53 +0200466 }
467
468 vma->vm_pgoff = vm_pgoff;
469 } else {
470 pgprot_t prot = vm_get_page_prot(vma->vm_flags);
471
472 vma->vm_flags |= VM_MIXEDMAP;
473 vma->vm_flags &= ~VM_PFNMAP;
474
475 vma->vm_page_prot = pgprot_writecombine(prot);
Thierry Reding53ea7212014-09-24 16:14:04 +0200476 }
477
Thierry Reding53ea7212014-09-24 16:14:04 +0200478 return 0;
Arto Merilainende2ba662013-03-22 16:34:08 +0200479}
Thierry Reding38003912013-12-12 10:00:43 +0100480
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200481int tegra_drm_mmap(struct file *file, struct vm_area_struct *vma)
482{
483 struct drm_gem_object *gem;
484 int err;
485
486 err = drm_gem_mmap(file, vma);
487 if (err < 0)
488 return err;
489
490 gem = vma->vm_private_data;
491
Thierry Reding04c07462018-02-07 18:45:55 +0100492 return __tegra_gem_mmap(gem, vma);
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200493}
494
Thierry Reding38003912013-12-12 10:00:43 +0100495static struct sg_table *
496tegra_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
497 enum dma_data_direction dir)
498{
499 struct drm_gem_object *gem = attach->dmabuf->priv;
500 struct tegra_bo *bo = to_tegra_bo(gem);
501 struct sg_table *sgt;
502
503 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
504 if (!sgt)
505 return NULL;
506
Thierry Redingdf06b752014-06-26 21:41:53 +0200507 if (bo->pages) {
508 struct scatterlist *sg;
509 unsigned int i;
510
511 if (sg_alloc_table(sgt, bo->num_pages, GFP_KERNEL))
512 goto free;
513
514 for_each_sg(sgt->sgl, sg, bo->num_pages, i)
515 sg_set_page(sg, bo->pages[i], PAGE_SIZE, 0);
516
517 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0)
518 goto free;
519 } else {
520 if (sg_alloc_table(sgt, 1, GFP_KERNEL))
521 goto free;
522
523 sg_dma_address(sgt->sgl) = bo->paddr;
524 sg_dma_len(sgt->sgl) = gem->size;
Thierry Reding38003912013-12-12 10:00:43 +0100525 }
526
Thierry Reding38003912013-12-12 10:00:43 +0100527 return sgt;
Thierry Redingdf06b752014-06-26 21:41:53 +0200528
529free:
530 sg_free_table(sgt);
531 kfree(sgt);
532 return NULL;
Thierry Reding38003912013-12-12 10:00:43 +0100533}
534
535static void tegra_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
536 struct sg_table *sgt,
537 enum dma_data_direction dir)
538{
Thierry Redingdf06b752014-06-26 21:41:53 +0200539 struct drm_gem_object *gem = attach->dmabuf->priv;
540 struct tegra_bo *bo = to_tegra_bo(gem);
541
542 if (bo->pages)
543 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
544
Thierry Reding38003912013-12-12 10:00:43 +0100545 sg_free_table(sgt);
546 kfree(sgt);
547}
548
549static void tegra_gem_prime_release(struct dma_buf *buf)
550{
551 drm_gem_dmabuf_release(buf);
552}
553
Thierry Reding27e92f12017-12-13 12:25:14 +0100554static int tegra_gem_prime_begin_cpu_access(struct dma_buf *buf,
555 enum dma_data_direction direction)
556{
557 struct drm_gem_object *gem = buf->priv;
558 struct tegra_bo *bo = to_tegra_bo(gem);
559 struct drm_device *drm = gem->dev;
560
561 if (bo->pages)
562 dma_sync_sg_for_cpu(drm->dev, bo->sgt->sgl, bo->sgt->nents,
563 DMA_FROM_DEVICE);
564
565 return 0;
566}
567
568static int tegra_gem_prime_end_cpu_access(struct dma_buf *buf,
569 enum dma_data_direction direction)
570{
571 struct drm_gem_object *gem = buf->priv;
572 struct tegra_bo *bo = to_tegra_bo(gem);
573 struct drm_device *drm = gem->dev;
574
575 if (bo->pages)
576 dma_sync_sg_for_device(drm->dev, bo->sgt->sgl, bo->sgt->nents,
577 DMA_TO_DEVICE);
578
579 return 0;
580}
581
Thierry Reding38003912013-12-12 10:00:43 +0100582static void *tegra_gem_prime_kmap(struct dma_buf *buf, unsigned long page)
583{
584 return NULL;
585}
586
587static void tegra_gem_prime_kunmap(struct dma_buf *buf, unsigned long page,
588 void *addr)
589{
590}
591
592static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma)
593{
Thierry Redinga8bc8c62017-08-15 15:42:40 +0200594 struct drm_gem_object *gem = buf->priv;
595 int err;
596
597 err = drm_gem_mmap_obj(gem, gem->size, vma);
598 if (err < 0)
599 return err;
600
Thierry Reding04c07462018-02-07 18:45:55 +0100601 return __tegra_gem_mmap(gem, vma);
Thierry Reding38003912013-12-12 10:00:43 +0100602}
603
Thierry Redingd40326f2014-01-29 20:32:33 +0100604static void *tegra_gem_prime_vmap(struct dma_buf *buf)
605{
606 struct drm_gem_object *gem = buf->priv;
607 struct tegra_bo *bo = to_tegra_bo(gem);
608
609 return bo->vaddr;
610}
611
612static void tegra_gem_prime_vunmap(struct dma_buf *buf, void *vaddr)
613{
614}
615
Thierry Reding38003912013-12-12 10:00:43 +0100616static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = {
617 .map_dma_buf = tegra_gem_prime_map_dma_buf,
618 .unmap_dma_buf = tegra_gem_prime_unmap_dma_buf,
619 .release = tegra_gem_prime_release,
Thierry Reding27e92f12017-12-13 12:25:14 +0100620 .begin_cpu_access = tegra_gem_prime_begin_cpu_access,
621 .end_cpu_access = tegra_gem_prime_end_cpu_access,
Logan Gunthorpef9b67f02017-04-19 13:36:10 -0600622 .map = tegra_gem_prime_kmap,
623 .unmap = tegra_gem_prime_kunmap,
Thierry Reding38003912013-12-12 10:00:43 +0100624 .mmap = tegra_gem_prime_mmap,
Thierry Redingd40326f2014-01-29 20:32:33 +0100625 .vmap = tegra_gem_prime_vmap,
626 .vunmap = tegra_gem_prime_vunmap,
Thierry Reding38003912013-12-12 10:00:43 +0100627};
628
629struct dma_buf *tegra_gem_prime_export(struct drm_device *drm,
630 struct drm_gem_object *gem,
631 int flags)
632{
Sumit Semwald8fbe342015-01-23 12:53:43 +0530633 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
634
Thierry Reding4bd91a52018-05-16 18:49:04 +0200635 exp_info.exp_name = KBUILD_MODNAME;
636 exp_info.owner = drm->driver->fops->owner;
Sumit Semwald8fbe342015-01-23 12:53:43 +0530637 exp_info.ops = &tegra_gem_prime_dmabuf_ops;
638 exp_info.size = gem->size;
639 exp_info.flags = flags;
640 exp_info.priv = gem;
641
Chris Wilsona4fce9c2016-10-05 13:21:44 +0100642 return drm_gem_dmabuf_export(drm, &exp_info);
Thierry Reding38003912013-12-12 10:00:43 +0100643}
644
645struct drm_gem_object *tegra_gem_prime_import(struct drm_device *drm,
646 struct dma_buf *buf)
647{
648 struct tegra_bo *bo;
649
650 if (buf->ops == &tegra_gem_prime_dmabuf_ops) {
651 struct drm_gem_object *gem = buf->priv;
652
653 if (gem->dev == drm) {
Cihangir Akturk7664b2f2017-08-11 15:33:07 +0300654 drm_gem_object_get(gem);
Thierry Reding38003912013-12-12 10:00:43 +0100655 return gem;
656 }
657 }
658
659 bo = tegra_bo_import(drm, buf);
660 if (IS_ERR(bo))
661 return ERR_CAST(bo);
662
663 return &bo->gem;
664}