video: tegra: nvmap: implement fast api functions
Kirill Artamonov [Thu, 25 Oct 2012 09:36:11 +0000 (12:36 +0300)]
Implement optimized nvmap_pin_array() for fast pinning
of array of nvmap_handles.

Implement fast nvmap functions which skip slow validation
step if it is possible:

_nvmap_duplicate_handle_id()
_nvmap_free()
_nvmap_pin()

bug 1158533

Signed-off-by: Kirill Artamonov <kartamonov@nvidia.com>
Change-Id: I4639973bf9cf507ed68a03087ec6a91449f8453e
Reviewed-on: http://git-master/r/147505
Reviewed-by: Automatic_Commit_Validation_User
Reviewed-by: Terje Bergstrom <tbergstrom@nvidia.com>
Reviewed-by: Krishna Reddy <vdumpa@nvidia.com>

drivers/video/tegra/nvmap/nvmap.c
drivers/video/tegra/nvmap/nvmap.h
drivers/video/tegra/nvmap/nvmap_dev.c
drivers/video/tegra/nvmap/nvmap_handle.c
include/linux/nvmap.h

index f088553..c510ca1 100644 (file)
@@ -71,15 +71,12 @@ static int pin_locked(struct nvmap_client *client, struct nvmap_handle *h)
 {
        struct tegra_iovmm_area *area;
        BUG_ON(!h->alloc);
-
-       nvmap_mru_lock(client->share);
        if (atomic_inc_return(&h->pin) == 1) {
                if (h->heap_pgalloc && !h->pgalloc.contig) {
                        area = nvmap_handle_iovmm_locked(client, h);
                        if (!area) {
                                /* no race here, inside the pin mutex */
                                atomic_dec(&h->pin);
-                               nvmap_mru_unlock(client->share);
                                return -ENOMEM;
                        }
                        if (area != h->pgalloc.area)
@@ -88,7 +85,6 @@ static int pin_locked(struct nvmap_client *client, struct nvmap_handle *h)
                }
        }
        trace_handle_pin(client, h, atomic_read(&h->pin));
-       nvmap_mru_unlock(client->share);
        return 0;
 }
 
@@ -133,6 +129,7 @@ static int handle_unpin(struct nvmap_client *client,
        return ret;
 }
 
+
 static int pin_array_locked(struct nvmap_client *client,
                struct nvmap_handle **h, int count)
 {
@@ -140,11 +137,13 @@ static int pin_array_locked(struct nvmap_client *client,
        int i;
        int err = 0;
 
+       nvmap_mru_lock(client->share);
        for (pinned = 0; pinned < count; pinned++) {
                err = pin_locked(client, h[pinned]);
                if (err)
                        break;
        }
+       nvmap_mru_unlock(client->share);
 
        if (err) {
                /* unpin pinned handles */
@@ -167,11 +166,14 @@ static int pin_array_locked(struct nvmap_client *client,
                 * We have to do pinning again here since there might be is
                 * no more incoming pin_wait wakeup calls from unpin
                 * operations */
+               nvmap_mru_lock(client->share);
                for (pinned = 0; pinned < count; pinned++) {
                        err = pin_locked(client, h[pinned]);
                        if (err)
                                break;
                }
+               nvmap_mru_unlock(client->share);
+
                if (err) {
                        pr_err("Pinning in empty iovmm failed!!!\n");
                        BUG_ON(1);
@@ -329,7 +331,7 @@ out:
        if (ret) {
                nvmap_ref_lock(client);
                for (i = 0; i < nr; i++) {
-                       if(!ids[i])
+                       if (!ids[i])
                                continue;
 
                        ref = _nvmap_validate_id_locked(client, ids[i]);
@@ -352,6 +354,155 @@ out:
        return ret;
 }
 
+static int nvmap_validate_get_pin_array(struct nvmap_client *client,
+                               unsigned long *h,
+                               long unsigned id_type_mask,
+                               long unsigned id_type,
+                               int nr,
+                               struct nvmap_handle **unique_handles,
+                               struct nvmap_handle_ref **unique_handle_refs)
+{
+       int i;
+       int err = 0;
+       int count = 0;
+       unsigned long last_h = 0;
+       struct nvmap_handle_ref *last_ref = 0;
+
+       nvmap_ref_lock(client);
+
+       for (i = 0; i < nr; i++) {
+               struct nvmap_handle_ref *ref;
+
+               if ((h[i] & id_type_mask) != id_type)
+                       continue;
+
+               if (last_h == h[i])
+                       continue;
+
+               ref = _nvmap_validate_id_locked(client, h[i]);
+
+               if (!ref)
+                       nvmap_err(client, "failed to validate id\n");
+               else if (!ref->handle)
+                       nvmap_err(client, "id had no associated handle\n");
+               else if (!ref->handle->alloc)
+                       nvmap_err(client, "handle had no allocation\n");
+
+               if (!ref || !ref->handle || !ref->handle->alloc) {
+                       err = -EPERM;
+                       break;
+               }
+
+               last_h = h[i];
+               last_ref = ref;
+               /* a handle may be referenced multiple times in arr, but
+                * it will only be pinned once; this ensures that the
+                * minimum number of sync-queue slots in the host driver
+                * are dedicated to storing unpin lists, which allows
+                * for greater parallelism between the CPU and graphics
+                * processor */
+               if (ref->handle->flags & NVMAP_HANDLE_VISITED)
+                       continue;
+
+               ref->handle->flags |= NVMAP_HANDLE_VISITED;
+
+               unique_handles[count] = nvmap_handle_get(ref->handle);
+
+               /* Duplicate handle */
+               atomic_inc(&ref->dupes);
+               nvmap_handle_get(ref->handle);
+               unique_handle_refs[count] = ref;
+
+               BUG_ON(!unique_handles[count]);
+               count++;
+       }
+
+       nvmap_ref_unlock(client);
+
+       if (err) {
+               for (i = 0; i < count; i++) {
+                       unique_handles[i]->flags &= ~NVMAP_HANDLE_VISITED;
+                       /* pin ref */
+                       nvmap_handle_put(unique_handles[i]);
+                       /* ref count */
+                       atomic_dec(&unique_handle_refs[i]->dupes);
+                       nvmap_handle_put(unique_handles[i]);
+               }
+       }
+
+       return err ? err : count;
+}
+
+/*
+ * @client:       nvmap_client which should be used for validation;
+ *                should be owned by the process which is submitting
+ *                command buffers
+ * @ids:          array of nvmap_handles to pin
+ * @id_type_mask: bitmask which defines handle type field in handle id.
+ * @id_type:      only handles with of this type will be pinned. Handles with
+ *                other type are ignored.
+ * @nr:           number of entries in arr
+ * @unique_arr:   list of nvmap_handle objects which were pinned by
+ *                nvmap_pin_array. Must be unpinned after use
+ * @unique_arr_ref: list of duplicated nvmap_handle_refs corresponding
+ *                  to unique_arr. Must be freed after use.
+ */
+int nvmap_pin_array(struct nvmap_client *client,
+               unsigned long   *ids,
+               long unsigned id_type_mask,
+               long unsigned id_type,
+               int nr,
+               struct nvmap_handle **unique_arr,
+               struct nvmap_handle_ref **unique_arr_refs)
+{
+       int count = 0;
+       int ret = 0;
+       int i;
+
+       if (mutex_lock_interruptible(&client->share->pin_lock)) {
+               nvmap_err(client, "%s interrupted when acquiring pin lock\n",
+                          current->group_leader->comm);
+               return -EINTR;
+       }
+
+       count = nvmap_validate_get_pin_array(client, ids,
+                       id_type_mask, id_type, nr,
+                       unique_arr, unique_arr_refs);
+
+       if (count < 0) {
+               mutex_unlock(&client->share->pin_lock);
+               nvmap_warn(client, "failed to validate pin array\n");
+               return count;
+       }
+
+       for (i = 0; i < count; i++)
+               unique_arr[i]->flags &= ~NVMAP_HANDLE_VISITED;
+
+       ret = wait_pin_array_locked(client, unique_arr, count);
+
+       mutex_unlock(&client->share->pin_lock);
+
+       if (WARN_ON(ret)) {
+               for (i = 0; i < count; i++) {
+                       /* pin ref */
+                       nvmap_handle_put(unique_arr[i]);
+                       /* remove duplicate */
+                       atomic_dec(&unique_arr_refs[i]->dupes);
+                       nvmap_handle_put(unique_arr[i]);
+               }
+               return ret;
+       } else {
+               for (i = 0; i < count; i++) {
+                       if (unique_arr[i]->heap_pgalloc &&
+                           unique_arr[i]->pgalloc.dirty)
+                               map_iovmm_area(unique_arr[i]);
+
+                       atomic_inc(&unique_arr_refs[i]->pin);
+               }
+       }
+       return count;
+}
+
 static phys_addr_t handle_phys(struct nvmap_handle *h)
 {
        phys_addr_t addr;
@@ -368,21 +519,36 @@ static phys_addr_t handle_phys(struct nvmap_handle *h)
        return addr;
 }
 
-phys_addr_t nvmap_pin(struct nvmap_client *client,
+/*
+ * Get physical address of the handle. Handle should be
+ * already validated and pinned.
+ */
+phys_addr_t _nvmap_get_addr_from_id(u32 id)
+{
+       struct nvmap_handle *h = (struct nvmap_handle *)id;
+       return handle_phys(h);
+}
+
+/*
+ * Pin handle without slow validation step
+ */
+phys_addr_t _nvmap_pin(struct nvmap_client *client,
                        struct nvmap_handle_ref *ref)
 {
+       int ret = 0;
        struct nvmap_handle *h;
        phys_addr_t phys;
-       int ret = 0;
-       unsigned long ref_id;
 
        if (!ref)
                return -EINVAL;
-       ref_id = nvmap_ref_to_id(ref);
-       h = nvmap_get_handle_id(client, ref_id);
+
+       h = ref->handle;
+
        if (WARN_ON(!h))
                return -EINVAL;
 
+       h = nvmap_handle_get(h);
+
        atomic_inc(&ref->pin);
 
        if (WARN_ON(mutex_lock_interruptible(&client->share->pin_lock))) {
@@ -404,6 +570,25 @@ phys_addr_t nvmap_pin(struct nvmap_client *client,
        return ret ?: phys;
 }
 
+phys_addr_t nvmap_pin(struct nvmap_client *client,
+                       struct nvmap_handle_ref *ref)
+{
+       struct nvmap_handle *h;
+
+       if (!ref)
+               return -EINVAL;
+       if (WARN_ON(!ref->handle))
+               return -EINVAL;
+
+       nvmap_ref_lock(client);
+       ref = _nvmap_validate_id_locked(client, (unsigned long)ref->handle);
+       if (ref)
+               h = ref->handle;
+       nvmap_ref_unlock(client);
+
+       return _nvmap_pin(client, ref);
+}
+
 phys_addr_t nvmap_handle_address(struct nvmap_client *c, unsigned long id)
 {
        struct nvmap_handle *h;
@@ -654,7 +839,11 @@ struct nvmap_handle_ref *nvmap_alloc_iovm(struct nvmap_client *client,
        err = mutex_lock_interruptible(&client->share->pin_lock);
        if (WARN_ON(err))
                goto fail;
+
+       nvmap_mru_lock(client->share);
        err = pin_locked(client, h);
+       nvmap_mru_unlock(client->share);
+
        mutex_unlock(&client->share->pin_lock);
        if (err)
                goto fail;
index dda58c8..e475758 100644 (file)
@@ -76,6 +76,12 @@ struct nvmap_handle {
        size_t orig_size;       /* original (as-requested) size */
        size_t align;
        struct nvmap_client *owner;
+       struct nvmap_handle_ref *owner_ref; /* use this ref to avoid spending
+                       time on validation in some cases.
+                       if handle was duplicated by other client and
+                       original client destroy ref, this field
+                       has to be set to zero. In this case ref should be
+                       obtained through validation */
        struct nvmap_device *dev;
        union {
                struct nvmap_pgalloc pgalloc;
index 39b7bd3..8847d79 100644 (file)
@@ -694,8 +694,10 @@ static void destroy_client(struct nvmap_client *client)
                smp_rmb();
                pins = atomic_read(&ref->pin);
 
-               if (ref->handle->owner == client)
+               if (ref->handle->owner == client) {
                        ref->handle->owner = NULL;
+                       ref->handle->owner_ref = NULL;
+               }
 
                while (pins--)
                        nvmap_unpin_handles(client, &ref->handle, 1);
index 2a80f87..8a18aeb 100644 (file)
@@ -866,6 +866,25 @@ out:
        return err;
 }
 
+/*
+ * Free handle without slow validation step
+ */
+void _nvmap_free(struct nvmap_client *client, struct nvmap_handle_ref *r)
+{
+       nvmap_ref_lock(client);
+       if (r->handle->owner == client && atomic_read(&r->dupes) > 1) {
+               atomic_dec(&r->dupes);
+               nvmap_ref_unlock(client);
+               nvmap_handle_put(r->handle);
+               return;
+       } else {
+               /* slow path */
+               nvmap_ref_unlock(client);
+               nvmap_free(client, r);
+               return;
+       }
+}
+
 void nvmap_free_handle_id(struct nvmap_client *client, unsigned long id)
 {
        struct nvmap_handle_ref *ref;
@@ -913,8 +932,10 @@ void nvmap_free_handle_id(struct nvmap_client *client, unsigned long id)
        while (pins--)
                nvmap_unpin_handles(client, &ref->handle, 1);
 
-       if (h->owner == client)
+       if (h->owner == client) {
                h->owner = NULL;
+               h->owner_ref = NULL;
+       }
 
        kfree(ref);
 
@@ -969,6 +990,7 @@ struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
        atomic_set(&h->ref, 1);
        atomic_set(&h->pin, 0);
        h->owner = client;
+       h->owner_ref = ref;
        h->dev = client->dev;
        BUG_ON(!h->owner);
        h->size = h->orig_size = size;
@@ -985,6 +1007,29 @@ struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
        return ref;
 }
 
+/*
+ * Duplicate handle without slow validation step.
+ */
+struct nvmap_handle_ref *_nvmap_duplicate_handle_id(struct nvmap_client *client,
+                                                  unsigned long id)
+{
+       struct nvmap_handle *h = (struct nvmap_handle *)id;
+       struct nvmap_handle_ref *ref = h->owner_ref;
+
+       nvmap_ref_lock(client);
+       if (client == h->owner) {
+               /* handle already duplicated in client; just increment
+                * the reference count rather than re-duplicating it */
+               atomic_inc(&ref->dupes);
+               nvmap_handle_get(h);
+               nvmap_ref_unlock(client);
+               return ref;
+       } else {
+               nvmap_ref_unlock(client);
+               return nvmap_duplicate_handle_id(client, id);
+       }
+}
+
 struct nvmap_handle_ref *nvmap_duplicate_handle_id(struct nvmap_client *client,
                                                   unsigned long id)
 {
index ca907db..12949e7 100644 (file)
@@ -102,8 +102,12 @@ struct nvmap_handle_ref *nvmap_alloc(struct nvmap_client *client, size_t size,
                                     size_t align, unsigned int flags,
                                     unsigned int heap_mask);
 
+phys_addr_t _nvmap_get_addr_from_id(u32 id);
+
 void nvmap_free(struct nvmap_client *client, struct nvmap_handle_ref *r);
 
+void _nvmap_free(struct nvmap_client *client, struct nvmap_handle_ref *r);
+
 void *nvmap_mmap(struct nvmap_handle_ref *r);
 
 void nvmap_munmap(struct nvmap_handle_ref *r, void *addr);
@@ -120,6 +124,8 @@ struct nvmap_client *nvmap_client_get(struct nvmap_client *client);
 void nvmap_client_put(struct nvmap_client *c);
 
 phys_addr_t nvmap_pin(struct nvmap_client *c, struct nvmap_handle_ref *r);
+phys_addr_t _nvmap_pin(struct nvmap_client *c, struct nvmap_handle_ref *r);
+
 
 phys_addr_t nvmap_handle_address(struct nvmap_client *c, unsigned long id);
 
@@ -130,6 +136,17 @@ void nvmap_unpin_handles(struct nvmap_client *client,
 
 struct nvmap_handle_ref *nvmap_duplicate_handle_id(struct nvmap_client *client,
                                                   unsigned long id);
+struct nvmap_handle_ref *_nvmap_duplicate_handle_id(struct nvmap_client *client,
+                                                  unsigned long id);
+
+int nvmap_pin_array(struct nvmap_client *client,
+               unsigned long    *ids,
+               long unsigned id_type_mask,
+               long unsigned id_type,
+               int nr,
+               struct nvmap_handle **unique_arr,
+               struct nvmap_handle_ref **unique_arr_refs);
+
 
 struct nvmap_platform_carveout {
        const char *name;