*
* LOCKING:
*
- * The subsystem keeps two global lists, dma_device_list and dma_client_list.
- * Both of these are protected by a mutex, dma_list_mutex.
+ * The subsystem keeps a global list of dma_device structs it is protected by a
+ * mutex, dma_list_mutex.
+ *
+ * A subsystem can get access to a channel by calling dmaengine_get() followed
+ * by dma_find_channel(), or if it has need for an exclusive channel it can call
+ * dma_request_channel(). Once a channel is allocated a reference is taken
+ * against its corresponding driver to disable removal.
*
* Each device has a channels list, which runs unlocked but is never modified
* once the device is registered, it's just setup by the driver.
*
- * Each client is responsible for keeping track of the channels it uses. See
- * the definition of dma_event_callback in dmaengine.h.
- *
- * Each device has a kref, which is initialized to 1 when the device is
- * registered. A kref_get is done for each device registered. When the
- * device is released, the corresponding kref_put is done in the release
- * method. Every time one of the device's channels is allocated to a client,
- * a kref_get occurs. When the channel is freed, the corresponding kref_put
- * happens. The device's release function does a completion, so
- * unregister_device does a remove event, device_unregister, a kref_put
- * for the first reference, then waits on the completion for all other
- * references to finish.
- *
- * Each channel has an open-coded implementation of Rusty Russell's "bigref,"
- * with a kref and a per_cpu local_t. A dma_chan_get is called when a client
- * signals that it wants to use a channel, and dma_chan_put is called when
- * a channel is removed or a client using it is unregistered. A client can
- * take extra references per outstanding transaction, as is the case with
- * the NET DMA client. The release function does a kref_put on the device.
- * -ChrisL, DanW
+ * See Documentation/dmaengine.txt for more details
*/
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/jiffies.h>
#include <linux/rculist.h>
+#include <linux/idr.h>
static DEFINE_MUTEX(dma_list_mutex);
static LIST_HEAD(dma_device_list);
-static LIST_HEAD(dma_client_list);
static long dmaengine_ref_count;
+static struct idr dma_idr;
/* --- sysfs implementation --- */
+/**
+ * dev_to_dma_chan - convert a device pointer to the its sysfs container object
+ * @dev - device node
+ *
+ * Must be called under dma_list_mutex
+ */
+static struct dma_chan *dev_to_dma_chan(struct device *dev)
+{
+ struct dma_chan_dev *chan_dev;
+
+ chan_dev = container_of(dev, typeof(*chan_dev), device);
+ return chan_dev->chan;
+}
+
static ssize_t show_memcpy_count(struct device *dev, struct device_attribute *attr, char *buf)
{
- struct dma_chan *chan = to_dma_chan(dev);
+ struct dma_chan *chan;
unsigned long count = 0;
int i;
+ int err;
- for_each_possible_cpu(i)
- count += per_cpu_ptr(chan->local, i)->memcpy_count;
+ mutex_lock(&dma_list_mutex);
+ chan = dev_to_dma_chan(dev);
+ if (chan) {
+ for_each_possible_cpu(i)
+ count += per_cpu_ptr(chan->local, i)->memcpy_count;
+ err = sprintf(buf, "%lu\n", count);
+ } else
+ err = -ENODEV;
+ mutex_unlock(&dma_list_mutex);
- return sprintf(buf, "%lu\n", count);
+ return err;
}
static ssize_t show_bytes_transferred(struct device *dev, struct device_attribute *attr,
char *buf)
{
- struct dma_chan *chan = to_dma_chan(dev);
+ struct dma_chan *chan;
unsigned long count = 0;
int i;
+ int err;
- for_each_possible_cpu(i)
- count += per_cpu_ptr(chan->local, i)->bytes_transferred;
+ mutex_lock(&dma_list_mutex);
+ chan = dev_to_dma_chan(dev);
+ if (chan) {
+ for_each_possible_cpu(i)
+ count += per_cpu_ptr(chan->local, i)->bytes_transferred;
+ err = sprintf(buf, "%lu\n", count);
+ } else
+ err = -ENODEV;
+ mutex_unlock(&dma_list_mutex);
- return sprintf(buf, "%lu\n", count);
+ return err;
}
static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf)
{
- struct dma_chan *chan = to_dma_chan(dev);
+ struct dma_chan *chan;
+ int err;
+
+ mutex_lock(&dma_list_mutex);
+ chan = dev_to_dma_chan(dev);
+ if (chan)
+ err = sprintf(buf, "%d\n", chan->client_count);
+ else
+ err = -ENODEV;
+ mutex_unlock(&dma_list_mutex);
- return sprintf(buf, "%d\n", chan->client_count);
+ return err;
}
static struct device_attribute dma_attrs[] = {
__ATTR_NULL
};
-static void dma_async_device_cleanup(struct kref *kref);
-
-static void dma_dev_release(struct device *dev)
+static void chan_dev_release(struct device *dev)
{
- struct dma_chan *chan = to_dma_chan(dev);
- kref_put(&chan->device->refcount, dma_async_device_cleanup);
+ struct dma_chan_dev *chan_dev;
+
+ chan_dev = container_of(dev, typeof(*chan_dev), device);
+ if (atomic_dec_and_test(chan_dev->idr_ref)) {
+ mutex_lock(&dma_list_mutex);
+ idr_remove(&dma_idr, chan_dev->dev_id);
+ mutex_unlock(&dma_list_mutex);
+ kfree(chan_dev->idr_ref);
+ }
+ kfree(chan_dev);
}
static struct class dma_devclass = {
.name = "dma",
.dev_attrs = dma_attrs,
- .dev_release = dma_dev_release,
+ .dev_release = chan_dev_release,
};
/* --- client and device registration --- */
/* allocate upon first client reference */
if (chan->client_count == 1 && err == 0) {
- int desc_cnt = chan->device->device_alloc_chan_resources(chan, NULL);
+ int desc_cnt = chan->device->device_alloc_chan_resources(chan);
if (desc_cnt < 0) {
err = desc_cnt;
chan->device->device_free_chan_resources(chan);
}
-/**
- * dma_client_chan_alloc - try to allocate channels to a client
- * @client: &dma_client
- *
- * Called with dma_list_mutex held.
- */
-static void dma_client_chan_alloc(struct dma_client *client)
-{
- struct dma_device *device;
- struct dma_chan *chan;
- enum dma_state_client ack;
-
- /* Find a channel */
- list_for_each_entry(device, &dma_device_list, global_node) {
- if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
- continue;
- if (!dma_device_satisfies_mask(device, client->cap_mask))
- continue;
-
- list_for_each_entry(chan, &device->channels, device_node) {
- if (!chan->client_count)
- continue;
- ack = client->event_callback(client, chan,
- DMA_RESOURCE_AVAILABLE);
-
- /* we are done once this client rejects
- * an available resource
- */
- if (ack == DMA_NAK)
- return;
- }
- }
-}
-
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
{
enum dma_status status;
}
EXPORT_SYMBOL(dma_sync_wait);
-/**
- * dma_chan_cleanup - release a DMA channel's resources
- * @kref: kernel reference structure that contains the DMA channel device
- */
-void dma_chan_cleanup(struct kref *kref)
-{
- struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
- kref_put(&chan->device->refcount, dma_async_device_cleanup);
-}
-EXPORT_SYMBOL(dma_chan_cleanup);
-
-static void dma_chan_free_rcu(struct rcu_head *rcu)
-{
- struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu);
-
- kref_put(&chan->refcount, dma_chan_cleanup);
-}
-
-static void dma_chan_release(struct dma_chan *chan)
-{
- call_rcu(&chan->rcu, dma_chan_free_rcu);
-}
-
/**
* dma_cap_mask_all - enable iteration over all operation types
*/
return err;
}
-subsys_initcall(dma_channel_table_init);
+arch_initcall(dma_channel_table_init);
/**
* dma_find_channel - find a channel to carry out the operation
}
}
-static struct dma_chan *private_candidate(dma_cap_mask_t *mask, struct dma_device *dev)
+static struct dma_chan *private_candidate(dma_cap_mask_t *mask, struct dma_device *dev,
+ dma_filter_fn fn, void *fn_param)
{
struct dma_chan *chan;
- struct dma_chan *ret = NULL;
if (!__dma_device_satisfies_mask(dev, mask)) {
pr_debug("%s: wrong capabilities\n", __func__);
list_for_each_entry(chan, &dev->channels, device_node) {
if (chan->client_count) {
pr_debug("%s: %s busy\n",
- __func__, dev_name(&chan->dev));
+ __func__, dma_chan_name(chan));
continue;
}
- ret = chan;
- break;
+ if (fn && !fn(chan, fn_param)) {
+ pr_debug("%s: %s filter said false\n",
+ __func__, dma_chan_name(chan));
+ continue;
+ }
+ return chan;
}
- return ret;
+ return NULL;
}
/**
{
struct dma_device *device, *_d;
struct dma_chan *chan = NULL;
- enum dma_state_client ack;
int err;
/* Find a channel */
mutex_lock(&dma_list_mutex);
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
- chan = private_candidate(mask, device);
- if (!chan)
- continue;
-
- if (fn)
- ack = fn(chan, fn_param);
- else
- ack = DMA_ACK;
-
- if (ack == DMA_ACK) {
+ chan = private_candidate(mask, device, fn, fn_param);
+ if (chan) {
/* Found a suitable channel, try to grab, prep, and
* return it. We first set DMA_PRIVATE to disable
* balance_ref_count as this channel will not be
if (err == -ENODEV) {
pr_debug("%s: %s module removed\n", __func__,
- dev_name(&chan->dev));
+ dma_chan_name(chan));
list_del_rcu(&device->global_node);
} else if (err)
pr_err("dmaengine: failed to get %s: (%d)\n",
- dev_name(&chan->dev), err);
+ dma_chan_name(chan), err);
else
break;
- } else if (ack == DMA_DUP) {
- pr_debug("%s: %s filter said DMA_DUP\n",
- __func__, dev_name(&chan->dev));
- } else if (ack == DMA_NAK) {
- pr_debug("%s: %s filter said DMA_NAK\n",
- __func__, dev_name(&chan->dev));
- break;
- } else
- WARN_ONCE(1, "filter_fn: unknown response?\n");
- chan = NULL;
+ chan = NULL;
+ }
}
mutex_unlock(&dma_list_mutex);
pr_debug("%s: %s (%s)\n", __func__, chan ? "success" : "fail",
- chan ? dev_name(&chan->dev) : NULL);
+ chan ? dma_chan_name(chan) : NULL);
return chan;
}
}
EXPORT_SYMBOL_GPL(dma_release_channel);
-/**
- * dma_chans_notify_available - broadcast available channels to the clients
- */
-static void dma_clients_notify_available(void)
-{
- struct dma_client *client;
-
- mutex_lock(&dma_list_mutex);
-
- list_for_each_entry(client, &dma_client_list, global_node)
- dma_client_chan_alloc(client);
-
- mutex_unlock(&dma_list_mutex);
-}
-
/**
* dmaengine_get - register interest in dma_channels
*/
break;
} else if (err)
pr_err("dmaengine: failed to get %s: (%d)\n",
- dev_name(&chan->dev), err);
+ dma_chan_name(chan), err);
}
}
}
EXPORT_SYMBOL(dmaengine_put);
-/**
- * dma_async_client_chan_request - send all available channels to the
- * client that satisfy the capability mask
- * @client - requester
- */
-void dma_async_client_chan_request(struct dma_client *client)
-{
- mutex_lock(&dma_list_mutex);
- dma_client_chan_alloc(client);
- mutex_unlock(&dma_list_mutex);
-}
-EXPORT_SYMBOL(dma_async_client_chan_request);
-
/**
* dma_async_device_register - registers DMA devices found
* @device: &dma_device
*/
int dma_async_device_register(struct dma_device *device)
{
- static int id;
int chancnt = 0, rc;
struct dma_chan* chan;
+ atomic_t *idr_ref;
if (!device)
return -ENODEV;
BUG_ON(!device->device_issue_pending);
BUG_ON(!device->dev);
- init_completion(&device->done);
- kref_init(&device->refcount);
-
+ idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
+ if (!idr_ref)
+ return -ENOMEM;
+ atomic_set(idr_ref, 0);
+ idr_retry:
+ if (!idr_pre_get(&dma_idr, GFP_KERNEL))
+ return -ENOMEM;
mutex_lock(&dma_list_mutex);
- device->dev_id = id++;
+ rc = idr_get_new(&dma_idr, NULL, &device->dev_id);
mutex_unlock(&dma_list_mutex);
+ if (rc == -EAGAIN)
+ goto idr_retry;
+ else if (rc != 0)
+ return rc;
/* represent channels in sysfs. Probably want devs too */
list_for_each_entry(chan, &device->channels, device_node) {
chan->local = alloc_percpu(typeof(*chan->local));
if (chan->local == NULL)
continue;
+ chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
+ if (chan->dev == NULL) {
+ free_percpu(chan->local);
+ continue;
+ }
chan->chan_id = chancnt++;
- chan->dev.class = &dma_devclass;
- chan->dev.parent = device->dev;
- dev_set_name(&chan->dev, "dma%dchan%d",
+ chan->dev->device.class = &dma_devclass;
+ chan->dev->device.parent = device->dev;
+ chan->dev->chan = chan;
+ chan->dev->idr_ref = idr_ref;
+ chan->dev->dev_id = device->dev_id;
+ atomic_inc(idr_ref);
+ dev_set_name(&chan->dev->device, "dma%dchan%d",
device->dev_id, chan->chan_id);
- rc = device_register(&chan->dev);
+ rc = device_register(&chan->dev->device);
if (rc) {
- chancnt--;
free_percpu(chan->local);
chan->local = NULL;
goto err_out;
}
-
- /* One for the channel, one of the class device */
- kref_get(&device->refcount);
- kref_get(&device->refcount);
- kref_init(&chan->refcount);
chan->client_count = 0;
- chan->slow_ref = 0;
- INIT_RCU_HEAD(&chan->rcu);
}
device->chancnt = chancnt;
dma_channel_rebalance();
mutex_unlock(&dma_list_mutex);
- dma_clients_notify_available();
-
return 0;
err_out:
list_for_each_entry(chan, &device->channels, device_node) {
if (chan->local == NULL)
continue;
- kref_put(&device->refcount, dma_async_device_cleanup);
- device_unregister(&chan->dev);
- chancnt--;
+ mutex_lock(&dma_list_mutex);
+ chan->dev->chan = NULL;
+ mutex_unlock(&dma_list_mutex);
+ device_unregister(&chan->dev->device);
free_percpu(chan->local);
}
return rc;
}
EXPORT_SYMBOL(dma_async_device_register);
-/**
- * dma_async_device_cleanup - function called when all references are released
- * @kref: kernel reference object
- */
-static void dma_async_device_cleanup(struct kref *kref)
-{
- struct dma_device *device;
-
- device = container_of(kref, struct dma_device, refcount);
- complete(&device->done);
-}
-
/**
* dma_async_device_unregister - unregister a DMA device
* @device: &dma_device
+ *
+ * This routine is called by dma driver exit routines, dmaengine holds module
+ * references to prevent it being called while channels are in use.
*/
void dma_async_device_unregister(struct dma_device *device)
{
WARN_ONCE(chan->client_count,
"%s called while %d clients hold a reference\n",
__func__, chan->client_count);
- device_unregister(&chan->dev);
- dma_chan_release(chan);
+ mutex_lock(&dma_list_mutex);
+ chan->dev->chan = NULL;
+ mutex_unlock(&dma_list_mutex);
+ device_unregister(&chan->dev->device);
}
-
- kref_put(&device->refcount, dma_async_device_cleanup);
- wait_for_completion(&device->done);
}
EXPORT_SYMBOL(dma_async_device_unregister);
return DMA_SUCCESS;
WARN_ONCE(tx->parent, "%s: speculatively walking dependency chain for"
- " %s\n", __func__, dev_name(&tx->chan->dev));
+ " %s\n", __func__, dma_chan_name(tx->chan));
/* poll through the dependency chain, return when tx is complete */
do {
static int __init dma_bus_init(void)
{
+ idr_init(&dma_idr);
mutex_init(&dma_list_mutex);
return class_register(&dma_devclass);
}
-subsys_initcall(dma_bus_init);
+arch_initcall(dma_bus_init);
Code Review / linux-2.6.git / blobdiff