Commit 19242d72 authored by Dan Williams's avatar Dan Williams

async_tx: fix multiple dependency submission

Shrink struct dma_async_tx_descriptor and introduce
async_tx_channel_switch to properly inject a channel switch interrupt in
the descriptor stream.  This simplifies the locking model as drivers no
longer need to handle dma_async_tx_descriptor.lock.
Acked-by: default avatarShannon Nelson <shannon.nelson@intel.com>
Signed-off-by: default avatarDan Williams <dan.j.williams@intel.com>
parent 1c62979e
...@@ -89,13 +89,19 @@ dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) ...@@ -89,13 +89,19 @@ dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
iter = tx; iter = tx;
/* find the root of the unsubmitted dependency chain */ /* find the root of the unsubmitted dependency chain */
while (iter->cookie == -EBUSY) { do {
parent = iter->parent; parent = iter->parent;
if (parent && parent->cookie == -EBUSY) if (!parent)
iter = iter->parent;
else
break; break;
} else
iter = parent;
} while (parent);
/* there is a small window for ->parent == NULL and
* ->cookie == -EBUSY
*/
while (iter->cookie == -EBUSY)
cpu_relax();
status = dma_sync_wait(iter->chan, iter->cookie); status = dma_sync_wait(iter->chan, iter->cookie);
} while (status == DMA_IN_PROGRESS || (iter != tx)); } while (status == DMA_IN_PROGRESS || (iter != tx));
...@@ -111,24 +117,33 @@ EXPORT_SYMBOL_GPL(dma_wait_for_async_tx); ...@@ -111,24 +117,33 @@ EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
void void
async_tx_run_dependencies(struct dma_async_tx_descriptor *tx) async_tx_run_dependencies(struct dma_async_tx_descriptor *tx)
{ {
struct dma_async_tx_descriptor *dep_tx, *_dep_tx; struct dma_async_tx_descriptor *next = tx->next;
struct dma_device *dev;
struct dma_chan *chan; struct dma_chan *chan;
list_for_each_entry_safe(dep_tx, _dep_tx, &tx->depend_list, if (!next)
depend_node) { return;
chan = dep_tx->chan;
dev = chan->device; tx->next = NULL;
/* we can't depend on ourselves */ chan = next->chan;
BUG_ON(chan == tx->chan);
list_del(&dep_tx->depend_node); /* keep submitting up until a channel switch is detected
tx->tx_submit(dep_tx); * in that case we will be called again as a result of
* processing the interrupt from async_tx_channel_switch
/* we need to poke the engine as client code does not
* know about dependency submission events
*/ */
dev->device_issue_pending(chan); while (next && next->chan == chan) {
struct dma_async_tx_descriptor *_next;
spin_lock_bh(&next->lock);
next->parent = NULL;
_next = next->next;
next->next = NULL;
spin_unlock_bh(&next->lock);
next->tx_submit(next);
next = _next;
} }
chan->device->device_issue_pending(chan);
} }
EXPORT_SYMBOL_GPL(async_tx_run_dependencies); EXPORT_SYMBOL_GPL(async_tx_run_dependencies);
...@@ -397,6 +412,92 @@ static void __exit async_tx_exit(void) ...@@ -397,6 +412,92 @@ static void __exit async_tx_exit(void)
} }
#endif #endif
/**
* async_tx_channel_switch - queue an interrupt descriptor with a dependency
* pre-attached.
* @depend_tx: the operation that must finish before the new operation runs
* @tx: the new operation
*/
static void
async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
struct dma_async_tx_descriptor *tx)
{
struct dma_chan *chan;
struct dma_device *device;
struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
/* first check to see if we can still append to depend_tx */
spin_lock_bh(&depend_tx->lock);
if (depend_tx->parent && depend_tx->chan == tx->chan) {
tx->parent = depend_tx;
depend_tx->next = tx;
intr_tx = NULL;
}
spin_unlock_bh(&depend_tx->lock);
if (!intr_tx)
return;
chan = depend_tx->chan;
device = chan->device;
/* see if we can schedule an interrupt
* otherwise poll for completion
*/
if (dma_has_cap(DMA_INTERRUPT, device->cap_mask))
intr_tx = device->device_prep_dma_interrupt(chan);
else
intr_tx = NULL;
if (intr_tx) {
intr_tx->callback = NULL;
intr_tx->callback_param = NULL;
tx->parent = intr_tx;
/* safe to set ->next outside the lock since we know we are
* not submitted yet
*/
intr_tx->next = tx;
/* check if we need to append */
spin_lock_bh(&depend_tx->lock);
if (depend_tx->parent) {
intr_tx->parent = depend_tx;
depend_tx->next = intr_tx;
async_tx_ack(intr_tx);
intr_tx = NULL;
}
spin_unlock_bh(&depend_tx->lock);
if (intr_tx) {
intr_tx->parent = NULL;
intr_tx->tx_submit(intr_tx);
async_tx_ack(intr_tx);
}
} else {
if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
panic("%s: DMA_ERROR waiting for depend_tx\n",
__func__);
tx->tx_submit(tx);
}
}
/**
* submit_disposition - while holding depend_tx->lock we must avoid submitting
* new operations to prevent a circular locking dependency with
* drivers that already hold a channel lock when calling
* async_tx_run_dependencies.
* @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
* @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
* @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
*/
enum submit_disposition {
ASYNC_TX_SUBMITTED,
ASYNC_TX_CHANNEL_SWITCH,
ASYNC_TX_DIRECT_SUBMIT,
};
void void
async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx, enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,
...@@ -405,28 +506,54 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, ...@@ -405,28 +506,54 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
tx->callback = cb_fn; tx->callback = cb_fn;
tx->callback_param = cb_param; tx->callback_param = cb_param;
/* set this new tx to run after depend_tx if: if (depend_tx) {
* 1/ a dependency exists (depend_tx is !NULL) enum submit_disposition s;
* 2/ the tx can not be submitted to the current channel
*/ /* sanity check the dependency chain:
if (depend_tx && depend_tx->chan != chan) { * 1/ if ack is already set then we cannot be sure
/* if ack is already set then we cannot be sure
* we are referring to the correct operation * we are referring to the correct operation
* 2/ dependencies are 1:1 i.e. two transactions can
* not depend on the same parent
*/ */
BUG_ON(depend_tx->ack); BUG_ON(depend_tx->ack || depend_tx->next || tx->parent);
tx->parent = depend_tx; /* the lock prevents async_tx_run_dependencies from missing
* the setting of ->next when ->parent != NULL
*/
spin_lock_bh(&depend_tx->lock); spin_lock_bh(&depend_tx->lock);
list_add_tail(&tx->depend_node, &depend_tx->depend_list); if (depend_tx->parent) {
if (depend_tx->cookie == 0) { /* we have a parent so we can not submit directly
struct dma_chan *dep_chan = depend_tx->chan; * if we are staying on the same channel: append
struct dma_device *dep_dev = dep_chan->device; * else: channel switch
dep_dev->device_dependency_added(dep_chan); */
if (depend_tx->chan == chan) {
tx->parent = depend_tx;
depend_tx->next = tx;
s = ASYNC_TX_SUBMITTED;
} else
s = ASYNC_TX_CHANNEL_SWITCH;
} else {
/* we do not have a parent so we may be able to submit
* directly if we are staying on the same channel
*/
if (depend_tx->chan == chan)
s = ASYNC_TX_DIRECT_SUBMIT;
else
s = ASYNC_TX_CHANNEL_SWITCH;
} }
spin_unlock_bh(&depend_tx->lock); spin_unlock_bh(&depend_tx->lock);
/* schedule an interrupt to trigger the channel switch */ switch (s) {
async_trigger_callback(ASYNC_TX_ACK, depend_tx, NULL, NULL); case ASYNC_TX_SUBMITTED:
break;
case ASYNC_TX_CHANNEL_SWITCH:
async_tx_channel_switch(depend_tx, tx);
break;
case ASYNC_TX_DIRECT_SUBMIT:
tx->parent = NULL;
tx->tx_submit(tx);
break;
}
} else { } else {
tx->parent = NULL; tx->parent = NULL;
tx->tx_submit(tx); tx->tx_submit(tx);
......
...@@ -600,8 +600,6 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, ...@@ -600,8 +600,6 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
{ {
tx->chan = chan; tx->chan = chan;
spin_lock_init(&tx->lock); spin_lock_init(&tx->lock);
INIT_LIST_HEAD(&tx->depend_node);
INIT_LIST_HEAD(&tx->depend_list);
} }
EXPORT_SYMBOL(dma_async_tx_descriptor_init); EXPORT_SYMBOL(dma_async_tx_descriptor_init);
......
...@@ -63,7 +63,6 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, ...@@ -63,7 +63,6 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
struct iop_adma_chan *iop_chan, dma_cookie_t cookie) struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
{ {
BUG_ON(desc->async_tx.cookie < 0); BUG_ON(desc->async_tx.cookie < 0);
spin_lock_bh(&desc->async_tx.lock);
if (desc->async_tx.cookie > 0) { if (desc->async_tx.cookie > 0) {
cookie = desc->async_tx.cookie; cookie = desc->async_tx.cookie;
desc->async_tx.cookie = 0; desc->async_tx.cookie = 0;
...@@ -101,7 +100,6 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, ...@@ -101,7 +100,6 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
/* run dependent operations */ /* run dependent operations */
async_tx_run_dependencies(&desc->async_tx); async_tx_run_dependencies(&desc->async_tx);
spin_unlock_bh(&desc->async_tx.lock);
return cookie; return cookie;
} }
...@@ -275,8 +273,11 @@ iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) ...@@ -275,8 +273,11 @@ iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
static void iop_adma_tasklet(unsigned long data) static void iop_adma_tasklet(unsigned long data)
{ {
struct iop_adma_chan *chan = (struct iop_adma_chan *) data; struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
__iop_adma_slot_cleanup(chan);
spin_lock(&iop_chan->lock);
__iop_adma_slot_cleanup(iop_chan);
spin_unlock(&iop_chan->lock);
} }
static struct iop_adma_desc_slot * static struct iop_adma_desc_slot *
......
...@@ -221,11 +221,9 @@ typedef void (*dma_async_tx_callback)(void *dma_async_param); ...@@ -221,11 +221,9 @@ typedef void (*dma_async_tx_callback)(void *dma_async_param);
* @callback: routine to call after this operation is complete * @callback: routine to call after this operation is complete
* @callback_param: general parameter to pass to the callback routine * @callback_param: general parameter to pass to the callback routine
* ---async_tx api specific fields--- * ---async_tx api specific fields---
* @depend_list: at completion this list of transactions are submitted * @next: at completion submit this descriptor
* @depend_node: allow this transaction to be executed after another
* transaction has completed, possibly on another channel
* @parent: pointer to the next level up in the dependency chain * @parent: pointer to the next level up in the dependency chain
* @lock: protect the dependency list * @lock: protect the parent and next pointers
*/ */
struct dma_async_tx_descriptor { struct dma_async_tx_descriptor {
dma_cookie_t cookie; dma_cookie_t cookie;
...@@ -236,8 +234,7 @@ struct dma_async_tx_descriptor { ...@@ -236,8 +234,7 @@ struct dma_async_tx_descriptor {
dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx); dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
dma_async_tx_callback callback; dma_async_tx_callback callback;
void *callback_param; void *callback_param;
struct list_head depend_list; struct dma_async_tx_descriptor *next;
struct list_head depend_node;
struct dma_async_tx_descriptor *parent; struct dma_async_tx_descriptor *parent;
spinlock_t lock; spinlock_t lock;
}; };
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment