[PATCH] expose dma_addr_t in urbs

This patch exposes DMA addresses in URBs.  It exposes new APIs that
let drivers be a bit smarter in terms of DMA, reducing USB overhead
on some platforms (but not commodity pcs).  As discussed with DaveM,
and on the usb-devel list.

Supporting patches are still needed.  Of course, there's teaching HCDs
to use _these_ addresses when they're provided (easy).  There's also
teaching drivers (like hid) to use the new usb_buffer_alloc() support,;
can happen incrementally.  And adding scatterlist support, which will
be desirable for usb-storage and hpusbscanner.  But this is the start
needed to get all of that going.

Documentation/DocBook/kernel-api.tmpl

@@ -299,6 +299,8 @@ KAO -->
     EHCI, OHCI, or UHCI.
     </para>
 !Edrivers/usb/core/hcd.c
+!Edrivers/usb/core/hcd-pci.c
+!Edrivers/usb/core/buffer.c
     </sect1>
 
   </chapter>

drivers/usb/core/buffer.c

+/*
+ * DMA memory management for framework level HCD code (hc_driver)
+ *
+ * This implementation plugs in through generic "usb_bus" level methods,
+ * and works with real PCI, or when "pci device == null" makes sense.
+ */
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+
+
+#ifdef CONFIG_USB_DEBUG
+	#define DEBUG
+#else
+	#undef DEBUG
+#endif
+
+#include <linux/usb.h>
+#include "hcd.h"
+
+
+/*
+ * DMA-Consistent Buffers
+ */
+
+/* FIXME tune these based on pool statistics ... */
+static const size_t	pool_max [HCD_BUFFER_POOLS] = {
+	32,
+	128,
+	512,
+	PAGE_SIZE / 2
+	/* bigger --> allocate pages */
+};
+
+
+/* SETUP primitives */
+
+/**
+ * hcd_buffer_create - initialize buffer pools
+ * @hcd: the bus whose buffer pools are to be initialized
+ *
+ * Call this as part of initializing a host controller that uses the pci dma
+ * memory allocators.  It initializes some pools of dma-consistent memory that
+ * will be shared by all drivers using that controller, or returns a negative
+ * errno value on error.
+ *
+ * Call hcd_buffer_destroy() to clean up after using those pools.
+ */
+int hcd_buffer_create (struct usb_hcd *hcd)
+{
+	char		name [16];
+	int 		i, size;
+
+	for (i = 0; i < HCD_BUFFER_POOLS; i++) { 
+		if (!(size = pool_max [i]))
+			continue;
+		snprintf (name, sizeof name, "buffer-%d", size);
+		hcd->pool [i] = pci_pool_create (name, hcd->pdev,
+				size, size, 0, SLAB_KERNEL);
+		if (!hcd->pool [i]) {
+			hcd_buffer_destroy (hcd);
+			return -ENOMEM;
+		}
+	}
+	return 0;
+}
+EXPORT_SYMBOL (hcd_buffer_create);
+
+
+/**
+ * hcd_buffer_destroy - deallocate buffer pools
+ * @hcd: the bus whose buffer pools are to be destroyed
+ *
+ * This frees the buffer pools created by hcd_buffer_create().
+ */
+void hcd_buffer_destroy (struct usb_hcd *hcd)
+{
+	int		i;
+
+	for (i = 0; i < HCD_BUFFER_POOLS; i++) { 
+		struct pci_pool		*pool = hcd->pool [i];
+		if (pool) {
+			pci_pool_destroy (pool);
+			hcd->pool [i] = 0;
+		}
+	}
+}
+EXPORT_SYMBOL (hcd_buffer_destroy);
+
+
+/* sometimes alloc/free could use kmalloc with SLAB_DMA, for
+ * better sharing and to leverage mm/slab.c intelligence.
+ */
+
+void *hcd_buffer_alloc (
+	struct usb_bus 		*bus,
+	size_t			size,
+	int			mem_flags,
+	dma_addr_t		*dma
+)
+{
+	struct usb_hcd		*hcd = bus->hcpriv;
+	int 			i;
+
+	for (i = 0; i < HCD_BUFFER_POOLS; i++) {
+		if (size <= pool_max [i])
+			return pci_pool_alloc (hcd->pool [i], mem_flags, dma);
+	}
+	return pci_alloc_consistent (hcd->pdev, size, dma);
+}
+
+void hcd_buffer_free (
+	struct usb_bus 		*bus,
+	size_t			size,
+	void 			*addr,
+	dma_addr_t		dma
+)
+{
+	struct usb_hcd		*hcd = bus->hcpriv;
+	int 			i;
+
+	for (i = 0; i < HCD_BUFFER_POOLS; i++) {
+		if (size <= pool_max [i]) {
+			pci_pool_free (hcd->pool [i], addr, dma);
+			return;
+		}
+	}
+	pci_free_consistent (hcd->pdev, size, addr, dma);
+}
+
+
+/*
+ * DMA-Mappings for arbitrary memory buffers
+ */
+
+int hcd_buffer_map (
+	struct usb_bus	*bus,
+	void		*addr,
+	dma_addr_t	*dma,
+	size_t		size,
+	int		direction
+) {
+	struct usb_hcd	*hcd = bus->hcpriv;
+
+	// FIXME pci_map_single() has no standard failure mode!
+	*dma = pci_map_single (hcd->pdev, addr, size,
+			(direction == USB_DIR_IN)
+				? PCI_DMA_FROMDEVICE
+				: PCI_DMA_TODEVICE);
+	return 0;
+}
+
+void hcd_buffer_dmasync (
+	struct usb_bus	*bus,
+	dma_addr_t	dma,
+	size_t		size,
+	int		direction
+) {
+	struct usb_hcd *hcd = bus->hcpriv;
+
+	pci_dma_sync_single (hcd->pdev, dma, size,
+			(direction == USB_DIR_IN)
+				? PCI_DMA_FROMDEVICE
+				: PCI_DMA_TODEVICE);
+}
+
+void hcd_buffer_unmap (
+	struct usb_bus	*bus,
+	dma_addr_t	dma,
+	size_t		size,
+	int		direction
+) {
+	struct usb_hcd *hcd = bus->hcpriv;
+
+	pci_unmap_single (hcd->pdev, dma, size,
+			(direction == USB_DIR_IN)
+				? PCI_DMA_FROMDEVICE
+				: PCI_DMA_TODEVICE);
+}
+
+
+// FIXME DMA-Mappings for struct scatterlist

drivers/usb/core/hcd-pci.c

@@ -130,10 +130,19 @@ int usb_hcd_pci_probe (struct pci_dev *dev, const struct pci_device_id *id)
 			return retval;
 		}
 	}
-	pci_set_drvdata(dev, hcd);
+	pci_set_drvdata (dev, hcd);
 	hcd->driver = driver;
 	hcd->description = driver->description;
 	hcd->pdev = dev;
+	hcd->self.bus_name = dev->slot_name;
+	hcd->product_desc = dev->name;
+
+	if ((retval = hcd_buffer_create (hcd)) != 0) {
+clean_3:
+		driver->hcd_free (hcd);
+		goto clean_2;
+	}
+
 	info ("%s @ %s, %s", hcd->description,  dev->slot_name, dev->name);
 
 	pci_read_config_byte (dev, PCI_LATENCY_TIMER, &latency);
@@ -154,8 +163,7 @@ int usb_hcd_pci_probe (struct pci_dev *dev, const struct pci_device_id *id)
 			!= 0) {
 		err ("request interrupt %s failed", bufp);
 		retval = -EBUSY;
-		driver->hcd_free (hcd);
-		goto clean_2;
+		goto clean_3;
 	}
 	hcd->irq = dev->irq;
 
@@ -168,8 +176,6 @@ int usb_hcd_pci_probe (struct pci_dev *dev, const struct pci_device_id *id)
 	usb_bus_init (&hcd->self);
 	hcd->self.op = &usb_hcd_operations;
 	hcd->self.hcpriv = (void *) hcd;
-	hcd->self.bus_name = dev->slot_name;
-	hcd->product_desc = dev->name;
 
 	INIT_LIST_HEAD (&hcd->dev_list);
 
@@ -216,6 +222,7 @@ void usb_hcd_pci_remove (struct pci_dev *dev)
 	usb_disconnect (&hub);
 
 	hcd->driver->stop (hcd);
+	hcd_buffer_destroy (hcd);
 	hcd->state = USB_STATE_HALT;
 
 	free_irq (hcd->irq, hcd);

drivers/usb/core/hcd.c

@@ -1245,6 +1245,11 @@ struct usb_operations usb_hcd_operations = {
 	.submit_urb =		hcd_submit_urb,
 	.unlink_urb =		hcd_unlink_urb,
 	.deallocate =		hcd_free_dev,
+	.buffer_alloc =		hcd_buffer_alloc,
+	.buffer_free =		hcd_buffer_free,
+	.buffer_map =		hcd_buffer_map,
+	.buffer_dmasync =	hcd_buffer_dmasync,
+	.buffer_unmap =		hcd_buffer_unmap,
 };
 EXPORT_SYMBOL (usb_hcd_operations);
 

drivers/usb/core/hcd.h

@@ -58,6 +58,9 @@ struct usb_hcd {	/* usb_bus.hcpriv points to this */
 	atomic_t		resume_count;	/* multiple resumes issue */
 #endif
 
+#define HCD_BUFFER_POOLS	4
+	struct pci_pool		*pool [HCD_BUFFER_POOLS];
+
 	int			state;
 #	define	__ACTIVE		0x01
 #	define	__SLEEPY		0x02
@@ -109,6 +112,25 @@ struct usb_operations {
 	int (*get_frame_number) (struct usb_device *usb_dev);
 	int (*submit_urb) (struct urb *urb, int mem_flags);
 	int (*unlink_urb) (struct urb *urb);
+
+	/* allocate dma-consistent buffer for URB_DMA_NOMAPPING */
+	void *(*buffer_alloc)(struct usb_bus *bus, size_t size,
+			int mem_flags,
+			dma_addr_t *dma);
+	void (*buffer_free)(struct usb_bus *bus, size_t size,
+			void *addr, dma_addr_t dma);
+
+	int (*buffer_map) (struct usb_bus *bus,
+		void *addr, dma_addr_t *dma,
+		size_t size, int direction);
+	void (*buffer_dmasync) (struct usb_bus *bus,
+		dma_addr_t dma,
+		size_t size, int direction);
+	void (*buffer_unmap) (struct usb_bus *bus,
+		dma_addr_t dma,
+		size_t size, int direction);
+
+	// FIXME  also: buffer_sg_map (), buffer_sg_unmap ()
 };
 
 /* each driver provides one of these, and hardware init support */
@@ -181,6 +203,25 @@ extern int usb_hcd_pci_resume (struct pci_dev *dev);
 
 #endif /* CONFIG_PCI */
 
+/* pci-ish (pdev null is ok) buffer alloc/mapping support */
+int hcd_buffer_create (struct usb_hcd *hcd);
+void hcd_buffer_destroy (struct usb_hcd *hcd);
+
+void *hcd_buffer_alloc (struct usb_bus *bus, size_t size,
+	int mem_flags, dma_addr_t *dma);
+void hcd_buffer_free (struct usb_bus *bus, size_t size,
+	void *addr, dma_addr_t dma);
+
+int hcd_buffer_map (struct usb_bus *bus,
+	void *addr, dma_addr_t *dma,
+	size_t size, int direction);
+void hcd_buffer_dmasync (struct usb_bus *bus,
+	dma_addr_t dma,
+	size_t size, int direction);
+void hcd_buffer_unmap (struct usb_bus *bus,
+	dma_addr_t dma,
+	size_t size, int direction);
+
 /* generic bus glue, needed for host controllers that don't use PCI */
 extern struct usb_operations usb_hcd_operations;
 extern void usb_hcd_irq (int irq, void *__hcd, struct pt_regs *r);

drivers/usb/core/usb.c

@@ -1461,6 +1461,152 @@ int usb_new_device(struct usb_device *dev)
 }
 
 
+/**
+ * usb_buffer_alloc - allocate dma-consistent buffer for URB_NO_DMA_MAP
+ * @dev: device the buffer will be used with
+ * @size: requested buffer size
+ * @mem_flags: affect whether allocation may block
+ * @dma: used to return DMA address of buffer
+ *
+ * Return value is either null (indicating no buffer could be allocated), or
+ * the cpu-space pointer to a buffer that may be used to perform DMA to the
+ * specified device.  Such cpu-space buffers are returned along with the DMA
+ * address (through the pointer provided).
+ *
+ * These buffers are used with URB_NO_DMA_MAP set in urb->transfer_flags to
+ * avoid behaviors like using "DMA bounce buffers", or tying down I/O mapping
+ * hardware for long idle periods.  The implementation varies between
+ * platforms, depending on details of how DMA will work to this device.
+ *
+ * When the buffer is no longer used, free it with usb_buffer_free().
+ */
+void *usb_buffer_alloc (
+	struct usb_device *dev,
+	size_t size,
+	int mem_flags,
+	dma_addr_t *dma
+)
+{
+	if (!dev || !dev->bus || !dev->bus->op || !dev->bus->op->buffer_alloc)
+		return 0;
+	return dev->bus->op->buffer_alloc (dev->bus, size, mem_flags, dma);
+}
+
+/**
+ * usb_buffer_free - free memory allocated with usb_buffer_alloc()
+ * @dev: device the buffer was used with
+ * @size: requested buffer size
+ * @addr: CPU address of buffer
+ * @dma: DMA address of buffer
+ *
+ * This reclaims an I/O buffer, letting it be reused.  The memory must have
+ * been allocated using usb_buffer_alloc(), and the parameters must match
+ * those provided in that allocation request. 
+ */
+void usb_buffer_free (
+	struct usb_device *dev,
+	size_t size,
+	void *addr,
+	dma_addr_t dma
+)
+{
+	if (!dev || !dev->bus || !dev->bus->op || !dev->bus->op->buffer_free)
+	    	return;
+	dev->bus->op->buffer_free (dev->bus, size, addr, dma);
+}
+
+/**
+ * usb_buffer_map - create DMA mapping(s) for an urb
+ * @urb: urb whose transfer_buffer will be mapped
+ *
+ * Return value is either null (indicating no buffer could be mapped), or
+ * the parameter.  URB_NO_DMA_MAP is added to urb->transfer_flags if the
+ * operation succeeds.
+ *
+ * This call would normally be used for an urb which is reused, perhaps
+ * as the target of a large periodic transfer, with usb_buffer_dmasync()
+ * calls to synchronize memory and dma state.  It may not be used for
+ * control requests.
+ *
+ * Reverse the effect of this call with usb_buffer_unmap().
+ */
+struct urb *usb_buffer_map (struct urb *urb)
+{
+	struct usb_bus		*bus;
+	struct usb_operations	*op;
+
+	if (!urb
+			|| usb_pipecontrol (urb->pipe)
+			|| !urb->dev
+			|| !(bus = urb->dev->bus)
+			|| !(op = bus->op)
+			|| !op->buffer_map)
+		return 0;
+
+	if (op->buffer_map (bus,
+			urb->transfer_buffer,
+			&urb->transfer_dma,
+			urb->transfer_buffer_length,
+			usb_pipein (urb->pipe)
+				? USB_DIR_IN
+				: USB_DIR_OUT))
+		return 0;
+	urb->transfer_flags |= URB_NO_DMA_MAP;
+	return urb;
+}
+
+/**
+ * usb_buffer_dmasync - synchronize DMA and CPU view of buffer(s)
+ * @urb: urb whose transfer_buffer will be synchronized
+ */
+void usb_buffer_dmasync (struct urb *urb)
+{
+	struct usb_bus		*bus;
+	struct usb_operations	*op;
+
+	if (!urb
+			|| !(urb->transfer_flags & URB_NO_DMA_MAP)
+			|| !urb->dev
+			|| !(bus = urb->dev->bus)
+			|| !(op = bus->op)
+			|| !op->buffer_dmasync)
+		return;
+
+	op->buffer_dmasync (bus,
+			urb->transfer_dma,
+			urb->transfer_buffer_length,
+			usb_pipein (urb->pipe)
+				? USB_DIR_IN
+				: USB_DIR_OUT);
+}
+
+/**
+ * usb_buffer_unmap - free DMA mapping(s) for an urb
+ * @urb: urb whose transfer_buffer will be unmapped
+ *
+ * Reverses the effect of usb_buffer_map().
+ */
+void usb_buffer_unmap (struct urb *urb)
+{
+	struct usb_bus		*bus;
+	struct usb_operations	*op;
+
+	if (!urb
+			|| !(urb->transfer_flags & URB_NO_DMA_MAP)
+			|| !urb->dev
+			|| !(bus = urb->dev->bus)
+			|| !(op = bus->op)
+			|| !op->buffer_unmap)
+		return;
+
+	op->buffer_unmap (bus,
+			urb->transfer_dma,
+			urb->transfer_buffer_length,
+			usb_pipein (urb->pipe)
+				? USB_DIR_IN
+				: USB_DIR_OUT);
+}
+
 #ifdef CONFIG_PROC_FS
 struct list_head *usb_driver_get_list(void)
 {
@@ -1540,4 +1686,11 @@ EXPORT_SYMBOL(__usb_get_extra_descriptor);
 
 EXPORT_SYMBOL(usb_get_current_frame_number);
 
+EXPORT_SYMBOL (usb_buffer_alloc);
+EXPORT_SYMBOL (usb_buffer_free);
+
+EXPORT_SYMBOL (usb_buffer_map);
+EXPORT_SYMBOL (usb_buffer_dmasync);
+EXPORT_SYMBOL (usb_buffer_unmap);
+
 MODULE_LICENSE("GPL");

include/linux/usb.h

@@ -734,6 +734,7 @@ extern void usb_deregister_dev(int num_minors, int start_minor);
  */
 #define URB_SHORT_NOT_OK	0x0001	/* report short reads as errors */
 #define USB_ISO_ASAP		0x0002	/* iso-only, urb->start_frame ignored */
+#define URB_NO_DMA_MAP		0x0004	/* urb->*_dma are valid on submit */
 #define USB_ASYNC_UNLINK	0x0008	/* usb_unlink_urb() returns asap */
 #define USB_NO_FSBR		0x0020	/* UHCI-specific */
 #define USB_ZERO_PACKET		0x0040	/* Finish bulk OUTs with short packet */
@@ -771,11 +772,15 @@ typedef void (*usb_complete_t)(struct urb *);
  * @transfer_flags: A variety of flags may be used to affect how URB
  *	submission, unlinking, or operation are handled.  Different
  *	kinds of URB can use different flags.
- * @transfer_buffer: For non-iso transfers, this identifies the buffer
- *	to (or from) which the I/O request will be performed.  This
- *	buffer must be suitable for DMA; allocate it with kmalloc()
+ * @transfer_buffer:  This identifies the buffer to (or from) which
+ * 	the I/O request will be performed (unless URB_NO_DMA_MAP is set).
+ *	This buffer must be suitable for DMA; allocate it with kmalloc()
  *	or equivalent.  For transfers to "in" endpoints, contents of
- *	this buffer will be modified.
+ *	this buffer will be modified.  This buffer is used for data
+ *	phases of control transfers.
+ * @transfer_dma: When transfer_flags includes URB_NO_DMA_MAP, the device
+ * 	driver is saying that it provided this DMA address, which the host
+ * 	controller driver should use instead of the transfer_buffer.
  * @transfer_buffer_length: How big is transfer_buffer.  The transfer may
  *	be broken up into chunks according to the current maximum packet
  *	size for the endpoint, which is a function of the configuration
@@ -789,6 +794,11 @@ typedef void (*usb_complete_t)(struct urb *);
  * @setup_packet: Only used for control transfers, this points to eight bytes
  *	of setup data.  Control transfers always start by sending this data
  *	to the device.  Then transfer_buffer is read or written, if needed.
+ *	(Not used when URB_NO_DMA_MAP is set.)
+ * @setup_dma: For control transfers with URB_NO_DMA_MAP set, the device
+ * 	driver has provided this DMA address for the setup packet.  The
+ * 	host controller driver should use instead of setup_buffer.
+ * 	If there is a data phase, its buffer is identified by transfer_dma.
  * @start_frame: Returns the initial frame for interrupt or isochronous
  *	transfers.
  * @number_of_packets: Lists the number of ISO transfer buffers.
@@ -811,6 +821,23 @@ typedef void (*usb_complete_t)(struct urb *);
  * are submitted using usb_submit_urb(), and pending requests may be canceled
  * using usb_unlink_urb().
  *
+ * Data Transfer Buffers:
+ *
+ * Normally drivers provide I/O buffers allocated with kmalloc() or otherwise
+ * taken from the general page pool.  That is provided by transfer_buffer
+ * (control requests also use setup_packet), and host controller drivers
+ * perform a dma mapping (and unmapping) for each buffer transferred.  Those
+ * mapping operations can be expensive on some platforms (such using a dma
+ * bounce buffer), although they're cheap on commodity x86 and ppc hardware.
+ *
+ * Alternatively, drivers may pass the URB_NO_DMA_MAP transfer flag, which
+ * tells the host controller driver that no such mapping is needed since
+ * the device driver is DMA-aware.  For example, they might allocate a DMA
+ * buffer with usb_buffer_alloc(), or call usb_buffer_map().
+ * When this transfer flag is provided, host controller drivers will use the
+ * dma addresses found in the transfer_dma and/or setup_dma fields rather than
+ * determing a dma address themselves.
+ *
  * Initialization:
  *
  * All URBs submitted must initialize dev, pipe,
@@ -818,10 +845,10 @@ typedef void (*usb_complete_t)(struct urb *);
  * The USB_ASYNC_UNLINK transfer flag affects later invocations of
  * the usb_unlink_urb() routine.
  *
- * All non-isochronous URBs must also initialize 
+ * All URBs must also initialize 
  * transfer_buffer and transfer_buffer_length.  They may provide the
  * URB_SHORT_NOT_OK transfer flag, indicating that short reads are
- * to be treated as errors.
+ * to be treated as errors; that flag is invalid for write requests.
  *
  * Bulk URBs may
  * use the USB_ZERO_PACKET transfer flag, indicating that bulk OUT transfers
@@ -896,10 +923,12 @@ struct urb
 	int status;			/* (return) non-ISO status */
 	unsigned int transfer_flags;	/* (in) URB_SHORT_NOT_OK | ...*/
 	void *transfer_buffer;		/* (in) associated data buffer */
+	dma_addr_t transfer_dma;	/* (in) dma addr for transfer_buffer */
 	int transfer_buffer_length;	/* (in) data buffer length */
 	int actual_length;		/* (return) actual transfer length */
 	int bandwidth;			/* bandwidth for INT/ISO request */
 	unsigned char *setup_packet;	/* (in) setup packet (control only) */
+	dma_addr_t setup_dma;		/* (in) dma addr for setup_packet */
 	int start_frame;		/* (modify) start frame (INT/ISO) */
 	int number_of_packets;		/* (in) number of ISO packets */
 	int interval;			/* (in) transfer interval (INT/ISO) */
@@ -910,6 +939,8 @@ struct urb
 	struct usb_iso_packet_descriptor iso_frame_desc[0];	/* (in) ISO ONLY */
 };
 
+/* -------------------------------------------------------------------------- */
+
 /**
  * usb_fill_control_urb - initializes a control urb
  * @urb: pointer to the urb to initialize.
@@ -1032,6 +1063,16 @@ extern struct urb *usb_get_urb(struct urb *urb);
 extern int usb_submit_urb(struct urb *urb, int mem_flags);
 extern int usb_unlink_urb(struct urb *urb);
 
+#define HAVE_USB_BUFFERS
+void *usb_buffer_alloc (struct usb_device *dev, size_t size,
+	int mem_flags, dma_addr_t *dma);
+void usb_buffer_free (struct usb_device *dev, size_t size,
+	void *addr, dma_addr_t dma);
+
+struct urb *usb_buffer_map (struct urb *urb);
+void usb_buffer_dmasync (struct urb *urb);
+void usb_buffer_unmap (struct urb *urb);
+
 /*-------------------------------------------------------------------*
  *                         SYNCHRONOUS CALL SUPPORT                  *
  *-------------------------------------------------------------------*/