dlan17/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'ioat-md-accel-for-linus' of git://lost.foo-projects.org/~dwillia2/git/iop

Browse Source 
* 'ioat-md-accel-for-linus' of git://lost.foo-projects.org/~dwillia2/git/iop: (28 commits)
  ioatdma: add the unisys "i/oat" pci vendor/device id
  ARM: Add drivers/dma to arch/arm/Kconfig
  iop3xx: surface the iop3xx DMA and AAU units to the iop-adma driver
  iop13xx: surface the iop13xx adma units to the iop-adma driver
  dmaengine: driver for the iop32x, iop33x, and iop13xx raid engines
  md: remove raid5 compute_block and compute_parity5
  md: handle_stripe5 - request io processing in raid5_run_ops
  md: handle_stripe5 - add request/completion logic for async expand ops
  md: handle_stripe5 - add request/completion logic for async read ops
  md: handle_stripe5 - add request/completion logic for async check ops
  md: handle_stripe5 - add request/completion logic for async compute ops
  md: handle_stripe5 - add request/completion logic for async write ops
  md: common infrastructure for running operations with raid5_run_ops
  md: raid5_run_ops - run stripe operations outside sh->lock
  raid5: replace custom debug PRINTKs with standard pr_debug
  raid5: refactor handle_stripe5 and handle_stripe6 (v3)
  async_tx: add the async_tx api
  xor: make 'xor_blocks' a library routine for use with async_tx
  dmaengine: make clients responsible for managing channels
  dmaengine: refactor dmaengine around dma_async_tx_descriptor
  ...
This commit is contained in:
Linus Torvalds
2007-07-13 10:52:27 -07:00
parent 12a2296054 3039f0735a
commit e030dbf91a
45 changed files with 7367 additions and 1753 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drivers/dma/Kconfig
+10 -2
View File
@@ -8,8 +8,8 @@ menu "DMA Engine support"
config DMA_ENGINE
bool "Support for DMA engines"
---help---
DMA engines offload copy operations from the CPU to dedicated
hardware, allowing the copies to happen asynchronously.
DMA engines offload bulk memory operations from the CPU to dedicated
hardware, allowing the operations to happen asynchronously.
comment "DMA Clients"
@@ -32,4 +32,12 @@ config INTEL_IOATDMA
---help---
Enable support for the Intel(R) I/OAT DMA engine.
config INTEL_IOP_ADMA
tristate "Intel IOP ADMA support"
depends on DMA_ENGINE && (ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX)
select ASYNC_CORE
default m
---help---
Enable support for the Intel(R) IOP Series RAID engines.
endmenu
drivers/dma/Makefile
+1
View File
@@ -1,3 +1,4 @@
obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
obj-$(CONFIG_NET_DMA) += iovlock.o
obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
drivers/dma/dmaengine.c
+314 -109
View File
@@ -37,11 +37,11 @@
* Each device has a channels list, which runs unlocked but is never modified
* once the device is registered, it's just setup by the driver.
*
* Each client has a channels list, it's only modified under the client->lock
* and in an RCU callback, so it's safe to read under rcu_read_lock().
* Each client is responsible for keeping track of the channels it uses. See
* the definition of dma_event_callback in dmaengine.h.
*
* Each device has a kref, which is initialized to 1 when the device is
* registered. A kref_put is done for each class_device registered. When the
* registered. A kref_get is done for each class_device registered. When the
* class_device is released, the coresponding kref_put is done in the release
* method. Every time one of the device's channels is allocated to a client,
* a kref_get occurs. When the channel is freed, the coresponding kref_put
@@ -51,14 +51,17 @@
* references to finish.
*
* Each channel has an open-coded implementation of Rusty Russell's "bigref,"
* with a kref and a per_cpu local_t. A single reference is set when on an
* ADDED event, and removed with a REMOVE event. Net DMA client takes an
* extra reference per outstanding transaction. The relase function does a
* kref_put on the device. -ChrisL
* with a kref and a per_cpu local_t. A dma_chan_get is called when a client
* signals that it wants to use a channel, and dma_chan_put is called when
* a channel is removed or a client using it is unregesitered. A client can
* take extra references per outstanding transaction, as is the case with
* the NET DMA client. The release function does a kref_put on the device.
* -ChrisL, DanW
*/
#include <linux/init.h>
#include <linux/module.h>
#include <linux/mm.h>
#include <linux/device.h>
#include <linux/dmaengine.h>
#include <linux/hardirq.h>
@@ -66,6 +69,7 @@
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/mutex.h>
#include <linux/jiffies.h>
static DEFINE_MUTEX(dma_list_mutex);
static LIST_HEAD(dma_device_list);
@@ -100,8 +104,19 @@ static ssize_t show_bytes_transferred(struct class_device *cd, char *buf)
static ssize_t show_in_use(struct class_device *cd, char *buf)
{
struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
int in_use = 0;
return sprintf(buf, "%d\n", (chan->client ? 1 : 0));
if (unlikely(chan->slow_ref) &&
atomic_read(&chan->refcount.refcount) > 1)
in_use = 1;
else {
if (local_read(&(per_cpu_ptr(chan->local,
get_cpu())->refcount)) > 0)
in_use = 1;
put_cpu();
}
return sprintf(buf, "%d\n", in_use);
}
static struct class_device_attribute dma_class_attrs[] = {
@@ -127,44 +142,73 @@ static struct class dma_devclass = {
/* --- client and device registration --- */
#define dma_chan_satisfies_mask(chan, mask) \
__dma_chan_satisfies_mask((chan), &(mask))
static int
__dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want)
{
dma_cap_mask_t has;
bitmap_and(has.bits, want->bits, chan->device->cap_mask.bits,
DMA_TX_TYPE_END);
return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
}
/**
* dma_client_chan_alloc - try to allocate a channel to a client
* dma_client_chan_alloc - try to allocate channels to a client
* @client: &dma_client
*
* Called with dma_list_mutex held.
*/
static struct dma_chan *dma_client_chan_alloc(struct dma_client *client)
static void dma_client_chan_alloc(struct dma_client *client)
{
struct dma_device *device;
struct dma_chan *chan;
unsigned long flags;
int desc; /* allocated descriptor count */
enum dma_state_client ack;
/* Find a channel, any DMA engine will do */
list_for_each_entry(device, &dma_device_list, global_node) {
/* Find a channel */
list_for_each_entry(device, &dma_device_list, global_node)
list_for_each_entry(chan, &device->channels, device_node) {
if (chan->client)
if (!dma_chan_satisfies_mask(chan, client->cap_mask))
continue;
desc = chan->device->device_alloc_chan_resources(chan);
if (desc >= 0) {
kref_get(&device->refcount);
kref_init(&chan->refcount);
chan->slow_ref = 0;
INIT_RCU_HEAD(&chan->rcu);
chan->client = client;
spin_lock_irqsave(&client->lock, flags);
list_add_tail_rcu(&chan->client_node,
&client->channels);
spin_unlock_irqrestore(&client->lock, flags);
return chan;
ack = client->event_callback(client,
chan,
DMA_RESOURCE_AVAILABLE);
/* we are done once this client rejects
* an available resource
*/
if (ack == DMA_ACK) {
dma_chan_get(chan);
kref_get(&device->refcount);
} else if (ack == DMA_NAK)
return;
}
}
}
return NULL;
}
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
{
enum dma_status status;
unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
dma_async_issue_pending(chan);
do {
status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
printk(KERN_ERR "dma_sync_wait_timeout!\n");
return DMA_ERROR;
}
} while (status == DMA_IN_PROGRESS);
return status;
}
EXPORT_SYMBOL(dma_sync_wait);
/**
* dma_chan_cleanup - release a DMA channel's resources
* @kref: kernel reference structure that contains the DMA channel device
@@ -173,7 +217,6 @@ void dma_chan_cleanup(struct kref *kref)
{
struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
chan->device->device_free_chan_resources(chan);
chan->client = NULL;
kref_put(&chan->device->refcount, dma_async_device_cleanup);
}
EXPORT_SYMBOL(dma_chan_cleanup);
@@ -189,7 +232,7 @@ static void dma_chan_free_rcu(struct rcu_head *rcu)
kref_put(&chan->refcount, dma_chan_cleanup);
}
static void dma_client_chan_free(struct dma_chan *chan)
static void dma_chan_release(struct dma_chan *chan)
{
atomic_add(0x7FFFFFFF, &chan->refcount.refcount);
chan->slow_ref = 1;
@@ -197,41 +240,42 @@ static void dma_client_chan_free(struct dma_chan *chan)
}
/**
* dma_chans_rebalance - reallocate channels to clients
*
* When the number of DMA channel in the system changes,
* channels need to be rebalanced among clients.
* dma_chans_notify_available - broadcast available channels to the clients
*/
static void dma_chans_rebalance(void)
static void dma_clients_notify_available(void)
{
struct dma_client *client;
struct dma_chan *chan;
unsigned long flags;
mutex_lock(&dma_list_mutex);
list_for_each_entry(client, &dma_client_list, global_node)
dma_client_chan_alloc(client);
mutex_unlock(&dma_list_mutex);
}
/**
* dma_chans_notify_available - tell the clients that a channel is going away
* @chan: channel on its way out
*/
static void dma_clients_notify_removed(struct dma_chan *chan)
{
struct dma_client *client;
enum dma_state_client ack;
mutex_lock(&dma_list_mutex);
list_for_each_entry(client, &dma_client_list, global_node) {
while (client->chans_desired > client->chan_count) {
chan = dma_client_chan_alloc(client);
if (!chan)
break;
client->chan_count++;
client->event_callback(client,
chan,
DMA_RESOURCE_ADDED);
}
while (client->chans_desired < client->chan_count) {
spin_lock_irqsave(&client->lock, flags);
chan = list_entry(client->channels.next,
struct dma_chan,
client_node);
list_del_rcu(&chan->client_node);
spin_unlock_irqrestore(&client->lock, flags);
client->chan_count--;
client->event_callback(client,
chan,
DMA_RESOURCE_REMOVED);
dma_client_chan_free(chan);
ack = client->event_callback(client, chan,
DMA_RESOURCE_REMOVED);
/* client was holding resources for this channel so
* free it
*/
if (ack == DMA_ACK) {
dma_chan_put(chan);
kref_put(&chan->device->refcount,
dma_async_device_cleanup);
}
}
@@ -239,28 +283,14 @@ static void dma_chans_rebalance(void)
}
/**
* dma_async_client_register - allocate and register a &dma_client
* @event_callback: callback for notification of channel addition/removal
* dma_async_client_register - register a &dma_client
* @client: ptr to a client structure with valid 'event_callback' and 'cap_mask'
*/
struct dma_client *dma_async_client_register(dma_event_callback event_callback)
void dma_async_client_register(struct dma_client *client)
{
struct dma_client *client;
client = kzalloc(sizeof(*client), GFP_KERNEL);
if (!client)
return NULL;
INIT_LIST_HEAD(&client->channels);
spin_lock_init(&client->lock);
client->chans_desired = 0;
client->chan_count = 0;
client->event_callback = event_callback;
mutex_lock(&dma_list_mutex);
list_add_tail(&client->global_node, &dma_client_list);
mutex_unlock(&dma_list_mutex);
return client;
}
EXPORT_SYMBOL(dma_async_client_register);
@@ -272,40 +302,42 @@ EXPORT_SYMBOL(dma_async_client_register);
*/
void dma_async_client_unregister(struct dma_client *client)
{
struct dma_device *device;
struct dma_chan *chan;
enum dma_state_client ack;
if (!client)
return;
rcu_read_lock();
list_for_each_entry_rcu(chan, &client->channels, client_node)
dma_client_chan_free(chan);
rcu_read_unlock();
mutex_lock(&dma_list_mutex);
/* free all channels the client is holding */
list_for_each_entry(device, &dma_device_list, global_node)
list_for_each_entry(chan, &device->channels, device_node) {
ack = client->event_callback(client, chan,
DMA_RESOURCE_REMOVED);
if (ack == DMA_ACK) {
dma_chan_put(chan);
kref_put(&chan->device->refcount,
dma_async_device_cleanup);
}
}
list_del(&client->global_node);
mutex_unlock(&dma_list_mutex);
kfree(client);
dma_chans_rebalance();
}
EXPORT_SYMBOL(dma_async_client_unregister);
/**
* dma_async_client_chan_request - request DMA channels
* @client: &dma_client
* @number: count of DMA channels requested
*
* Clients call dma_async_client_chan_request() to specify how many
* DMA channels they need, 0 to free all currently allocated.
* The resulting allocations/frees are indicated to the client via the
* event callback.
* dma_async_client_chan_request - send all available channels to the
* client that satisfy the capability mask
* @client - requester
*/
void dma_async_client_chan_request(struct dma_client *client,
unsigned int number)
void dma_async_client_chan_request(struct dma_client *client)
{
client->chans_desired = number;
dma_chans_rebalance();
mutex_lock(&dma_list_mutex);
dma_client_chan_alloc(client);
mutex_unlock(&dma_list_mutex);
}
EXPORT_SYMBOL(dma_async_client_chan_request);
@@ -316,12 +348,31 @@ EXPORT_SYMBOL(dma_async_client_chan_request);
int dma_async_device_register(struct dma_device *device)
{
static int id;
int chancnt = 0;
int chancnt = 0, rc;
struct dma_chan* chan;
if (!device)
return -ENODEV;
/* validate device routines */
BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
!device->device_prep_dma_memcpy);
BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
!device->device_prep_dma_xor);
BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
!device->device_prep_dma_zero_sum);
BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
!device->device_prep_dma_memset);
BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
!device->device_prep_dma_interrupt);
BUG_ON(!device->device_alloc_chan_resources);
BUG_ON(!device->device_free_chan_resources);
BUG_ON(!device->device_dependency_added);
BUG_ON(!device->device_is_tx_complete);
BUG_ON(!device->device_issue_pending);
BUG_ON(!device->dev);
init_completion(&device->done);
kref_init(&device->refcount);
device->dev_id = id++;
@@ -338,17 +389,38 @@ int dma_async_device_register(struct dma_device *device)
snprintf(chan->class_dev.class_id, BUS_ID_SIZE, "dma%dchan%d",
device->dev_id, chan->chan_id);
rc = class_device_register(&chan->class_dev);
if (rc) {
chancnt--;
free_percpu(chan->local);
chan->local = NULL;
goto err_out;
}
kref_get(&device->refcount);
class_device_register(&chan->class_dev);
kref_init(&chan->refcount);
chan->slow_ref = 0;
INIT_RCU_HEAD(&chan->rcu);
}
mutex_lock(&dma_list_mutex);
list_add_tail(&device->global_node, &dma_device_list);
mutex_unlock(&dma_list_mutex);
dma_chans_rebalance();
dma_clients_notify_available();
return 0;
err_out:
list_for_each_entry(chan, &device->channels, device_node) {
if (chan->local == NULL)
continue;
kref_put(&device->refcount, dma_async_device_cleanup);
class_device_unregister(&chan->class_dev);
chancnt--;
free_percpu(chan->local);
}
return rc;
}
EXPORT_SYMBOL(dma_async_device_register);
@@ -371,32 +443,165 @@ static void dma_async_device_cleanup(struct kref *kref)
void dma_async_device_unregister(struct dma_device *device)
{
struct dma_chan *chan;
unsigned long flags;
mutex_lock(&dma_list_mutex);
list_del(&device->global_node);
mutex_unlock(&dma_list_mutex);
list_for_each_entry(chan, &device->channels, device_node) {
if (chan->client) {
spin_lock_irqsave(&chan->client->lock, flags);
list_del(&chan->client_node);
chan->client->chan_count--;
spin_unlock_irqrestore(&chan->client->lock, flags);
chan->client->event_callback(chan->client,
chan,
DMA_RESOURCE_REMOVED);
dma_client_chan_free(chan);
}
dma_clients_notify_removed(chan);
class_device_unregister(&chan->class_dev);
dma_chan_release(chan);
}
dma_chans_rebalance();
kref_put(&device->refcount, dma_async_device_cleanup);
wait_for_completion(&device->done);
}
EXPORT_SYMBOL(dma_async_device_unregister);
/**
* dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
* @chan: DMA channel to offload copy to
* @dest: destination address (virtual)
* @src: source address (virtual)
* @len: length
*
* Both @dest and @src must be mappable to a bus address according to the
* DMA mapping API rules for streaming mappings.
* Both @dest and @src must stay memory resident (kernel memory or locked
* user space pages).
*/
dma_cookie_t
dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
void *src, size_t len)
{
struct dma_device *dev = chan->device;
struct dma_async_tx_descriptor *tx;
dma_addr_t addr;
dma_cookie_t cookie;
int cpu;
tx = dev->device_prep_dma_memcpy(chan, len, 0);
if (!tx)
return -ENOMEM;
tx->ack = 1;
tx->callback = NULL;
addr = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
tx->tx_set_src(addr, tx, 0);
addr = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
tx->tx_set_dest(addr, tx, 0);
cookie = tx->tx_submit(tx);
cpu = get_cpu();
per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
per_cpu_ptr(chan->local, cpu)->memcpy_count++;
put_cpu();
return cookie;
}
EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
/**
* dma_async_memcpy_buf_to_pg - offloaded copy from address to page
* @chan: DMA channel to offload copy to
* @page: destination page
* @offset: offset in page to copy to
* @kdata: source address (virtual)
* @len: length
*
* Both @page/@offset and @kdata must be mappable to a bus address according
* to the DMA mapping API rules for streaming mappings.
* Both @page/@offset and @kdata must stay memory resident (kernel memory or
* locked user space pages)
*/
dma_cookie_t
dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
unsigned int offset, void *kdata, size_t len)
{
struct dma_device *dev = chan->device;
struct dma_async_tx_descriptor *tx;
dma_addr_t addr;
dma_cookie_t cookie;
int cpu;
tx = dev->device_prep_dma_memcpy(chan, len, 0);
if (!tx)
return -ENOMEM;
tx->ack = 1;
tx->callback = NULL;
addr = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
tx->tx_set_src(addr, tx, 0);
addr = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
tx->tx_set_dest(addr, tx, 0);
cookie = tx->tx_submit(tx);
cpu = get_cpu();
per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
per_cpu_ptr(chan->local, cpu)->memcpy_count++;
put_cpu();
return cookie;
}
EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
/**
* dma_async_memcpy_pg_to_pg - offloaded copy from page to page
* @chan: DMA channel to offload copy to
* @dest_pg: destination page
* @dest_off: offset in page to copy to
* @src_pg: source page
* @src_off: offset in page to copy from
* @len: length
*
* Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
* address according to the DMA mapping API rules for streaming mappings.
* Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
* (kernel memory or locked user space pages).
*/
dma_cookie_t
dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
unsigned int dest_off, struct page *src_pg, unsigned int src_off,
size_t len)
{
struct dma_device *dev = chan->device;
struct dma_async_tx_descriptor *tx;
dma_addr_t addr;
dma_cookie_t cookie;
int cpu;
tx = dev->device_prep_dma_memcpy(chan, len, 0);
if (!tx)
return -ENOMEM;
tx->ack = 1;
tx->callback = NULL;
addr = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
tx->tx_set_src(addr, tx, 0);
addr = dma_map_page(dev->dev, dest_pg, dest_off, len, DMA_FROM_DEVICE);
tx->tx_set_dest(addr, tx, 0);
cookie = tx->tx_submit(tx);
cpu = get_cpu();
per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
per_cpu_ptr(chan->local, cpu)->memcpy_count++;
put_cpu();
return cookie;
}
EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
struct dma_chan *chan)
{
tx->chan = chan;
spin_lock_init(&tx->lock);
INIT_LIST_HEAD(&tx->depend_node);
INIT_LIST_HEAD(&tx->depend_list);
}
EXPORT_SYMBOL(dma_async_tx_descriptor_init);
static int __init dma_bus_init(void)
{
mutex_init(&dma_list_mutex);
drivers/dma/ioatdma.c
+178 -191
View File
@@ -32,16 +32,17 @@
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include "ioatdma.h"
#include "ioatdma_io.h"
#include "ioatdma_registers.h"
#include "ioatdma_hw.h"
#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
/* internal functions */
static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
static void ioat_shutdown(struct pci_dev *pdev);
static void __devexit ioat_remove(struct pci_dev *pdev);
static int enumerate_dma_channels(struct ioat_device *device)
@@ -51,8 +52,8 @@ static int enumerate_dma_channels(struct ioat_device *device)
int i;
struct ioat_dma_chan *ioat_chan;
device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET);
xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET);
device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
for (i = 0; i < device->common.chancnt; i++) {
@@ -71,13 +72,79 @@ static int enumerate_dma_channels(struct ioat_device *device)
INIT_LIST_HEAD(&ioat_chan->used_desc);
/* This should be made common somewhere in dmaengine.c */
ioat_chan->common.device = &device->common;
ioat_chan->common.client = NULL;
list_add_tail(&ioat_chan->common.device_node,
&device->common.channels);
}
return device->common.chancnt;
}
static void
ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
{
struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
pci_unmap_addr_set(desc, src, addr);
list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
iter->hw->src_addr = addr;
addr += ioat_chan->xfercap;
}
}
static void
ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
{
struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
pci_unmap_addr_set(desc, dst, addr);
list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
iter->hw->dst_addr = addr;
addr += ioat_chan->xfercap;
}
}
static dma_cookie_t
ioat_tx_submit(struct dma_async_tx_descriptor *tx)
{
struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
int append = 0;
dma_cookie_t cookie;
struct ioat_desc_sw *group_start;
group_start = list_entry(desc->async_tx.tx_list.next,
struct ioat_desc_sw, node);
spin_lock_bh(&ioat_chan->desc_lock);
/* cookie incr and addition to used_list must be atomic */
cookie = ioat_chan->common.cookie;
cookie++;
if (cookie < 0)
cookie = 1;
ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
/* write address into NextDescriptor field of last desc in chain */
to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
group_start->async_tx.phys;
list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
ioat_chan->pending += desc->tx_cnt;
if (ioat_chan->pending >= 4) {
append = 1;
ioat_chan->pending = 0;
}
spin_unlock_bh(&ioat_chan->desc_lock);
if (append)
writeb(IOAT_CHANCMD_APPEND,
ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
return cookie;
}
static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
struct ioat_dma_chan *ioat_chan,
gfp_t flags)
@@ -99,8 +166,13 @@ static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
}
memset(desc, 0, sizeof(*desc));
dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
desc_sw->async_tx.tx_set_src = ioat_set_src;
desc_sw->async_tx.tx_set_dest = ioat_set_dest;
desc_sw->async_tx.tx_submit = ioat_tx_submit;
INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
desc_sw->hw = desc;
desc_sw->phys = phys;
desc_sw->async_tx.phys = phys;
return desc_sw;
}
@@ -123,7 +195,7 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
* In-use bit automatically set by reading chanctrl
* If 0, we got it, if 1, someone else did
*/
chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
return -EBUSY;
@@ -132,12 +204,12 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
IOAT_CHANCTRL_ERR_INT_EN |
IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
IOAT_CHANCTRL_ERR_COMPLETION_EN;
ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET);
chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
if (chanerr) {
printk("IOAT: CHANERR = %x, clearing\n", chanerr);
ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr);
writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
}
/* Allocate descriptors */
@@ -161,10 +233,10 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
&ioat_chan->completion_addr);
memset(ioat_chan->completion_virt, 0,
sizeof(*ioat_chan->completion_virt));
ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW,
((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF);
ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH,
((u64) ioat_chan->completion_addr) >> 32);
writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
writel(((u64) ioat_chan->completion_addr) >> 32,
ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
ioat_start_null_desc(ioat_chan);
return i;
@@ -182,18 +254,20 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)
ioat_dma_memcpy_cleanup(ioat_chan);
ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET);
writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
spin_lock_bh(&ioat_chan->desc_lock);
list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
in_use_descs++;
list_del(&desc->node);
pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
pci_pool_free(ioat_device->dma_pool, desc->hw,
desc->async_tx.phys);
kfree(desc);
}
list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
list_del(&desc->node);
pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
pci_pool_free(ioat_device->dma_pool, desc->hw,
desc->async_tx.phys);
kfree(desc);
}
spin_unlock_bh(&ioat_chan->desc_lock);
@@ -210,50 +284,30 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)
ioat_chan->last_completion = ioat_chan->completion_addr = 0;
/* Tell hw the chan is free */
chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
}
/**
* do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
* @ioat_chan: IOAT DMA channel handle
* @dest: DMA destination address
* @src: DMA source address
* @len: transaction length in bytes
*/
static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
dma_addr_t dest,
dma_addr_t src,
size_t len)
static struct dma_async_tx_descriptor *
ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)
{
struct ioat_desc_sw *first;
struct ioat_desc_sw *prev;
struct ioat_desc_sw *new;
dma_cookie_t cookie;
struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
struct ioat_desc_sw *first, *prev, *new;
LIST_HEAD(new_chain);
u32 copy;
size_t orig_len;
dma_addr_t orig_src, orig_dst;
unsigned int desc_count = 0;
unsigned int append = 0;
if (!ioat_chan || !dest || !src)
return -EFAULT;
int desc_count = 0;
if (!len)
return ioat_chan->common.cookie;
return NULL;
orig_len = len;
orig_src = src;
orig_dst = dest;
first = NULL;
prev = NULL;
spin_lock_bh(&ioat_chan->desc_lock);
while (len) {
if (!list_empty(&ioat_chan->free_desc)) {
new = to_ioat_desc(ioat_chan->free_desc.next);
@@ -270,141 +324,36 @@ static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
new->hw->size = copy;
new->hw->ctl = 0;
new->hw->src_addr = src;
new->hw->dst_addr = dest;
new->cookie = 0;
new->async_tx.cookie = 0;
new->async_tx.ack = 1;
/* chain together the physical address list for the HW */
if (!first)
first = new;
else
prev->hw->next = (u64) new->phys;
prev->hw->next = (u64) new->async_tx.phys;
prev = new;
len -= copy;
dest += copy;
src += copy;
list_add_tail(&new->node, &new_chain);
desc_count++;
}
list_splice(&new_chain, &new->async_tx.tx_list);
new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
new->hw->next = 0;
new->tx_cnt = desc_count;
new->async_tx.ack = 0; /* client is in control of this ack */
new->async_tx.cookie = -EBUSY;
/* cookie incr and addition to used_list must be atomic */
cookie = ioat_chan->common.cookie;
cookie++;
if (cookie < 0)
cookie = 1;
ioat_chan->common.cookie = new->cookie = cookie;
pci_unmap_addr_set(new, src, orig_src);
pci_unmap_addr_set(new, dst, orig_dst);
pci_unmap_len_set(new, src_len, orig_len);
pci_unmap_len_set(new, dst_len, orig_len);
/* write address into NextDescriptor field of last desc in chain */
to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
list_splice_init(&new_chain, ioat_chan->used_desc.prev);
ioat_chan->pending += desc_count;
if (ioat_chan->pending >= 20) {
append = 1;
ioat_chan->pending = 0;
}
spin_unlock_bh(&ioat_chan->desc_lock);
if (append)
ioatdma_chan_write8(ioat_chan,
IOAT_CHANCMD_OFFSET,
IOAT_CHANCMD_APPEND);
return cookie;
return new ? &new->async_tx : NULL;
}
/**
* ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
* @chan: IOAT DMA channel handle
* @dest: DMA destination address
* @src: DMA source address
* @len: transaction length in bytes
*/
static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
void *dest,
void *src,
size_t len)
{
dma_addr_t dest_addr;
dma_addr_t src_addr;
struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
dest_addr = pci_map_single(ioat_chan->device->pdev,
dest, len, PCI_DMA_FROMDEVICE);
src_addr = pci_map_single(ioat_chan->device->pdev,
src, len, PCI_DMA_TODEVICE);
return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
}
/**
* ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
* @chan: IOAT DMA channel handle
* @page: pointer to the page to copy to
* @offset: offset into that page
* @src: DMA source address
* @len: transaction length in bytes
*/
static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
struct page *page,
unsigned int offset,
void *src,
size_t len)
{
dma_addr_t dest_addr;
dma_addr_t src_addr;
struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
dest_addr = pci_map_page(ioat_chan->device->pdev,
page, offset, len, PCI_DMA_FROMDEVICE);
src_addr = pci_map_single(ioat_chan->device->pdev,
src, len, PCI_DMA_TODEVICE);
return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
}
/**
* ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
* @chan: IOAT DMA channel handle
* @dest_pg: pointer to the page to copy to
* @dest_off: offset into that page
* @src_pg: pointer to the page to copy from
* @src_off: offset into that page
* @len: transaction length in bytes. This is guaranteed not to make a copy
* across a page boundary.
*/
static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
struct page *dest_pg,
unsigned int dest_off,
struct page *src_pg,
unsigned int src_off,
size_t len)
{
dma_addr_t dest_addr;
dma_addr_t src_addr;
struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
dest_addr = pci_map_page(ioat_chan->device->pdev,
dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
src_addr = pci_map_page(ioat_chan->device->pdev,
src_pg, src_off, len, PCI_DMA_TODEVICE);
return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
}
/**
* ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
@@ -417,9 +366,8 @@ static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
if (ioat_chan->pending != 0) {
ioat_chan->pending = 0;
ioatdma_chan_write8(ioat_chan,
IOAT_CHANCMD_OFFSET,
IOAT_CHANCMD_APPEND);
writeb(IOAT_CHANCMD_APPEND,
ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
}
}
@@ -449,7 +397,7 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
printk("IOAT: Channel halted, chanerr = %x\n",
ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET));
readl(chan->reg_base + IOAT_CHANERR_OFFSET));
/* TODO do something to salvage the situation */
}
@@ -467,8 +415,8 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
* exceeding xfercap, perhaps. If so, only the last one will
* have a cookie, and require unmapping.
*/
if (desc->cookie) {
cookie = desc->cookie;
if (desc->async_tx.cookie) {
cookie = desc->async_tx.cookie;
/* yes we are unmapping both _page and _single alloc'd
regions with unmap_page. Is this *really* that bad?
@@ -483,14 +431,19 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
PCI_DMA_TODEVICE);
}
if (desc->phys != phys_complete) {
/* a completed entry, but not the last, so cleanup */
list_del(&desc->node);
list_add_tail(&desc->node, &chan->free_desc);
if (desc->async_tx.phys != phys_complete) {
/* a completed entry, but not the last, so cleanup
* if the client is done with the descriptor
*/
if (desc->async_tx.ack) {
list_del(&desc->node);
list_add_tail(&desc->node, &chan->free_desc);
} else
desc->async_tx.cookie = 0;
} else {
/* last used desc. Do not remove, so we can append from
it, but don't look at it next time, either */
desc->cookie = 0;
desc->async_tx.cookie = 0;
/* TODO check status bits? */
break;
@@ -506,6 +459,17 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
spin_unlock(&chan->cleanup_lock);
}
static void ioat_dma_dependency_added(struct dma_chan *chan)
{
struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
spin_lock_bh(&ioat_chan->desc_lock);
if (ioat_chan->pending == 0) {
spin_unlock_bh(&ioat_chan->desc_lock);
ioat_dma_memcpy_cleanup(ioat_chan);
} else
spin_unlock_bh(&ioat_chan->desc_lock);
}
/**
* ioat_dma_is_complete - poll the status of a IOAT DMA transaction
* @chan: IOAT DMA channel handle
@@ -553,6 +517,8 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
static struct pci_device_id ioat_pci_tbl[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
{ PCI_DEVICE(PCI_VENDOR_ID_UNISYS,
PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
{ 0, }
};
@@ -560,6 +526,7 @@ static struct pci_driver ioat_pci_driver = {
.name = "ioatdma",
.id_table = ioat_pci_tbl,
.probe = ioat_probe,
.shutdown = ioat_shutdown,
.remove = __devexit_p(ioat_remove),
};
@@ -569,21 +536,21 @@ static irqreturn_t ioat_do_interrupt(int irq, void *data)
unsigned long attnstatus;
u8 intrctrl;
intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET);
intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
return IRQ_NONE;
if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
return IRQ_NONE;
}
attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET);
attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
return IRQ_HANDLED;
}
@@ -607,19 +574,17 @@ static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
desc->hw->next = 0;
desc->async_tx.ack = 1;
list_add_tail(&desc->node, &ioat_chan->used_desc);
spin_unlock_bh(&ioat_chan->desc_lock);
#if (BITS_PER_LONG == 64)
ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys);
#else
ioatdma_chan_write32(ioat_chan,
IOAT_CHAINADDR_OFFSET_LOW,
(u32) desc->phys);
ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0);
#endif
ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START);
writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
writel(((u64) desc->async_tx.phys) >> 32,
ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
}
/*
@@ -633,6 +598,8 @@ static int ioat_self_test(struct ioat_device *device)
u8 *src;
u8 *dest;
struct dma_chan *dma_chan;
struct dma_async_tx_descriptor *tx;
dma_addr_t addr;
dma_cookie_t cookie;
int err = 0;
@@ -658,7 +625,15 @@ static int ioat_self_test(struct ioat_device *device)
goto out;
}
cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
async_tx_ack(tx);
addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
DMA_TO_DEVICE);
ioat_set_src(addr, tx, 0);
addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
DMA_FROM_DEVICE);
ioat_set_dest(addr, tx, 0);
cookie = ioat_tx_submit(tx);
ioat_dma_memcpy_issue_pending(dma_chan);
msleep(1);
@@ -748,19 +723,20 @@ static int __devinit ioat_probe(struct pci_dev *pdev,
device->reg_base = reg_base;
ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN);
writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET);
pci_set_master(pdev);
INIT_LIST_HEAD(&device->common.channels);
enumerate_dma_channels(device);
dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
device->common.device_memcpy_complete = ioat_dma_is_complete;
device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
device->common.device_is_tx_complete = ioat_dma_is_complete;
device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
device->common.device_dependency_added = ioat_dma_dependency_added;
device->common.dev = &pdev->dev;
printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
device->common.chancnt);
@@ -787,9 +763,20 @@ err_request_regions:
err_set_dma_mask:
pci_disable_device(pdev);
err_enable_device:
printk(KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n");
return err;
}
static void ioat_shutdown(struct pci_dev *pdev)
{
struct ioat_device *device;
device = pci_get_drvdata(pdev);
dma_async_device_unregister(&device->common);
}
static void __devexit ioat_remove(struct pci_dev *pdev)
{
struct ioat_device *device;
@@ -818,7 +805,7 @@ static void __devexit ioat_remove(struct pci_dev *pdev)
}
/* MODULE API */
MODULE_VERSION("1.7");
MODULE_VERSION("1.9");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Intel Corporation");
drivers/dma/ioatdma.h
+6 -10
View File
@@ -30,9 +30,6 @@
#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
extern struct list_head dma_device_list;
extern struct list_head dma_client_list;
/**
* struct ioat_device - internal representation of a IOAT device
* @pdev: PCI-Express device
@@ -105,21 +102,20 @@ struct ioat_dma_chan {
/**
* struct ioat_desc_sw - wrapper around hardware descriptor
* @hw: hardware DMA descriptor
* @node:
* @cookie:
* @phys:
* @node: this descriptor will either be on the free list,
* or attached to a transaction list (async_tx.tx_list)
* @tx_cnt: number of descriptors required to complete the transaction
* @async_tx: the generic software descriptor for all engines
*/
struct ioat_desc_sw {
struct ioat_dma_descriptor *hw;
struct list_head node;
dma_cookie_t cookie;
dma_addr_t phys;
int tx_cnt;
DECLARE_PCI_UNMAP_ADDR(src)
DECLARE_PCI_UNMAP_LEN(src_len)
DECLARE_PCI_UNMAP_ADDR(dst)
DECLARE_PCI_UNMAP_LEN(dst_len)
struct dma_async_tx_descriptor async_tx;
};
#endif /* IOATDMA_H */
drivers/dma/ioatdma_io.h
-118
View File
@@ -1,118 +0,0 @@
/*
* Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 59
* Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* The full GNU General Public License is included in this distribution in the
* file called COPYING.
*/
#ifndef IOATDMA_IO_H
#define IOATDMA_IO_H
#include <asm/io.h>
/*
* device and per-channel MMIO register read and write functions
* this is a lot of anoying inline functions, but it's typesafe
*/
static inline u8 ioatdma_read8(struct ioat_device *device,
unsigned int offset)
{
return readb(device->reg_base + offset);
}
static inline u16 ioatdma_read16(struct ioat_device *device,
unsigned int offset)
{
return readw(device->reg_base + offset);
}
static inline u32 ioatdma_read32(struct ioat_device *device,
unsigned int offset)
{
return readl(device->reg_base + offset);
}
static inline void ioatdma_write8(struct ioat_device *device,
unsigned int offset, u8 value)
{
writeb(value, device->reg_base + offset);
}
static inline void ioatdma_write16(struct ioat_device *device,
unsigned int offset, u16 value)
{
writew(value, device->reg_base + offset);
}
static inline void ioatdma_write32(struct ioat_device *device,
unsigned int offset, u32 value)
{
writel(value, device->reg_base + offset);
}
static inline u8 ioatdma_chan_read8(struct ioat_dma_chan *chan,
unsigned int offset)
{
return readb(chan->reg_base + offset);
}
static inline u16 ioatdma_chan_read16(struct ioat_dma_chan *chan,
unsigned int offset)
{
return readw(chan->reg_base + offset);
}
static inline u32 ioatdma_chan_read32(struct ioat_dma_chan *chan,
unsigned int offset)
{
return readl(chan->reg_base + offset);
}
static inline void ioatdma_chan_write8(struct ioat_dma_chan *chan,
unsigned int offset, u8 value)
{
writeb(value, chan->reg_base + offset);
}
static inline void ioatdma_chan_write16(struct ioat_dma_chan *chan,
unsigned int offset, u16 value)
{
writew(value, chan->reg_base + offset);
}
static inline void ioatdma_chan_write32(struct ioat_dma_chan *chan,
unsigned int offset, u32 value)
{
writel(value, chan->reg_base + offset);
}
#if (BITS_PER_LONG == 64)
static inline u64 ioatdma_chan_read64(struct ioat_dma_chan *chan,
unsigned int offset)
{
return readq(chan->reg_base + offset);
}
static inline void ioatdma_chan_write64(struct ioat_dma_chan *chan,
unsigned int offset, u64 value)
{
writeq(value, chan->reg_base + offset);
}
#endif
#endif /* IOATDMA_IO_H */
drivers/dma/iop-adma.c
+1467
View File
File diff suppressed because it is too large Load Diff
drivers/md/Kconfig
+2
View File
@@ -109,6 +109,8 @@ config MD_RAID10
config MD_RAID456
tristate "RAID-4/RAID-5/RAID-6 mode"
depends on BLK_DEV_MD
select ASYNC_MEMCPY
select ASYNC_XOR
---help---
A RAID-5 set of N drives with a capacity of C MB per drive provides
the capacity of C * (N - 1) MB, and protects against a failure
drivers/md/Makefile
+2 -2
View File
@@ -18,7 +18,7 @@ raid456-objs := raid5.o raid6algos.o raid6recov.o raid6tables.o \
hostprogs-y := mktables
# Note: link order is important. All raid personalities
# and xor.o must come before md.o, as they each initialise
# and must come before md.o, as they each initialise
# themselves, and md.o may use the personalities when it
# auto-initialised.
@@ -26,7 +26,7 @@ obj-$(CONFIG_MD_LINEAR) += linear.o
obj-$(CONFIG_MD_RAID0) += raid0.o
obj-$(CONFIG_MD_RAID1) += raid1.o
obj-$(CONFIG_MD_RAID10) += raid10.o
obj-$(CONFIG_MD_RAID456) += raid456.o xor.o
obj-$(CONFIG_MD_RAID456) += raid456.o
obj-$(CONFIG_MD_MULTIPATH) += multipath.o
obj-$(CONFIG_MD_FAULTY) += faulty.o
obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
drivers/md/md.c
+1 -1
View File
@@ -5814,7 +5814,7 @@ static __exit void md_exit(void)
}
}
module_init(md_init)
subsys_initcall(md_init);
module_exit(md_exit)
static int get_ro(char *buffer, struct kernel_param *kp)
drivers/md/raid5.c
+1676 -1043
View File
File diff suppressed because it is too large Load Diff
drivers/md/xor.c
-154
View File
@@ -1,154 +0,0 @@
/*
* xor.c : Multiple Devices driver for Linux
*
* Copyright (C) 1996, 1997, 1998, 1999, 2000,
* Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson.
*
* Dispatch optimized RAID-5 checksumming functions.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2, or (at your option)
* any later version.
*
* You should have received a copy of the GNU General Public License
* (for example /usr/src/linux/COPYING); if not, write to the Free
* Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#define BH_TRACE 0
#include <linux/module.h>
#include <linux/raid/md.h>
#include <linux/raid/xor.h>
#include <asm/xor.h>
/* The xor routines to use. */
static struct xor_block_template *active_template;
void
xor_block(unsigned int count, unsigned int bytes, void **ptr)
{
unsigned long *p0, *p1, *p2, *p3, *p4;
p0 = (unsigned long *) ptr[0];
p1 = (unsigned long *) ptr[1];
if (count == 2) {
active_template->do_2(bytes, p0, p1);
return;
}
p2 = (unsigned long *) ptr[2];
if (count == 3) {
active_template->do_3(bytes, p0, p1, p2);
return;
}
p3 = (unsigned long *) ptr[3];
if (count == 4) {
active_template->do_4(bytes, p0, p1, p2, p3);
return;
}
p4 = (unsigned long *) ptr[4];
active_template->do_5(bytes, p0, p1, p2, p3, p4);
}
/* Set of all registered templates. */
static struct xor_block_template *template_list;
#define BENCH_SIZE (PAGE_SIZE)
static void
do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
{
int speed;
unsigned long now;
int i, count, max;
tmpl->next = template_list;
template_list = tmpl;
/*
* Count the number of XORs done during a whole jiffy, and use
* this to calculate the speed of checksumming. We use a 2-page
* allocation to have guaranteed color L1-cache layout.
*/
max = 0;
for (i = 0; i < 5; i++) {
now = jiffies;
count = 0;
while (jiffies == now) {
mb();
tmpl->do_2(BENCH_SIZE, b1, b2);
mb();
count++;
mb();
}
if (count > max)
max = count;
}
speed = max * (HZ * BENCH_SIZE / 1024);
tmpl->speed = speed;
printk(" %-10s: %5d.%03d MB/sec\n", tmpl->name,
speed / 1000, speed % 1000);
}
static int
calibrate_xor_block(void)
{
void *b1, *b2;
struct xor_block_template *f, *fastest;
b1 = (void *) __get_free_pages(GFP_KERNEL, 2);
if (! b1) {
printk("raid5: Yikes! No memory available.\n");
return -ENOMEM;
}
b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE;
/*
* If this arch/cpu has a short-circuited selection, don't loop through all
* the possible functions, just test the best one
*/
fastest = NULL;
#ifdef XOR_SELECT_TEMPLATE
fastest = XOR_SELECT_TEMPLATE(fastest);
#endif
#define xor_speed(templ) do_xor_speed((templ), b1, b2)
if (fastest) {
printk(KERN_INFO "raid5: automatically using best checksumming function: %s\n",
fastest->name);
xor_speed(fastest);
} else {
printk(KERN_INFO "raid5: measuring checksumming speed\n");
XOR_TRY_TEMPLATES;
fastest = template_list;
for (f = fastest; f; f = f->next)
if (f->speed > fastest->speed)
fastest = f;
}
printk("raid5: using function: %s (%d.%03d MB/sec)\n",
fastest->name, fastest->speed / 1000, fastest->speed % 1000);
#undef xor_speed
free_pages((unsigned long)b1, 2);
active_template = fastest;
return 0;
}
static __exit void xor_exit(void) { }
EXPORT_SYMBOL(xor_block);
MODULE_LICENSE("GPL");
module_init(calibrate_xor_block);
module_exit(xor_exit);