linux - 如何使用 DMA 在一个线程中同时写入不同磁盘上的多个文件？

Question

我使用 aio 在一个线程中将多个文件写入不同的磁盘。当我使用缓冲写入时，IO 处理是并发的。但是cpu负载很高。当我使用 DIRECT 标志打开文件时，IO 处理不是并发的。

如何使用 DMA 在一个线程中同时写入不同磁盘上的多个文件？

#include <malloc.h>
#include <stdio.h>
#include <string.h>
#include <iostream>
#include <sstream>
#include <inttypes.h>

#include <unistd.h>
#include <fcntl.h>
#include <sys/syscall.h>
#include <linux/aio_abi.h>

using namespace std;

long double timeDiff(timespec start, timespec end) {
const long double s = start.tv_sec + start.tv_nsec * 1.0e-9;
const long double e = end.tv_sec   + end.tv_nsec   * 1.0e-9;
return e - s;
}

// nr: maximum number of requests that can simultaneously reside in the context.
inline int io_setup(unsigned nr, aio_context_t *ctxp) {
return syscall(__NR_io_setup, nr, ctxp);
}

inline int io_destroy(aio_context_t ctx) {
return syscall(__NR_io_destroy, ctx);
}

// Every I/O request that is submitted to 
inline int io_submit(aio_context_t ctx, long nr, struct iocb **iocbpp) {
return syscall(__NR_io_submit, ctx, nr, iocbpp);
}

// For every completed I/O request kernel creates an io_event structure.
// minimal number of events one wants to get.
// maximum number of events one wants to get.
inline int io_getevents(aio_context_t ctx, long min_nr, long max_nr,
    struct io_event *events, struct timespec *timeout) {
return syscall(__NR_io_getevents, ctx, min_nr, max_nr, events, timeout);
}

int main(int argc, char *argv[]) {

// prepare data
const unsigned int kAlignment = 4096;
const long data_size = 1600 * 1024 * 12 / 8; 
//const long data_size = 2448 * 1344 * 12 / 8; 
void * data = memalign(kAlignment, data_size);
memset(data, 0, data_size);
//for (int i = 0; i < data_size; ++i)
//   data[i] = 'A';

// prepare fd
//const int file_num = 3;
const int file_num = 2;
int fd_arr[file_num];
for (int i = 0; i < file_num; ++i) {
    ostringstream filename;
    if (i == 0) {
        //filename << "/data/test";
        filename << "/test";
    } else {
        filename << "/data" << i << "/test";
    }
    //filename << "/data/test" << i;
    int fd = open(filename.str().c_str(), O_WRONLY | O_NONBLOCK | O_CREAT | O_DIRECT | O_APPEND, 0644);
    //int fd = open(filename.str().c_str(), O_WRONLY | O_NONBLOCK | O_CREAT | O_DIRECT, 0644);
    //int fd = open(filename.str().c_str(), O_WRONLY | O_NONBLOCK | O_CREAT, 0644);
    if (fd < 0) {
        perror("open");
        return -1;
    }
    fd_arr[i] = fd;
}

aio_context_t ctx;
struct io_event events[file_num];
int ret;
ctx = 0;

ret = io_setup(1000, &ctx);
if (ret < 0) {
    perror("io_setup");
    return -1;
}

struct iocb cbs[file_num];
for (int i = 0; i < file_num; ++i) {
    memset(&cbs[i], 0, sizeof(cbs[i]));
}
struct iocb * cbs_pointer[file_num];
for (int i = 0; i < file_num; ++i) {
    /* setup I/O control block */
    cbs_pointer[i] = &cbs[i];
    cbs[i].aio_fildes = fd_arr[i];
    cbs[i].aio_lio_opcode = IOCB_CMD_PWRITE; // IOCV_CMD
    cbs[i].aio_nbytes = data_size;
}

timespec tStart, tCurr;
clock_gettime(CLOCK_REALTIME, &tStart);

const int frame_num = 10000;
for (int k = 0; k < frame_num; ++k) {

    for (int i = 0; i < file_num; ++i) {
        /* setup I/O control block */
        cbs[i].aio_buf = (uint64_t)data;
        //cbs[i].aio_offset = k * data_size;
    }

    ret = io_submit(ctx, file_num, cbs_pointer);
    if (ret < 0) {
        perror("io_submit");
        return -1;
    }

    /* get reply */
    ret = io_getevents(ctx, file_num, file_num, events, NULL);
    //printf("events: %d, k: %d\n", ret, k);
}

clock_gettime(CLOCK_REALTIME, &tCurr);
cout << "frame: " << frame_num << " time: " << timeDiff(tStart, tCurr) << endl;

ret = io_destroy(ctx);
if (ret < 0) {
    perror("io_destroy");
    return -1;
}

// close fd
for (int i = 0; i < file_num; ++i) {
    fsync(fd_arr[i]);
    close(fd_arr[i]);
}
return 0;
}

Answer 1

当且仅当正在写入的物理范围已经在磁盘上分配时，Linux 才能使写入实际上是异步的。否则它必须先获取一个互斥量并进行分配，这样一切就变得同步了。

请注意，将文件截断为新长度通常不会实际分配底层范围。您需要先预写内容。此后，重写相同的范围现在将异步完成，从而变为并发。

正如您可能了解到的那样，Linux 上的异​​步文件输入/输出并不是很好，尽管它会随着时间的推移不断改进。 Windows 或 FreeBSD 的实现要好得多。甚至 OS X 也不可怕。请改用其中任何一个。