c - 循环中的文件 mmap

Question

我正在尝试使用 mmap 循环读取文件，我的文件包含有关 3 个部分的信息，第一部分的大小为 3*sizeof(double)，第二个也是 3*sizeof(double) 的大小，第三个也是 sizeof(double) 的大小。在文件的第一部分，我有 HEADER 大小为 32768 字节。组织的文件:

标题||部分(1),部分(1)....部分(1)||部分(2),部分(2)....部分(2)||部分(3),部分(3)....第(3)部分|

每个部分我有100次。我想每次处理 30 个部分(每个部分 10 个部分)。

我试过这段代码:

void readingFile(FILE *file, double *a, double *b, double *c, int start, int end, int chunksz, long total)
{
    int i = 0;
    int size = end - start + 1;
    int fd;
    fd = fileno(file);
    off_t fullsize = lseek(fd,SEEK_CUR,SEEK_END); //getting the file size
    fullsize-=1;//the lseek gives one more byte, its ok!
    unsigned long summ = (unsigned long)(start-1)*chunksz; //chunk is 56
    summ+=(unsigned long)HEADER_SIZE;//offset the header size
    unsigned long paramm=(unsigned long)((unsigned long)summ/(unsigned long)(sysconf(_SC_PAGE_SIZE)));
    unsigned long param = floor(paramm);
    void *buf=NULL;
    buf =mmap(NULL,fullsize , PROT_READ, MAP_PRIVATE , fd, param*sysconf(_SC_PAGE_SIZE));
    if(buf==MAP_FAILED)
    {
        printf("we have an error\n");
    }
    unsigned long gapp = (sysconf(_SC_PAGE_SIZE))*param;
    unsigned long gap =summ-gapp;
    buf+=gap;
    memcpy(a,buf,3*sizeof(double)*size);
    buf+=(unsigned long)((long)total-(start-1))*3*sizeof(double);
    buf+=((start-1)*3*sizeof(double));
    memcpy(b,buf,3*sizeof(double)*size);
    buf+=(unsigned long)((long)total-(start-1))*3*sizeof(double);
    buf+=((start-1)*sizeof(double));
    memcpy(c,buf,sizeof(double)*size);
    munmap(buf, fullsize);
    return;
}

在某个地方我有溢出和程序崩溃!每次调用该函数时，都会将新内存正确分配给 a,b,c。这里有什么？该进程在第 14 次迭代时崩溃:

memcpy(c,buf,sizeof(double)*size);

谢谢!

Answer 1

我知道用源代码回答一个问题并不熟悉。但我试图说明 mmap 有什么用处。基本上 mmap 使用内核功能将文件内容加载(并写回)到内存区域。所以我们不需要频繁调用read/seek 可以让你的应用程序更有效。另一方面，它是一种直接访问您的数据的舒适解决方案，只需查看代码:

#include <unistd.h>
#include <sys/mman.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

struct mapping
{
        void* start_addr;
        size_t length;
        int fd;
};

struct mapping* map_file(const char* file)
{
        struct mapping* ret = malloc(sizeof(struct mapping));
        if(NULL == ret)
        {
                printf("Can't allocate memory for struct mapping.\n");
                return NULL;
        }

        ret->fd = open(file, O_RDONLY);
        if(0 > ret->fd)
        {
                perror("can't open specified file.");
                free(ret);
                return NULL;
        }

        struct stat fs;
        if(0 != fstat(ret->fd, &fs))
        {
                perror("can't specify file size.");
                close(ret->fd);
                free(ret);
                return NULL;
        }

        ret->length = fs.st_size;

        //offset means offset in file
        ret->start_addr = mmap(NULL, ret->length, PROT_READ, MAP_PRIVATE, ret->fd, 0);
        if(MAP_FAILED == ret->start_addr)
        {
                perror("Mapping file failed.");
                close(ret->fd);
                free(ret);
                return NULL;
        }

        return ret;
}

//returns zero on success and free the `struct mapping` data
int unmap_file(struct mapping* mmf)
{
        //note that now we use read only mapping
        //if you want to write this memory pages
        //before detach maybe you have to call:
        //msync(mmf->start_addr, mmf->length, MS_SYNC);
        // avoid data loss (write all dirty page into file).

        if(NULL != mmf->start_addr)
        {
                if(0 != munmap(mmf->start_addr, mmf->length))
                {
                        perror("Can't munmap file.");
                        return 1;
                }
        }

        mmf->start_addr = NULL;
        if(-1 != mmf->fd)
        {
                if(0 != close(mmf->fd))
                {
                        perror("can't close file descriptor.");
                        return 2;
                }
        }

        free(mmf);

        return 0;
}

// for test#define  MAGIC_START_INDEX 0
#define  MAGIC_START_INDEX 32768

int main(int arg_length, char** args)
{
        if(arg_length < 2)
        {
                printf("No input file specified.\n");
                exit(1);
        }

        int i = 0;
        //first argument is the name of program
        while(++i < arg_length)
        {
                struct mapping* mmf = map_file(args[i]);
                if(NULL == mmf)
                {
                        printf("can't use %s for input file\n", args[i]);
                        continue;
                }

                if(mmf->length >  MAGIC_START_INDEX)
                {
                        //upper base
                        int max_index = (mmf->length - MAGIC_START_INDEX) / sizeof(double);

                        //an offset alias for start memory address
                        double* data = ((double*)(mmf->start_addr + MAGIC_START_INDEX));

                        int ni = 0;
                        while(ni+2 < max_index)
                        {
                                printf("num0: %f, num1: %f, num2: %f\n", data[ni], data[ni+1], data[ni+2]);
                                ni += 3;
                        }
                }
                else
                {
                        printf("File: %s has no valuable data.", args[i]);
                }

                unmap_file(mmf);
        }
}

总的来说，你看，我们可以直接使用内存地址而不是重复读取操作。这是我复制的示例代码(存储在结构中的映射相关数据和相关函数负责创建/释放文件映射)。读取可以更偷懒，打开文件，读取大小(fstat)如果我有有值(value)的数据，使用mmap的offset参数跳过文件中的header部分:

double[] data = (double*)(mmap(NULL, file_length, PROT_READ, MAP_PRIVATE, fd, MAGIC_START_INDEX)); //TODO check null.

您可以“即时”访问数据。