c - malloc 有时会失败。或者 pread 有时会失败

Question

所以我有一个测试程序，它将大量数据读入缓冲区并 mallocs相应地缓冲。然而，它 malloc 在大尺寸上失败。

有办法解决这个问题吗？

感谢任何回复

设备/dev/sdc 是一个 2TB 的磁盘。

这里准备了2个编译代码:

#define          _FILE_OFFSET_BITS                            64     
#define          BLKGETSIZE64                                _IOR(0x12,114,size_t)    
#define          _POSIX_C_SOURCE                             200809L

#include <stdio.h>    
#include <inttypes.h>    
#include <stdlib.h>    
#include <fcntl.h>    
#include <unistd.h>

int readdata(int fp,uint64_t seekpoint, uint64_t seekwidth) {

    int16_t  *buf;

    buf=(int16_t *)malloc(seekwidth*sizeof(int16_t)+1);
    if (buf==0) {
      printf("ERROR malloc(%"PRIu64")\n",seekwidth*(sizeof(int16_t)));
      return 3;
    }

    if (pread(fp,buf,seekwidth,seekpoint)==seekwidth) {
        printf("SUCCES READING AT: %"PRIu64"| WITH READ WIDTH: %"PRIu64"\n",seekpoint,seekwidth);
        free(buf);
        return 1;
    } else {
        printf("ERROR READING AT: %"PRIu64"| WITH READ WIDTH: %"PRIu64"\n",seekpoint,seekwidth);
        free(buf);
        return 2;
    }

}



int main() {
    uint64_t    readwith,
                offset;
    int         fp=open("/dev/sdc",O_RDWR);

    readwith=10000;   offset=0;
    readdata(fp,offset,readwith);
    readwith=100000;   offset=0;
    readdata(fp,offset,readwith);
    readwith=1000000;   offset=0;
    readdata(fp,offset,readwith);
    readwith=10000000;   offset=0;
    readdata(fp,offset,readwith);
    readwith=100000000;   offset=0;
    readdata(fp,offset,readwith);
    readwith=1000000000;   offset=0;
    readdata(fp,offset,readwith);
    readwith=10000000000;   offset=0;
    readdata(fp,offset,readwith);
    close(fp);

}

我系统的输出是:

SUCCES READING AT: 0| WITH READ WIDTH: 10000
SUCCES READING AT: 0| WITH READ WIDTH: 100000
SUCCES READING AT: 0| WITH READ WIDTH: 1000000
SUCCES READING AT: 0| WITH READ WIDTH: 10000000
SUCCES READING AT: 0| WITH READ WIDTH: 100000000
SUCCES READING AT: 0| WITH READ WIDTH: 1000000000
ERROR READING AT: 0| WITH READ WIDTH: 10000000000
ERROR READING AT: 0| WITH READ WIDTH: 100000000000
ERROR READING AT: 0| WITH READ WIDTH: 1000000000000
ERROR READING AT: 0| WITH READ WIDTH: 10000000000000
ERROR READING AT: 0| WITH READ WIDTH: 100000000000000
ERROR READING AT: 0| WITH READ WIDTH: 1000000000000000

有时

ERROR malloc(2000000000)
ERROR READING AT: 0| WITH READ WIDTH: 10000000000

Answer 1

好吧，我会说你不能分配大小为 10000000000 的内存，因为那是 9536 MiB，我猜你没有那么多 RAM。

您可以考虑使用 STXXL .

The core of STXXL is an implementation of the C++ standard template library STL for external memory (out-of-core) computations, i. e., STXXL implements containers and algorithms that can process huge volumes of data that only fit on disks.