c++ - 读取文本文件并随机播放

Question

我有一个很大的 txt 文件(100MB，2300 万行)，我想逐行打开它并像 linux 中的 GNU shuf 命令一样随机播放它。我在 Windows 平台上工作，安装了 Visual Studio 2015 并开始使用 C++ 进行编程。我第一次尝试使用我的旧 C++ 代码，但它太慢了，我切换到 boost 库。我不得不承认，它确实很快，但我不知道如何将结果放入数组并打乱它们(数组必须容纳最多 100.000.000 个索引)。

这是我的尝试

#include <boost/iostreams/device/mapped_file.hpp> // for mmap
#include <algorithm>  // for std::find
#include <iostream>   // for std::cout
#include <cstring>

#include <fstream>
#include <sstream>
#include <string>

int main()
{
    boost::iostreams::mapped_file mmap("input.txt", boost::iostreams::mapped_file::readonly);
    auto f = mmap.const_data();
    auto l = f + mmap.size();

    uintmax_t m_numLines = 0;
    int inc1 = 0;

    char ** ip = NULL;

    boost::array<char, sizeof(int)> send_buf; <-- error here
    /*
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      namespace "boost" has no member "array" hshuffle    c:\path_to_the\main.cpp 21  
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      type name is not allowed    hshuffle    c:\path_to_the\main.cpp 21  
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      identifier "send_buf" is undefined  hshuffle    c:\path_to_the\main.cpp 21  
    Severity    Code    Description Project File    Line    Suppression State
    Error (active)      a value of type "const char *" cannot be assigned to an entity of type "char *" hshuffle    c:\path_to_the\main.cpp 29  
    */

    while (f && f != l)
    {
        if ((f = static_cast<const char*>(memchr(f, '\n', l - f))))
        {
            if ((m_numLines % 1000000) == 0)
            {
                ip[m_numLines] = l;
                std::cout << m_numLines << "\n";
            }


            m_numLines++, f++;
        }
    }

    std::cout << "m_numLines = " << m_numLines << "\n";




    printf("endfille\n");

    char a;
    std::cin >> a;
}

旧的 C++ 程序

puts("reading ips file [./i]");

if((fp=fopen("i","r")) == NULL)
{ 
   printf("FATAL: Cant find i\n");
   return -1;
}

int increment_ips = 0;
indIP = 0;
while (fgets(nutt,2024,fp))
{
    while (t = strchr (nutt,'\n'))
        *t = ' ';

    temp = strtok (nutt, " ");

    if (temp != NULL) {
        string = strdup (temp);
        indIP++;

        while (temp = strtok (NULL, " "))
        {
            indIP++;
        }
    }

    increment_ips++;
}
fclose(fp);




if((fp=fopen("i","r")) == NULL)
{ 
   printf("FATAL: Cant find i\n");
   return -1;
}

increment_ips = 0;
ip = new char*[indIP];
indIP = 0;

while (fgets(nutt,2024,fp))
{
    while (t = strchr (nutt,'\n'))
        *t = ' ';

    temp = strtok (nutt, " ");

    if (temp != NULL) {
        string = strdup (temp);     
        ip[indIP++]=string;

        while (temp = strtok (NULL, " "))
        {
            string = strdup (temp);

            ip[indIP++]=string;
        }
    }

    increment_ips++;
}
fclose(fp);

// shuffle
printf("Loaded [%d] ips\n",increment_ips);

puts("Shuffeling ips");
srand(time(NULL));
for(int i = 0; i <= increment_ips; i++)
{
    int randnum = rand() % increment_ips + 1;
    char* tempval;
    tempval = ip[i];

    ip[i] = ip[randnum];
    ip[randnum] = tempval;
}
puts("Shuffeled");

有什么解决办法吗？我更喜欢 boost 因此它真的很快。

谢谢。

Answer 1

“旧”程序读取输入文件两次，第一次计算空格分隔的单词(似乎不是行)，第二次实际将数据存储在数组中。使用 std::string 的 std::vector 不需要事先知道元素的确切数量，可以预留一些空间并让内存管理到标准库。

从 C++11 开始，也可以使用 std::shuffle做OP需要的事情。但是，很难想象对于如此大的数组(数百万个元素)，Fisher-Yates(或 Knuth)洗牌算法的缓存友好实现。

I don't know how to put results into an array and shuffle them

一个可能的解决方案(没有 Boost)可能是:

#include <iostream>
#include <fstream>
#include <vector>
#include <string>
#include <algorithm>
#include <random>

using std::string;
using std::vector;
using std::cout;

int main() {
    // initialize random number generator
    std::random_device rd;
    std::mt19937 g(rd());

    // open input file  
    string file_name{"input.txt"};
    std::ifstream in_file{file_name};
    if ( !in_file ) {
        std::cerr << "Error: Failed to open file \"" << file_name << "\"\n";
        return -1;
    }

    vector<string> words;
    // if you want to avoid too many reallocations:
    const int expected = 100000000;
    words.reserve(expected);

    string word;
    while ( in_file >> word ) {
        words.push_back(word);
    }

    std::cout << "Number of elements read: " << words.size() << '\n';
    std::cout << "Beginning shuffle..." << std::endl;

    std::shuffle(words.begin(),words.end(),g);

    std::cout << "Shuffle done." << std::endl;

    // do whatever you need to do with the shuffled vector...

    return 0;
}