c++ - 使用 Boost 处理流

Question

我有一些 *.bz2 文件，其中包含 *.csv 文本文件。

我需要处理它 - 解压它并更改它的编码。

现在我有了代码，可以解压这个文件并通过将原始文件读取到另一个文件来更改 *.csv 文件编码。但是这一刻我“遇到了情况”- *.csv 文件很大(5 GB)，但服务器只有 8 GB。 IE，如果我将通过将文件复制到另一个文件来更改文件编码，我将没有硬盘空间。

因此，我需要“即时”更改文件的编码。

目前，我有两段代码:

解压

bool NetIO::unBZ2(string PathToBZ2, string PathToCSV) {
       try {

             boost::locale::generator gen;

             ofstream outFile("c:/temp/sqlShare/qqq.csv", std::ofstream::out | std::ofstream::binary);

             ifstream file(PathToBZ2, ios_base::in | ios_base::binary);

             filtering_streambuf<input> in;
             in.push(bzip2_decompressor());
             in.push(file);
             boost::iostreams::copy(in, outFile);

             file.close();                    
             outFile.close();
       }
       catch (const bzip2_error& exception) {
             int error = exception.error();

             if (error == boost::iostreams::bzip2::data_error) {
                    // compressed data stream is corrupted
                    cout << "compressed data stream is corrupted";
             }
             else if (error == boost::iostreams::bzip2::data_error_magic)
             {
                    // compressed data stream does not begin with the 'magic' sequence 'B' 'Z' 'h'
                    cout << "compressed data stream does not begin with the 'magic' sequence 'B' 'Z' 'h'";
             }
             else if (error == boost::iostreams::bzip2::config_error) {
                    // libbzip2 has been improperly configured for the current platform
                    cout << "libbzip2 has been improperly configured for the current platform";
             }
             return false;
       }

       return true;
}

和改变文件编码的代码

bool NetIO::replaceLineBreaks(const string& inFilePath, const string& searchStr, const string& replacement) {

       try {     

             boost::locale::generator gen;

             ifstream iss(inFilePath.c_str(), std::ios::in | std::ios::binary);

             std::locale lru = gen("ru_RU.CP1251");
             std::locale lru2 = gen("ru_RU.UTF-8");
             iss.imbue(lru2);    

             //string tempFilePath2 = inFilePath + ".tmp3";
             string tempFilePath2 = "d:/temp/qqq.tmp3";
             ofstream os(tempFilePath2, std::ios::out | std::ios::binary);
             os.imbue(lru);

             const int buffSize= 500000;
             char qqq[buffSize];
             //wchar_t wqqq[buffSize];
             while (iss) {
                    iss.read(qqq, buffSize);
             //     MultiByteToWideChar(CP_UTF8, 0, qqq, buffSize, wqqq, buffSize);
                    os << qqq;
                    //cnt++;
                    /*if (cnt > 80) {
                           break;
                    }*/
             }

             os.close();



             /*boost::iostreams::mapped_file istm(inFilePath.c_str());
             if (!istm.is_open())
                    return false;

             string tempFilePath = inFilePath + ".tmp";
             ofstream ostm(tempFilePath.c_str(), std::ios::out | std::ios::binary);
             if (!ostm.is_open()) {
                    istm.close();
                    return false;
             }

             boost::regex regexp(searchStr);
             ostreambuf_iterator<char> it(ostm);
             boost::regex_replace(it, istm.begin(), istm.end(), regexp, replacement, boost::match_default | boost::format_all);

             istm.close();
             ostm.close();

             boost::filesystem::rename(tempFilePath, inFilePath);*/
       }
       catch (exception ex) {
             cout << "Problem in line endings replacer" << std::endl;
             cout << ex.what() << std::endl;
             cout << "-----------------" << std::endl;
             return false;
       }

       return true;
}

我想知道，我怎样才能将这两段代码结合起来？

据我所知，可能的解决方案是创建一个自定义的 ofstream 对象，它将接收数据并转换它的编码，并使用 boost::iostreams 将数据传递给它::复制 .

如何合并这两段代码？

非常感谢!

Answer 1

你在第二个函数中的代码有问题，你在哪里做

iss.read(qqq, buffSize);
os << qqq;

它不仅假定了 qqq 的完整大小总是读，也是qqq从 char(&)[500000] 衰减至 char*并将被解释为以 NUL 结尾的字符串。这至少打破了嵌入的 NUL 字符。喜欢:

auto bytes_read = iss.read(qqq, buffSize);
os.write(qqq, bytes_read);

Note: boost::iostreams::copy already does the correct thing, using the default buffersize (if unspecified)

#ifndef BOOST_IOSTREAMS_DEFAULT_DEVICE_BUFFER_SIZE
# define BOOST_IOSTREAMS_DEFAULT_DEVICE_BUFFER_SIZE 4096
#endif

接下来，循环很容易组合成类似的东西

// input
std::ifstream file(PathToBZ2, std::ios::binary);

bio::filtering_stream<bio::input> in;
in.push(bio::bzip2_decompressor());
in.push(file);

std::locale lru2 = gen("ru_RU.UTF-8");
in.imbue(lru2);

// output
std::ofstream outFile("c:/temp/sqlShare/qqq.csv", std::ios::binary);
std::locale lru = gen("ru_RU.CP1251");
outFile.imbue(lru);

// copy
const int buffSize = 500000;
/*auto totalWritten =*/ boost::iostreams::copy(in, outFile, buffSize);

注意这里使用了 filtering_stream , 不是 filtering_streambuf

更新

使用 code_converter<> 强制进行所需的转换:

Live On Coliru

#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/bzip2.hpp>
#include <boost/iostreams/stream.hpp>
#include <boost/iostreams/copy.hpp>
#include <boost/iostreams/code_converter.hpp>
#include <boost/locale.hpp>
#include <iostream>
#include <fstream>

namespace bio = boost::iostreams;

bool unBZ2(std::string PathToBZ2, std::string PathToCSV) {
    try {
        boost::locale::generator gen;

        // input
        std::ifstream file(PathToBZ2, std::ios::binary);

        bio::filtering_stream<bio::input> in;
        in.push(bio::bzip2_decompressor());
        in.push(file);

        std::locale lru2 = gen("ru_RU.UTF-8");
        in.imbue(lru2);

        // output
        std::ofstream outFile(PathToCSV, std::ios::binary);
        std::locale lru = gen("ru_RU.CP1251");
        outFile.imbue(lru);

        {
            bio::code_converter<bio::filtering_stream<bio::input> > win(in);
            bio::code_converter<decltype(outFile)> wout(outFile);

            win.imbue(lru2);
            wout.imbue(lru);
            const int buffSize = 500000;
            /*auto totalWritten =*/ boost::iostreams::copy(win, wout, buffSize);
        }
    } catch (const bio::bzip2_error &exception) {
        int error = exception.error();

        if (error == boost::iostreams::bzip2::data_error) {
            // compressed data stream is corrupted
            std::cout << "compressed data stream is corrupted";
        } else if (error == boost::iostreams::bzip2::data_error_magic) {
            // compressed data stream does not begin with the 'magic' sequence 'B' 'Z'
            // 'h'
            std::cout << "compressed data stream does not begin with the 'magic' sequence " "'B' 'Z' 'h'";
        } else if (error == boost::iostreams::bzip2::config_error) {
            // libbzip2 has been improperly configured for the current platform
            std::cout << "libbzip2 has been improperly configured for the current platform";
        }
        return false;
    }

    return true;
}

int main() {
    unBZ2("input.txt.bz2", "output.txt");
}

翻译 "Lorem Ipsum".ru进入

00000000: cbee f0e5 ec20 e8ef f1f3 ec20 e4ee ebee  ..... ..... ....
00000010: f020 f1e8 f220 e0ec e5f2 2c20 f1ee ebf3  . ... ...., ....
00000020: ec20 eff0 e8ec e8f1 20e0 ed20 f1e5 e42e  . ...... .. ....
00000030: 20cd e5f6 20e5 f220 eee4 e8ee 20e4 e8f1   ... .. .... ...
00000040: eff3 f2e0 ede4 ee2c 20ef f0e8 20ed e50a  ......., ... ...
00000050: f4ee f0e5 edf1 e8e1 f3f1 20e2 eeeb f3ef  .......... .....
00000060: f2e0 f2e8 e1f3 f12e 20c5 efe8 f6f3 f0e8  ........ .......
00000070: 20f6 eeec ecf3 ede5 20f1 f3f1 f6e8 efe8   ....... .......
00000080: f220 e5f5 20ef f0ee 2e20 cff0 eee1 ee20  . .. .... ..... 
00000090: f4e5 f3e3 e0e8 f20a f0e5 f6f2 e5ff f3e5  ................
000000a0: 20f3 f220 eff0 e82c 20ec e5e8 20f2 eeeb   .. ..., ... ...
000000b0: ebe8 f220 ecee ebe5 f1f2 e8e5 20e4 e8f1  ... ........ ...
000000c0: eff3 f2e0 ede4 ee20 f6f3 2e20 d1e8 edf2  ....... ... ....
000000d0: 20e1 f0f3 f2e5 20ec e0e7 e8ec 20e5 eef1   ..... ..... ...
000000e0: 20e5 f32c 0ae1 f0f3 f2e5 20e0 ebf2 e5f0   ..,...... .....
000000f0: e020 eced e5f1 e0f0 f7f3 ec20 f6f3 20ec  . ......... .. .
00000100: e5eb 2c20 e5e8 20f5 e0f1 20eb f3ef f2e0  .., .. ... .....
00000110: f2f3 ec20 f6ee edf1 e5f6 f2e5 f2f3 e5f0  ... ............
00000120: 2e0a 0acd e520 e0eb e8e8 20ff f3e0 f120  ..... .... .... 
00000130: e0f1 f1e5 edf2 e8ee f020 e2e8 f12c 20e5  ......... ..., .
00000140: e820 e2e8 ec20 ecf3 f6e8 f3f1 20ec e5e4  . ... ...... ...
00000150: e8ee f6f0 e8f2 e0f2 e5ec 2c20 eff0 ee20  .........., ... 
00000160: e5e0 20e0 e5ff f3e5 20eb e0ee f0e5 e5f2  .. ..... .......
00000170: 0aef f5e0 e5e4 f0f3 ec2e 20c5 eef1 20f1  .......... ... .
00000180: f3ec ee20 e0e4 f5f3 f620 f6ee edf1 e5ff  ... ..... ......
00000190: f3e0 f220 e5f5 2c20 e8e4 20f5 e0f1 20eb  ... .., .. ... .