javascript - MongoDB NodeJS 进程内存不足

Question

我正在尝试使用 NodeJS 程序在 MongoDB 集合中插入一百万条以上的记录(虚拟)。但不幸的是我的进程内存不足:

这是我用 JavaScript 编写并通过 Node 运行的代码

var MongoClient = require('mongodb').MongoClient;

MongoClient.connect('mongodb://localhost:27017/course', function(err, db) {
    if(err) throw err;

    db.collection('students').drop();

    var types = ['exam', 'quiz', 'homework', 'homework'];

    // For 1 Million Records
    for (var i = 0; i < 1000000; i++) {
        // Each student taking 10 classes
        for (var class_counter = 0; class_counter < 10; class_counter ++) {
            scores = [];
            // Each Class has 4 grades
            // and each class has 4 grades
            for (var j = 0; j < 4; j++) {
            scores.push({'type':types[j],'score':Math.random()*100});
            }

            // // there are 500 different classes that they can take
           class_id = Math.floor(Math.random()*501); // get a class id between 0 and 500

            record = {'student_id':i, 'scores':scores, 'class_id':class_id};
           db.collection('students').insert(record);
        }
    }
});

这是我得到的错误跟踪:

AMAC02PC0PHG3QP:25_Indexes_Insert macadmin$ node app.js

<--- Last few GCs --->

   28373 ms: Scavenge 1397.8 (1457.4) -> 1397.8 (1457.4) MB, 1.1 / 0 ms (+ 151.3 ms in 1 steps since last GC) [allocation failure] [incremental marking delaying mark-sweep].
   29444 ms: Mark-sweep 1397.8 (1457.4) -> 1397.7 (1457.4) MB, 1071.5 / 0 ms (+ 427.1 ms in 14 steps since start of marking, biggest step 202.5 ms) [last resort gc].
   30486 ms: Mark-sweep 1397.7 (1457.4) -> 1397.6 (1457.4) MB, 1041.4 / 0 ms [last resort gc].


<--- JS stacktrace --->

==== JS stack trace =========================================

Security context: 0x23473037399 <JS Object>
    1: /* anonymous */(aka /* anonymous */) [/Users/macadmin/Desktop/NodeJS_MongoDB/25_Indexes_Insert/app.js:~3] [pc=0x3f5d2b92c716] (this=0x23473004131 <undefined>,err=0x23473004131 <undefined>,db=0x1f851bb90029 <JS Object>)
    2: /* anonymous */(aka /* anonymous */) [/Users/macadmin/Desktop/NodeJS_MongoDB/25_Indexes_Insert/node_modules/mongodb/lib/mongo_client.js:455] [pc=0x3f5d2b4da8bd] (thi...

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory
Abort trap: 6

我的笔记本配置:

// Macbook Pro
// OS X 10.9.5
// 2.5 Ghz Intel Core i7
// 16 GB Ram DDR3
// SSD

Answer 1

如果您的 MongoDB 服务器是 2.6 或更新版本，最好利用写入命令 Bulk API 允许执行 bulk insert 操作是服务器之上的简单抽象，可以轻松构建批量操作。这些批量操作主要有两种形式:

• 订购批量操作。这些操作按顺序执行所有操作，并在第一次写入错误时出错。
• 无序批量操作。这些操作并行执行所有操作并汇总所有错误。无序批量操作不保证执行顺序。

请注意，对于低于 2.6 的旧服务器，API 将对操作进行下转换。但是，不可能 100% 下转换，因此可能存在一些无法正确报告正确数字的边缘情况。

在您的情况下，您可以实现 Bulk API 像这样:

var MongoClient = require('mongodb').MongoClient;

MongoClient.connect("mongodb://localhost:27017/course", function(err, db) {
    // Handle error
    if(err) throw err;

    // Get the collection and bulk api artefacts
    var col = db.collection('students'),
        types = ['exam', 'quiz', 'homework', 'homework'],
        bulk = col.initializeOrderedBulkOp(), // Initialize the Ordered Batch
        counter = 0;

    // Drop the collection
    col.drop();


    // Representing a long loop with 1 Million Records
    for (var i = 0; i < 1000000; i++) {
        var scores = [],
            class_id = 0,
            record = {};

        // Each student taking 10 classes
        for (var class_counter = 0; class_counter < 10; class_counter ++) {

            // Each Class has 4 grades
            // and each class has 4 grades
            for (var j = 0; j < 4; j++) {
                scores.push({ 'type': types[j], 'score': Math.random()*100 });
            }

            // there are 500 different classes that they can take
            class_id = Math.floor(Math.random() * 501); // get a class id between 0 and 500    

            record['student_id'] = i;
            record['scores'] = scores;
            record['class_id'] = class_id;
        }

        bulk.insert(record);            
        counter++;

        if (counter % 1000 == 0 ) {
            bulk.execute(function(err, result) {  
                // re-initialise batch operation           
                bulk = col.initializeOrderedBulkOp();
            });
        }
    }

    if (counter % 1000 != 0 ){
        bulk.execute(function(err, result) {
            // do something with result
            db.close();
        }); 
    } 
});

-- 更新--

感谢@MarkusWMahlberg，为了生成虚拟内容，您可能想尝试包 mgenerate .