javascript - NodeJS、MongoDB、Mongoose 非阻塞保存大数据

Question

我目前正在使用 NodeJS、ExpressJS(带有 EJS)、MongoDB 和 Mongoose 开发一个简单的应用程序。以下是我面临的问题的简要说明，并寻求一些建议

场景

1) 在特定事件中调用使用 SOAP 的 Web 服务并提取数据。

2) API 一次返回大约百万行数据

3) 拉取的数据使用mongoose保存到MongoDB

代码

数据库模型 - (myModel.js)

var mongoose = require('mongoose')
var Schema = mongoose.Schema

var prodSchema = new Schema({
    rowIndex: {
        type: Number,
    },
    prodId: {
        type: String,
    },
    prodDesc: {
        type: String,
    },
    prodCategory: {
        type: String,
    }
});

module.exports = mongoose.model('Product', prodSchema);

拉取附加到 Controller 的数据 - (app.js)

/**
 * Module dependencies.
 */

/* Express */
var express = require('express');
var http = require('http');
var path = require('path');
var fs = require('fs');
var bcrypt = require('bcrypt-nodejs');
var moment = require('moment');
var os = require('os');

var config = require('./config');

/* Models */
var Product = require('./models/myModel');

var soap = require('soap');

var app = express();

/// Include the express body parser
app.configure(function () {
    app.use(express.bodyParser());
});

/* all environments */
app.engine('.html', require('ejs').__express);

app.set('port', process.env.PORT || 3000);
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'html');

app.use(express.favicon());
app.use(express.logger('dev'));
app.use(express.json());
app.use(express.urlencoded());
app.use(express.methodOverride());
app.use(express.cookieParser('your secret here'));
app.use(express.session());
app.use(app.router);
app.use(express.static(path.join(__dirname, 'public')));

/* DB Connect */
mongoose.connect( 'mongodb://localhost:27017/productDB', function(err){
    if ('development' == app.get('env')) {
        if (err) throw err;
        console.log('Successfully connected to MongoDB globally');
    }
} );

/* Close DB gracefully */
var gracefulExit = function() { 
  mongoose.connection.close(function () {
    console.log('Mongoose default connection with DB is disconnected through app termination');
    process.exit(0);
  });
}

// If the Node process ends, close the Mongoose connection
process.on('SIGINT', gracefulExit).on('SIGTERM', gracefulExit);

/* development only */
if ('development' == app.get('env')) {
    app.use(express.errorHandler());
}

/********************************************************/
/***** GET *****/
/********************************************************/

app.get('/getproducts', getProducts);

/* If GET on http://localhost:3000/getproducts the call the below function to get data from web service */
function getProducts(req, res){
    var post = req.body;
    var url = 'http://www.example.com/?wsdl';
    soap.createClient(url, function(err, client) {
        client.setSecurity(new soap.BasicAuthSecurity(User, Pass));
        client.someMethod(function(err, result) {
            var product = result.DATA.item;
            for(var i=0; i<product.length; i++) {
                var saveData = new Product({
                    rowIndex: product.ROW_INDEX,
                    prodId: product.PROD_ID,
                    prodDesc: product.PROD_DESC,
                    prodCategory: product.PROD_CATEGORY,
                 });
                 saveData.save();
            }           
        });
    });   
}

/* Create Server */
http.createServer(app).listen(app.get('port'), function(){    
    console.log('Express server listening on port ' + app.get('port') + ' in ' + app.get('env') + ' mode');
});

从网络服务返回的数据

[ { ROW_INDEX: '1',
    PROD_ID: 'A1',
    PROD_DESC: 'New product',
    PROD_CATEGORY: 'Clothes' },
  { ROW_INDEX: '2',
    PROD_ID: 'A2',
    PROD_DESC: 'New product 2',
    PROD_CATEGORY: 'Clothes' },
  { ROW_INDEX: '3',
    PROD_ID: 'A3',
    PROD_DESC: 'New product 3',
    PROD_CATEGORY: 'shoes' },
  .
  .
  . millions of rows
]

需要问题/建议

我面临的问题是，在所有数据都保存到数据库之前，服务器已被阻塞，并且没有其他功能(如为并发用户呈现页面或保存更多数据)执行。

我正在创建一个 View ，该 View 也将返回保存的数据。这些又将是数百万行数据 - 但这次是从 MongoDB 获取并传递给 EJS 中的 View 。

对于优化运行并行进程和计算大量数据的性能的任何帮助/建议，我们将不胜感激。

Answer 1

您的保存不是异步的。此行正在阻塞:

saveData.save();

相反，异步保存模型(保存完成后传递一个函数运行):

function getProducts(req, res){
    var post = req.body;
    var url = 'http://www.example.com/?wsdl';
    soap.createClient(url, function(err, client) {
        client.setSecurity(new soap.BasicAuthSecurity(User, Pass));
        client.someMethod(function(err, result) {
            var product = result.DATA.item;
            for(var i=0; i<product.length; i++) {
                var saveData = new Product({
                    rowIndex: product.ROW_INDEX,
                    prodId: product.PROD_ID,
                    prodDesc: product.PROD_DESC,
                    prodCategory: product.PROD_CATEGORY,
                });
                saveData.save( function (err, data) {
                    // any statements here will run when this row is saved,
                    // but the loop will continue onto the next product without
                    // waiting for the save to finish
                });
            }           
        });
    });   
    res.send("Getting the data! You can browse the site while you wait...");
}

这样，整个循环将(几乎)立即运行，并且数据将在数据进入时得到保存。同时，您的 Node 进程可以自由地处理其他网络请求。