node.js - MongoError : server instance pool was destroyed

Question

我在 this question 的帮助下构建了这个应用程序我以前做过。

app.js:

var mongolib = require('./middlewares/db.js');
var downloaderCoverageWho = require('./routers/downloaderCoverageWho.js');
var downloaderCoverageIta = require('./routers/downloaderCoverageIta.js');

const start = async function() {
    const conn = await mongolib.connectToMongoDb();
    const coverages = await mongolib.createACollection('coverages');
    const isPageHasUpdates = true; 
    if(isPageHasUpdates) {
        await downloadCoverageIta();
        await downloadCoverageWho();
    }
    await mongolib.closeConnection();
    await console.log('d3.js creation...');
    return 'FINISH';
}
start()
.then(res => console.log(res)) 
.catch(err => console.log(err)); 

async function downloadCoverageWho() {
    await downloaderCoverageWho.download();
    console.log('Finish');
}
async function downloadCoverageIta() {
    await downloaderCoverageIta.download();
    console.log('Finish');
}

db.js:

var fs = require('fs');
var MongoClient = require('mongodb').MongoClient;
const url = 'mongodb://localhost:27017/';
const dbName = 'db';
var collCovName = 'coverages';
var collCov;
let myClient;
let myConn;

var methods = {};

methods.getConnection = function() {
    return myConn;
}

methods.connectToMongoDb = async function() {
    return MongoClient.connect(url + dbName)
    .then(function(conn) {
        console.log('Connected to MongoDB');
        myClient = conn;
        myConn = myClient.db(dbName);
        return myConn;
    })
    .catch(function(err) {
        console.log('Error during connection');
        throw err;
    });
}

methods.closeConnection = async function() {
    myClient.close()
    .then(function() {
        console.log('Connection closed');
    })
    .catch(function(err) {
        console.log('Error closing connection');
        throw err;
    });
}

methods.createACollection = async function(collectionName) {
    return myConn.createCollection(collectionName)
    .then(function() {
        console.log('Collection', collectionName, 'created');
    })
    .catch(function(err) {
        console.log('Error during creation of collection', collectionName);
        throw err;
    });
}

methods.insert = async function(collectionName, obj) {
    return myConn.collection(collectionName).updateOne(obj, {$set: obj}, {upsert: true})
    .then(function(res) {
        console.log('Inserted 1 element in', collectionName);
    })
    .catch(function(err) {
        console.log('Error during insertion in', collectionName);
        throw err;
    });
}

module.exports = methods;

下载CoverageIta.js:

var cheerio = require('cheerio');
var express = require('express');
var fs = require('fs');
var request = require('request');
var textract = require('textract');

var util = require('../helpers/util.js');
var mongolib = require('../middlewares/db.js');

var methods = {};

var outDir = './output/';
var finalFilename = outDir + 'coverage-ita.json'

var urls = [
    {year: '2013', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_0_fileAllegati_itemFile_1_file.pdf'}, 
    {year: '2012', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_5_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2011', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_6_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2010', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_7_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2009', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_8_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2008', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_15_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2007', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_14_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2006', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_13_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2005', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_12_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2004', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_11_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2003', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_10_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2002', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_9_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2001', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_1_fileAllegati_itemFile_0_file.pdf'}, 
    {year: '2000', link: 'http://www.salute.gov.it/imgs/C_17_tavole_20_allegati_iitemAllegati_0_fileAllegati_itemFile_0_file.pdf'}
];

var jsons = [];

methods.download = async function(req, res) {
    jsons = await extractText()
    .then(function() {
        console.log('Extract text success');
    })
    .catch(function() {
        console.log('Extract text error');
    });
}; 

async function extractText() {
    var config = {
        preserveLineBreaks: true
    };

    //await extractTextTest();

    await urls.forEach(async function(url) {
        await textract.fromUrl(url.link, config, async function(error, text) {
            if(error) {
                throw error;
            }
            switch(url.year) {
                case '2000': 
                case '2001': 
                case '2002': 
                case '2003':
                case '2004': 
                case '2005': 
                case '2006': 
                case '2007': 
                case '2008': 
                case '2009': 
                case '2010': 
                case '2011': 
                case '2012':
                    await extractTextType1(url, text)
                    .then(function() {
                        console.log('extractTextType1 success');
                    })
                    .catch(function() {
                        console.log('extractTextType1 error');
                    });
                    break;

                case '2013': 
                    extractTextType2(url, text)
                    .then(function() {
                        console.log('extractTextType2 success');
                    })
                    .catch(function() {
                        console.log('extractTextType2 error');
                    });
                    break;

                default:
                    console.log('Error: no case');
            }
        });
    });
}

async function extractTextTest() { // THIS WORKS
    var obj = {A: 'aa', B: 'bb', C: 'cc'};
    await mongolib.insert('coverages', obj);
}

async function extractTextType1(url, text) {
    var matrix = [];
    var map = [];
    var vaccines = [];
    var regionsTemp = [];
    var regions = [];
    var regionLength = [1, 2, 1, 2, 3, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]; 

    // text to matrix
    var textArray = text.split('\n');
    for(var i = 0; i < 23; i++) {
        matrix[i] = textArray[i].split(' ');
    }

    matrix[0].shift();
    vaccines = matrix[0];
    map[0] = vaccines;

    for(var i = 0; i < regionLength.length; i++) {
        var j = i + 1; 
        var indexToRemove = 0;
        var numberToRemove = regionLength[i];
        var region = matrix[j].splice(indexToRemove, numberToRemove);
        regionsTemp.push(region);
        map[j+1] = matrix[j];
    }

    for(var i = 0; i < regionsTemp.length; i++) {
        var region = '';
        if(regionLength[i] > 1) {
            region = regionsTemp[i].join(' ');
        }
        else {
            region = regionsTemp[i].join('');
        }
        regions.push(region);
    }
    map[1] = regions;

    vaccines = map.shift(); 
    regions = map.shift(); 

    var thisJson = await map.reduce(function(result, v, i) {
        v.forEach(function(o, k) {
            var obj = util.createJsonObjectCoverage(url.year, 'Italy', vaccines[k], regions[i], o);
            // save on db
            mongolib.insert('coverages', obj); // HERE
            result.push(obj);
        });
        return result;
    }, jsons);

    util.printOnFile(jsons, finalFilename);
}

function extractTextType2(url, text) {
    var matrix = [];
    var map = [];
    var vaccines = [];
    var regions = [];

    var textArray = text.split('\n');
    for(var i = 0; i < 36; i++) {
        matrix[i] = textArray[i].split(' ');
    }

    vaccines.push(matrix[0][1].replace(/\(a\)/g, '').replace(/\(b\)/g, '').replace(/\(c\)/g, '').replace(/\r/g, ''));
    for(var i = 1; i < 10; i++) {
        vaccines.push(matrix[i][0].replace(/\(a\)/g, '').replace(/\(b\)/g, '').replace(/\(c\)/g, '').replace(/\r/g, ''));
    }
    var meningo = ''.concat(matrix[10][0], matrix[11]).replace(/\(a\)/g, '').replace(/\(b\)/g, '').replace(/\(c\)/g, '').replace(/\r/g, '');
    vaccines.push(meningo);
    var pneumo = ''.concat(matrix[12][0], ' ', matrix[13]).replace(/\(a\)/g, '').replace(/\(b\)/g, '').replace(/\(c\)/g, '').replace(/\r/g, '');
    vaccines.push(pneumo);
    map[0] = vaccines;

    for(var i = 14; i < matrix.length; i++) { 
        regions.push(matrix[i][0]);
    }
    map[1] = regions;
    for(var i = 14; i < matrix.length; i++) {
        matrix[i].shift();
        map.push(matrix[i]);
    }

    vaccines = map.shift(); 
    regions = map.shift(); 

    var thisJson = map.reduce(function(result, v, i) {
        v.forEach(function(o, k) {
            var obj = util.createJsonObjectCoverage(url.year, 'Italy', vaccines[k], regions[i], o);
            // save on db
            mongolib.insert('coverages', obj); // HERE
            result.push(obj);
        });
        return result;
    }, jsons);

    util.printOnFile(jsons, finalFilename);
}

module.exports = methods;

downloaderCoverageWho.js:

var cheerio = require('cheerio');
var express = require('express');
var fs = require('fs');
var request = require('request');

var util = require('../helpers/util.js');
var mongolib = require('../middlewares/db.js');

var methods = {};
var countries = {
    'Albania': 'ALB',
    'Austria': 'AUT'
};

var outDir = './output/';
var finalData = outDir + 'coverage-eu.json'
var jsons = [];

methods.download = async function(req, res) {
    for(country in countries) {
        var url = 'http://apps.who.int/immunization_monitoring/globalsummary/coverages?c=' + countries[country];
        request(url, (function(country) {
            var thisCountry = country;

            return function(error, res, html) {
                if(error) {
                    throw error;
                }
                $ = cheerio.load(html);
                var years = [];
                var vaccines = [];
                var coverages = [];
                $('.ts .year').each(function() {
                    years.push($(this).text().trim());
                });
                $('.ts .odd td a, .ts .even td a').each(function() {
                    vaccines.push($(this).text().trim());
                });
                $('.ts .odd .statistics_small, .ts .even .statistics_small').each(function() {
                    coverages.push($(this).text().trim());
                });

                const numYears = years.length;
                const numVaccines = vaccines.length;
                for(var vaccineIdx = 0; vaccineIdx < numVaccines; vaccineIdx++) {
                    for(var yearIdx = 0; yearIdx < numYears; yearIdx++) {
                        let obj = {
                            year: years[yearIdx],
                            country: country,
                            region: "",
                            vaccine: vaccines[vaccineIdx],
                            coverage: coverages[vaccineIdx*numYears + yearIdx]
                        }
                        jsons.push(obj);
                        // save on db
                        mongolib.insert('coverages', obj); // PROBLEM HERE
                    } 
                }
                util.printOnFile(jsons, finalData);
            } 
        })(country)); 
    } 
}; 

module.exports = methods;

当我运行代码时，我得到了这个错误:

(node:11952) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 2950): MongoError: server instance pool was destroyed

我认为这两个文件(downloaderCoverageIta 和 downloaderCoverageWho)存在相同的问题。

我读过 here可能我在 inserts 完成之前调用了 db.close() ，但这不是真的。我不知道如何解决它。

这是我第一次使用 async/await。我该如何解决？

Answer 1

我找到了问题...

我忘记了 javaScript 是异步的这一事实 :)mongo.close() 是问题所在。我评论了它，它起作用了