node.js - 使用 Node.js 分块/流 API 数据

Question

我们有一个要求，需要编写一个 Node 应用程序，它可以从数据库中读取图像的 URL(大约超过一百万)。使用 image-size npm 包检索图像元数据，如高度、宽度。这里应该有一个可以列出结果的API。

我能够控制台日志数据，但是当我将其转换为 API 时，我需要对数据进行分块，以便它可以开始出现在浏览器上，但我无法做到这一点并需要帮助。这是我的代码

var express = require('express');

var url = require('url');
var http = require('http');
var sizeOf = require('image-size');
const sql = require('mssql');
var app = express();
var port = process.env.PORT || 3000;
const hostname = 'localhost';

var config1 = {
    user: '*********',
    password: '*********',
    server: '*********', 
    database: '*******',
    port: 1433,
    debug: true,
    options: {
        encrypt: false // Use this if you're on Windows Azure
    }
};

app.get('/', function(req, res){

    //res.writeHead(200, { 'Content-Type': 'application/json' });

var finalResult = [];
sql.close();
sql.connect(config1, function (err) {
        
        if (err) console.log(err);
        const request = new sql.Request()
        var myQuery = `select imagename from media`;

        request.stream = true;
        request.query(myQuery);
        request.on('row', row => {
            
            //console.log('Image : ' + row.ImageUrl);    
            if (row.ImageUrl != ''){
                if (row.ImageUrl.indexOf('http') < 0)
                    row.ImageUrl = "http:" + row.ImageUrl;
                    
                var options = url.parse(row.ImageUrl);          
                
                
                http.get(options, function (response) {
                    
                    if (response.statusCode == 200)
                    {
                        var chunks = [];
                        response.on('data', function (chunk) {
                        chunks.push(chunk);
                        }).on('end', function() {
                            var buffer = Buffer.concat(chunks);
                            //console.log(options.href);
                            //console.log(sizeOf(buffer).height);
                            var result = {};
                            result.MaskUrl = row.MaskUrl;
                            result.ImageUrl = options.href;
                            result.Height = sizeOf(buffer).height;    
                            result.Width = sizeOf(buffer).width;    
                            result.statusCode = 200;
                            finalResult.push(result);
                            //console.log(result);
                            console.log(finalResult);
                            res.write(result, function(){
                                res.end();
                            });
                            });
                    }
                    else
                    {
                        var result = {};
                        result.MaskUrl = row.MaskUrl;
                        result.ImageUrl = options.href;
                        result.Height = 0;
                        result.Width = 0;
                        result.statusCode = response.statusCode;
                        finalResult.push(result);
                        console.log(result);
                        res.write(result, function(){
                            res.end();
                        });
                        
                }

                    
                });

            }
        })

        request.on('error', err => {
            
            console.log ('Error for ' + row.ImageUrl );
        })

        request.on('done', err => {
            console.log('Last Time' + finalResult.length);
        })

        // request.query(myQuery,(err,result) =>{
        //     console.log(result);
        // });
        
    });
    console.log('Last Time' + finalResult.length);
    res.send(finalResult);
});

app.listen(port, hostname, function(){
    console.log('ImageSize running on PORT: ' + port);
});

 

我尝试了 res.write、res.end 但没有成功。

Answer 1

您的问题的可能原因如下:

res.write(result, function(){
    res.end();
});

您在读取第一张图像后立即结束并关闭请求。

我会稍微重写代码并使用一些功能框架，例如 scramjet ，直接从数据库传输数据。正如尼古拉斯指出的那样，运行代码并不容易，所以我盲目地编写 - 但如果你修复了我的任何明显错误，这应该可以工作:

第一:

npm install scramjet JSONStream node-fetch

接下来，尝试以下代码:

var express = require('express');
var sizeOf = require('image-size');
const sql = require('mssql');
var app = express();
var port = process.env.PORT || 3000;
const hostname = 'localhost';
const {DataStream} = require('scramjet');
const fetch = require('node-fetch');
const JSONStream = require('JSONStream');

var config1 = {
    user: '*********',
    password: '*********',
    server: '*********',
    database: '*******',
    port: 1433,
    debug: true,
    options: {
        encrypt: false // Use this if you're on Windows Azure
    }
};

app.get('/', function(req, res, next){

    // you should consider not doing these two lines on each request, 
    // but I don't want to mess you code...
    sql.close(); 
    sql.connect(config1, function (err) {
        if (err) next(err);

        res.writeHead(200, { 'Content-Type': 'application/json' });

        const request = new sql.Request();
        var myQuery = `select imagename from media`;

        request.stream = true;
        request.query(myQuery);

        const stream = new DataStream();
        request.on('row', row => stream.write(row));

        stream.filter(
                row => row.ImageUrl !== ''
            )
            .map(
                async row => {
                    if (row.ImageUrl.indexOf('http') !== 0) // url must start with http.
                        row.ImageUrl = "http:" + row.ImageUrl;

                    const response = await fetch(row.ImageUrl);
                    let size = {width:0, height:0};

                    if (response.status === 200) {
                        const buffer = await response.buffer();
                        size = sizeOf(buffer);
                    }

                    return {
                        MaskUrl: row.MaskUrl,
                        ImageUrl: row.ImageUrl,
                        Height: size.height,
                        Width: size.width,
                        statusCode: response.status
                    };

                }
            )
            .pipe(
                JSONStream.stringify()
            ).pipe(
                res
            );


        request.on('error', () => {
            res.writeHead(500, { 'Content-Type': 'application/json' });
            stream.end("{error:true}");
        });

        request.on('done', () => stream.end());

    });
});

app.listen(port, hostname, function(){
    console.log('ImageSize running on PORT: ' + port);
});