javascript - NODE.JS 'request' 未在数组中添加响应

Question

所以我正在尝试抓取 Steam 社区市场的数据。它适用于少量元素，但当我尝试 CS:GO(平均:13400 件元素)时却不起作用。我使用 Steam 的 JSON，它允许我一次获取 100 个项目。该代码执行以下操作:在循环内它会随机暂停(我知道在那段时间暂停的丑陋方法，但另一种方法卡住了我的循环)，然后我发送第一个请求。我检查是否收到响应，如果响应代码为 200(= 成功)，则我从 json 数组中获取每个项目，并检查它是否已在我的数组中，如果没有，我将添加它。

但问题就在这里。它似乎执行了所有请求循环，然后在最后它似乎执行了添加到数组中的所有项目。不执行请求，而是添加项目，然后启动新请求并继续。

const colors = require('colors/safe');
const request = require('request');
const random = require('random');
var fs = require('fs');

var itemNames = new Array();

console.log(colors.green(time() + 'Starting Steam Market Monitor...'));

request('https://steamcommunity.com/market/search/render/?search_descriptions=0&sort_column=default&sort_dir=desc&appid=730&norender=1&start=0&count=100', function(err, res, body) {
    if (res.statusCode == 200 && res.statusCode != undefined) {
    var itemsObj = JSON.parse(body);
    var totalItems = itemsObj['total_count'];
    console.log(colors.green(time() + 'Total of items found: ' + totalItems));
    getItems(totalItems);
    } else {
        console.log(colors.red(time() + 'Response code: ' + res.statusCode));
    }
});

function getItems(totalItems) {
    var status = 200; //default success message
    console.log(colors.green(time() + "Started item grabbing."));


    for (var i = 0; i < totalItems; i+= 100) {

        var options = {
            url: 'https://steamcommunity.com/market/search/render/?search_descriptions=0&sort_column=default&sort_dir=desc&appid=730&norender=1&start='+ i +'&count=100'
        };

        var dt = new Date();
        var randomSeconds = random.int(115, 125);
        dt.setSeconds( dt.getSeconds() + randomSeconds );
        while(dt > new Date()) {}
            console.log(colors.green(time() + "Getting items " + i + " out of " + totalItems + " items"));
                var dt = new Date();
                var randomSeconds = random.int(55, 65);
                dt.setSeconds( dt.getSeconds() + randomSeconds );
                while(dt > new Date()) {}
                request(options, function(err, res, body) {
                    if(res) {
                        if (res.statusCode == 200 && res.statusCode != undefined) {
                            var itemsObj = JSON.parse(body);
                            var itemsArr = itemsObj['results'];

                            for (var j = 0; j < 100; j++) {
                                var item = itemsArr[j]['hash_name'];
                                if (itemNames.indexOf(item) > -1) {
                                    if (j == 99) {
                                    }
                                } else {
                                    console.log(item);
                                    itemNames.push(item);
                                    if (j == 99) {
                                    }
                                }
                            }
                        } else {
                            console.log(colors.red(time() + 'Response code: ' + res.statusCode));
                        }
                    } else {
                        console.log(colors.red(time() + 'Undefined response...'));
                    }
                });
    }
}

Answer 1

您可以使用request-promise相反，您可以以异步方式运行请求，因此您的循环将等到请求结束

所以你的代码可以通过这种方式改进:

const colors = require('colors/safe');
const rp = require('request-promise');
const random = require('random');
var fs = require('fs');

var itemNames = new Array();

console.log(colors.green(time() + 'Starting Steam Market Monitor...'));

rp('https://steamcommunity.com/market/search/render/?search_descriptions=0&sort_column=default&sort_dir=desc&appid=730&norender=1&start=0&count=100',)
.then(function (res) {
    if (res.statusCode == 200 && res.statusCode != undefined) {
    var itemsObj = JSON.parse(body);
    var totalItems = itemsObj['total_count'];
    console.log(colors.green(time() + 'Total of items found: ' + totalItems));
    await getItems(totalItems);
    } else {
        console.log(colors.red(time() + 'Response code: ' + res.statusCode));
    }
})
.catch(function (err) {
        console.log(err);
    });

async function getItems(totalItems) {
    var status = 200; //default success message
    console.log(colors.green(time() + "Started item grabbing."));


    for (var i = 0; i < totalItems; i+= 100) {

        var options = {
            uri: 'https://steamcommunity.com/market/search/render/?search_descriptions=0&sort_column=default&sort_dir=desc&appid=730&norender=1&start='+ i +'&count=100'
        };

            console.log(colors.green(time() + "Getting items " + i + " out of " + totalItems + " items"));
                await rp(options)
               .then(function (res) {
                    if(res) {
                        if (res.statusCode == 200 && res.statusCode != undefined) {
                            var itemsObj = JSON.parse(body);
                            var itemsArr = itemsObj['results'];

                            for (var j = 0; j < 100; j++) {
                                var item = itemsArr[j]['hash_name'];
                                if (itemNames.indexOf(item) > -1) {
                                    if (j == 99) {
                                    }
                                } else {
                                    console.log(item);
                                    itemNames.push(item);
                                    if (j == 99) {
                                    }
                                }
                            }
                        } else {
                            console.log(colors.red(time() + 'Response code: ' + res.statusCode));
                        }
                    } else {
                        console.log(colors.red(time() + 'Undefined response...'));
                    }
                }).catch(function (err) {
                   console.log(err);
                });
    }
}