javascript - 如何在 nodejs 中的 async.each 中进行同步 http 调用

Question

我想向 API 发出 http 请求，为每个用户收集数据并插入到 mongodb 中。

我遇到的问题是，它同时执行所有请求，似乎卡在某个地方，我不知道发生了什么。

虽然我正在使用异步库并在每次迭代中添加 request() 方法，但我不知道这是否正确，这是代码:

  function iterateThruAllStudents(from, to) {
    Student.find({status: 'student'})
        .populate('user')
        .exec(function (err, students) {
            if (err) {
                throw err;
            }

            async.forEach(students, function iteratee(student, callback) {
                if (student.worksnap.user != null) {
                    var options = {
                        url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
                        headers: {
                            'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
                        }
                    };
                    request(options, getTimeEntriesFromWorksnap);
                }
                callback(); // tell async that the iterator has completed
            }, function (err) {
                console.log('iterating done');
            });
        });
}

    function getTimeEntriesFromWorksnap(error, response, body) {
        console.log(response.statusCode);
        if (!error && response.statusCode == 200) {
            parser.parseString(body, function (err, results) {
                var json_string = JSON.stringify(results.time_entries);
                var timeEntries = JSON.parse(json_string);
                _.forEach(timeEntries, function (timeEntry) {
                    _.forEach(timeEntry, function (item) {
                        saveTimeEntry(item);
                    });
                });
            });
        }
    }

    function saveTimeEntry(item) {
        Student.findOne({
                'worksnap.user.user_id': item.user_id[0]
            })
            .populate('user')
            .exec(function (err, student) {
                if (err) {
                    throw err;
                }
                student.timeEntries.push(item);
                student.save(function (err) {
                    if (err) {
                        console.log(err);
                    } else {
                        console.log('item inserted...');
                    }
                });

            });
    }

var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);

我是 JavaScript 的新手，尤其是在处理异步时。

有什么帮助吗？

Answer 1

使用 Async.eachLimit() 向 API 发出批量请求...试试这个 iterateThruAllStudents() 函数。

我之前已经有过同样的问题here

限制教程见here .虽然我将限制设置为 5，但你可以做任何你想做的事情(10,50 等)。

function iterateThruAllStudents(from, to) {
  Student.find({status: 'student'})
    .populate('user')
    .exec(function (err, students) {
      if (err) {
        throw err;
      }
      async.eachLimit(students,5,function iteratee(student, callback) {
        if (student.worksnap.user != null) {
          var options = {
            url: 'https://api.worksnaps.com/api/projects/' + project_id + '/time_entries.xml?user_ids=' + student.worksnap.user.user_id + '&from_timestamp=' + from + '&to_timestamp=' + to,
            headers: {
              'Authorization': 'Basic bGhNSVJkVUFwOE1DS2loOFVyZkFyOENEZEhPSXdCdUlHdElWMHo0czo='
            }
          };
          request(options,getTimeEntriesFromWorksnap(callback));
        }
      }, function (err) {
        console.log(err);
        console.log('iterating done');
      });
    });
}

function getTimeEntriesFromWorksnap(cb) {
  return function(error, response, body){
    console.log(response.statusCode);
    if (!error && response.statusCode == 200) {
      parser.parseString(body, function (err, results) {
        var json_string = JSON.stringify(results.time_entries);
        var timeEntries = JSON.parse(json_string);
        async.each(timeEntries,function(timeEntry,cb1){
          async.each(timeEntry,function(item,cb2){
            saveTimeEntry(item,cb2);
          },function(err){
            if(err)
              cb1(err);
            else
              cb1();
          })
        },function(err){
          if(err)
            cb(err);
          else
            cb();
        });
        //_.forEach(timeEntries, function (timeEntry) {
        //  _.forEach(timeEntry, function (item) {
        //    saveTimeEntry(item);
        //  });
        //});
      });
    }
    cb(null);
  }
}

function saveTimeEntry(item,cb2) {
  Student.findOne({
      'worksnap.user.user_id': item.user_id[0]
    })
    .populate('user')
    .exec(function (err, student) {
      if (err) {
        return cb2(err);
      }
      student.timeEntries.push(item);
      student.save(function (err) {
        if (err) {
          console.log(err);
          //return cb2(err);//Do it if you wanna throw an error.
        } else {
          console.log('item inserted...');
        }
        cb2();
      });
    });
}

var from = new Date(startDate).getTime() / 1000;
startDate.setDate(startDate.getDate() + 30);
var to = new Date(startDate).getTime() / 1000;
iterateThruAllStudents(from, to);