javascript - 在 NodeJS 上实现高性能代理

Question

我正在使用 NodeJS Luminati代理，但是当发生 502 错误时我需要 request another ip 。他们的文档本身对此有一个非常完整的示例，但它是如此完整，以至于让我感到困惑。

我无法使该示例适应我的场景。

这是他们的示例，也处理 502 错误:

var request = require('request-promise');
var promise = require('bluebird'); // promises lib used by request-promise
var lookup = promise.promisify(require('dns').lookup);
var http = require('http');
var username = 'myusername';
var password = 'mypassword';
var port = 22225;
var at_req = 0;
var n_total_req = 1000;
var n_parallel_exit_nodes = 100;
var switch_ip_every_n_req = 50;
var max_failures = 3;
var req_timeout = 60*1000;

function main(){
    http.Agent.defaultMaxSockets = Infinity;
    for (var i=0; i < n_parallel_exit_nodes; i++)
        new Session(i).start();
}

function Session(id){
    this.id = id;
    this.n_req_for_exit_node = 0;
    this.fail_count = 0;
    this.switch_session_id();
}

var proto = Session.prototype;

proto.start = proto.next = function(){
    if (at_req >= n_total_req)
        return this.cleanup(); // all done
    at_req++;
    var _this = this;
    promise.try(function(){
        if (!_this.have_good_super_proxy())
            return _this.switch_super_proxy();
    }).then(function(){
        if (_this.n_req_for_exit_node==switch_ip_every_n_req)
            _this.switch_session_id();
        var options = {
            url: 'http://lumtest.com/myip.json',
            timeout: req_timeout,
            pool: _this.pool,
            forever: true,
            proxy: _this.super_proxy_url,
        };
        return request(options);
    }).then(function success(res){
        console.log(res);
        _this.fail_count = 0;
        _this.n_req_for_exit_node++;
    }, function error(err){
        if (err.statusCode
            && !status_code_requires_exit_node_switch(err.statusCode))
        {
            // this could be 404 or other website error
            _this.n_req_for_exit_node++;
            return;
        }
        _this.switch_session_id();
        _this.fail_count++;
    }).finally(function(){
        _this.next();
    });
};

proto.have_good_super_proxy = function(){
    return this.super_proxy_host && this.fail_count < max_failures;
};

proto.update_super_proxy_url = function(){
    this.super_proxy_url = 'http://'+username+
        '-session-'+
        this.session_id+':'+password+'@'+this.super_proxy_host+':'+port;
};

proto.switch_session_id = function(){
    connection_pool_cleanup(this.pool);
    this.pool = {};
    this.session_id = (1000000 * Math.random())|0;
    this.n_req_for_exit_node = 0;
    this.update_super_proxy_url();
};

proto.switch_super_proxy = function(){
    var _this = this;
    this.switch_session_id();
    return promise.try(function(){
        return lookup('session-'+_this.session_id+
            '.'+
            'zproxy.luminati.io');
    }).then(function success(res){
        _this.super_proxy_host = res;
        _this.update_super_proxy_url();
    });
};

proto.cleanup = function(){
    connection_pool_cleanup(this.pool);
};

function connection_pool_cleanup(pool){
    if (!pool)
        return;
    for (let key in pool)
    {
        var sockets = pool[key].sockets;
        for (let name in sockets)
            sockets[name].forEach(s=>s.destroy());
    }
}

function status_code_requires_exit_node_switch(status_code){
    return [403, 429, 502, 503].indexOf(status_code)>=0;
}

main();

我的代码:

function add(req, res, next) {
  request({ url: 'myurl', proxy: 'myproxy' }, (err, response, body) => {
    if (err) {
      return next(err);
    }

    res.json(body);
  });
};

function list(req, res, next) {
  request({ url: 'myurl', proxy: 'myproxy' }, (err, response, body) => {
    if (err) {
      return next(err);
    }

    res.json(body);
  });
};

如何调整我的代码以与示例相同的方式请求另一个 IP？

Answer 1

代码可以这样编写以在失败时重试:

    const request = require('request-promise');
    const co = require('co');

    function add(req, res, next) {
         co(function*(){
             let last_err;
             const max_try = 10, max_fail = 3;
             for (let count=0, fail = 0; count<max_try; count++)
             {
                try {
                    let body = yield request({url: 'myurl', proxy: 'myproxy'});
                    res.json(body);
                    return;
                } catch(err){
                    last_err = err;
                    fail++;
                    if (fail==max_fail)
                    {
                        fail = 0;
                        proxy = 'myproxy2'; // switch proxy
                    }
                 }
              }
              next(last_err);
           });
    };