python - 如何从 pycurl 多 curl 请求中获取响应正文

Question

在执行 curl 多请求时，我只能得到空响应。没有抛出异常，但响应值没有内容(在下面的代码段中注释)

这是我的代码的简化版本:

from StringIO import StringIO

import pycurl


class CurlStream(object):
    curl_count = 0
    curl_storage = []

    def __init__(self):
        self.curl_multi = pycurl.CurlMulti()

    def add_request(self, request, post_fields=None):
        self.curl_count += 1
        curl = self._create_curl(request, post_fields)
        self.curl_multi.add_handle(curl)

    def perform(self):
        while self.curl_count:
            while True:
                response, self.curl_count = self.curl_multi.perform()
                if response != pycurl.E_CALL_MULTI_PERFORM:
                    break
            self.curl_multi.select(1.0)

    def read_all(self):
        for response in self.curl_storage:
            print response.getvalue() # this does nothing --prints blank lines

    def close(self):
        self.curl_multi.close()

    def _create_curl(self, request, post_fields):
        curl = pycurl.Curl()
        output = StringIO()
        self.curl_storage.append(output)
        curl.setopt(curl.URL, request)
        curl.setopt(curl.WRITEFUNCTION, output.write)
        curl.setopt(curl.TIMEOUT, 20)
        return curl


def main():
    curl_stream = CurlStream()
    curl_stream.add_request('http://www.google.com')
    curl_stream.add_request('http://www.example.com')
    curl_stream.perform()
    curl_stream.read_all()
    curl_stream.close()

if __name__ == '__main__':
    main()

我在没有使用 curl multi 的情况下使用相同的选项发出了单个请求并且它有效。

Answer 1

好的，当我将 _create_curl 方法更改为此(添加 write_out 用于调试)时，我发现它有效:

def _create_curl(self, request, post_fields):
    curl = pycurl.Curl()
    curl.setopt(curl.URL, request)
    curl.setopt(curl.WRITEFUNCTION, self.write_out)
    curl.setopt(curl.TIMEOUT, 20)

    # Below is the important bit, I am now adding each curl object to a list
    self.curl_storage.append(curl)
    return curl

def write_out(self, data):
        print data
        return len(data)

问题是在将 curl 对象添加到 multicurl 对象时，我没有保留对单个 curl 对象的任何引用，因此它自动关闭了。

根据pycurl docs对于 curl close() 方法:

Corresponds to curl_easy_cleanup in libcurl. This method is automatically called by pycurl when a Curl object no longer has any references to it, but can also be called explicitly.