python - gevent和线程之间的效率比较

Question

最近，我正在开发一个gevent演示，我试图比较gevent和线程之间的效率。一般而言，gevent代码应比线程代码更高效。但是，当我使用time命令对程序进行概要分析时，得到了异常的结果(我的命令是time python FILENAME.py 50 1000，最后两个参数表示池号或线程号，因此我在下表中更改了两个号)。结果表明该线程比gevent代码更有效，所以我想知道为什么会发生这种情况以及我的程序出了什么问题？谢谢。

gevent VS线程

我的代码如下(主要思想是使用线程或gevent发送多个HTTP请求):

******这是线程版本代码******

# _*_ coding: utf-8 _*_
import sys
reload(sys)
sys.setdefaultencoding("utf8")
import requests
import threading
import time
import urllib2

finished = 0


def GetUrl(pagenum):
    url = 'http://opendata.baidu.com/zhaopin/s?p=mini&wd=%B0%D9%B6%C8&pn=' + \
        str(pagenum*20) + '&rn=20'
    return url


def setUrlSet():
    for i in xrange(requestnum):
        urlnum = i % 38
        urlset.append(GetUrl(urlnum))


def GetResponse(pagenum):
    try:
        r = requests.get(urlset[pagenum])
    except Exception, e:
        print e
    pass


def DigJobByPagenum(pagenum, requestnum):
    init_num = pagenum
    print '%d begin' % init_num
    while pagenum < requestnum:
        GetResponse(pagenum)
        pagenum += threadnum
    print '%d over' % init_num


def NormalThread(threadnum):
    startime = time.time()
    print "%s is running..." % threading.current_thread().name
    threads = []
    global finished, requestnum
    for i in xrange(threadnum):
        thread = threading.Thread(target=DigJobByPagenum, args=(i, requestnum))
        threads.append(thread)
    for t in threads:
        t.daemon = True
        t.start()
    for t in threads:
        t.join()
        finished += 1
    endtime = time.time()
    print "%s is stop.The total time is %0.2f" % \
        (threading.current_thread().name, (endtime - startime))


def GetAvageTime(array):
    alltime = 0.0
    for i in array:
        alltime += i
    avageTime = alltime/len(array)
    return avageTime

if __name__ == '__main__':
    threadnum = int(sys.argv[1])
    requestnum = int(sys.argv[2])
    print 'threadnum : %s,requestnum %s ' % (threadnum, requestnum)
    originStartTime = time.time()
    urlset = []
    setUrlSet()
    NormalThread(threadnum)

******这是gevent版本代码******

# _*_ coding: utf-8 _*_
import sys
reload(sys)
sys.setdefaultencoding("utf8")
from gevent import monkey
monkey.patch_all()
import gevent
from gevent import pool
import requests
import time

finished = 0


def GetUrl(pagenum):
    url = 'http://opendata.baidu.com/zhaopin/s?p=mini&wd=%B0%D9%B6%C8&pn=' + \
        str(pagenum*20) + '&rn=20'
    return url


def setUrlSet():
    for i in xrange(requestnum):
        urlnum = i % 38
        urlset.append(GetUrl(urlnum))


def GetResponse(url):
    startime = time.time()
    r = requests.get(url)
    print url
    endtime = time.time()
    spendtime = endtime - startime
    NormalSpendTime.append(spendtime)
    global finished
    finished += 1
    print finished


def GetAvageTime(array):
    alltime = 0.0
    for i in array:
        alltime += i
    avageTime = alltime/len(array)
    return avageTime


def RunAsyncJob():
    jobpool = pool.Pool(concurrent)
    for url in urlset:
        jobpool.spawn(GetResponse, url)
    jobpool.join()
    endtime = time.time()
    allSpendTime = endtime - originStartime
    print 'Total spend time is %0.3f, total request num is %s within %s \
            seconds' % (allSpendTime, finished, timeoutNum)
    print 'Each request time is %0.3f' % (GetAvageTime(NormalSpendTime))


if __name__ == '__main__':
    concurrent = int(sys.argv[1])
    requestnum = int(sys.argv[2])
    timeoutNum = 100
    NormalSpendTime = []
    urlset = []
    urlActionList = []
    setUrlSet()
    originStartime = time.time()
    RunAsyncJob()

Answer 1

尝试

gevent.monkey.patch_all(httplib=True)

似乎默认情况下，gevent不会修补httplib(请查看 http://www.gevent.org/gevent.monkey.html: httplib=False)，因此您实际上正在执行阻塞请求，并且失去了异步框架的所有优势。尽管我不确定请求是否使用httplib。

如果这不起作用，请查看以下库:

https://github.com/kennethreitz/grequests