Python 队列问题 - queue.get 不删除对象

Question

我正在创建一个线程化的 python 脚本，它将一些主机/ips 放入队列中，然后未知数量的线程(默认为 10)从队列中获取这些主机并执行 snmp 查询。 Snmp 查询正在运行，但每个线程都在查询所有主机。主机只想接收来自随机线程的单个调用。在这里，我从每个线程获得对主机的调用。

#!/usr/bin/env python
import Queue
import threading
import urllib2
import time
import datetime
import netsnmp
import pprint

queue = Queue.Queue()

class ThreadUrl(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
    def run(self):
        while True:
      #grabs host from queue
            item = self.queue.get()
            print item

      #grabs urls of hosts and prints first 1024 bytes of page
            dane = get_rittal(item)
            obj_in_que = self.queue.qsize()
            #print obj_in_que 
            print dane + "\n"

      #signals to queue job is done
            self.queue.task_done()

def log(message):
    now = datetime.datetime.now().strftime("%H:%M:%S")
    print "%s %s\n" % (now, message)



def convert(val,div):
    string = str(val)
    f = '%05.2f' % (float(string)*div)
    return f

def get_rittal(rittal):
    x = netsnmp.Session(Version=1,DestHost=rittal,Community='',Timeout=25000, Retries=3)
    oid_loadA = netsnmp.Varbind('.1.3.6.1.4.1.2606.100.1.2.2.1.7.1',1)
    oid_loadB = netsnmp.Varbind('.1.3.6.1.4.1.2606.100.1.2.2.1.7.1',2)
    oid_loadC = netsnmp.Varbind('.1.3.6.1.4.1.2606.100.1.2.2.1.7.1',3)

    all_oids = netsnmp.VarList(oid_loadA, oid_loadB, oid_loadC)

    get = list(x.get(all_oids))

    loadA = convert(get[0],0.01)
    loadB = convert(get[1],0.01)
    loadC = convert(get[2],0.01)

    res = 'Host:' + rittal + '\t\tLA:' + loadA + 'A LB:' + loadB + 'A LC:' + loadC + 'A'
    return res

start = time.time()
def main():
    list=[]
    for line in open ("test", "r").readlines():
        for h in line.rstrip('\n').split(' '):
            #print h
            list.append(h)
    x = str(len(list))
    print "Ilosc rittali = " + x
    print list;

  #spawn a pool of threads, and pass them queue instance 
    for i in range(10):
        t = ThreadUrl(queue)
        t.setDaemon(True)
        t.start()

 #populate queue with data   
        for item in list:
            queue.put(item)

 #wait on the queue until everything has been processed     
        queue.join()

main()
print "Elapsed Time: %s" % (time.time() - start)

示例输出:

['host1', 'host2', 'host3']
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.50A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host3      LA:00.75A LB:01.13A LC:09.63A
Host:host1      LA:00.00A LB:00.00A LC:00.38A
Host:host2      LA:00.00A LB:00.00A LC:00.00A
Host:host3      LA:00.75A LB:01.13A LC:09.63A

Elapsed Time: 0.634697914124


# python.py | grep Host | sort | uniq -c
     10 Host:host1      LA:00.00A LB:00.00A LC:00.38A
     10 Host:host2      LA:00.00A LB:00.00A LC:00.00A
      5 Host:host3      LA:00.75A LB:01.13A LC:09.50A
      5 Host:host3      LA:00.75A LB:01.13A LC:09.63A

Answer 1

仔细检查你的缩进。

每次初始化一个新的工作线程时，您都会将整个主机列表排入队列。此外，您可以通过 join()ing 队列来序列化这些线程:

for i in xrange(10):
  t = ThreadURL(q)
  ...
  for item in host_list:   # oops
    q.put(item)
  q.join()                 # oops