python - multiprocessing:每个任务的进度条

Question

我有一个实现迭代算法的类。由于执行需要一段时间，而且我需要对多次执行的结果进行平均，所以我决定使用 multiprocessing。问题是我希望每次执行都有一个进度条(或不那么花哨的东西)。像这样的东西:

experiment 1 [##########] 60%
experiment 2 [#############] 70%
experiment 3 [###] 20%

我的类看起来像这样(请注意，我已经使用了进度条，我想将其保留在那里，以便在不并行化时继续工作):

from __future__ import division
from time import sleep

class Algo():

    def __init__(self, total_iters, arg1, arg2, name):
        self.total_iters = total_iters
        self.arg1 = arg1
        self.arg2 = arg2
        self.name = name

    def step(self, iteration):
            """
            One iteration of Algorithm
            """
            # Progress bar
            completed = 100*iteration/self.total_iters
            if completed == 0: print ""
            print '\r {2} [{0}] {1}%'.format('#'*(int(completed/5)), completed, self.name),

            # Do some stuff
            sleep(0.001)

    def run(self):
        for i in xrange(self.total_iters):
            self.step(i)

        #  Output the final result in unique file

这是我的尝试:

import multiprocessing as mp 

if __name__ == "__main__":

    algo1 = Algo(200, 0,0, "test1")
    pool = mp.Pool(processes=3)
    for i in xrange(3):
        pool.apply_async(algo1.run) # in real life run will be passed N arguments

    pool.close()
    pool.join()

有什么想法吗？

PS:我正在努力避免诅咒

Answer 1

快速而肮脏，在 python 3 中，但你会明白的；)

import random
import time
import multiprocessing
import os
import collections


class Algo(multiprocessing.Process):
    def __init__(self, steps, name, status_queue):
        multiprocessing.Process.__init__(self)
        self.steps = steps
        self.name = name
        self.status_queue = status_queue

    def step(self, step):
            # Progress bar
            self.status_queue.put((self.name, (step+1.0)/self.steps))
            # Do some stuff
            time.sleep(0.1)

    def run(self):
        for i in range(self.steps):
            self.step(i)


def print_progress(progress):
    # Windows:
    os.system('cls') 
    for name, percent in progress.items():
        percent = int(percent * 100)
        bar = ('#' * int(percent/10)) + (' ' * (10 - int(percent/10)))
        print("{}: [{}] {}%".format(name, bar, percent))

if __name__ == "__main__":
    status = multiprocessing.Queue()
    progress = collections.OrderedDict()
    algos = [Algo(random.randrange(100, 200), "test" + str(i), status) for i in range(3)]

    for a in algos:
        a.start()

    while any([a.is_alive() for a in algos]):
        while not status.empty():
            name, percent = status.get()
            progress[name] = percent
            print_progress(progress)

        time.sleep(0.1)