python - 如何使用多线程更新进度条以及下载文件

Question

我正在尝试使用多线程的请求模块下载文件，同时还需要更新进度栏。但是在执行此操作时，每当我按下提交按钮时，我的 GUI 就会停止工作，并且在后台文件开始下载，并且只有在下载之后，我的进度栏才会更新并且 GUI 会继续正常工作。

我也尝试将多线程应用于GUI，以便显示下载文件的进度，但它只是在整个进程结束后更新进度。

from datetime import datetime
from threading import Thread
import dask.dataframe as dd
from pubsub import pub
from queue import Queue
import pandas as pd
import requests
import shutil
import sqlite3
import time
import os
import csv
import wx


class MainPage(wx.Dialog):
    def __init__(self):
        wx.Dialog.__init__(self, None, title="File Selector")

        user_sizer = wx.BoxSizer(wx.HORIZONTAL)
        user_label = wx.StaticText(self, label='Number of Threads:')
        user_sizer.Add(user_label, 0, wx.ALL | wx.CENTER, 5)
        self.threads = wx.TextCtrl(self)
        user_sizer.Add(self.threads, 5, wx.ALL, 5)

        button_sizer = wx.BoxSizer(wx.HORIZONTAL)
        file_label = wx.StaticText(self, label='Select File:')
        button_sizer.Add(file_label, 0, wx.ALL | wx.CENTER, 5)
        self.select = wx.TextCtrl(self)
        button_sizer.Add(self.select, 0, wx.ALL | wx.CENTER, 10)
        file_select = wx.Button(self, label='Browse')
        file_select.Bind(wx.EVT_BUTTON, self.file)
        button_sizer.Add(file_select, 0, wx.ALL | wx.CENTER, 5)

        link_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
        link_choice_label = wx.StaticText(self, label='Link Column:')
        link_choice_sizer.Add(link_choice_label, 0, wx.ALL | wx.CENTER, 5)
        self.link_choice = wx.TextCtrl(self)
        link_choice_sizer.Add(self.link_choice, 0, wx.ALL | wx.CENTER, 5)

        file_choice_sizer = wx.BoxSizer(wx.HORIZONTAL)
        file_choice_label = wx.StaticText(self, label='File column:')
        file_choice_sizer.Add(file_choice_label, 0, wx.ALL | wx.CENTER, 5)
        self.file_choice = wx.TextCtrl(self)
        file_choice_sizer.Add(self.file_choice, 0, wx.ALL | wx.CENTER, 5)

        submit_sizer = wx.BoxSizer(wx.HORIZONTAL)
        submit_button = wx.Button(self, label='Submit')
        submit_button.Bind(wx.EVT_BUTTON, self.process)
        submit_sizer.Add(submit_button, 0, wx.TOP, 20)

        main_box = wx.BoxSizer(wx.VERTICAL)
        main_box.Add(user_sizer, 0, wx.ALL, 3)
        main_box.Add(button_sizer, 0, wx.LEFT, 43)
        main_box.Add(link_choice_sizer, 0, wx.LEFT, 39)
        main_box.Add(file_choice_sizer, 0, wx.LEFT, 43)
        main_box.Add(submit_sizer, 0, wx.LEFT, 135)

        self.user_sizer = user_sizer
        self.button_sizer = button_sizer
        self.SetSizer(main_box)
        self.Center()


    def file(self, event):
        dialog = wx.FileDialog(None, "Choose a File:", wildcard="CSV files (*.csv)|*.csv", style=wx.DD_DEFAULT_STYLE | wx.DD_NEW_DIR_BUTTON)
        if dialog.ShowModal() == wx.ID_OK:
            print(dialog.GetPath())
            path = dialog.GetPath()
            self.select.ChangeValue(path)
            print(path)
        dialog.Destroy()


    def process(self, event):
        selected_path = self.select.GetValue()
        thread = self.threads.GetValue()
        link = self.link_choice.GetValue()
        file = self.file_choice.GetValue()
        print(thread)
        print(link)
        print(file)
        if (thread != ""):
            if (selected_path != "") :
                if (link != "") :
                    if (file != "") :
                        if (thread.isdigit()) and (link.isdigit()) and (file.isdigit()) :
                            if (int(link) != int(file)):
                                if selected_path.endswith('.csv'):
                                    self.threads.Clear()
                                    self.select.Clear()
                                    self.link_choice.Clear()
                                    self.file_choice.Clear()
                                    print("out")
                                    print(file)
                                    wx.LogMessage("File processed Sucessfully")
                                    # self.csv_processing(selected_path, thread, link, file)
                                    self.progress_show(selected_path, thread, link, file)
                                else:
                                    wx.LogError("File Extension Not Valid")
                            else:
                                wx.LogError("Both Link Column and File Column Can't be Same")
                        else:
                            wx.LogError("Please Enter Valid Number")
                    else:
                        wx.LogError("Please Enter Number in File Column")
                else:
                    wx.LogError("Please Enter Number in Link Column")
            else:
                wx.LogError("Please Select File")
        else:
            wx.LogError("Please Enter Number in Thread")

    def progress_show(self, selected_path, thread, link, file):
        pub.sendMessage('frameListener', message='show', selected_path = selected_path, thread = thread, link = link, file = file)

class Download_file():
    def download_image(self, url, file_path, data_load):
        print("In Class")
        try:
            wx.CallAfter(self.new_update, data_load)
            response = requests.get(url)
            if response.status_code == 200:
                if (response.raw) != None:
                    with open(file_path, 'wb') as out_file:
                        out_file.write(response.content)
                    del response
            else:
                print("not found")
        except requests.exceptions.RequestException as e:
            print(e)



class DownloadWorker(Thread):
    def __init__(self, queue):
        Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            print("In queue")
            url, file_path, data_load = self.queue.get()
            download = Download_file
            try:
                # pub.sendMessage('progress', message='show', data_load=data_load)
                # print((url))
                download.download_image(self, url, file_path, data_load)
                # wx.CallAfter(self.new_update, data_load)
                time.sleep(1)
            finally:
                self.queue.task_done()
                print("Downloaded")

    def new_update(self, data_load):
        print("upload")
        print(data_load)
        pub.sendMessage('progress', message='show', data_load=data_load)


class Progressbar(wx.Dialog):
    def __init__(self, selected_path, thread, link, file):
        wx.Dialog.__init__(self, None, title='Progess Bar',  size=(300,100))
        print("INnnn")
        progress_sizer = wx.BoxSizer(wx.HORIZONTAL)
        self.progress_bar = wx.Gauge(self, range=100, size=(250,25), style= wx.GA_HORIZONTAL)
        progress_sizer.Add(self.progress_bar, 0, wx.CENTER)
        progress_box = wx.BoxSizer(wx.VERTICAL)
        progress_box.Add(progress_sizer, wx.ALIGN_CENTRE, 25)
        print(selected_path, thread, link, file)
        self.SetSizer(progress_box)
        self.Center()
        self.Show(True)
        self.call_progress(selected_path, thread, link, file)
        pub.subscribe(self.progressListener, 'progress')
        self.Destroy()

    def call_progress(self, selected_path, thread, link, file):
        start = datetime.now()
        count = 1
        thread = int(thread)
        dask_data = dd.read_csv(selected_path, sep=',', error_bad_lines=False, quoting=csv.QUOTE_NONE, dtype="unicode")
        data_length = len(dask_data)
        data = pd.read_csv(selected_path, delimiter=',', chunksize=thread)
        count = 1
        date = str(datetime.date(datetime.now()))
        mydir = os.getcwd()
        image_dir = os.path.join(mydir, "Downloaded_Images")
        if not os.path.exists(image_dir):
            os.mkdir(image_dir)
        date_dir = os.path.join(image_dir, date)
        if not os.path.exists(date_dir):
            os.mkdir(date_dir)
        link_name =  'Link_column' + '___' + str(link)
        link_column = os.path.join(date_dir, link_name)
        if not os.path.exists(link_column):
            os.mkdir(link_column)
        queue = Queue()
        for x in range(thread):
            worker = DownloadWorker(queue)
            worker.daemon = True
            worker.start()
        thread_length = thread
        for df in data:
            data_load = round((thread_length / data_length) * 100)
            if data_load > 100:
                data_load = 100
            for i in range(thread):
                try:
                    print(df.iloc[i][int(link) - 1])
                    url = (df.iloc[i][int(link) - 1])
                    print(df.iloc[i][int(file) - 1])
                    file_name = (df.iloc[i][int(file) - 1])
                    print(count)
                    # print(data_load)
                    file_path = os.path.join(link_column, file_name)
                    queue.put((url, file_path, data_load))
                    count += 1
                except IndexError:
                    pass
            print("-------------------------")

            queue.join()
            thread_length += thread

            # data_load = round((thread / data_length) * 100)
            # if data_load > 100:
            #     data_load = 100
            # self.progress_bar.SetValue(data_load)

        # queue.join()
        print((datetime.now()) - start)

    def progressListener(self, message, data_load):
        print("listener")
        self.progress_bar.SetValue(data_load)

class MyPanel(wx.Panel):
    def __init(self, parent):
        wx.Panel.__init__(self, parent)

class MainFrame(wx.Frame):
    def __init__(self):
        wx.Frame.__init__(self, None)
        MyPanel(self)
        pub.subscribe(self.myListener, 'frameListener')
        self.Center()
        dlg = MainPage()
        dlg.ShowModal()

    def myListener(self, message, selected_path, thread, link, file):
        print(selected_path, thread, link, file)
        # self.center()
        dlg = Progressbar(selected_path = selected_path, thread = thread, link = link, file = file)
        dlg.ShowModal()

def myapp():
    app = wx.App(False)
    MainFrame()
    # app.MainLoop()


if __name__ == '__main__':
    myapp()

预期结果是应该下载文件，同时进度栏也应该更新。

Answer 1

向 Mike 劫持他的骨架代码表示歉意，这里是一个使用 urlretrieve 和 wxPython event 来执行以下操作的粗略版本正在更新。

import os
import wx
import time
from threading import Thread
from urllib.request import FancyURLopener
from urllib.request import urlopen
import wx.lib.newevent

urlretrieve = FancyURLopener().retrieve
progress_event, EVT_PROGRESS_EVENT = wx.lib.newevent.NewEvent()

class DownloadThread(Thread):

    def __init__(self, parent, url, fsize):
        """Constructor"""
        Thread.__init__(self)
        self.fsize = fsize
        self.url = url
        self.parent = parent
        self.start()

    def run(self):
        local_fname = os.path.basename(self.url)
        urlretrieve(self.url, local_fname, self.reporthook)

    def reporthook(self,blocknum, blocksize, totalsize):
        readsofar = blocknum * blocksize
        if totalsize > 0:
            percent = int((readsofar / totalsize) * 100)
        else: # total size is unknown
            percent = 0
        evt = progress_event(count=percent)
        #Send back current count for the progress bar
        try:
            wx.PostEvent(self.parent, evt)
        except: # The parent frame has probably been destroyed
            pass

class MyPanel(wx.Panel):
    def __init__(self, parent):
        """Constructor"""
        wx.Panel.__init__(self, parent)

        self.lbl = wx.StaticText(self, label="Download URL:")
        self.dl_txt = wx.TextCtrl(self)
        btn = wx.Button(self, label="Download")
        self.gauge = wx.Gauge(self, -1, range=100)

        self.main_sizer = wx.BoxSizer(wx.VERTICAL)
        dl_sizer = wx.BoxSizer(wx.HORIZONTAL)

        dl_sizer.Add(self.dl_txt, 1, wx.EXPAND|wx.ALL, 5)
        dl_sizer.Add(btn, 0, wx.ALL, 5)

        self.main_sizer.Add(dl_sizer, 0, wx.EXPAND)
        self.main_sizer.Add(self.lbl, 0, wx.ALL|wx.CENTER, 5)
        self.main_sizer.Add(self.gauge, 0, wx.ALL|wx.EXPAND, 5)

        self.SetSizer(self.main_sizer)

        #Bind to the progress event issued by the thread
        self.Bind(EVT_PROGRESS_EVENT, self.OnProgress)
        btn.Bind(wx.EVT_BUTTON, self.OnDownload)

    def OnDownload(self, event):
        url = self.dl_txt.GetValue()
        self.lbl.SetLabel(os.path.basename(url))
        try:
            header = urlopen(url)
            fsize = int(header.headers['Content-Length'])
            # start thread
            d_thread = DownloadThread(self, url, fsize)
            while d_thread.isAlive():
                time.sleep(0.1)
                wx.GetApp().Yield()
                continue
            # Finished re-set
            self.dl_txt.SetValue("")
            self.lbl.SetLabel("Download URL:")
            self.gauge.SetValue(0)

        except Exception as e:
            print ("Error: ", e)

    def OnProgress(self, event):
        """"""
        self.gauge.SetValue(event.count)

class DownloaderFrame(wx.Frame):

    def __init__(self):
        """Constructor"""
        wx.Frame.__init__(self, None, title="Downloader", size=(800, 400))
        panel = MyPanel(self)
        self.Show()

if __name__ == "__main__":
    app = wx.App(False)
    frame = DownloaderFrame()
    app.MainLoop()