python - 使用 Web 服务时无法 pickle _thread.RLock 对象

Question

我使用的是python 3.6

我正在尝试从下面显示的名称 SubmitJobsUsingMultiProcessing() 的类方法内部使用多重处理，该方法进一步依次调用另一个类方法。

我不断遇到此错误:类型错误:无法 pickle _thread.RLock 对象。

我不知道这意味着什么。我怀疑下面尝试建立与网络服务器 API 的连接的行可能是负责任的，但我完全不明白为什么。

我不是一个合适的程序员(作为投资组合建模团队的一部分进行编码)，所以如果这是一个明显的问题，请原谅我的无知，并提前非常感谢。

import multiprocessing as mp,functools

def SubmitJobsUsingMultiProcessing(self,PartitionsOfAnalysisDates,PickleTheJobIdsDict = True):
    if (self.ExportSetResult == "SUCCESS"):
        NumPools = mp.cpu_count()
        PoolObj =  mp.Pool(NumPools)   
        userId,clientId,password,expSetName = self.userId , self.clientId , self.password , self.expSetName
        PartialFunctor = functools.partial(self.SubmitJobsAsOfDate,userId = userId,clientId = clientId,password = password,expSetName = expSetName)
        Result = PoolObj.map(self.SubmitJobsAsOfDate, PartitionsOfAnalysisDates)
        BatchJobIDs = OrderedDict((key, val) for Dct in Result for key, val in Dct.items())
        f_pickle = open(self.JobIdPickleFileName, 'wb')
        pickle.dump(BatchJobIDs, f_pickle, -1)
        f_pickle.close()


 def SubmitJobsAsOfDate(self,ListOfDatesForBatchJobs,userId,clientId,password,expSetName):

    client = Client(self.url, proxy=self.proxysettings)
    if (self.ExportSetResult != "SUCCESS"):
        print("The export set creation was not successful...exiting")
        sys.exit()

    BatchJobIDs = OrderedDict()
    NumJobsSubmitted = 0
    CurrentProcessID = mp.current_process()

    for AnalysisDate in ListOfDatesForBatchJobs:
        jobName = "Foo_" + str(AnalysisDate)
        print('Sending job from process : ', CurrentProcessID, ' : ', jobName)
        jobId = client.service.SubmitExportJob(userId,clientId,password,expSetName, AnalysisDate, jobName, False)
        BatchJobIDs[AnalysisDate] = jobId
        NumJobsSubmitted += 1

        'Sleep for 30 secs every 100 jobs'
        if (NumJobsSubmitted % 100 == 0):
            print('100 jobs have been submitted thus far from process : ', CurrentProcessID,'---Sleeping for 30 secs to avoid the SSL time out error')
            time.sleep(30)
    self.BatchJobIDs = BatchJobIDs
    return BatchJobIDs

下面是跟踪::

    Traceback (most recent call last):
  File "C:\Program Files\JetBrains\PyCharm 2017.2.3\helpers\pydev\pydevd.py", line 1599, in <module>
    globals = debugger.run(setup['file'], None, None, is_module)
  File "C:\Program Files\JetBrains\PyCharm 2017.2.3\helpers\pydev\pydevd.py", line 1026, in run
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm 2017.2.3\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/trpff85/PycharmProjects/QuantEcon/BDTAPIMultiProcUsingPathos.py", line 289, in <module>
    BDTProcessObj.SubmitJobsUsingMultiProcessing(Partitions)
  File "C:/Users/trpff85/PycharmProjects/QuantEcon/BDTAPIMultiProcUsingPathos.py", line 190, in SubmitJobsUsingMultiProcessing
    Result = PoolObj.map(self.SubmitJobsAsOfDate, PartitionsOfAnalysisDates)
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 266, in map
    return self._map_async(func, iterable, mapstar, chunksize).get()
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 644, in get
    raise self._value
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 424, in _handle_tasks
    put(task)
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\connection.py", line 206, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "C:\Users\trpff85\AppData\Local\Continuum\anaconda3\lib\multiprocessing\reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
TypeError: can't pickle _thread.RLock objects

Answer 1

我也遇到了类似的问题。 <=3.5 中存在一个错误，即 _thread.RLock 对象在 pickle 时不会引发错误(它们不能)为了使 Pool 对象正常工作，必须从主进程向其传递函数和参数，这依赖于 pickle (pickling 是序列化对象的一种方法)在我的例子中，RLock 对象位于日志记录模块中的某个位置。我怀疑你的代码在 3.5 上可以正常工作。祝你好运。请参阅this bug resolution.