python - 在 python 中使用多个暗网模型并行执行 GPU

Question

我正在使用来自 https://github.com/pjreddie/darknet 的 darknet 存储库

使用暗网，我可以在模型运行时更改我的 GPU。

例如，以下代码在 GPU 0 上加载暗网模型权重，我对 10 张图像运行预测。

from darknet import *
import time

# most likely this will be called when the server starts
set_gpu(0) # running on GPU 0, can change this
net1 = load_net(b"cfg/yolov3-lp_vehicles.cfg", b"backup/yolov3-lp_vehicles.backup", 0)
meta1 = load_meta(b"data/lp_vehicles.data")

# each time a request is sent
for i in range(10):
    a = cv2.imread("lp_tester/bug1.jpg")
    t1 = time.time()
    r = detect_np_lp(net1, meta1, a) # detect_np_lp is just a custom function written to load the image as numpy array and pass to detector and get the predictions

    t2 = time.time()
    print(f"FPS: {1/(t2-t1)}")

暗网.py:https://github.com/pjreddie/darknet/blob/master/python/darknet.py

现在，我有 2 个 GPU，我想并行运行两个模型。对于初学者，我意识到的一件事是，当程序/服务器启动时，我需要在两个不同的 GPU 中加载两个模型并将它们保持在那里，因为我无法在每次发送请求时加载模型，加载很昂贵。

我应该如何处理这个问题？我不确定在这种情况下多处理或多线程是否是正确的选择(不需要任何手动同步)，这里大部分计算将在 GPU 上完成，但我需要同时运行两个模型两个独立的 GPU，在它们都完成后，我需要将结果合并到一个数组中。

任何演示类似行为的 python 代码片段都可以，我可以从中改编，不一定要完整或考虑 darknet 场景。

Answer 1

所以，我做了一个基准测试，似乎 Thread 是可行的方法，因为每次收到请求时创建一个多处理池的成本很高。

下面的代码实现了我所需要的，我在两个独立的 GPU 中加载了两个模型，每次收到请求时，我都使用 ThreadPoolExecutor 并行运行两个模型，从而加快预测速度2.3 倍(最少 1.8 倍)[time.time() 不是那么准确，但我会粗略估计] 这对我有 2 个 GPU 的人来说已经足够了。

from darknet import *
import concurrent.futures
import time

set_gpu(0) # running on GPU 0
net1 = load_net(b"cfg/yolov3-lp_vehicles.cfg", b"backup/yolov3-lp_vehicles.backup", 0)
meta1 = load_meta(b"data/lp_vehicles.data")

set_gpu(1) # running on GPU 0
net2 = load_net(b"cfg/yolov3-lp_vehicles.cfg", b"backup/yolov3-lp_vehicles.backup", 0)
meta2 = load_meta(b"data/lp_vehicles.data")


def f(x):
    if x[0] == 0: # gpu 0
        return detect_np_lp(net1, meta1, x[1])
    else:
        return detect_np_lp(net2, meta2, x[1])

def func1(): # without threading
    a = cv2.imread("lp_tester/bug1.jpg")
    r1 = f( (0, a) )
    r2 = f(  (1, a) )
    print('out f1')
    #return [r1, r2]


def func2(): # with threading
    a = cv2.imread("lp_tester/bug1.jpg")
    nums = [(0, a), (1, a)]
    with concurrent.futures.ThreadPoolExecutor() as executor:
        r_m = [val for val in executor.map(f, nums)]
    print('out f2')
    #return r_m

t1 = time.time()
func1()
t2 = time.time()
print(t2-t1)

t1 = time.time()
func2()
t2 = time.time()
print(t2-t1)

...
...
OPs
  103 conv    128  1 x 1 / 1    52 x  52 x 256   ->    52 x  52 x 128  0.177 BFLOPs
  104 conv    256  3 x 3 / 1    52 x  52 x 128   ->    52 x  52 x 256  1.595 BFLOPs
  105 conv     39  1 x 1 / 1    52 x  52 x 256   ->    52 x  52 x  39  0.054 BFLOPs
  106 yolo
Loading weights from backup/yolov3-lp_vehicles.backup...Done!

out f1
0.05610060691833496
out f2
0.02923417091369629

更多:

av_fps = 0.
for _ in range(100):
    t1 = time.time()
    func1()
    t2 = time.time()
    print(f'fps: {1/(t2-t1)}')
    av_fps += (1/(t2-t1))/100.

print(f'Average: {av_fps}')

av_fps = 0.
for _ in range(100):
    t1 = time.time()
    func2()
    t2 = time.time()
    print(f'fps: {1/(t2-t1)}')
    av_fps += (1/(t2-t1))/100.

print(f'Average: {av_fps}')

fps: 18.02846347931863
fps: 18.257139747970488
fps: 18.246814434559415
fps: 18.471007376417482
fps: 18.579090514453785
fps: 18.611902944674206
fps: 18.37777300669947
fps: 18.59416325680163
fps: 18.582547671368825
fps: 18.57695101426167
fps: 18.57966661793955
fps: 18.583947362591108
fps: 18.583535666814356
fps: 18.555418904451386
fps: 18.584770808870772
fps: 18.090515029048827
fps: 18.580983560078145
fps: 18.482646784058662
fps: 18.544671556728698
fps: 18.578432154215502
fps: 18.546229560388053
fps: 18.58921868005726
fps: 18.592020283957677
fps: 18.594575423601075
fps: 18.48248389399561
fps: 18.1624280635509
fps: 18.362164599577095
fps: 18.561824007364006
fps: 18.600595140425646
fps: 18.61140742449925
fps: 18.589713019390583
fps: 18.574318459603564
fps: 18.552546256363982
fps: 18.627525347852927
fps: 18.371574742448665
fps: 18.57489426717743
fps: 18.42046921799928
fps: 18.593091708631817
fps: 18.59663653171707
fps: 18.594740295437216
fps: 18.58106587516059
fps: 18.58954823669153
fps: 18.584029703935418
fps: 18.126635233308413
fps: 18.57127549823112
fps: 18.587323956145248
fps: 18.574400715642728
fps: 18.59507004788083
fps: 18.59490517019711
fps: 18.59292686602892
fps: 18.582547671368825
fps: 18.609260475269313
fps: 18.025751663199877
fps: 18.13133675414669
fps: 18.349391897803834
fps: 18.254279260655174
fps: 18.57933741157293
fps: 18.594987608673485
fps: 18.591855460352217
fps: 18.569137797454346
fps: 18.595152487819153
fps: 18.417557325651856
fps: 18.593009286965003
fps: 18.564535189947375
fps: 18.56815133230332
fps: 18.580983560078145
fps: 18.439662183846902
fps: 18.300793675033923
fps: 18.327182476393556
fps: 18.294966413678793
fps: 18.333671364129103
fps: 18.376323687265877
fps: 18.57769155471695
fps: 18.551397446161058
fps: 18.260319118831493
fps: 18.5768687356332
fps: 18.50262255885869
fps: 18.583041655959523
fps: 18.575223316105774
fps: 18.58271232998095
fps: 18.144906470089463
fps: 18.166754736267638
fps: 18.284837414500387
fps: 18.570206586322623
fps: 18.583041655959523
fps: 18.56215259337936
fps: 18.59836821567932
fps: 18.53221694465923
fps: 18.575305580159434
fps: 17.912816942912908
fps: 18.596471626253088
fps: 18.564124353799308
fps: 18.54180223511105
fps: 18.124990276997536
fps: 18.573660437516605
fps: 18.584112046009402
fps: 18.554762220747623
fps: 18.55911361655243
fps: 18.29871778651298
fps: 18.561249009828696
Average: 18.482654425676994
fps: 33.9106292496382
fps: 34.70468404808989
fps: 36.1313175690227
fps: 30.45640634644011
fps: 36.19867263892844
fps: 36.1387890850501
fps: 35.846301107616576
fps: 36.19180091637832
fps: 36.100219477557346
fps: 36.018067840274796
fps: 36.05274286991353
fps: 36.105191574344275
fps: 36.06452278589854
fps: 36.03508741784441
fps: 34.441083247113696
fps: 34.590200978087864
fps: 36.04716559524219
fps: 36.150625307051186
fps: 34.85291209293436
fps: 33.35775467841606
fps: 36.19773543220105
fps: 36.11980503263809
fps: 36.11482891043414
fps: 34.30811261799205
fps: 36.100219477557346
fps: 36.10550237586943
fps: 34.2711094406223
fps: 36.121049277459136
fps: 35.92643922327768
fps: 36.14595218807632
fps: 36.1120304441785
fps: 36.336342372000345
fps: 34.004377928753264
fps: 33.852878981097355
fps: 34.7919106790318
fps: 34.90279684782518
fps: 35.76042084082906
fps: 35.77811329767724
fps: 34.25599477295002
fps: 35.76316507503411
fps: 34.2996957901279
fps: 35.7616404484802
fps: 35.89784320438206
fps: 36.05615205412329
fps: 35.75462884031779
fps: 34.27362984874609
fps: 35.77201047325823
fps: 36.10798898071625
fps: 36.174009038534514
fps: 35.93136356237846
fps: 34.56938926893596
fps: 36.04220946619462
fps: 36.11856087353392
fps: 36.167146675864444
fps: 36.14657520079975
fps: 35.922131533645654
fps: 36.17244918199616
fps: 36.09866597813925
fps: 36.12602711408933
fps: 36.01837714364228
fps: 35.531098046524235
fps: 35.02491816420603
fps: 36.0016823601109
fps: 35.045989304812835
fps: 36.096491303562054
fps: 36.06545310713088
fps: 36.09058993598128
fps: 35.70866429988337
fps: 34.61275148954431
fps: 36.09555938037866
fps: 36.05987189958303
fps: 36.155922969501574
fps: 34.903668197856334
fps: 34.91877851410303
fps: 34.362078288083104
fps: 35.97358354632314
fps: 36.190551792570865
fps: 35.945528559797744
fps: 36.17650508883906
fps: 36.00291847998695
fps: 34.41932069030601
fps: 36.03353951890034
fps: 36.04035126914022
fps: 36.211485996477535
fps: 35.85549419548975
fps: 34.85667747029004
fps: 35.36154857855866
fps: 34.92197660380501
fps: 36.032610843363145
fps: 36.181186111710154
fps: 36.204921924228955
fps: 35.138474427177144
fps: 34.33170172710158
fps: 36.10488077816992
fps: 36.073207651025186
fps: 36.18056190533698
fps: 34.95224206464946
fps: 35.90491110026794
fps: 36.1493790238479
fps: 36.119493984826434
Average: 35.55810025312704

• 我将在未来进行更多更新(并行暗网推理)的存储库:https://github.com/zabir-nabil/darknet-multi-gpu-parallel