制作GPU版本Paddle Serving推理镜像
更新时间：2022-03-29
前言
前文重点介绍了通过BIE将Paddle Serving部署至边缘，为了通用起见，使用了Paddle Serving的CPU镜像。本文重点介绍如何制作Paddle Serving的GPU镜像，并同样以yolov3模型进行验证。
实验设备
本文所使用的实验设备是一台GPU服务器，参数如下：
- 操作系统：ubuntu 18.04 desktop
- CPU：intel i9
- GPU：NVIDIA 3090
- cuda 11.4
Tue Dec 21 20:45:17 2021
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 470.86 Driver Version: 470.86 CUDA Version: 11.4 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|===============================+======================+======================|
| 0 NVIDIA GeForce ... Off | 00000000:65:00.0 Off | N/A |
| 0% 32C P8 9W / 350W | 10441MiB / 24259MiB | 0% Default |
| | | N/A |
+-------------------------------+----------------------+----------------------+镜像准备
# 下载镜像
docker pull registry.baidubce.com/paddlepaddle/serving:0.7.0-cuda11.2-cudnn8-devel
# 运行镜像
docker run --rm -dit --name pipeline_serving_demo registry.baidubce.com/paddlepaddle/serving:0.7.0-cuda11.2-cudnn8-devel bash
# copy Paddle Serving代码包至容器镜像
docker cp Serving-0.7.0.zip pipeline_serving_demo:/home/
# 进入容器镜像
docker exec -it pipeline_serving_demo bash
# 解压代码包
unzip Serving-0.7.0.zip
# 进入代码目录
cd Serving-0.7.0
# 安装paddle-serving-gpu
pip3 install paddle-serving-client==0.7.0 -i https://pypi.tuna.tsinghua.edu.cn/simple
pip3 install paddle-serving-server-gpu==0.7.0.post112 -i https://pypi.tuna.tsinghua.edu.cn/simple
pip3 install paddle-serving-app==0.7.0 -i https://pypi.tuna.tsinghua.edu.cn/simple
# 因为本地cuda版本并非10.2，需要现在指定whl包来安装paddlepaddle-gpu
docker cp paddlepaddle_gpu-2.2.1.post112-cp36-cp36m-linux_x86_64.whl pipeline_serving_demo:/home/
pip3 install paddlepaddle_gpu-2.2.1.post112-cp36-cp36m-linux_x86_64.whl
# 保存镜像
docker commit pipeline_serving_demo paddle_serving:0.7.0-cuda11.2-cudnn8-py36模型准备
模型依然使用上文制作的模型paddle_serving_yolov3_darknet53_270e_coco.zip，可直接下载。
针对GPU设备，需要单独修改config.yml文件，主要修改如下：
- device_type: 1
- devices: '0'
dag:
#op资源类型, True, 为线程模型；False，为进程模型
is_thread_op: false
#使用性能分析, True，生成Timeline性能数据，对性能有一定影响；False为不使用
tracer:
interval_s: 30
#http端口, rpc_port和http_port不允许同时为空。当rpc_port可用且http_port为空时，不自动生成http_port
http_port: 18082
op:
yolov3:
#并发数，is_thread_op=True时，为线程并发；否则为进程并发
concurrency: 10
local_service_conf:
#client类型，包括brpc, grpc和local_predictor.local_predictor不启动Serving服务，进程内预测
client_type: local_predictor
# device_type, 0=cpu, 1=gpu, 2=tensorRT, 3=arm cpu, 4=kunlun xpu
device_type: 1
#计算硬件ID，当devices为""或不写时为CPU预测；当devices为"0", "0,1,2"时为GPU预测，表示使用的GPU卡
devices: '0'
#Fetch结果列表，以bert_seq128_model中fetch_var的alias_name为准, 如果没有设置则全部返回
fetch_list:
- save_infer_model/scale_0.tmp_1
#模型路径
model_config: serving_server/
#rpc端口, rpc_port和http_port不允许同时为空。当rpc_port为空且http_port不为空时，会自动将rpc_port设置为http_port+1
rpc_port: 9998
#worker_num, 最大并发数。当build_dag_each_worker=True时, 框架会创建worker_num个进程，每个进程内构建grpcSever和DAG
#当build_dag_each_worker=False时，框架会设置主线程grpc线程池的max_workers=worker_num
worker_num: 20测试验证
服务端
# 使用nvidia-docker运行之前commit的镜像
nvidia-docker run --rm -dit --name pipeline_serving_demo paddle_serving:0.7.0-cuda11.2-cudnn8-py36 bash
# copy模型文件到gpu镜像当中
docker cp yolov3 pipeline_serving_demo:/home/work
# 使用nvidia-docker进入容器
nvidia-docker exec -it pipeline_serving_demo bash
# 进入模型目录
cd /home/work/yolov3
# 运行模型服务
python3 web_service.py
# 查看端口监听，看到18082端口已经开启监听，这个就是http服务端口。
λ 3660ea4b3380 /home netstat -nlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:9998 0.0.0.0:* LISTEN 1851/python3
tcp6 0 0 :::18082 :::* LISTEN 2104/python3
Active UNIX domain sockets (only servers)
Proto RefCnt Flags Type State I-Node PID/Program name Path
unix 2 [ ACC ] SEQPACKET LISTENING 24640668 1969/python3 @cuda-uvmfd-4026533080-1969@
unix 2 [ ACC ] STREAM LISTENING 24624779 1886/python3 /tmp/pymp-hb_gnzrz/listener-9c_ys160
unix 2 [ ACC ] STREAM LISTENING 24636935 1897/python3 /tmp/pymp-im3nxm72/listener-9esgp3vf
unix 2 [ ACC ] SEQPACKET LISTENING 24639912 2053/python3 @cuda-uvmfd-4026533080-2053@
unix 2 [ ACC ] SEQPACKET LISTENING 24624837 1961/python3 @cuda-uvmfd-4026533080-1961@
unix 2 [ ACC ] SEQPACKET LISTENING 24635815 1992/python3 @cuda-uvmfd-4026533080-1992@
unix 2 [ ACC ] SEQPACKET LISTENING 24633669 1980/python3 @cuda-uvmfd-4026533080-1980@
unix 2 [ ACC ] SEQPACKET LISTENING 24634676 2065/python3 @cuda-uvmfd-4026533080-2065@
unix 2 [ ACC ] SEQPACKET LISTENING 24638784 2016/python3 @cuda-uvmfd-4026533080-2016@
unix 2 [ ACC ] SEQPACKET LISTENING 24622982 2029/python3 @cuda-uvmfd-4026533080-2029@
unix 2 [ ACC ] SEQPACKET LISTENING 24628818 2005/python3 @cuda-uvmfd-4026533080-2005@
unix 2 [ ACC ] SEQPACKET LISTENING 24626960 2043/python3 @cuda-uvmfd-4026533080-2043@
# 查看2104进程对应的程序为python3 web_service.py
λ 3660ea4b3380 /home ps -ef | grep 2104
grep: warning: GREP_OPTIONS is deprecated; please use an alias or script
root 2104 1851 0 Dec20 pts/1 00:00:04 python3 web_service.py
root 2947 2925 0 02:33 pts/2 00:00:00 grep 2104客户端
# 使用nvidia-docker进入容器
nvidia-docker exec -it pipeline_serving_demo bash
# 进入模型目录
cd /home/work/yolov3
# 运行模型服务
python3 pipeline_http_client.py执行以后，得到返回如下：
{'err_no': 0, 'err_msg': '', 'key': ['bbox_result'], 'value': ["[{'category_id': 0, 'bbox': [215.15884399414062, 438.11700439453125, 43.3013916015625, 186.95361328125], 'score': 0.9860507845878601}, {'category_id': 0, 'bbox': [404.8855895996094, 463.1451110839844, 50.00469970703125, 174.96017456054688], 'score': 0.9721941947937012}, {'category_id': 0, 'bbox': [259.8423156738281, 458.6835021972656, 47.04754638671875, 154.56033325195312], 'score': 0.9670807719230652}, {'category_id': 0, 'bbox': [438.2522888183594, 491.8885803222656, 68.91619873046875, 145.52993774414062], 'score': 0.9092322587966919}, {'category_id': 0, 'bbox': [156.28872680664062, 505.4490661621094, 57.74395751953125, 55.969207763671875], 'score': 0.7772055864334106}, {'category_id': 0, 'bbox': [28.40646743774414, 451.0560302734375, 27.932456970214844, 113.3223876953125], 'score': 0.7687524557113647}, {'category_id': 0, 'bbox': [297.420654296875, 511.60821533203125, 59.17889404296875, 76.46728515625], 'score': 0.7284822463989258}, {'category_id': 0, 'bbox': [498.0848693847656, 504.2113037109375, 35.236419677734375, 134.76318359375], 'score': 0.610772430896759}, {'category_id': 0, 'bbox': [522.0022583007812, 479.5626220703125, 63.4168701171875, 156.0999755859375], 'score': 0.5191351175308228}, {'category_id': 0, 'bbox': [139.90017700195312, 483.0073547363281, 21.446075439453125, 69.74172973632812], 'score': 0.43744343519210815}, {'category_id': 0, 'bbox': [91.0887680053711, 494.5196228027344, 27.570602416992188, 55.927032470703125], 'score': 0.4109981656074524}, {'category_id': 0, 'bbox': [9.614776611328125, 462.5898132324219, 21.809978485107422, 82.99966430664062], 'score': 0.34510746598243713}, {'category_id': 0, 'bbox': [395.7677307128906, 460.096435546875, 13.8270263671875, 51.9024658203125], 'score': 0.3054893910884857}, {'category_id': 0, 'bbox': [120.70944213867188, 496.8636474609375, 25.2960205078125, 54.2845458984375], 'score': 0.21390226483345032}, {'category_id': 0, 'bbox': [595.8245239257812, 466.6754150390625, 6.9317626953125, 22.8021240234375], 'score': 0.21250930428504944}, {'category_id': 0, 'bbox': [624.6636352539062, 466.89056396484375, 6.6029052734375, 23.391845703125], 'score': 0.13077197968959808}, {'category_id': 0, 'bbox': [71.7218246459961, 493.44879150390625, 23.067031860351562, 58.1712646484375], 'score': 0.11481819301843643}, {'category_id': 0, 'bbox': [93.5288314819336, 494.7815246582031, 18.050064086914062, 29.621185302734375], 'score': 0.09744449704885483}, {'category_id': 0, 'bbox': [633.906494140625, 466.9317932128906, 5.0838623046875, 22.54248046875], 'score': 0.08925836533308029}, {'category_id': 0, 'bbox': [616.2350463867188, 467.5209655761719, 6.663330078125, 22.73736572265625], 'score': 0.0747247114777565}, {'category_id': 0, 'bbox': [330.21624755859375, 460.6182861328125, 14.3553466796875, 50.36187744140625], 'score': 0.05385982245206833}, {'category_id': 0, 'bbox': [623.5626220703125, 521.3427124023438, 15.657470703125, 118.65728759765625], 'score': 0.051674000918865204}, {'category_id': 0, 'bbox': [118.22461700439453, 493.11761474609375, 15.310569763183594, 46.677490234375], 'score': 0.05103645846247673}, {'category_id': 0, 'bbox': [76.89342498779297, 489.4928894042969, 14.863052368164062, 36.152740478515625], 'score': 0.050921157002449036}, {'category_id': 0, 'bbox': [612.20947265625, 466.2704162597656, 6.8406982421875, 24.81988525390625], 'score': 0.049481648951768875}, {'category_id': 0, 'bbox': [459.984619140625, 466.306640625, 11.69122314453125, 34.4151611328125], 'score': 0.04908357560634613}, {'category_id': 0, 'bbox': [0.4049067497253418, 462.48992919921875, 10.412243366241455, 61.63555908203125], 'score': 0.048190776258707047}, {'category_id': 0, 'bbox': [163.00997924804688, 501.29937744140625, 14.07818603515625, 28.6597900390625], 'score': 0.043660055845975876}, {'category_id': 0, 'bbox': [42.41655731201172, 456.065673828125, 17.122093200683594, 96.2384033203125], 'score': 0.04359209164977074}, {'category_id': 0, 'bbox': [124.5979995727539, 495.2655029296875, 14.956932067871094, 29.2493896484375], 'score': 0.03233138099312782}, {'category_id': 0, 'bbox': [322.88623046875, 510.54901123046875, 35.19482421875, 58.5223388671875], 'score': 0.03211212158203125}, {'category_id': 0, 'bbox': [499.7281188964844, 472.06951904296875, 6.73486328125, 23.5274658203125], 'score': 0.03186612203717232}, {'category_id': 0, 'bbox': [635.6022338867188, 472.9559326171875, 4.26025390625, 17.8455810546875], 'score': 0.031331829726696014}, {'category_id': 0, 'bbox': [13.47189712524414, 509.23486328125, 17.924728393554688, 59.2049560546875], 'score': 0.02480548433959484}, {'category_id': 0, 'bbox': [492.8508605957031, 474.7364807128906, 5.197998046875, 15.80572509765625], 'score': 0.024477316066622734}, {'category_id': 0, 'bbox': [141.70169067382812, 491.1126708984375, 16.06195068359375, 34.2840576171875], 'score': 0.017450490966439247}, {'category_id': 0, 'bbox': [111.90899658203125, 493.113037109375, 13.244583129882812, 30.58624267578125], 'score': 0.016772815957665443}, {'category_id': 0, 'bbox': [70.63213348388672, 495.75823974609375, 15.301864624023438, 42.4739990234375], 'score': 0.016078853979706764}, {'category_id': 0, 'bbox': [499.5570373535156, 506.3521423339844, 30.213226318359375, 69.96072387695312], 'score': 0.014187620021402836}, {'category_id': 2, 'bbox': [500.8840637207031, 460.91815185546875, 13.796478271484375, 11.3297119140625], 'score': 0.05759419873356819}, {'category_id': 2, 'bbox': [492.96539306640625, 461.0340881347656, 16.81353759765625, 11.0130615234375], 'score': 0.05016255006194115}, {'category_id': 2, 'bbox': [566.1678466796875, 461.9084777832031, 14.2823486328125, 11.172119140625], 'score': 0.04674103483557701}, {'category_id': 2, 'bbox': [582.1033935546875, 463.1783752441406, 11.5361328125, 9.767578125], 'score': 0.039195213466882706}, {'category_id': 2, 'bbox': [573.3780517578125, 463.0284118652344, 15.52001953125, 9.6724853515625], 'score': 0.03403704985976219}, {'category_id': 2, 'bbox': [600.5579223632812, 462.50634765625, 7.7847900390625, 8.21417236328125], 'score': 0.025833962485194206}, {'category_id': 2, 'bbox': [486.5694274902344, 463.896728515625, 12.26434326171875, 7.6646728515625], 'score': 0.02206607721745968}, {'category_id': 25, 'bbox': [437.30438232421875, 545.134033203125, 38.206298828125, 87.2926025390625], 'score': 0.01893363706767559}, {'category_id': 26, 'bbox': [29.909515380859375, 486.89495849609375, 16.0562744140625, 22.83349609375], 'score': 0.04065336659550667}, {'category_id': 26, 'bbox': [581.0613403320312, 620.1487426757812, 32.03466796875, 18.63525390625], 'score': 0.03942570090293884}, {'category_id': 26, 'bbox': [385.1342468261719, 517.0342407226562, 18.2425537109375, 13.5084228515625], 'score': 0.02212528884410858}, {'category_id': 26, 'bbox': [590.1001586914062, 559.3577270507812, 13.1656494140625, 43.9593505859375], 'score': 0.01538659818470478}, {'category_id': 26, 'bbox': [582.7340698242188, 560.33203125, 26.6346435546875, 76.6241455078125], 'score': 0.014935318380594254}, {'category_id': 33, 'bbox': [460.78826904296875, 336.1343994140625, 17.67626953125, 23.15081787109375], 'score': 0.8833640217781067}, {'category_id': 33, 'bbox': [319.20867919921875, 214.5591278076172, 142.62786865234375, 70.58403015136719], 'score': 0.861716091632843}, {'category_id': 33, 'bbox': [233.2698211669922, 29.54220962524414, 9.237030029296875, 9.06732177734375], 'score': 0.8449435234069824}, {'category_id': 33, 'bbox': [175.91339111328125, 314.95343017578125, 79.73098754882812, 29.95263671875], 'score': 0.8208757042884827}, {'category_id': 33, 'bbox': [504.2532958984375, 261.37615966796875, 18.8287353515625, 24.36346435546875], 'score': 0.8136603832244873}, {'category_id': 33, 'bbox': [328.57733154296875, 86.04251861572266, 61.0360107421875, 27.47052001953125], 'score': 0.8073546290397644}, {'category_id': 33, 'bbox': [363.26824951171875, 161.76214599609375, 12.36004638671875, 10.897674560546875], 'score': 0.804345428943634}, {'category_id': 33, 'bbox': [222.80226135253906, 350.0331115722656, 162.12986755371094, 79.55096435546875], 'score': 0.7994755506515503}, {'category_id': 33, 'bbox': [279.991943359375, 124.65776824951172, 45.864990234375, 16.149635314941406], 'score': 0.7841681838035583}, {'category_id': 33, 'bbox': [98.05258178710938, 419.7305908203125, 65.68304443359375, 62.5074462890625], 'score': 0.7423993349075317}, {'category_id': 33, 'bbox': [242.89691162109375, 166.6405487060547, 20.291534423828125, 14.351043701171875], 'score': 0.6260237097740173}, {'category_id': 33, 'bbox': [86.47010803222656, 137.08712768554688, 48.22509765625, 19.8271484375], 'score': 0.593645453453064}, {'category_id': 33, 'bbox': [430.31268310546875, 40.187400817871094, 18.87237548828125, 19.073143005371094], 'score': 0.5935146808624268}, {'category_id': 33, 'bbox': [392.0285339355469, 394.45513916015625, 26.3792724609375, 23.6346435546875], 'score': 0.5019822120666504}, {'category_id': 33, 'bbox': [271.4106140136719, 299.24444580078125, 149.43499755859375, 59.80413818359375], 'score': 0.4226715862751007}, {'category_id': 33, 'bbox': [282.48858642578125, 281.7738952636719, 164.79986572265625, 56.425537109375], 'score': 0.4141180217266083}, {'category_id': 33, 'bbox': [114.94334411621094, 237.03826904296875, 22.5118408203125, 17.688507080078125], 'score': 0.4054967761039734}, {'category_id': 33, 'bbox': [160.3792266845703, 257.0617370605469, 10.084808349609375, 8.29144287109375], 'score': 0.3284390866756439}, {'category_id': 33, 'bbox': [260.01031494140625, 44.601375579833984, 4.965576171875, 5.313316345214844], 'score': 0.2895190715789795}, {'category_id': 33, 'bbox': [216.72500610351562, 238.3572540283203, 180.0152587890625, 60.69700622558594], 'score': 0.2510143518447876}, {'category_id': 33, 'bbox': [58.31549835205078, 340.76739501953125, 7.572578430175781, 5.217529296875], 'score': 0.23151512444019318}, {'category_id': 33, 'bbox': [2.2926530838012695, 424.1970520019531, 17.95302104949951, 10.41741943359375], 'score': 0.2101164609193802}, {'category_id': 33, 'bbox': [281.13623046875, 130.5193634033203, 8.10443115234375, 9.677093505859375], 'score': 0.13483838737010956}, {'category_id': 33, 'bbox': [170.76173400878906, 425.6689758300781, 60.929779052734375, 52.5660400390625], 'score': 0.11661253869533539}, {'category_id': 33, 'bbox': [218.15464782714844, 236.66712951660156, 86.75819396972656, 36.82374572753906], 'score': 0.09920619428157806}, {'category_id': 33, 'bbox': [57.5946044921875, 340.63653564453125, 11.537132263183594, 10.23974609375], 'score': 0.09504096955060959}, {'category_id': 33, 'bbox': [83.42386627197266, 300.5022888183594, 4.1671905517578125, 3.516357421875], 'score': 0.05351868271827698}, {'category_id': 33, 'bbox': [85.8399887084961, 137.02593994140625, 7.16094970703125, 8.848388671875], 'score': 0.044700782746076584}, {'category_id': 33, 'bbox': [412.984619140625, 38.16804504394531, 32.95037841796875, 45.055572509765625], 'score': 0.04125674441456795}, {'category_id': 33, 'bbox': [596.878173828125, 1.0675864219665527, 35.042724609375, 10.81494951248169], 'score': 0.037190232425928116}, {'category_id': 33, 'bbox': [218.40818786621094, 239.1458740234375, 37.583831787109375, 23.478668212890625], 'score': 0.03471671789884567}, {'category_id': 33, 'bbox': [22.161663055419922, 430.1984558105469, 15.832508087158203, 46.72918701171875], 'score': 0.03379001468420029}, {'category_id': 33, 'bbox': [51.58502197265625, 443.5048828125, 30.810791015625, 35.0467529296875], 'score': 0.028409656137228012}, {'category_id': 33, 'bbox': [436.8493347167969, 541.2244873046875, 41.42230224609375, 93.6212158203125], 'score': 0.016602490097284317}, {'category_id': 33, 'bbox': [577.9293212890625, 0.0, 54.5059814453125, 48.892173767089844], 'score': 0.0161479152739048}, {'category_id': 33, 'bbox': [3.0973596572875977, 438.25433349609375, 20.413020133972168, 28.048095703125], 'score': 0.01403715182095766}, {'category_id': 56, 'bbox': [567.2759399414062, 559.4578857421875, 37.158447265625, 77.5042724609375], 'score': 0.38048961758613586}, {'category_id': 56, 'bbox': [72.79810333251953, 514.731201171875, 21.9168701171875, 37.7213134765625], 'score': 0.08128189295530319}, {'category_id': 56, 'bbox': [508.6864929199219, 559.0671997070312, 93.94400024414062, 78.7864990234375], 'score': 0.06235785409808159}, {'category_id': 56, 'bbox': [174.7191925048828, 497.6101989746094, 17.550689697265625, 22.370819091796875], 'score': 0.04737205058336258}, {'category_id': 56, 'bbox': [90.97484588623047, 507.9281921386719, 28.048797607421875, 45.361968994140625], 'score': 0.0391169935464859}, {'category_id': 56, 'bbox': [377.0031433105469, 491.51287841796875, 20.13720703125, 34.9970703125], 'score': 0.03886980563402176}, {'category_id': 56, 'bbox': [503.7839660644531, 566.7869873046875, 28.499969482421875, 72.0155029296875], 'score': 0.03676376864314079}, {'category_id': 56, 'bbox': [119.07807922363281, 507.2283630371094, 28.09521484375, 46.024688720703125], 'score': 0.029960021376609802}, {'category_id': 56, 'bbox': [70.57154846191406, 499.4459533691406, 15.872421264648438, 51.188751220703125], 'score': 0.02471238747239113}, {'category_id': 56, 'bbox': [389.9171447753906, 493.0232849121094, 14.80792236328125, 33.047027587890625], 'score': 0.019949335604906082}, {'category_id': 56, 'bbox': [60.86650848388672, 502.6177673339844, 12.64495849609375, 27.787689208984375], 'score': 0.018412427976727486}, {'category_id': 56, 'bbox': [529.2908325195312, 566.8237915039062, 40.314208984375, 71.199951171875], 'score': 0.0175805501639843}]"], 'tensors': []}