cd /home/work/

wget https://github.com/PaddlePaddle/PaddleClas/archive/refs/heads/release/2.3.zip && unzip 2.3.zip

python -m pip install paddlepaddle-gpu==2.1.3.post101 -f https://www.paddlepaddle.org.cn/whl/linux/mkl/avx/stable.html

图像相对路径 图像的label_id（数字类别）（注意：中间有空格）

import os
import json
import glob
import codecs
import random

def parse_label_list(src_data_dir, save_dir):
    """
    遍历标注文件，获取label_list
    :param src_data_dir:
    :return:
    """
    label_list = []
    anno_files = glob.glob(src_data_dir + "*.json")
    for anno_f in anno_files:
        annos = json.loads(codecs.open(anno_f).read())
        for object in annos["labels"]:
            label_list.append(object["name"])
    label_list = list(set(label_list))
    with codecs.open(os.path.join(save_dir, "label_list.txt"), 'w', encoding="utf-8") as f:
        for id, label in enumerate(label_list):
            f.writelines("%s:%s\n" % (id, label))
    return len(label_list), label_list


def trans_split_data(src_data_dir, save_dir):
    """转换数据格式，并3/7分切分数据"""
    image_list = glob.glob(src_data_dir + "*.[jJPpBb][PpNnMm]*")
    image_label_list = []
    for image_file in image_list:
        json_file = image_file.split('.')[0]+".json"
        if os.path.isfile(json_file):
            annos = json.loads(codecs.open(json_file).read())
            label = annos["labels"][0]["name"]
            image_label_list.append("{} {}\n".format(os.path.basename(image_file), label_list.index(label)))
    
    random.shuffle(image_label_list)
    split_nums = int(len(image_label_list) * 0.3)
    val_list = image_label_list[:split_nums]
    train_list = image_label_list[split_nums:]
    with open(os.path.join(save_dir, "train.txt"), 'w') as f:
        f.writelines(train_list)
    with open(os.path.join(save_dir, "val.txt"), 'w') as f:
        f.writelines(val_list)
  
class_nums, label_list = parse_label_list("/home/work/data/${dataset_id}/", "/home/work/PretrainedModel/")
trans_split_data("/home/work/data/${dataset_id}/", "/home/work/PretrainedModel/")

python coversion.py

cd /path/to/PaddleClas

/home/work/PaddleClas-release-2.3/ppcls/configs/ImageNet/ResNet/ResNet50_vd.yaml

# global configs
Global:
  checkpoints: null
  pretrained_model: null
  output_dir: ./output/
  # 使用GPU训练
  device: gpu
  # 每几个轮次保存一次
  save_interval: 1
  eval_during_train: True
  # 每几个轮次验证一次
  eval_interval: 1
  # 训练轮次
  epochs: 10
  print_batch_step: 10
  use_visualdl: False #是否开启可视化
  # used for static mode and model export
  # 图像大小
  image_shape: [3, 224, 224]
  save_inference_dir: ./inference

# model architecture
Arch:
  # 采用的网络
  name: ResNet50_vd
  # 类别数
  class_num: 1000
 
# loss function config for traing/eval process
Loss:
  Train:
    - CELoss:
        weight: 1.0
        epsilon: 0.1
  Eval:
    - CELoss:
        weight: 1.0


Optimizer:
  name: Momentum
  momentum: 0.9
  lr:
    name: Cosine
    learning_rate: 0.1
  regularizer:
    name: 'L2'
    coeff: 0.00007


# data loader for train and eval
DataLoader:
  Train:
    dataset:
      name: ImageNetDataset
      # 数据集根路径
      image_root: /home/work/data/${dataset_id}
      # 前面生产得到的训练集列表文件路径
      cls_label_path: /home/work/PretrainedModel/train_list.txt
      # 数据预处理
      transform_ops:
        - DecodeImage:
            to_rgb: True
            channel_first: False
        - RandCropImage:
            size: 224
        - RandFlipImage:
            flip_code: 1
        - NormalizeImage:
            scale: 1.0/255.0
            mean: [0.485, 0.456, 0.406]
            std: [0.229, 0.224, 0.225]
            order: ''
      batch_transform_ops:
        - MixupOperator:
            alpha: 0.2

    sampler:
      name: DistributedBatchSampler
      batch_size: 64
      drop_last: False
      shuffle: True
    loader:
      num_workers: 0
      use_shared_memory: True

  Eval:
    dataset: 
      name: ImageNetDataset
      # 数据集根路径
      image_root: /home/work/data/${dataset_id}
      # 前面生产得到的训练集列表文件路径
      cls_label_path: /home/work/PretrainedModel/val_list.txt
      transform_ops:
        - DecodeImage:
            to_rgb: True
            channel_first: False
        - ResizeImage:
            resize_short: 256
        - CropImage:
            size: 224
        - NormalizeImage:
            scale: 1.0/255.0
            mean: [0.485, 0.456, 0.406]
            std: [0.229, 0.224, 0.225]
            order: ''
    sampler:
      name: DistributedBatchSampler
      batch_size: 64
      drop_last: False
      shuffle: False
    loader:
      num_workers: 0
      use_shared_memory: True

Infer:
  infer_imgs: docs/images/whl/demo.jpg
  batch_size: 10
  transforms:
    - DecodeImage:
        to_rgb: True
        channel_first: False
    - ResizeImage:
        resize_short: 256
    - CropImage:
        size: 224
    - NormalizeImage:
        scale: 1.0/255.0
        mean: [0.485, 0.456, 0.406]
        std: [0.229, 0.224, 0.225]
        order: ''
    - ToCHWImage:
  PostProcess:
    # 输出的可能性最高的前topk个
    name: Topk
    topk: 5
    # 前面得到标签文件
    class_id_map_file: /home/work/PretrainedModel/label_list.txt

Metric:
  Train:
  Eval:
    - TopkAcc:
        topk: [1, 5]

修改类别数
20行：class_num: 5

修改训练集的路径(数据集id根据您自己的情况调整)
49行：image_root: /home/work/data/302273
50行：cls_label_path: /home/work/PretrainedModel/train.txt

修改训练GPU
74行：num_workers: 0

修改验证集的路径(数据集id根据您自己的情况调整)
80行：image_root: /home/work/data/302273
81行：cls_label_path: /home/work/PretrainedModel/val.txt

修改验证GPU
101行：num_workers: 0

修改标签文件地址
124行：class_id_map_file: /home/work/PretrainedModel/label_list.txt

cd PaddleClas-release-2.3/
python tools/train.py -c ./ppcls/configs/ImageNet/ResNet/ResNet50_vd.yaml

python tools/infer.py \
    -c ./ppcls/configs/ImageNet/ResNet/ResNet50_vd.yaml \
    -o Infer.infer_imgs=/home/work/data/${dataset_id}/xxx.jpg \
    -o Global.pretrained_model=output/ResNet50_vd/best_model

python tools/export_model.py \
    -c ./ppcls/configs/ImageNet/ResNet/ResNet50_vd.yaml \
    -o Global.pretrained_model=output/ResNet50_vd/best_model \
    -o Global.save_inference_dir=/home/work/PretrainedModel/

mv /home/work/PretrainedModel/inference.pdiparams /home/work/PretrainedModel/model.pdiparams
mv /home/work/PretrainedModel/inference.pdmodel /home/work/PretrainedModel/model.pdmodel
mv /home/work/PretrainedModel/inference.pdiparams.info /home/work/PretrainedModel/model.pdiparams.info

Global:
  inference_model_dir: "/home/work/PretrainedModel/"
  batch_size: 1
  use_gpu: True
  enable_mkldnn: True
  cpu_num_threads: 10
  enable_benchmark: True
  use_fp16: False
  ir_optim: True
  use_tensorrt: False
  gpu_mem: 8000
  enable_profile: False
PreProcess:
  transform_ops:
    - ResizeImage:
        resize_short: 256
    - CropImage:
        size: 224
    - NormalizeImage:
        scale: 0.00392157
        mean: [0.485, 0.456, 0.406]
        std: [0.229, 0.224, 0.225]
        order: ''
        channel_num: 3
    - ToCHWImage:

  - 可以看下平台预置代码文件，以及各个类及函数的注释了解实现细节。这里贴了`PaddleClass套件`对应的脚本文件，整个代码比较长，但大部分内容都拷贝于[PaddleClas套件的推理示例](https://github.com/PaddlePaddle/PaddleClas/blob/develop/deploy/python/predict_cls.py)。如果自身训练模型比较特殊，当前脚本支持不了，需要自己去PaddleClas套件中寻找逻辑，并更新到该脚本中。
  - 对于处理脚本里面的预处理`preprocess`方法，第一个返回参数`input_info`为字典类型，其中字典的key 为模型的输入节点名称，需要根据模型修改。
   ![图片](http://agroup.baidu-int.com/file/stream/bj/bj-70c34318f1ccd776e0c1a1c755ad0a3e6b11420a)
  如果不知道训练模型的输入节点名称是什么，可以先用下面提供的脚本进行一次模型验证，验证日志里面查看对应的输入名称。
  ![图片](http://agroup.baidu-int.com/file/stream/bj/bj-0dd53385ff6d94e0397f063ac12ea3ba700253bf)
  ![图片](http://agroup.baidu-int.com/file/stream/bj/bj-2219a7252b9fda78281faa7bcb47955519b718e9)

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# *******************************************************************************
#
# Copyright (c) 2020 Baidu.com, Inc. All Rights Reserved
#
# *******************************************************************************

# 注意事项：
# 该脚本用于将通过notebook训练产出的模型发布成模型服务
# 用户可以通过修改PredictWrapper的preprocess和postprocess方法来实现自定义的请求预处理和预测结果后处理
# 当在EasyDL提交该脚本时，系统会根据用户选择的模型文件和脚本内容，来验证是否可以启动模型服务，如果验证通过，即可进行模型效果校验以及部署模型服务
# 下面是修改脚本的一些限制和建议：
# 1. CustomException必须存在且是异常类；在自定义的逻辑中，建议当处理进入错误的分支时,抛出CustomException并指定message，指定的message在请求回包中会作为error_msg返回；
# 2. PredictWrapper类必须存在，且必需包含preprocess和postprocess两个方法；
# 3. PredictWrapper的preprocess和postprocess方法，是用户自定义模型服务请求预处理和预测结果后处理的入口；
# 4. preprocess方法接收的第一个参数为用户请求的json字典，对于图像类服务，传入图像的参数key必须是"image"，且传入的是图片的base64编码
# 5. 系统会根据postprocess方法的返回结果`result`类型的不同，做以下处理：
#       dict: 不作修改
#       list: 转换为 {"results": result}
#       其他: 转换为 {"result": result}
#    处理结果将转换为json字符串，以确保系统能正常将结果作为请求回包返回。

import base64
import math
import random

import cv2
import logging
import numpy as np
import six
import yaml

from functools import partial
from PIL import Image

DEFAULT_TOP_NUM = 6


class CustomException(RuntimeError):
    """
    进行模型验证和部署服务必需的异常类，缺少该类在代码验证时将会失败
    在处理异常数据或者请求时，推荐在`PredictWrapper`中的自定义预处理preprocess和后处理postprocess函数中抛出`CustomException`类，
    并为`message`指定准确可读的错误信息，以便在服务响应包中的`error_msg`参数中返回。
    """
    def __init__(self, message, orig_error=None):
        """ 根据`message`初始化 """
        super(CustomException, self).__init__(message)
        self.orig_error = orig_error


class PredictWrapper(object):
    """ 模型服务预测封装类，支持用户自定义对服务请求数据的预处理和模型预测结果的后处理函数 """

    def __init__(self, model_path):
        """
        根据`model_path`初始化`PredictWrapper`类，如解析label_list.txt，加载模型输出标签id和标签名称的映射关系
        :param model_path: 该目录下存放了用户选择的模型版本中包含的所有文件
        """
        # 加载推理配置文件，获取【预处理配置】及【标签id和名称的映射关系】
        ops_config = [
            {"ResizeImage": {"resize_short": 256}},
            {"CropImage": {"size": 224}},
            {"NormalizeImage": {"scale": 0.00392157, "mean": [0.485, 0.456, 0.406],
                                "std": [0.229, 0.224, 0.225], "order": '', "channel_num": 3}},
            {"ToCHWImage": {}},
        ]
        conf_path = '{model_path}/{conf_file}'.format(model_path=model_path, conf_file='infer_cfg.yml')
        try:
            with open(conf_path) as conf_fin:
                infer_conf = yaml.load(conf_fin, Loader=yaml.FullLoader)
                if "PreProcess" in infer_conf and "transform_ops" in infer_conf["PreProcess"]:
                    ops_config = infer_conf["PreProcess"]["transform_ops"]
        except:
            pass

        self._preprocess_ops = create_operators(ops_config)

        try:
            label_path = '{model_path}/{label_file}'.format(model_path=model_path, label_file='label_list.txt')
            self._label_list = {}
            with open(label_path) as label_list_f:
                for index, line in enumerate(label_list_f):
                    line = line.strip()
                    label_info = line.split(":", 1)
                    if len(label_info) == 2:
                        self._label_list[int(label_info[0])] = label_info[1]
                    else:
                        self._label_list[index] = line
        except:
            pass

    def preprocess(self, request_body, **preprocess_args):
        """
        自定义对请求体的预处理，针对图像类模型服务，包括对图片对图像的解析、转化等
        :param request_body: 请求体的json字典
        :param preprocess_args: 从`{model_path}/preprocess_args.json`中加载的预处理参数字典，json文件不存在时，传入为空字典
        :return:
            data: 用于模型预测的输入。注意：data结构为dict，key为模型输入节点的名称，value为对应需要喂入的值，batch只能为1
            infer_args: 用于模型预测的其他参数
            request_context: 透传给自定义后处理函数`postprocess`的参数，例如指定返回预测结果的top N，过滤低score的阈值threshold.
        """
        # decode image from base64 string in request
        try:
            image_b64 = request_body['image']
            img_bin = base64.b64decode(image_b64)
        except KeyError:
            raise CustomException('Missing required parameter')
        except Exception:
            raise CustomException('Invalid BASE64')

        data = np.frombuffer(img_bin, dtype='uint8')
        im = cv2.imdecode(data, 1)  # BGR mode, but need RGB mode
        im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)

        # paddle cls
        # code: https://github.com/PaddlePaddle/PaddleClas/blob/develop/deploy/python/predict_cls.py
        try:
            for preprocess_op in self._preprocess_ops:
                im = preprocess_op(im)

            input_info = {"x": im[np.newaxis, ...]}
        except Exception:
            raise CustomException('Failed decoding input')

        return input_info, {}, {"top_num": request_body.get("top_num", DEFAULT_TOP_NUM)}

    def postprocess(self, infer_result, request_context, **postprocess_args):
        """
        自定义对图像分类模型输出结果的后处理，例如根据score对label进行排序，获取top N分类结果等
        :param infer_result: fluid模型的预测结果
        :param request_context: 自定义预处理函数中返回的`request context`
        :param postprocess_args: 从`{model_path}/postprocess_args.json`中加载的后处理参数字典，json文件不存在时，传入为空字典
        :return: request results 请求的处理结果
        """
        top_num = request_context["top_num"]
        result = infer_result[0]
        result = get_result_list(result)

        indices = np.flip(np.argsort(result), 0)[:top_num]
        top_results = list()
        for item in indices:
            item = int(item)
            top_results.append({
                'score': float(result[item]),
                'name': self._label_list.get(item, str(item))
            })
        return top_results


def create_operators(params):
    """
    create operators based on the config
    Args:
        params(list): a dict list, used to create some operators
    """
    assert isinstance(params, list), ('operator config should be a list')
    op_mapping = {
        "UnifiedResize": UnifiedResize,
        "DecodeImage": DecodeImage,
        "ResizeImage": ResizeImage,
        "CropImage": CropImage,
        "RandCropImage": RandCropImage,
        "RandFlipImage": RandFlipImage,
        "NormalizeImage": NormalizeImage,
        "ToCHWImage": ToCHWImage,
    }
    ops = []
    for operator in params:
        assert isinstance(operator,
                          dict) and len(operator) == 1, "yaml format error"
        op_name = list(operator)[0]
        param = {} if operator[op_name] is None else operator[op_name]
        op = op_mapping[op_name](**param)
        ops.append(op)

    return ops


class UnifiedResize(object):
    def __init__(self, interpolation=None, backend="cv2"):
        _cv2_interp_from_str = {
            'nearest': cv2.INTER_NEAREST,
            'bilinear': cv2.INTER_LINEAR,
            'area': cv2.INTER_AREA,
            'bicubic': cv2.INTER_CUBIC,
            'lanczos': cv2.INTER_LANCZOS4
        }
        _pil_interp_from_str = {
            'nearest': Image.NEAREST,
            'bilinear': Image.BILINEAR,
            'bicubic': Image.BICUBIC,
            'box': Image.BOX,
            'lanczos': Image.LANCZOS,
            'hamming': Image.HAMMING
        }

        def _pil_resize(src, size, resample):
            pil_img = Image.fromarray(src)
            pil_img = pil_img.resize(size, resample)
            return np.asarray(pil_img)

        if backend.lower() == "cv2":
            if isinstance(interpolation, str):
                interpolation = _cv2_interp_from_str[interpolation.lower()]
            # compatible with opencv < version 4.4.0
            elif interpolation is None:
                interpolation = cv2.INTER_LINEAR
            self.resize_func = partial(cv2.resize, interpolation=interpolation)
        elif backend.lower() == "pil":
            if isinstance(interpolation, str):
                interpolation = _pil_interp_from_str[interpolation.lower()]
            self.resize_func = partial(_pil_resize, resample=interpolation)
        else:
            logging.warning(
                "The backend of Resize only support \"cv2\" or \"PIL\". \"%s\" is unavailable. Use \"cv2\" instead.",
                backend
            )
            self.resize_func = cv2.resize

    def __call__(self, src, size):
        return self.resize_func(src, size)


class OperatorParamError(ValueError):
    """ OperatorParamError
    """
    pass


class DecodeImage(object):
    """ decode image """

    def __init__(self, to_rgb=True, to_np=False, channel_first=False):
        self.to_rgb = to_rgb
        self.to_np = to_np  # to numpy
        self.channel_first = channel_first  # only enabled when to_np is True

    def __call__(self, img):
        if six.PY2:
            assert type(img) is str and len(
                img) > 0, "invalid input 'img' in DecodeImage"
        else:
            assert type(img) is bytes and len(
                img) > 0, "invalid input 'img' in DecodeImage"
        data = np.frombuffer(img, dtype='uint8')
        img = cv2.imdecode(data, 1)
        if self.to_rgb:
            assert img.shape[2] == 3, 'invalid shape of image[%s]' % (
                img.shape)
            img = img[:, :, ::-1]

        if self.channel_first:
            img = img.transpose((2, 0, 1))

        return img


class ResizeImage(object):
    """ resize image """

    def __init__(self,
                 size=None,
                 resize_short=None,
                 interpolation=None,
                 backend="cv2"):
        if resize_short is not None and resize_short > 0:
            self.resize_short = resize_short
            self.w = None
            self.h = None
        elif size is not None:
            self.resize_short = None
            self.w = size if type(size) is int else size[0]
            self.h = size if type(size) is int else size[1]
        else:
            raise OperatorParamError("invalid params for ReisizeImage for '\
                'both 'size' and 'resize_short' are None")

        self._resize_func = UnifiedResize(
            interpolation=interpolation, backend=backend)

    def __call__(self, img):
        img_h, img_w = img.shape[:2]
        if self.resize_short is not None:
            percent = float(self.resize_short) / min(img_w, img_h)
            w = int(round(img_w * percent))
            h = int(round(img_h * percent))
        else:
            w = self.w
            h = self.h
        return self._resize_func(img, (w, h))


class CropImage(object):
    """ crop image """

    def __init__(self, size):
        if type(size) is int:
            self.size = (size, size)
        else:
            self.size = size  # (h, w)

    def __call__(self, img):
        w, h = self.size
        img_h, img_w = img.shape[:2]

        if img_h < h or img_w < w:
            raise Exception(
                "The size({h}, {w}) of CropImage must be greater than size({img_h}, {img_w}) of image. "
                "Please check image original size and size of ResizeImage if used.".format(h=h, w=w, img_w=img_w,
                                                                                           img_h=img_h)
            )

        w_start = (img_w - w) // 2
        h_start = (img_h - h) // 2

        w_end = w_start + w
        h_end = h_start + h
        return img[h_start:h_end, w_start:w_end, :]


class RandCropImage(object):
    """ random crop image """

    def __init__(self,
                 size,
                 scale=None,
                 ratio=None,
                 interpolation=None,
                 backend="cv2"):
        if type(size) is int:
            self.size = (size, size)  # (h, w)
        else:
            self.size = size

        self.scale = [0.08, 1.0] if scale is None else scale
        self.ratio = [3. / 4., 4. / 3.] if ratio is None else ratio

        self._resize_func = UnifiedResize(
            interpolation=interpolation, backend=backend)

    def __call__(self, img):
        size = self.size
        scale = self.scale
        ratio = self.ratio

        aspect_ratio = math.sqrt(random.uniform(*ratio))
        w = 1. * aspect_ratio
        h = 1. / aspect_ratio

        img_h, img_w = img.shape[:2]

        bound = min((float(img_w) / img_h) / (w**2),
                    (float(img_h) / img_w) / (h**2))
        scale_max = min(scale[1], bound)
        scale_min = min(scale[0], bound)

        target_area = img_w * img_h * random.uniform(scale_min, scale_max)
        target_size = math.sqrt(target_area)
        w = int(target_size * w)
        h = int(target_size * h)

        i = random.randint(0, img_w - w)
        j = random.randint(0, img_h - h)

        img = img[j:j + h, i:i + w, :]

        return self._resize_func(img, size)


class RandFlipImage(object):
    """ random flip image
        flip_code:
            1: Flipped Horizontally
            0: Flipped Vertically
            -1: Flipped Horizontally & Vertically
    """

    def __init__(self, flip_code=1):
        assert flip_code in [-1, 0, 1
                             ], "flip_code should be a value in [-1, 0, 1]"
        self.flip_code = flip_code

    def __call__(self, img):
        if random.randint(0, 1) == 1:
            return cv2.flip(img, self.flip_code)
        else:
            return img


class NormalizeImage(object):
    """ normalize image such as substract mean, divide std
    """

    def __init__(self,
                 scale=None,
                 mean=None,
                 std=None,
                 order='chw',
                 output_fp16=False,
                 channel_num=3):
        if isinstance(scale, str):
            scale = eval(scale)
        assert channel_num in [
            3, 4
        ], "channel number of input image should be set to 3 or 4."
        self.channel_num = channel_num
        self.output_dtype = 'float16' if output_fp16 else 'float32'
        self.scale = np.float32(scale if scale is not None else 1.0 / 255.0)
        self.order = order
        mean = mean if mean is not None else [0.485, 0.456, 0.406]
        std = std if std is not None else [0.229, 0.224, 0.225]

        shape = (3, 1, 1) if self.order == 'chw' else (1, 1, 3)
        self.mean = np.array(mean).reshape(shape).astype('float32')
        self.std = np.array(std).reshape(shape).astype('float32')

    def __call__(self, img):
        from PIL import Image
        if isinstance(img, Image.Image):
            img = np.array(img)

        assert isinstance(img,
                          np.ndarray), "invalid input 'img' in NormalizeImage"

        img = (img.astype('float32') * self.scale - self.mean) / self.std

        if self.channel_num == 4:
            img_h = img.shape[1] if self.order == 'chw' else img.shape[0]
            img_w = img.shape[2] if self.order == 'chw' else img.shape[1]
            pad_zeros = np.zeros(
                (1, img_h, img_w)) if self.order == 'chw' else np.zeros(
                    (img_h, img_w, 1))
            img = (np.concatenate(
                (img, pad_zeros), axis=0)
                   if self.order == 'chw' else np.concatenate(
                       (img, pad_zeros), axis=2))
        return img.astype(self.output_dtype)


class ToCHWImage(object):
    """ convert hwc image to chw image
    """

    def __init__(self):
        pass

    def __call__(self, img):
        from PIL import Image
        if isinstance(img, Image.Image):
            img = np.array(img)

        return img.transpose((2, 0, 1))


def get_result_list(results):
    """ 判断模型输出结果的shape，并返回一维的soft-maxed 数组"""
    shape = np.array(results).shape
    max_val = max(shape)
    max_dim = shape.index(max_val)
    ret = results
    for i in range(max_dim):
        ret = ret[0]
    result_arr = []
    for item in ret:
        real_item = item
        for i in range(max_dim + 1, len(shape)):
            real_item = real_item[0]
        result_arr.append(real_item)
    return np.array(result_arr)