1# concat_prompt
2# 变量名含义
3## latest_question_template：当前问题拼接规范
4## history_qa_template：历史问答拼接规范
5## prompt_template: 外层Prompt模板
6
7final_prompt = ""
8
9round = 1
10need_history_round = "{round}" in latest_question_template
11
12# 历史回话问答内容的拼接
13for i in range(0, len(messages) - 1, 2):
14    history_question = messages[i]["content"]
15    history_answer = messages[i + 1]["content"]
16    
17    if need_history_round:
18        final_prompt += history_qa_template.format(round=round, question=history_question, answer=history_answer)
19    else:
20        final_prompt += history_qa_template.format(question=history_question, answer=history_answer)
21    round += 1
22
23# 当前问题内的拼接
24latest_question = messages[-1]["content"]
25if need_history_round:
26    final_prompt += latest_question_template.format(round=round, question=latest_question)
27else:
28    final_prompt += latest_question_template.format(question=latest_question)
29
30# 外层Prompt模板拼接
31final_prompt = prompt_template.format(input=final_prompt)
32
33print("将要喂给模型的完整prompt如下：\n%s" % final_prompt)
34
35# 进行模型推理
36# inputs = tokenizer(final_prompt)
37# result = model.generate(**inputs)

1当前问题拼接规范：[Round {round}]\n\n问：{question}\n\n答：
2历史问答拼接规范：[Round {round}]\n\n问：{question}\n\n答：{answer}\n\n
3外层Prompt模板：{input}

1# 请求：
2{
3    "messages": [
4        {"role": "user", "content": "问题1"},
5        {"role": "assistant", "content": "答案1"},
6        {"role": "user", "content": "问题2"}
7    ]
8}
9
10# 将要喂给模型的完整prompt如下：
11[Round 1]
12
13问：问题1
14
15答：答案1
16
17[Round 2]
18
19问：问题2
20
21答：

1def build_model_input(model_tokenizer, messages=None, prompt=None, **kwargs):
2    """
3    build model generate input with model tokenizer, input messages(对话模式)
4    :param model_tokenizer: tokenizer loading from uploaded model
5    :param messages: input messages whose structure is same with the `messages` param in qianfan chat api
6    :param kwargs: other params for further future
7    :return: 1-dim token id list
8    """
9    token_ids = []
10    # convert input messages or prompt token ids
11    return token_ids

1# !/usr/bin/env python3
2# -*- coding: utf-8 -*-
3
4
5def build_model_input(model_tokenizer, messages=None, **kwargs):
6    return model_tokenizer.build_chat_input(messages[-1]["content"], history=messages[:-1]).input_ids[0].tolist()

1# !/usr/bin/env python3
2# -*- coding: utf-8 -*-
3from transformers.pipelines.conversational import Conversation
4
5
6def build_model_input(model_tokenizer, messages=None, **kwargs):
7    past_user_inputs, generated_responses = [], []
8    for i in range(0, len(messages) - 1, 2):
9        past_user_inputs.append(messages[i]["content"])
10        generated_responses.append(messages[i + 1]["content"])
11    conv = Conversation(messages[-1]["content"], past_user_inputs=past_user_inputs, generated_responses=generated_responses)
12    return model_tokenizer.apply_chat_template(conv)

1try:
2    try:
3        model = load_model_class.from_pretrained(model_path, trust_remote_code=True,
4                                               torch_dtype=infer_dtype, device_map="auto")
5    except RuntimeError as e:
6        model = load_model_class.from_pretrained(model_path, trust_remote_code=True,
7                                               torch_dtype=infer_dtype, device="cuda")
8except Exception as e:
9    model = load_model_class.from_pretrained(model_path, trust_remote_code=True, device="cuda")

1def load_model(model_path, **kwargs):
2    """
3    load model with model path
4    :param model_path: model path
5    :param kwargs: other params for further future
6    :return: instance of the model class
7    """
8    model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,
9                                       torch_dtype=torch.float16, device_map="auto")
10    return model

1# !/usr/bin/env python3
2# -*- coding: utf-8 -*-
3from transformers import AutoModelForCausalLM
4import torch
5
6def load_model(model_path, **kwargs):
7    model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,
8                                       torch_dtype=torch.float16, device_map="auto")
9    return model

1# !/usr/bin/env python3
2# -*- coding: utf-8 -*-
3from transformers import AutoModelForCausalLM, GenerationConfig
4import torch
5
6def load_model(model_path, **kwargs):
7    max_memory = {i: "75GB" for i in range(8)}
8    model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, device_map="sequential", torch_dtype=torch.bfloat16, max_memory=max_memory, attn_implementation="eager")
9    model.generation_config = GenerationConfig.from_pretrained(model_path)
10    model.generation_config.pad_token_id = model.generation_config.eos_token_id
11    return model