使用vLLM加速大模型推理

export VLLM_USE_MODELSCOPE=True

pip install modelscope -i https://pypi.tuna.tsinghua.edu.cn/simple
用法：
import os
from modelscope import snapshot_download
# cache_dir 指定你的保存模型的路径
model_dir = snapshot_download('shakechen/Llama-2-7b-hf',cache_dir="/home/rd/fanjikang/model")

安装并初始化 git-lfs
apt install git-lfs -y
git lfs install
用法：
git lfs clone https://www.modelscope.cn/qwen/Qwen1.5-1.8B-Chat.git

from vllm import LLM, SamplingParams

# 载入 LLM 和 SamplingParams
from vllm import LLM, SamplingParams
# 推理数据以List[str]格式组织
prompts = ["Hello, my name is","The president of the United States is","The capital of France is","The future of AI is",]
# 设置采样参数
sampling_params = SamplingParams(temperature=0.8,top_p=0.95)
# 加载模型
llm = LLM(model="/root/vllm/models/Qwen1.5-1.8B-Chat")
# 执行推理
outputs=llm.generate(prompts,sampling_params)
# 输出推理结果
for output in outputs:
    prompt=output.prompt
    generated_text=output.outputs[0].text
    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

python -m vllm.entrypoints.openai.api_server \
--model /root/vllm/models/Qwen1.5-1.8B-Chat \
--served-model-name qwen \
--host 0.0.0.0 \
--port 8000

python -m vllm.entrypoints.openai.api_server \
--model /root/vllm/models/Qwen1.5-1.8B-Chat \
--served-model-name qwen \
--host 0.0.0.0 \
--port 8000 \
--tensor-parallel-size 2

curl  http://120.48.131.39:8028/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "qwen",
"messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Who won the world series in 2020?"}
]
}'

python -m vllm.entrypoints.openai.api_server \
--model/root/vllm/models/Qwen1/5-1/8b-Chat \
--served-model-name qwen \
--host 0.0.0.0 \
--port 8000
 #--chat-template ./examples/template_chatml.jinja  目前并没有内置聊天模板文件，可自行到官网下载使用：https://github.com/vllm-project/vllm/tree/main/examples

curl http://120.48.131.39:8028/v1/completions \
-H "Content-Type: application/json" \
-d '{
"model": "qwen",
"prompt": "San Francisco is a",
"max_tokens": 7,
"temperature": 0
}'

from openai import OpenAI

# Modify OpenAI's API key and API base to use vLLM's API server.
openai_api_key = "EMPTY"
openai_api_base = "http://120.48.131.39:8028/v1"
client = OpenAI(
    api_key=openai_api_key,
    base_url=openai_api_base,
)
chat_response = client.chat.completions.create(
    model="qwen",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Tell me a joke."},
    ]
)
print(chat_response)