学员闯关手册:https://aicarrier.feishu.cn/wiki/QdhEwaIINietCak3Y1dcdbLJn3e
课程视频:https://www.bilibili.com/video/BV13U1VYmEUr/
课程文档:https://github.com/InternLM/Tutorial/tree/camp4/docs/L0/Python
关卡作业:https://github.com/InternLM/Tutorial/blob/camp4/docs/L0/Python/task.md
开发机平台:https://studio.intern-ai.org.cn/
开发机平台介绍:https://aicarrier.feishu.cn/wiki/GQ1Qwxb3UiQuewk8BVLcuyiEnHe
书生浦语官网:https://internlm.intern-ai.org.cn/
github网站:https://github.com/internLM/
InternThinker: https://internlm-chat.intern-ai.org.cn/internthinker
快速上手飞书文档:https://www.feishu.cn/hc/zh-CN/articles/945900971706-%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B%E6%96%87%E6%A1%A3
提交作业:https://aicarrier.feishu.cn/share/base/form/shrcnUqshYPt7MdtYRTRpkiOFJd;
作业批改结果:https://aicarrier.feishu.cn/share/base/query/shrcnkNtOS9gPPnC9skiBLlao2c
internLM-Chat 智能体:https://github.com/InternLM/InternLM/blob/main/agent/README_zh-CN.md
lagent:https://lagent.readthedocs.io/zh-cn/latest/tutorials/action.html#id2

LMDeploy 是一个高效且友好的 LLMs 模型部署工具箱,功能涵盖了量化、推理和服务
#1、环境安装
conda create -n lmdeploy python=3.8 -y
conda activate lmdeploy
pip install lmdeploy
#如果发生错误/版本冲突,可指定版本安装(下三行)
export LMDEPLOY_VERSION=0.7.2.post1
export PYTHON_VERSION=38
pip install https://github.com/InternLM/lmdeploy/releases/download/v${LMDEPLOY_VERSION}/lmdeploy-${LMDEPLOY_VERSION}+cu118-cp${PYTHON_VERSION}-cp${PYTHON_VERSION}-manylinux2014_x86_64.whl --extra-index-url https://download.pytorch.org/whl/cu118
#源码安装
git clone https://github.com/InternLM/lmdeploy.git
cd lmdeploy
pip install -e .
#从HF获取模型和acess token,https://huggingface.co,
export HF_TOKEN="your_copied_token_here" # 替换为你的 token
#2.1、本地推理bendituili.py
touch bendituili.py
python bendituili.py
#2.2、指定推理引擎
touch TurbomindEngineConfig.py
python TurbomindEngineConfig.py
#2.3、 VLM 推理(Visual Language Models)
touch VLM.py
#2.4、离线部署lixian.py
touch lixian.py
#2.5、 部署类OpenAI 服务-- 使用 lmdeploy cli 工具
lmdeploy serve api_server internlm/InternVL2-1B --server-port 23333
#2.6、 部署类OpenAI 服务-- - 使用类openai方式
touch none_api.py
#2.7、使用WebUI
# 启动server
# ApiServer+Turbomind api_server => AsyncEngine => TurboMind
lmdeploy serve api_server InternVL2-1B --server-name 0.0.0.0 --server-port 23333 --tp 1
# Gradio+ApiServer。必须先开启 Server,此时 Gradio 为 Client
lmdeploy serve gradio http://0.0.0.0:23333 --server-name 0.0.0.0 --server-port 6006
#2.8、部署Huggingface Demo
# app.py
#3、视觉-语言模型(VLMs)部署
#3.1、lixian_VLM.py
touch lixian_VLM.py
#3.2、token_VLM.py
touch token.py
IMAGE_TOKEN = '<IMAGE_TOKEN>'
#3.3、自定义对话self_lag_VLM.py
touch self_lag.py
#3.4、多图推理duotu_VLM.py
#3.5、多轮对话mlt_VLM.py
#2.1、bendituili.py
import lmdeploy
pipe = lmdeploy.pipeline("OpenGVLab/InternVL2_5-1B")
response = pipe(prompts=["Hi, pls intro yourself", "Shanghai is"],
gen_config=GenerationConfig(max_new_tokens=1024,
top_p=0.8,
top_k=40,
temperature=0.6))
print(response)
实践过程中报错了3次,1个错误是没有HF的token无法读模型,另外1个错误是HF上没有教程中的模型internlm/InternVL2-1B,搜索改为了模型OpenGVLab/InternVL2_5-1B,第3个错误是没有 GenerationConfig模块引出没有_turbomind模块,最终这个代码无法运行
2、模型部署
### 2.2、TurbomindEngineConfig推理引擎TurbomindEngineConfig.py
from lmdeploy import pipeline, TurbomindEngineConfig
pipe = pipeline('OpenGVLab/InternVL2_5-1B',
backend_config=TurbomindEngineConfig(
max_batch_size=32,
enable_prefix_caching=True,
cache_max_entry_count=0.8,
session_len=8192,
))
### PytorchEngineConfig推理引擎
from lmdeploy import pipeline, PytorchEngineConfig
pipe = pipeline('internlm/InternVL2-1B',
backend_config=PytorchEngineConfig(
max_batch_size=32,
enable_prefix_caching=True,
cache_max_entry_count=0.8,
session_len=8192,
))
#2.3、VLM.py
from lmdeploy import pipeline, VisionConfig
from lmdeploy.vl import load_image
pipe = pipeline('OpenGVLab/InternVL2-1B',
vision_config=VisionConfig(
max_batch_size=8
))
image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg')
response = pipe(('describe this image', image))
print(response)
#2.4、lixian.py
import lmdeploy
from lmdeploy import GenerationConfig
pipe = lmdeploy.pipeline("internlm/InternVL2-1B")
response = pipe(prompts=["Hi, pls intro yourself", "Shanghai is"],
gen_config=GenerationConfig(max_new_tokens=1024,
top_p=0.8,
top_k=40,
temperature=0.6))
print(response)
#2.6、none_api.py
from openai import OpenAI
client = OpenAI(
api_key='none',# 若未启用鉴权,可填任意值(如 "none")
base_url="http://0.0.0.0:23333/v1"
)
model_name = client.models.list().data[0].id
response = client.chat.completions.create(
model=model_name,
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": " provide three suggestions about time management"},
],
temperature=0.8,
top_p=0.8
)
print(response)
#2.8、app.py
from lmdeploy.serve.gradio.turbomind_coupled import run_local
from lmdeploy.messages import TurbomindEngineConfig
backend_config = TurbomindEngineConfig(max_batch_size=8)
model_path = 'InternVL2-1B'
run_local(model_path, backend_config=backend_config, server_name="huggingface-space")
3、视觉模型VLM模型部署
#3.1、lixian_VLM.py
from lmdeploy import pipeline
from lmdeploy.vl import load_image
pipe = pipeline('InternVL2-1B')
image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/tests/data/tiger.jpeg')
response = pipe(('describe this image', image))
print(response)
#3.2、token_VLM.py
from lmdeploy import pipeline
from lmdeploy.vl import load_image
from lmdeploy.vl.constants import IMAGE_TOKEN
pipe = pipeline('InternVL2-1B')
image = load_image('tiger.jpeg')
response = pipe((f'describe this image{IMAGE_TOKEN}', image))
print(response)
#3.3、自定义对话self_lag_VLM.py
from lmdeploy.model import MODELS, BaseChatTemplate
@MODELS.register_module(name='customized_model')
class CustomizedModel(BaseChatTemplate):
"""A customized chat template."""
def __init__(self,
system='<|im_start|>system\n',
meta_instruction='You are a robot developed by LMDeploy.',
user='<|im_start|>user\n',
assistant='<|im_start|>assistant\n',
eosys='<|im_end|>\n',
eoh='<|im_end|>\n',
eoa='<|im_end|>',
separator='\n',
stop_words=['<|im_end|>', '<|action_end|>']):
super().__init__(system=system,
meta_instruction=meta_instruction,
eosys=eosys,
user=user,
eoh=eoh,
assistant=assistant,
eoa=eoa,
separator=separator,
stop_words=stop_words)
from lmdeploy import ChatTemplateConfig, pipeline
messages = [{'role': 'user', 'content': 'who are you?'}]
pipe = pipeline('InternVL2-1B',
chat_template_config=ChatTemplateConfig('customized_model'))
for response in pipe.stream_infer(messages):
print(response.text, end='')
#3.4、多图推理duotu_VLM.py
from lmdeploy import pipeline, TurbomindEngineConfig
from lmdeploy.vl import load_image
pipe = pipeline('InternVL2-1B',
backend_config=TurbomindEngineConfig(session_len=8192))
image_urls=[
'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg',
'https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/det.jpg'
]
images = [load_image(img_url) for img_url in image_urls]
response = pipe(('describe these images', images))
print(response)
#3.5、多轮对话mlt_VLM.py
from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig
from lmdeploy.vl import load_image
pipe = pipeline('InternVL2-1B',
backend_config=TurbomindEngineConfig(session_len=8192))
image = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg')
gen_config = GenerationConfig(top_k=40, top_p=0.8, temperature=0.6)
sess = pipe.chat(('describe this image', image), gen_config=gen_config)
print(sess.response.text)
sess = pipe.chat('What is the woman doing?', session=sess, gen_config=gen_config)
print(sess.response.text)
4、模型量化
LMDeploy 量化模块目前仅支持 AWQ 量化算法。
量化是指将高精度数字转换为低精度数字。低精度实体可以存储在磁盘上很小的空间内,从而减少内存需求。
4.1、量化模型
#4.1、量化模型
export HF_MODEL=internlm/internlm2_5-7b-chat
export WORK_DIR=internlm/internlm2_5-7b-chat-4bit
lmdeploy lite auto_awq \
$HF_MODEL \
--calib-dataset 'ptb' \
--calib-samples 128 \
--calib-seqlen 2048 \
--w-bits 4 \
--w-group-size 128 \
--batch-size 1 \
--work-dir $WORK_DIR
4.2、模型评测
#4.2、模型评测
#在评测一个模型时,需要准备一份评测配置,指明评测集、模型和推理参数等信息。
cd /root/opencompass/opencompass/configs
touch configs/eval_internlm2_lmdeploy.py
#编辑文件eval_internlm2_lmdeploy.py
python run.py configs/eval_internlm2_lmdeploy.py -w outputs
/root/opencompass/opencompass/configs/eval_internlm2_lmdeploy.py
# configure the dataset
from mmengine.config import read_base
with read_base():
# choose a list of datasets
from .datasets.mmlu.mmlu_gen_a484b3 import mmlu_datasets
from .datasets.ceval.ceval_gen_5f30c7 import ceval_datasets
from .datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
from opencompass.configs.datasets.gsm8k.gsm8k_0shot_v2_gen_a58960 import \
gsm8k_datasets
# and output the results in a chosen format
from .summarizers.medium import summarizer
datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
# configure lmdeploy
from opencompass.models import TurboMindModelwithChatTemplate
# configure the model
models = [
dict(
type=TurboMindModelwithChatTemplate,
abbr=f'internlm2-chat-7b-lmdeploy',
# model path, which can be the address of a model repository on the Hugging Face Hub or a local path
path='internlm/internlm2-chat-7b',
# inference backend of LMDeploy. It can be either 'turbomind' or 'pytorch'.
# If the model is not supported by 'turbomind', it will fallback to
# 'pytorch'
backend='turbomind',
# For the detailed engine config and generation config, please refer to
# https://github.com/InternLM/lmdeploy/blob/main/lmdeploy/messages.py
engine_config=dict(tp=1),
gen_config=dict(do_sample=False),
# the max size of the context window
max_seq_len=7168,
# the max number of new tokens
max_out_len=1024,
# the max number of prompts that LMDeploy receives
# in `generate` function
batch_size=5000,
run_cfg=dict(num_gpus=1),
)
]
4.3 W8A8量化
#1.lmdeploy 提供了命令行工具 lmdeploy lite smooth_quant 实现量化功能。并且其中命令行参数 --quant-dtype 可以用来控制是进行8-bit整数还是浮点数类型的量化。更多命令行工具使用方式,请执行 lmdeploy lite smooth_quant --help 查看。
pip install lmdeploy[all]
#2. int8量化
lmdeploy lite smooth_quant internlm/internlm2_5-7b-chat --work-dir ./internlm2_5-7b-chat-int8 --quant-dtype int8
#3. fp8量化
lmdeploy lite smooth_quant internlm/internlm2_5-7b-chat --work-dir ./internlm2_5-7b-chat-fp8 --quant-dtype fp8
4.4 Key-Value(KV) Cache 量化
#Key-Value(KV) Cache 量化
#LMDeploy 规定 qant_policy=4 表示 kv int4 量化,quant_policy=8 表示 kv int8 量化。
from lmdeploy import pipeline, TurbomindEngineConfig
engine_config = TurbomindEngineConfig(quant_policy=8)
pipe = pipeline("internlm/internlm2_5-7b-chat", backend_config=engine_config)
response = pipe(["Hi, pls intro yourself", "Shanghai is"])
print(response)
305



被折叠的 条评论
为什么被折叠?



