From baf0afcf411b5c22868c097437368caf21108c78 Mon Sep 17 00:00:00 2001 From: horcam Date: Wed, 21 Jan 2026 18:19:24 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=8B=A5=E5=B9=B2?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/vllm_mindspore/docs/source_en/conf.py | 2 +- .../source_en/developer_guide/contributing.md | 2 +- .../docs/source_en/faqs/faqs.md | 4 ++-- .../installation/installation.md | 24 ++++++++++++------- .../quick_start/quick_start.md | 2 +- .../deepseek_r1_671b_w8a8_dp4_tp4_ep4.md | 2 +- .../qwen2.5_32b_multiNPU.md | 2 +- .../qwen2.5_7b_singleNPU.md | 2 +- .../qwen3_vl_8b_singleNPU.md | 2 +- docs/vllm_mindspore/docs/source_en/index.rst | 8 +++---- .../source_en/release_notes/release_notes.md | 11 +++++++-- docs/vllm_mindspore/docs/source_zh_cn/conf.py | 2 +- .../developer_guide/contributing.md | 2 +- .../docs/source_zh_cn/faqs/faqs.md | 4 ++-- .../installation/installation.md | 24 ++++++++++++------- .../quick_start/quick_start.md | 2 +- .../deepseek_r1_671b_w8a8_dp4_tp4_ep4.md | 2 +- .../qwen2.5_32b_multiNPU.md | 2 +- .../qwen2.5_7b_singleNPU.md | 2 +- .../qwen3_vl_8b_singleNPU.md | 2 +- .../docs/source_zh_cn/index.rst | 6 ++--- .../release_notes/release_notes.md | 9 +++++-- 22 files changed, 71 insertions(+), 47 deletions(-) diff --git a/docs/vllm_mindspore/docs/source_en/conf.py b/docs/vllm_mindspore/docs/source_en/conf.py index 964df2041b..f82c63fa1c 100644 --- a/docs/vllm_mindspore/docs/source_en/conf.py +++ b/docs/vllm_mindspore/docs/source_en/conf.py @@ -37,7 +37,7 @@ copyright = 'MindSpore' author = 'vLLM-MindSpore Plugin' # The full version, including alpha/beta/rc tags -release = '0.5.0' +release = '0.5.1' # -- General configuration --------------------------------------------------- diff --git a/docs/vllm_mindspore/docs/source_en/developer_guide/contributing.md b/docs/vllm_mindspore/docs/source_en/developer_guide/contributing.md index 52dda0064d..1a4deadc42 100644 --- a/docs/vllm_mindspore/docs/source_en/developer_guide/contributing.md +++ b/docs/vllm_mindspore/docs/source_en/developer_guide/contributing.md @@ -18,7 +18,7 @@ To support a new model for vLLM-MindSpore Plugin code repository, please note th - **Follow file format and location specifications.** Model code files should be placed under the `vllm_mindspore/model_executor` directory, organized in corresponding subfolders by model type. - **Implement models using MindSpore interfaces with jit static graph support.** Model definitions in vLLM-MindSpore Plugin must be implemented using MindSpore interfaces. Since MindSpore's static graph mode offers performance advantages, models should support execution via @jit static graphs. For reference, see the [Qwen2.5](https://gitee.com/mindspore/vllm-mindspore/blob/r0.5.0/vllm_mindspore/model_executor/models/qwen2.py) implementation. - **Register new models in vLLM-MindSpore Plugin.** After implementing the model structure, register it in vLLM-MindSpore Plugin by adding it to `_NATIVE_MODELS` in `vllm_mindspore/model_executor/models/registry.py`. -- **Write unit tests.** New models must include corresponding unit tests. Refer to the [Qwen2.5 testcases](https://gitee.com/mindspore/vllm-mindspore/blob/r0.5.0/tests/st/python/cases_parallel/vllm_qwen_7b.py) for examples. +- **Write unit tests.** New models must include corresponding unit tests. Refer to the [Qwen2.5 testcases](https://gitee.com/mindspore/vllm-mindspore/blob/master/tests/st/models/qwen2_5/test_vllm_qwen_7b_v1.py) for examples. ## Contribution Process diff --git a/docs/vllm_mindspore/docs/source_en/faqs/faqs.md b/docs/vllm_mindspore/docs/source_en/faqs/faqs.md index 24343451c8..fdb8fa7898 100644 --- a/docs/vllm_mindspore/docs/source_en/faqs/faqs.md +++ b/docs/vllm_mindspore/docs/source_en/faqs/faqs.md @@ -11,7 +11,7 @@ Execute the following command to install vLLM-MindSpore Plugin: ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore bash install_depend_pkgs.sh pip install . @@ -29,7 +29,7 @@ 2. Please check if the `pip` version is greater than or equal to 25.3. If so, please use the following command to compile and install vLLM-MindSpore Plugin: ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore bash install_depend_pkgs.sh pip install --no-build-isolation . diff --git a/docs/vllm_mindspore/docs/source_en/getting_started/installation/installation.md b/docs/vllm_mindspore/docs/source_en/getting_started/installation/installation.md index c185b14f7a..8774ce195d 100644 --- a/docs/vllm_mindspore/docs/source_en/getting_started/installation/installation.md +++ b/docs/vllm_mindspore/docs/source_en/getting_started/installation/installation.md @@ -10,18 +10,24 @@ This document will introduce the [Version Matching](#version-compatibility) of v ## Version Compatibility - OS: Linux-aarch64 -- Python:3.10 / 3.11 / 3.12 +- Python:3.9 / 3.10 / 3.11 - Depent Software version compatibility | Software | Version And Links | | ----- | ----- | | CANN | [8.3.RC1](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/83RC1/index/index.html) | - | MindSpore | [2.7.1.post1](https://repo.mindspore.cn/mindspore/mindspore/version/202512/20251225/r2.7.1_20251225204822_6eefcb41b2c30e42eb049908d5efea40c3425abc_newest/) | + | MindSpore | [2.7.1.post1](https://repo.mindspore.cn/mindspore/mindspore/version/202512/20251229/r2.7.1_20251229100307_d3099307fd78df39099f0e7801a6d2aa3121bf2c_newest/) | | MSAdapter| [0.0.5](https://repo.mindspore.cn/mindspore/msadapter/version/202510/20251011/r0.3.0_20251011095813_951a8218d4c29785e48f304e720212b57056573e_newest/) | - | MindSpore Transformers | [1.7.0](https://repo.mindspore.cn/mindspore/mindformers/version/202510/20251030/r1.7.0_20251030031507_8ccc49b3f6645d3d1abfab80b4c78f3cafe5c84e_newest/) | - | vLLM | [0.11.0](https://repo.mindspore.cn/mirrors/vllm/version/202511/20251113/v0.11.0/) | - | ms_custom_ops | [0.1.0](https://repo.mindspore.cn/mindspore/ms_custom_ops/version/202512/20251210/r0.1.0_20251210100500_15d84de93ca616c1a9880275c15213a34f2c0a39_newest/) | - | MindSpore ONE | [0.5.0](https://repo.mindspore.cn/mindspore-lab/mindone/version/202512/20251223/master_20251223115818_36c25653b3ebaff55eb40ca10bc742d603b21f0a_newest/) | + | MindSpore Transformers | [1.7.0](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1/MindFormers/any/mindformers-1.7.0-py3-none-any.whl) | + | vLLM | [0.11.0](https://repo.mindspore.cn/mirrors/vllm/version/202511/20251113/v0.11.0/any/vllm-0.11.0%2Bempty-py3-none-any.whl) | + | ms_custom_ops | [0.1.0](https://repo.mindspore.cn/mindspore/ms_custom_ops/version/202512/20251229/r0.1.0_20251229145109_5bd05d81cf153ce96c2244fbf5d0c03b9fba1ffe_newest/) | + | MindSpore ONE | [0.5.0](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/MindOne/any/mindone-0.5.0-py3-none-any.whl) | + +- Source code and download link of vLLM-MindSpore Plugin + + | Source Code Link | Package Link | + | ----- | ----- | + | [v0.5.1](https://gitee.com/mindspore/vllm-mindspore/tree/v0.5.1/) | [Python3.9](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/VllmMindSpore/ascend/aarch64/vllm_mindspore-0.5.1-cp39-cp39-linux_aarch64.whl),[Python3.10](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/VllmMindSpore/ascend/aarch64/vllm_mindspore-0.5.1-cp310-cp310-linux_aarch64.whl),[Python3.11](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/VllmMindSpore/ascend/aarch64/vllm_mindspore-0.5.1-cp311-cp311-linux_aarch64.whl) | ## Docker Installation @@ -32,7 +38,7 @@ We recommend using Docker for quick deployment of the vLLM-MindSpore Plugin envi User can execute the following commands to clone the vLLM-MindSpore Plugin code repository: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` To build the image according to your npu type, follow these steps: @@ -136,7 +142,7 @@ vLLM-MindSpore Plugin can be installed in the following two ways. **vLLM-MindSpo To install vLLM-MindSpore Plugin, user needs to pull the vLLM-MindSpore Plugin source code and then runs the following command to install the dependencies: ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore bash install_depend_pkgs.sh ``` @@ -206,7 +212,7 @@ vLLM-MindSpore Plugin can be installed in the following two ways. **vLLM-MindSpo User could also use source code to install vLLM-MindSpore Plugin. ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore pip install . ``` diff --git a/docs/vllm_mindspore/docs/source_en/getting_started/quick_start/quick_start.md b/docs/vllm_mindspore/docs/source_en/getting_started/quick_start/quick_start.md index 6b0b5f5c6e..48422e84d6 100644 --- a/docs/vllm_mindspore/docs/source_en/getting_started/quick_start/quick_start.md +++ b/docs/vllm_mindspore/docs/source_en/getting_started/quick_start/quick_start.md @@ -13,7 +13,7 @@ In this section, we recommend using docker to deploy the vLLM-MindSpore Plugin e User can execute the following commands to clone the vLLM-MindSpore Plugin code repository: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` To build the image according to your npu type, follow these steps: diff --git a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md index a22a7dec49..c0ae90dc0a 100644 --- a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md +++ b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md @@ -17,7 +17,7 @@ In this section, we recommend using Docker creation to quickly deploy the vLLM-M Users can execute the following commands to pull the vLLM-MindSpore plugin code repository and build the image: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git bash build_image.sh ``` diff --git a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md index f7847be4f5..738b35abd5 100644 --- a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md +++ b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md @@ -13,7 +13,7 @@ In this section, we recommend using Docker for quick deployment of the vLLM-Mind User can execute the following commands to clone the vLLM-MindSpore Plugin code repository: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` To build the image according to your npu type, follow these steps: diff --git a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md index c9dea90b98..7c326924fc 100644 --- a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md +++ b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md @@ -13,7 +13,7 @@ In this section, we recommend using Docker for quick deployment of the vLLM-Mind User can execute the following commands to clone the vLLM-MindSpore Plugin code repository: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` To build the image according to your npu type, follow these steps: diff --git a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md index 7549a292f6..386b0296cb 100644 --- a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md +++ b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md @@ -13,7 +13,7 @@ In this section, we recommend using Docker for quick deployment of the vLLM-Mind User can execute the following commands to clone the vLLM-MindSpore Plugin code repository: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` To build the image according to your npu type, follow these steps: diff --git a/docs/vllm_mindspore/docs/source_en/index.rst b/docs/vllm_mindspore/docs/source_en/index.rst index 351e36ab42..de5f5edb33 100644 --- a/docs/vllm_mindspore/docs/source_en/index.rst +++ b/docs/vllm_mindspore/docs/source_en/index.rst @@ -39,8 +39,8 @@ Prerequisites - Operating System: openEuler or Ubuntu Linux - Software: - * Python >= 3.10, <= 3.12 - * CANN >= 8.3.RC1 + * Python >= 3.9, <= 3.11 + * CANN == 8.3.RC1 * MindSpore (matched with the vllm-mindspore version) * vLLM (matched with the vllm-mindspore version) @@ -88,9 +88,9 @@ The following are the version branches: * - r0.4.0 - Maintained - Compatible with vLLM v0.9.1, and CI commitment for MindSpore 2.7.1 - * - r0.5.0 + * - v0.5.1 - Maintained - - Compatible with vLLM v0.11.0, and CI commitment for MindSpore 2.7.1.post1 + - Compatible with vLLM v0.11.0, and CI commitment for MindSpore 2.7.1.post1 SIG ----------------------------------------------------- diff --git a/docs/vllm_mindspore/docs/source_en/release_notes/release_notes.md b/docs/vllm_mindspore/docs/source_en/release_notes/release_notes.md index 0e75a38251..d95e6acbce 100644 --- a/docs/vllm_mindspore/docs/source_en/release_notes/release_notes.md +++ b/docs/vllm_mindspore/docs/source_en/release_notes/release_notes.md @@ -2,9 +2,9 @@ [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.7.1.post1/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/r2.7.1.post1/docs/vllm_mindspore/docs/source_en/release_notes/release_notes.md) -## vLLM-MindSpore Plugin 0.5.0 Release Notes +## vLLM-MindSpore Plugin 0.5.1 Release Notes -The vLLM-MindSpore Plugin 0.5.0 version is compatible with vLLM 0.11.0. Below are the new features and models supported in this release. +The vLLM-MindSpore Plugin 0.5.1 version is compatible with vLLM 0.11.0. Below are the new features and models supported in this release. ### New Features @@ -18,6 +18,13 @@ The vLLM-MindSpore Plugin 0.5.0 version is compatible with vLLM 0.11.0. Below ar - GLM Model Series: - GLM-4.1V Native Model +### Problem Fixes + +- Encoder Memory Leak Issue + - [Fix for vLLM](https://github.com/vllm-project/vllm/pull/31857) + - [Fix for vLLM-MindSpore Plugin](https://gitee.com/mindspore/vllm-mindspore/pulls/1447) + + ### Contributors Thanks to the following contributors for their efforts: diff --git a/docs/vllm_mindspore/docs/source_zh_cn/conf.py b/docs/vllm_mindspore/docs/source_zh_cn/conf.py index ca970f28a8..a2e5503c37 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/conf.py +++ b/docs/vllm_mindspore/docs/source_zh_cn/conf.py @@ -28,7 +28,7 @@ copyright = 'MindSpore' author = 'vLLM-MindSpore Plugin' # The full version, including alpha/beta/rc tags -release = '0.5.0' +release = '0.5.1' # -- General configuration --------------------------------------------------- diff --git a/docs/vllm_mindspore/docs/source_zh_cn/developer_guide/contributing.md b/docs/vllm_mindspore/docs/source_zh_cn/developer_guide/contributing.md index 2351177758..7da84784bf 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/developer_guide/contributing.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/developer_guide/contributing.md @@ -19,7 +19,7 @@ - **文件格式及位置要遵循规范。** 模型代码文件统一放置于`vllm_mindspore/model_executor`文件夹下,请根据不同模型将代码文件放置于对应的文件夹下。 - **模型基于MindSpore接口实现,支持jit静态图方式执行。** vLLM-MindSpore插件中的模型定义需基于MindSpore接口实现。由于MindSpore静态图模式执行性能有优势,因此模型需支持@jit静态图方式执行。详细可参考[Qwen2.5](https://gitee.com/mindspore/vllm-mindspore/blob/r0.5.0/vllm_mindspore/model_executor/models/qwen2.py)模型定义实现。 - **将新模型在vLLM-MindSpore插件代码中进行注册。** 模型结构定义实现后,需要将该模型注册到vLLM-MindSpore插件中,注册文件位于'vllm_mindspore/model_executor/models/registry.py'中,请将模型注册到`_NATIVE_MODELS`。 -- **编写单元测试。** 新增的模型需同步提交单元测试用例,用例编写请参考[Qwen2.5模型用例](https://gitee.com/mindspore/vllm-mindspore/blob/r0.5.0/tests/st/python/cases_parallel/vllm_qwen_7b.py)。 +- **编写单元测试。** 新增的模型需同步提交单元测试用例,用例编写请参考[Qwen2.5模型用例](https://gitee.com/mindspore/vllm-mindspore/blob/master/tests/st/models/qwen2_5/test_vllm_qwen_7b_v1.py)。 ## 贡献流程 diff --git a/docs/vllm_mindspore/docs/source_zh_cn/faqs/faqs.md b/docs/vllm_mindspore/docs/source_zh_cn/faqs/faqs.md index 71b34071d5..5df7f9c832 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/faqs/faqs.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/faqs/faqs.md @@ -11,7 +11,7 @@ 在执行以下命令安装vLLM-MindSpore Plugin ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore bash install_depend_pkgs.sh pip install . @@ -29,7 +29,7 @@ 2. 请检查`pip`版本是否大于等于25.3。如果是,则使用以下命令编译并安装vLLM-MindSpore Plugin: ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore bash install_depend_pkgs.sh pip install --no-build-isolation . diff --git a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/installation/installation.md b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/installation/installation.md index ab35b88911..3127218de3 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/installation/installation.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/installation/installation.md @@ -10,18 +10,24 @@ ## 版本配套 - OS:Linux-aarch64 -- Python:3.10 / 3.11 / 3.12 +- Python:3.9 / 3.10 / 3.11 - 依赖软件版本配套 | 软件 | 配套版本与下载链接 | | ----- | ----- | | CANN | [8.3.RC1](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/83RC1/index/index.html) | - | MindSpore | [2.7.1.post1](https://repo.mindspore.cn/mindspore/mindspore/version/202512/20251225/r2.7.1_20251225204822_6eefcb41b2c30e42eb049908d5efea40c3425abc_newest/) | + | MindSpore | [2.7.1.post1](https://repo.mindspore.cn/mindspore/mindspore/version/202512/20251229/r2.7.1_20251229100307_d3099307fd78df39099f0e7801a6d2aa3121bf2c_newest/) | | MSAdapter| [0.0.5](https://repo.mindspore.cn/mindspore/msadapter/version/202510/20251011/r0.3.0_20251011095813_951a8218d4c29785e48f304e720212b57056573e_newest/) | - | MindSpore Transformers | [1.7.0](https://repo.mindspore.cn/mindspore/mindformers/version/202510/20251030/r1.7.0_20251030031507_8ccc49b3f6645d3d1abfab80b4c78f3cafe5c84e_newest/) | - | vLLM | [0.11.0](https://repo.mindspore.cn/mirrors/vllm/version/202511/20251113/v0.11.0/) | - | ms_custom_ops | [0.1.0](https://repo.mindspore.cn/mindspore/ms_custom_ops/version/202512/20251210/r0.1.0_20251210100500_15d84de93ca616c1a9880275c15213a34f2c0a39_newest/) | - | MindSpore ONE | [0.5.0](https://repo.mindspore.cn/mindspore-lab/mindone/version/202512/20251223/master_20251223115818_36c25653b3ebaff55eb40ca10bc742d603b21f0a_newest/) | + | MindSpore Transformers | [1.7.0](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1/MindFormers/any/mindformers-1.7.0-py3-none-any.whl) | + | vLLM | [0.11.0](https://repo.mindspore.cn/mirrors/vllm/version/202511/20251113/v0.11.0/any/vllm-0.11.0%2Bempty-py3-none-any.whl) | + | ms_custom_ops | [0.1.0](https://repo.mindspore.cn/mindspore/ms_custom_ops/version/202512/20251229/r0.1.0_20251229145109_5bd05d81cf153ce96c2244fbf5d0c03b9fba1ffe_newest/) | + | MindSpore ONE | [0.5.0](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/MindOne/any/mindone-0.5.0-py3-none-any.whl) | + +- vLLM-MindSpore插件代码仓与下载链接 + + |代码仓链接 | 插件包下载链接 | + | ----- | ----- | + | [v0.5.1](https://gitee.com/mindspore/vllm-mindspore/tree/v0.5.1/) | [Python3.9](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/VllmMindSpore/ascend/aarch64/vllm_mindspore-0.5.1-cp39-cp39-linux_aarch64.whl),[Python3.10](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/VllmMindSpore/ascend/aarch64/vllm_mindspore-0.5.1-cp310-cp310-linux_aarch64.whl),[Python3.11](https://ms-release.obs.cn-north-4.myhuaweicloud.com/2.7.1.post1/VllmMindSpore/ascend/aarch64/vllm_mindspore-0.5.1-cp311-cp311-linux_aarch64.whl) | ## docker安装 @@ -32,7 +38,7 @@ 用户可执行以下命令,拉取vLLM-MindSpore插件代码仓库: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` 根据计算卡类型,构建镜像: @@ -136,7 +142,7 @@ vLLM-MindSpore插件有以下两种安装方式。**vLLM-MindSpore插件快速 采用快速安装脚本来安装vLLM-MindSpore插件,需要在拉取vLLM-MindSpore插件源码后,执行以下命令安装依赖包: ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore bash install_depend_pkgs.sh ``` @@ -206,7 +212,7 @@ vLLM-MindSpore插件有以下两种安装方式。**vLLM-MindSpore插件快速 也可以进行如下的源码编译安装: ```bash - git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git + git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git cd vllm-mindspore pip install . ``` diff --git a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/quick_start/quick_start.md b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/quick_start/quick_start.md index 1c4097ac33..4363e67426 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/quick_start/quick_start.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/quick_start/quick_start.md @@ -13,7 +13,7 @@ 用户可执行以下命令,拉取vLLM-MindSpore插件代码仓库: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` 根据计算卡类型,构建镜像: diff --git a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md index 441c7f00f9..0d843e6361 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/deepseek_parallel/deepseek_r1_671b_w8a8_dp4_tp4_ep4.md @@ -17,7 +17,7 @@ vLLM-MindSpore插件支持[张量并行(TP)](https://docs.vllm.ai/en/v0.11.0 用户可执行以下命令,拉取vLLM-MindSpore插件代码仓库,并构建镜像: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git bash build_image.sh ``` diff --git a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md index c4b9cc0d80..5f86f14695 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_32b_multiNPU/qwen2.5_32b_multiNPU.md @@ -13,7 +13,7 @@ 用户可执行以下命令,拉取vLLM-MindSpore插件代码仓库: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` 根据计算卡类型,构建镜像: diff --git a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md index aac0d4ef92..ce2d86e1e4 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen2.5_7b_singleNPU/qwen2.5_7b_singleNPU.md @@ -13,7 +13,7 @@ 用户可执行以下命令,拉取vLLM-MindSpore插件代码仓库: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` 根据计算卡类型,构建镜像: diff --git a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md index 929357b206..3921547504 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md @@ -13,7 +13,7 @@ 用户可执行以下命令,拉取vLLM-MindSpore插件代码仓库: ```bash -git clone -b r0.5.0 https://gitee.com/mindspore/vllm-mindspore.git +git clone -b v0.5.1 https://gitee.com/mindspore/vllm-mindspore.git ``` 根据计算卡类型,构建镜像: diff --git a/docs/vllm_mindspore/docs/source_zh_cn/index.rst b/docs/vllm_mindspore/docs/source_zh_cn/index.rst index d6ab0f650d..5f1a25db41 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/index.rst +++ b/docs/vllm_mindspore/docs/source_zh_cn/index.rst @@ -39,8 +39,8 @@ vLLM-MindSpore插件采用vLLM社区推荐的插件机制,实现能力注册 - 操作系统:openEuler或Ubuntu Linux - 软件: - * Python >= 3.10, <= 3.12 - * CANN >= 8.3.RC1 + * Python >= 3.9, <= 3.11 + * CANN == 8.3.RC1 * MindSpore(与vLLM-MindSpore插件版本配套) * vLLM(与vLLM-MindSpore插件版本配套) @@ -88,7 +88,7 @@ vLLM-MindSpore插件代码仓包含主干分支、开发分支、版本分支: * - r0.4.0 - Maintained - 基于vLLM v0.9.1版本和MindSpore 2.7.1版本CI看护 - * - r0.5.0 + * - v0.5.1 - Maintained - 基于vLLM v0.11.0版本和MindSpore 2.7.1.post1版本CI看护 diff --git a/docs/vllm_mindspore/docs/source_zh_cn/release_notes/release_notes.md b/docs/vllm_mindspore/docs/source_zh_cn/release_notes/release_notes.md index 8e628d26c7..ddd45a05a9 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/release_notes/release_notes.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/release_notes/release_notes.md @@ -2,9 +2,9 @@ [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.7.1.post1/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/r2.7.1.post1/docs/vllm_mindspore/docs/source_zh_cn/release_notes/release_notes.md) -## vLLM-MindSpore插件 0.5.0 Release Notes +## vLLM-MindSpore插件 0.5.1 Release Notes -vLLM MindSpore插件0.5.0版本,配套vLLM 0.11.0版本。以下为此版本支持的关键新功能和模型。 +vLLM MindSpore插件0.5.1版本,配套vLLM 0.11.0版本。以下为此版本支持的关键新功能和模型。 ### 新特性 @@ -18,6 +18,11 @@ vLLM MindSpore插件0.5.0版本,配套vLLM 0.11.0版本。以下为此版本 - GLM系列模型 - GLM-4.1V支持原生模型 +### 问题修复 +- 关于encoder的显存泄露问题 + - [对于vLLM的修复](https://github.com/vllm-project/vllm/pull/31857) + - [对于vLLM-MindSpore插件的修复](https://gitee.com/mindspore/vllm-mindspore/pulls/1447) + ### 贡献者 感谢以下人员做出的贡献: -- Gitee From e93b3598c86defb6cd9d4fe7194b61ef85ff4ff8 Mon Sep 17 00:00:00 2001 From: lvhaoyu Date: Mon, 26 Jan 2026 11:35:24 +0800 Subject: [PATCH 2/2] update vllm-mindspore multimodal docs --- .../qwen3_vl_8b_singleNPU.md | 64 ++++++++----------- .../models_list/models_list.md | 2 +- .../qwen3_vl_8b_singleNPU.md | 60 +++++++---------- .../models_list/models_list.md | 2 +- 4 files changed, 50 insertions(+), 78 deletions(-) diff --git a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md index 7549a292f6..a590b4189d 100644 --- a/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md +++ b/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md @@ -1,8 +1,8 @@ -# Multi-modal Single-Card Inference (Qwen2.5-7B) +# Multi-modal Single-Card Inference (Qwen3-VL-8B-Instruct) [![View Source On Gitee](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/r2.7.1.post1/resource/_static/logo_source_en.svg)](https://gitee.com/mindspore/docs/blob/r2.7.1.post1/docs/vllm_mindspore/docs/source_en/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md) -This document introduces single NPU inference process by vLLM-MindSpore Plugin. Taking the [Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) model as an example, user can configure the environment through the [Docker Installation](#docker-installation) or the [Installation Guide](../../installation/installation.md#installation-guide), and [downloading model weights](#downloading-model-weights). After [setting environment variables](#setting-environment-variables), user can perform [offline inference](#offline-inference) and [online inference](#online-inference) to experience single NPU inference abilities. +This document introduces single NPU multimodal inference process by vLLM-MindSpore Plugin. Taking the [Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) model as an example, user can configure the environment through the [Docker Installation](#docker-installation) or the [Installation Guide](../../installation/installation.md#installation-guide), and [downloading model weights](#downloading-model-weights). After [setting environment variables](#setting-environment-variables), user can perform [offline inference](#offline-inference) and [online inference](#online-inference) to experience single NPU inference abilities. ## Docker Installation @@ -92,22 +92,7 @@ docker exec -it $DOCKER_NAME bash ## Downloading Model Weights -User can download the model using either [Python Tool](#downloading-with-python-tool) or [git-lfs Tool](#downloading-with-git-lfs-tool). - -### Downloading with Python Tool - -Execute the following Python script to download the [Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct) weights and files from [Hugging Face](https://huggingface.co/): - -```python -from huggingface_hub import snapshot_download -snapshot_download( - repo_id="Qwen/Qwen3-VL-8B-Instruct", - local_dir="/path/to/save/Qwen3-VL-8B-Instruct", - local_dir_use_symlinks=False -) -``` - -`local_dir` is the user-specified model save path. Ensure sufficient disk space is available. +User can download the model using either Web or [git-lfs Tool](#downloading-with-git-lfs-tool). ### Downloading with git-lfs Tool @@ -137,7 +122,7 @@ For [Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct), th ```bash #set environment variables -export VLLM_MS_MODEL_BACKEND=MindFormers # use MindSpore TransFormers as model backend. +export VLLM_MS_MODEL_BACKEND=Native # use Native Model as model backend. ``` Here is an explanation of these variables: @@ -162,14 +147,11 @@ from vllm import LLM, SamplingParams # Sample prompts. PROMPT_TEMPLATE = ( "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" - "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" - "Is there anyone in the picture?<|im_end|>\n" + "\n<|im_start|>user\nDescribe the content of the image" + "<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n" "<|im_start|>assistant\n") -# example: -# https://tools.mindspore.cn/dataset/workspace/mindspore_dataset/images/houses_and_mountain.jpeg - -image_path = "/path/to/houses_and_mountain.jpeg" +image_path = "/path/to/image.jpg" def pil_image() -> Image.Image: return Image.open(image_path) @@ -184,13 +166,15 @@ inputs = [ ] # Create a sampling params object. -sampling_params = SamplingParams(temperature=0.0, top_p=0.95) +sampling_params = SamplingParams(max_tokens=512) model_path = "/path/to/save/Qwen3-VL-8B-Instruct" # Create a LLM -llm = LLM(model=model_path) +llm = LLM(model=model_path, + max_model_len=32768, + gpu_memory_utilization=0.85) # Generate texts from the prompts. The output is a list of RequestOutput objects # that contain the prompt, generated text, and other information. -outputs = llm.generate(prompts, sampling_params) +outputs = llm.generate(inputs, sampling_params) # Print the outputs. for output in outputs: prompt = output.prompt @@ -201,7 +185,7 @@ for output in outputs: If offline inference runs successfully, similar results will be obtained: ```text -Generated text: This is a breathtaking landscape photograph capturing a serene, idyllic scene in a mountainous region. Here's a breakdown of what's in the image:\n\n* **Foreground:** A vibrant green meadow dotted with wildflowers, including yellow dandelions and some +"Generated text: This image captures a Pallas's cat (also known as the manul or Otocolobus manul) walking through a snowy landscape.\n\nKey features of the cat:\n* Distinctive Appearance: The cat has a stocky, robust build with a very thick, dense coat of fur that is a mix of gray, brown, and buff colors. This thick fur, dusted with snowflakes, is a key adaptation for its cold, high-altitude habitat.\n* Round Head and Face: It has a wide, rounded head and a remarkably flattened face with short, broad muzzle and large, prominent ears.\n* Gaze: The cat is looking down and to its left, seemingly focused on something in the snow.\n* Posture: It is walking or stalking with one front paw lifted, indicating movement.\n* Paws: Its paws are large and furry, which help it walk on snow.\n\nEnvironment:\n* The cat is on a blanket of fresh white snow.\n* The background is composed of the characteristic white, peeling bark of birch trees and a dark, possibly wrought iron, fence.\n* The lighting suggests an overcast day, which is common in winter mountain environments where this species lives.\n\nIn summary, the image provides a clear, naturalistic view of a Pallas's cat navigating its snowy, wooded habitat, showcasing its unique and endearing physical characteristics." ``` ## Online Inference @@ -213,7 +197,7 @@ vLLM-MindSpore Plugin supports online inference deployment with the OpenAI API p Start the vLLM service with the following command: ```bash -nohup vllm-mindspore serve /path/to/save/Qwen3-VL-8B-Instruct & +nohup vllm-mindspore serve /path/to/save/Qwen3-VL-8B-Instruct --max-model-len 32768 --gpu-memory-utilization 0.85 & ``` User can also set the local model path as model tag. If the service starts successfully, similar output will be obtained: @@ -251,17 +235,13 @@ def send_request(): ) print(response.json()) -# The image for encoding -# example: -# https://tools.mindspore.cn/dataset/workspace/mindspore_dataset/images/houses_and_mountain.jpeg - -image_path = "/path/to/houses_and_mountain.jpeg" +image_path = "/path/to/image.jpg" base64_image = encode_image(image_path) # make a request payload = { - "model": "/home/ckpt/Qwen2.5-VL-7B-Instruct", + "model": "/home/ckpt/Qwen3-VL-8B-Instruct", "messages": [ { "role": "user", @@ -270,13 +250,19 @@ payload = { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"} }, - {"type": "text", "text": "Is there anyone in the picture?"} + {"type": "text", "text": "Describe the content of the image"} ] } ], - "max_tokens": 100, - "temperature": 0.1, + "max_tokens": 512, } + +send_request() ``` User needs to ensure that the `"model"` field matches the model tag in the service startup, and the request can successfully match the model. + +If online inference runs successfully, similar results will be obtained: +``` +{'id': 'chatcmpl-0f0a85dcbe7343f89539200e1a201e04', 'object': 'chat.completion', 'created': 1769408795, 'model': '/home/ckpt/Qwen3-VL-8B-Instruct', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': 'This is a photograph of a Pallas's cat (also known as a "manul") walking through a snowy landscape.\n\nHere are the key details:\n\n* Subject: The central focus is a Pallas's cat, a small wild feline native to Central Asia. It is covered in thick, fluffy fur that is a mix of gray, brown, and tan, with distinct dark markings around its eyes and on its cheeks.\n* Action: The cat is captured mid-stride, walking forward through the snow. Its body is low to the ground, and its front left paw is lifted, indicating movement.\n* Environment: The setting is a winter scene. The ground is covered in white snow, and the background consists of the distinctive white, peeling bark of birch trees. There are also some dark, vertical elements in the background, possibly a fence or another structure.\n* Atmosphere: The image conveys a sense of quiet wilderness. The cat's thick fur is dusted with snow, suggesting it has been moving through the snow for some time. The overall mood is calm and natural.\n\nThe Pallas's cat's unique, somewhat "pug-faced" appearance, with its large, rounded ears and dense coat, is clearly visible, making it a striking and memorable subject.', 'refusal': None, 'annotations': None, 'audio': None, 'function_call': None, 'tool_calls': [], 'reasoning_content': None}, 'logprobs': None, 'finish_reason': 'stop', 'stop_reason': None, 'token_ids': None}], 'service_tier': None, 'system_fingerprint': None, 'usage': {'prompt_tokens': 646, 'total_tokens': 917, 'completion_tokens': 271, 'prompt_tokens_details': None}, 'prompt_logprobs': None, 'prompt_token_ids': None, 'kv_transfer_params': None} +``` diff --git a/docs/vllm_mindspore/docs/source_en/user_guide/supported_models/models_list/models_list.md b/docs/vllm_mindspore/docs/source_en/user_guide/supported_models/models_list/models_list.md index 7b657dab5d..ef548f0fc5 100644 --- a/docs/vllm_mindspore/docs/source_en/user_guide/supported_models/models_list/models_list.md +++ b/docs/vllm_mindspore/docs/source_en/user_guide/supported_models/models_list/models_list.md @@ -10,7 +10,7 @@ | DeepSeek-R1 W8A4 | Supported | MindFormers | Atlas 800I A2 | [DeepSeek-R1-W8A4](https://modelers.cn/models/MindSpore-Lab/DeepSeek-R1-0528-gs-A8W4) | | Telechat2 | Supported | MindFormers | Atlas 800I A2 | [TeleChat2-7B-32K](https://www.modelscope.cn/models/TeleAI/TeleChat2-7B-32K), [TeleChat2-35B-32K](https://www.modelscope.cn/models/TeleAI/TeleChat2-35B-32K) | | GLM-4.5 | Supported | MindFormers | Atlas 800I A2 | [GLM-4.5](https://huggingface.co/zai-org/GLM-4.5), [GLM-4.5-Air](https://huggingface.co/zai-org/GLM-4.5-Air) | -| GLM-4.1V-Thinking | Supported | MindFormers | Atlas 800I A2 | [GLM-4.1V-9B-Thinking](https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking) | +| GLM-4.1V-Thinking | Supported | Native | Atlas 800I A2 | [GLM-4.1V-9B-Thinking](https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking) | | GLM-4 | Supported | MindFormers | Atlas 800I A2 | [GLM-4-9B-0414](https://huggingface.co/zai-org/GLM-4-9B-0414), [GLM-4-32B-0414](https://huggingface.co/zai-org/GLM-4-32B-0414) | | Qwen3-VL | Supported | Native | Atlas 800I A2 | [Qwen3-VL-2B](https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct), [Qwen3-VL-4B](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct), [Qwen3-VL-8B](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct), [Qwen3-VL-30B-A3B](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct), [Qwen3-VL-235B-A22B](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct) | | Qwen3 | Supported | Native, MindFormers | Atlas 800I A2, Atlas 300I Duo | [Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B), [Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B), [Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B), [Qwen3-8B](https://huggingface.co/Qwen/Qwen3-8B), [Qwen3-14B](https://modelers.cn/models/MindSpore-Lab/Qwen3-14B), [Qwen3-32B](https://modelers.cn/models/MindSpore-Lab/Qwen3-32B) | diff --git a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md index 929357b206..e8f6200de9 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md @@ -2,7 +2,7 @@ [![查看源文件](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/resource/_static/logo_source.svg)](https://gitee.com/mindspore/docs/blob/r2.7.1.post1/docs/vllm_mindspore/docs/source_zh_cn/getting_started/tutorials/qwen3_vl_8b_singleNPU/qwen3_vl_8b_singleNPU.md) -本文档将介绍使用vLLM-MindSpore插件进行单卡推理的流程。以[Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)模型为例,用户可通过以下[docker安装](#docker安装)章节或[安装指南](../../installation/installation.md#安装指南)章节进行环境配置,并[下载模型权重](#下载模型权重)。在[设置环境变量](#设置环境变量)之后,可进行[离线推理](#离线推理)与[在线推理](#在线推理),体验单卡推理功能。 +本文档将介绍使用vLLM-MindSpore插件进行多模态单卡推理的流程。以[Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)模型为例,用户可通过以下[docker安装](#docker安装)章节或[安装指南](../../installation/installation.md#安装指南)章节进行环境配置,并[下载模型权重](#下载模型权重)。在[设置环境变量](#设置环境变量)之后,可进行[离线推理](#离线推理)与[在线推理](#在线推理),体验单卡推理功能。 ## docker安装 @@ -92,22 +92,7 @@ docker exec -it $DOCKER_NAME bash ## 下载模型权重 -用户可采用[Python工具下载](#python工具下载)或[git-lfs工具下载](#git-lfs工具下载)两种方式,进行模型下载。 - -### Python工具下载 - -执行以下 Python 脚本,从[Hugging Face社区](https://huggingface.co/)下载[Qwen3-VL-8B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)权重及文件: - -```python -from openmind_hub import snapshot_download -snapshot_download( - repo_id="Qwen/Qwen3-VL-8B-Instruct", - local_dir="/path/to/save/Qwen3-VL-8B-Instruct", - local_dir_use_symlinks=False -) -``` - -其中`local_dir`为模型保存路径,由用户指定,请确保该路径下有足够的硬盘空间。 +用户可采用Hugging Face网页下载或者(#git-lfs工具下载)两种方式,进行模型下载。 ### git-lfs工具下载 @@ -162,14 +147,11 @@ from vllm import LLM, SamplingParams # Sample prompts. PROMPT_TEMPLATE = ( "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" - "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" - "Is there anyone in the picture?<|im_end|>\n" + "\n<|im_start|>user\nDescribe the content of the image" + "<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n" "<|im_start|>assistant\n") -# example: -# https://tools.mindspore.cn/dataset/workspace/mindspore_dataset/images/houses_and_mountain.jpeg - -image_path = "/path/to/houses_and_mountain.jpeg" +image_path = "/path/to/image.jpg" def pil_image() -> Image.Image: return Image.open(image_path) @@ -184,13 +166,15 @@ inputs = [ ] # Create a sampling params object. -sampling_params = SamplingParams(temperature=0.0, top_p=0.95) +sampling_params = SamplingParams(max_tokens=512) model_path = "/path/to/save/Qwen3-VL-8B-Instruct" # Create a LLM -llm = LLM(model=model_path) +llm = LLM(model=model_path, + max_model_len=32768, + gpu_memory_utilization=0.85) # Generate texts from the prompts. The output is a list of RequestOutput objects # that contain the prompt, generated text, and other information. -outputs = llm.generate(prompts, sampling_params) +outputs = llm.generate(inputs, sampling_params) # Print the outputs. for output in outputs: prompt = output.prompt @@ -201,7 +185,7 @@ for output in outputs: 若成功执行,则可以获得类似的执行结果: ```text -Generated text: This is a breathtaking landscape photograph capturing a serene, idyllic scene in a mountainous region. Here's a breakdown of what's in the image:\n\n* **Foreground:** A vibrant green meadow dotted with wildflowers, including yellow dandelions and some +"Generated text: This image captures a Pallas's cat (also known as the manul or Otocolobus manul) walking through a snowy landscape.\n\nKey features of the cat:\n* Distinctive Appearance: The cat has a stocky, robust build with a very thick, dense coat of fur that is a mix of gray, brown, and buff colors. This thick fur, dusted with snowflakes, is a key adaptation for its cold, high-altitude habitat.\n* Round Head and Face: It has a wide, rounded head and a remarkably flattened face with short, broad muzzle and large, prominent ears.\n* Gaze: The cat is looking down and to its left, seemingly focused on something in the snow.\n* Posture: It is walking or stalking with one front paw lifted, indicating movement.\n* Paws: Its paws are large and furry, which help it walk on snow.\n\nEnvironment:\n* The cat is on a blanket of fresh white snow.\n* The background is composed of the characteristic white, peeling bark of birch trees and a dark, possibly wrought iron, fence.\n* The lighting suggests an overcast day, which is common in winter mountain environments where this species lives.\n\nIn summary, the image provides a clear, naturalistic view of a Pallas's cat navigating its snowy, wooded habitat, showcasing its unique and endearing physical characteristics." ``` ## 在线推理 @@ -213,7 +197,7 @@ vLLM-MindSpore插件可使用OpenAI的API协议,部署在线推理。以下以 使用如下命令启动vLLM服务: ```bash -nohup vllm-mindspore serve /path/to/save/Qwen3-VL-8B-Instruct & +nohup vllm-mindspore serve /path/to/save/Qwen3-VL-8B-Instruct --max-model-len 32768 --gpu-memory-utilization 0.85 & ``` 用户可以通过指定模型保存的本地路径作为模型标签。若服务成功启动,则可以获得类似的执行结果: @@ -251,17 +235,13 @@ def send_request(): ) print(response.json()) -# 编码图片 -# example: -# https://tools.mindspore.cn/dataset/workspace/mindspore_dataset/images/houses_and_mountain.jpeg - -image_path = "/path/to/houses_and_mountain.jpeg" +image_path = "/path/to/image.jpg" base64_image = encode_image(image_path) # 构造请求 payload = { - "model": "/home/ckpt/Qwen2.5-VL-7B-Instruct", + "model": "/home/ckpt/Qwen3-VL-8B-Instruct", "messages": [ { "role": "user", @@ -270,13 +250,19 @@ payload = { "type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"} }, - {"type": "text", "text": "Is there anyone in the picture?"} + {"type": "text", "text": "Describe the content of the image"} ] } ], - "max_tokens": 100, - "temperature": 0.1, + "max_tokens": 512, } + +send_request() ``` 其中,用户需确认`"model"`字段与启动服务中的模型标签一致,请求才能成功匹配到模型。 + +若成功执行,则可以获得类似的执行结果: +``` +{'id': 'chatcmpl-0f0a85dcbe7343f89539200e1a201e04', 'object': 'chat.completion', 'created': 1769408795, 'model': '/home/ckpt/Qwen3-VL-8B-Instruct', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': 'This is a photograph of a Pallas's cat (also known as a "manul") walking through a snowy landscape.\n\nHere are the key details:\n\n* Subject: The central focus is a Pallas's cat, a small wild feline native to Central Asia. It is covered in thick, fluffy fur that is a mix of gray, brown, and tan, with distinct dark markings around its eyes and on its cheeks.\n* Action: The cat is captured mid-stride, walking forward through the snow. Its body is low to the ground, and its front left paw is lifted, indicating movement.\n* Environment: The setting is a winter scene. The ground is covered in white snow, and the background consists of the distinctive white, peeling bark of birch trees. There are also some dark, vertical elements in the background, possibly a fence or another structure.\n* Atmosphere: The image conveys a sense of quiet wilderness. The cat's thick fur is dusted with snow, suggesting it has been moving through the snow for some time. The overall mood is calm and natural.\n\nThe Pallas's cat's unique, somewhat "pug-faced" appearance, with its large, rounded ears and dense coat, is clearly visible, making it a striking and memorable subject.', 'refusal': None, 'annotations': None, 'audio': None, 'function_call': None, 'tool_calls': [], 'reasoning_content': None}, 'logprobs': None, 'finish_reason': 'stop', 'stop_reason': None, 'token_ids': None}], 'service_tier': None, 'system_fingerprint': None, 'usage': {'prompt_tokens': 646, 'total_tokens': 917, 'completion_tokens': 271, 'prompt_tokens_details': None}, 'prompt_logprobs': None, 'prompt_token_ids': None, 'kv_transfer_params': None} +``` diff --git a/docs/vllm_mindspore/docs/source_zh_cn/user_guide/supported_models/models_list/models_list.md b/docs/vllm_mindspore/docs/source_zh_cn/user_guide/supported_models/models_list/models_list.md index d50bf776cf..cc9ee7860a 100644 --- a/docs/vllm_mindspore/docs/source_zh_cn/user_guide/supported_models/models_list/models_list.md +++ b/docs/vllm_mindspore/docs/source_zh_cn/user_guide/supported_models/models_list/models_list.md @@ -10,7 +10,7 @@ | DeepSeek-R1 W8A4 | 已支持 | MindFormers | Atlas 800I A2 | [DeepSeek-R1-W8A4](https://modelers.cn/models/MindSpore-Lab/DeepSeek-R1-0528-gs-A8W4) | | Telechat2 | 已支持 | MindFormers | Atlas 800I A2 | [TeleChat2-7B-32K](https://www.modelscope.cn/models/TeleAI/TeleChat2-7B-32K)、[TeleChat2-35B-32K](https://www.modelscope.cn/models/TeleAI/TeleChat2-35B-32K) | | GLM-4.5 | 已支持 | MindFormers | Atlas 800I A2 | [GLM-4.5](https://huggingface.co/zai-org/GLM-4.5), [GLM-4.5-Air](https://huggingface.co/zai-org/GLM-4.5-Air) | -| GLM-4.1V-Thinking | 已支持 | MindFormers | Atlas 800I A2 | [GLM-4.1V-9B-Thinking](https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking) | +| GLM-4.1V-Thinking | 已支持 | Native | Atlas 800I A2 | [GLM-4.1V-9B-Thinking](https://huggingface.co/zai-org/GLM-4.1V-9B-Thinking) | | GLM-4 | 已支持 | MindFormers | Atlas 800I A2 | [GLM-4-9B-0414](https://huggingface.co/zai-org/GLM-4-9B-0414)、[GLM-4-32B-0414](https://huggingface.co/zai-org/GLM-4-32B-0414) | | Qwen3-VL | 已支持 | Native | Atlas 800I A2 | [ Qwen3-VL-2B](https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct)、[Qwen3-VL-4B](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct)、[Qwen3-VL-8B](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct)、[Qwen3-VL-30B-A3B](https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct)、[Qwen3-VL-235B-A22B](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct) | -- Gitee