大语言模型系列(8): Qwen2.5-omini-3B 端侧部署推理教程

1.1资源下载与准备

下载模型文件：访问网站下载对应平台的模型文件：

• Qwen2.5-omini-3B 骁龙 X Elite 平台 (8380) 模型下载
• Qwen2.5-omini-3B 骁龙 X2 Elite 平台(8480) 模型下载

将下载模型放置QAI AppBuilder\samples\genie\python\models目录下。

下载 Genie 服务程序：前往 GitHub Releases 页面下载

• 8380：GenieAPIService_Stable_QAIRT_v73.zip：Releases 下载页面。
• 8480：GenieAPIService_Stable_QAIRT_v81.zip：Releases 下载页面。

解压文件：将下载的压缩包解压至项目代码目录ai-engine-direct-helper\samples下。

1.2启动服务与运行示例

# 1. 进入目录 cd ai-engine-direct-helper\samples # 2. 启动 GenieAPI 服务 (加载配置文件) GenieAPIService_Stable_QAIRT_v73\GenieAPIService.exe -c "genie\python\models\qwen2.5_omini_8380-2.42\config.json" -l 成功启动会有日志 [INFO] "Allocated total size = 119406600 across 5 buffers" [W] load successfully! use second: 10.9317 [W] Model load successfully: qwen2.5_omini_8380-2.42 [W] GenieService::setupHttpServer start [W] GenieService::setupHttpServer end [A] [OK] Genie API Service IS Running. [A] [OK] Genie API Service -> http://0.0.0.0:8910 # 3. 运行客户端进行测试 (确保当前目录下有 test.png 图片) GenieAPIClient.exe --prompt "what is the image descript?" --img test.png --stream --model qwen2.5_omini_8380-2.42

注意:运行客户端命令前，请确保当前目录下存在名为test.png的测试图片文件。

第三部分：Python 调用指南

无论是在 Windows 运行GenieAPIService.exe还是在 Android 启动GenieAPIService.apk，服务启动成功后都会显示一个 IP 地址和端口（例如127.0.0.1:8910或手机IP）。我们可以使用 Python 通过 OpenAI 兼容接口调用该服务。

请确保已安装openai库。

pip install openai

创建一个 Python 脚本，并将以下代码复制进去。请注意根据实际情况修改 IP 地址。

import argparse import base64 import os import requests from openai import OpenAI DEFAULT_ADDR = "127.0.0.1:8910" DEFAULT_API_KEY = "123" SYSTEM_PROMPTS = { "llm": "You are a helpful assistant.", "vl": "You are a helpful assistant.", "omini": ( "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group. " "You are helpful and honest. You can perceive all types of information, including " "but not limited to text, images, and audio." ), } def encode_file(file_input: str) -> str: if file_input.startswith(("http://", "https://")): print(f" Downloading: {file_input}") resp = requests.get(file_input, timeout=30) resp.raise_for_status() return base64.b64encode(resp.content).decode("utf-8") if not os.path.exists(file_input): raise FileNotFoundError(f"File not found: {file_input}") with open(file_input, "rb") as f: return base64.b64encode(f.read()).decode("utf-8") def detect_mode(args) -> str: if args.mode: return args.mode if args.audio: return "omini" if args.image: return "vl" return "llm" def build_messages_llm(prompt: str, system: str) -> list: return [ {"role": "system", "content": system}, {"role": "user", "content": prompt}, ] def build_messages_vl(prompt: str, image_b64: str, system: str) -> list: return [ {"role": "system", "content": system}, { "role": "user", "content": [ {"type": "text", "text": prompt}, { "type": "image_url", "image_url": { "url": f"data:image/jpeg;base64,{image_b64}" }, }, ], }, ] def build_messages_omini( prompt: str, image_b64: str | None, audio_b64: str | None, system: str, ) -> list: content = {"question": prompt} if image_b64: content["image"] = image_b64 if audio_b64: content["audio"] = audio_b64 return [ {"role": "system", "content": system}, {"role": "user", "content": content}, ] def send_request(client: OpenAI, model: str, messages: list, stream: bool, extra_body: dict): if stream: response = client.chat.completions.create( model=model, stream=True, messages=messages, extra_body=extra_body, ) print("Response: ", end="") for chunk in response: if chunk.choices: delta = chunk.choices[0].delta.content if delta is not None: print(delta, end="", flush=True) print() else: response = client.chat.completions.create( model=model, messages=messages, extra_body=extra_body, ) if response.choices: print("Response:", response.choices[0].message.content) def main(): parser = argparse.ArgumentParser( description="Genie API Client — supports LLM / VL / Omini modes", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: # Pure text (LLM) python demo.py --prompt "What is AI?" # Image understanding (VL) python demo.py --prompt "Describe this image" --image photo.jpg # Image + Audio (Omini) python demo.py --prompt "What do you see and hear?" --audio speech.wav # Force Omini mode for text-only (uses embedding pipeline) python demo.py --prompt "Hello" --mode omini # Specify model name python demo.py --model qwen2.5_omini_8380-2.42 --prompt "Hi" """, ) parser.add_argument("--addr", type=str, default=DEFAULT_ADDR, help=f"Server address (default: {DEFAULT_ADDR})") parser.add_argument("--model", type=str, default="qwen2.5_omini_8380-2.42", help="Model name to use") parser.add_argument("--mode", type=str, choices=["llm", "vl", "omini"], default=None, help="Force mode (auto-detect if not set)") parser.add_argument("--prompt", type=str, default="Hello", help="Text prompt") parser.add_argument("--image", type=str, default=None, help="Image path or URL") parser.add_argument("--audio", type=str, default=None, help="Audio WAV path or URL (triggers Omini mode)") parser.add_argument("--stream", action="store_true", help="Enable streaming output") parser.add_argument("--system", type=str, default=None, help="Custom system prompt (overrides default)") parser.add_argument("--temp", type=float, default=0.7) parser.add_argument("--top_k", type=int, default=40) parser.add_argument("--top_p", type=float, default=0.9) parser.add_argument("--max_tokens", type=int, default=2048) args = parser.parse_args() mode = detect_mode(args) system = args.system or SYSTEM_PROMPTS[mode] print(f"=== Genie API Client ===") print(f" Server : {args.addr}") print(f" Model : {args.model}") print(f" Mode : {mode.upper()}") if args.image: print(f" Image : {args.image}") if args.audio: print(f" Audio : {args.audio}") print(f" Prompt : {args.prompt}") print() client = OpenAI(base_url=f"http://{args.addr}/v1", api_key=DEFAULT_API_KEY) extra_body = { "temp": args.temp, "top_k": args.top_k, "top_p": args.top_p, "size": args.max_tokens, } try: image_b64 = encode_file(args.image) if args.image else None audio_b64 = encode_file(args.audio) if args.audio else None except Exception as e: print(f"Error loading file: {e}") return if mode == "llm": messages = build_messages_llm(args.prompt, system) elif mode == "vl": if not image_b64: print("Error: VL mode requires --image") return messages = build_messages_vl(args.prompt, image_b64, system) elif mode == "omini": messages = build_messages_omini(args.prompt, image_b64, audio_b64, system) try: send_request(client, args.model, messages, args.stream, extra_body) except Exception as e: print(f"\nRequest failed: {e}") if __name__ == "__main__": main()

# Pure text (LLM) python demo.py --prompt "What is AI?" # Image understanding (VL) python demo.py --prompt "Describe this image" --image photo.jpg # Image + Audio (Omini) python demo.py --prompt "What do you see and hear?" --audio speech.wav # Specify model name python demo.py --model qwen2.5_omini_8380-2.42 --prompt "Hi"