端侧大模型部署：手机/Edge/IoT 全场景选型

import MLX
import MLXLMCommon

// 加载模型
let modelContainer = try await LLMModelFactory.shared.loadContainer(
    configuration: ModelConfiguration(
        id: "Qwen/Qwen3.5-3B-Instruct-MLX-4bit"
    )
)

// 生成回复
let result = try await modelContainer.perform { context in
    let input = try await context.processor.prepare(
        input: .init(prompt: "你好，请介绍一下自己")
    )
    let result = try MLXLMCommon.generate(
        input: input,
        parameters: GenerateParameters(temperature: 0.7),
        context: context
    ) { tokens in
        if tokens.count >= 256 { return .stop }
        return .more
    }
    return result.output
}

# 使用 MLC-LLM 编译部署
import tvm
from mlc_llm import MLCEngine

# 加载预编译模型
engine = MLCEngine(
    model="Qwen3.5-3B-Instruct-q4f16_1-MLC",
    device="android-adreno"  # GPU 加速
)

# 推理
response = engine.chat.completions.create(
    messages=[{"role": "user", "content": "你好"}],
    model="Qwen3.5-3B",
    max_tokens=128
)

# 安装 llama.cpp with CUDA
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
mkdir build && cd build
cmake .. -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=87
make -j4

# 运行 Qwen3.5-7B INT4
./llama-server \
    -m qwen3.5-7b-instruct-q4_k_m.gguf \
    -c 4096 -t 6 -ngl 99 \
    --port 8080

# 使用 llama.cpp ARM NEON 优化
cd llama.cpp/build
cmake .. -DGGML_ARM_CPU=ON
make -j4

# 运行 SmolLM3-3B INT3
./llama-server \
    -m smollm3-3b-q3_k_s.gguf \
    -c 2048 -t 4 \
    --port 8080

// ESP32-S3 上的 TinyLlama INT2 部署
#include "tiny_engine.h"

TinyModel* model = tiny_load_model("tinyllama_1.1b_int2.bin");
TinyTensor* input = tiny_tensor_from_text("打开客厅灯");
TinyTensor* output = tiny_forward(model, input);
char* response = tiny_decode(output);

用户语音 → ESP32(意图识别) → 简单指令本地执行
                          → 复杂请求转发云端