Mistral 模型家族指南：欧洲的 AI 雄心

# Mistral 7B 的 SWA（滑动窗口注意力）
class SlidingWindowAttention(nn.Module):
    def __init__(self, d_model, n_heads, window_size=4096):
        self.n_heads = n_heads
        self.window_size = window_size
        # 窗口内计算 attention，窗口外通过 KV Cache 传递
        # 计算复杂度从 O(n²) 降至 O(n * window_size)

    def forward(self, x):
        # 1. 窗口内注意力（局部）
        local_attn = attention(x, window=self.window_size)
        # 2. 通过堆叠多层，信息可以传递到更远
        #    L 层 SWA 的有效感受野 = L * window_size
        return local_attn

# Mixtral MoE 路由
class MixtralMoE(nn.Module):
    def __init__(self, d_model, n_experts=8, top_k=2):
        self.router = nn.Linear(d_model, n_experts)
        self.experts = nn.ModuleList([
            SwiGLUFFN(d_model) for _ in range(n_experts)
        ])
        self.top_k = top_k

    def forward(self, x):
        # 路由计算
        router_logits = self.router(x)  # [batch, seq, 8]
        # Top-2 选择
        scores, indices = softmax(router_logits, dim=-1).topk(self.top_k, dim=-1)
        # 归一化权重
        scores = scores / scores.sum(dim=-1, keepdim=True)
        # 加权求和
        output = torch.zeros_like(x)
        for i in range(self.top_k):
            for b in range(x.size(0)):
                expert_idx = indices[b, :, i]
                expert_out = self.experts[expert_idx[b]](x[b])
                output[b] += scores[b, :, i:i+1] * expert_out
        return output

# Codestral FIM 调用
from openai import OpenAI

client = OpenAI(
    api_key="your-mistral-api-key",
    base_url="https://codestral.mistral.ai/v1"
)

# 代码补全模式
response = client.completions.create(
    model="codestral-latest",
    prompt="""# 实现一个快速排序算法
def quicksort""",
    max_tokens=200,
    temperature=0.2
)

# FIM 模式
response = client.completions.create(
    model="codestral-latest",
    prompt="[PREFIX]def merge_sort(arr):\n    [SUFFIX]\n    return sorted_arr[MIDDLE]",
    max_tokens=200
)

# Pixtral 多图理解
response = client.chat.completions.create(
    model="pixtral-12b-latest",
    messages=[{
        "role": "user",
        "content": [
            {"type": "image_url", "image_url": {"url": "url1"}},
            {"type": "image_url", "image_url": {"url": "url2"}},
            {"type": "text", "text": "比较这两张图表的差异"}
        ]
    }]
)

# Mixtral 8x7B（双卡 A100）
python -m vllm.entrypoints.openai.api_server \
  --model mistralai/Mixtral-8x7B-Instruct-v0.1 \
  --tensor-parallel-size 2 \
  --gpu-memory-utilization 0.9 \
  --max-model-len 32768 \
  --port 8000

# Mixtral 8x22B（4卡 A100 80GB）
python -m vllm.entrypoints.openai.api_server \
  --model mistralai/Mixtral-8x22B-Instruct-v0.1 \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.95 \
  --max-model-len 65536 \
  --port 8000

# Ollama
ollama run mistral:7b

# vLLM 单卡
python -m vllm.entrypoints.openai.api_server \
  --model mistralai/Mistral-7B-Instruct-v0.3 \
  --gpu-memory-utilization 0.9 \
  --max-model-len 32768