Promptfoo：LLM 评测与 Red Teaming 实战指南

# 使用 npx（无需安装）
npx promptfoo@latest eval

# 或全局安装
npm install -g promptfoo

# 或使用 Homebrew
brew install promptfoo

# 或使用 pip
pip install promptfoo

# 初始化示例项目
npx promptfoo@latest init --example getting-started

# 进入目录，运行评测
cd getting-started
npx promptfoo@latest eval

# 查看结果
npx promptfoo@latest view

# 通过 CLI 向导创建配置
npx promptfoo@latest init

# 或通过 Web UI 配置
npx promptfoo@latest eval setup

export OPENAI_API_KEY=sk-abc123
export ANTHROPIC_API_KEY=sk-ant-xxx

# promptfooconfig.yaml
prompts:
  - 'Convert the following English text to {{language}}: {{input}}'

providers:
  - openai:gpt-4o

tests:
  - vars:
      language: French
      input: Hello world
    assert:
      - type: contains
        value: 'Bonjour le monde'

providers:
  # OpenAI
  - openai:gpt-4o
  - openai:gpt-4o-mini

  # Anthropic
  - anthropic:messages:claude-sonnet-4-20250514

  # Google
  - google:gemini-2.5-pro

  # 本地模型
  - ollama:chat:llama3

  # 自定义 Provider（JS/Python 文件）
  - file://path/to/custom_provider.py

providers:
  - id: openai:gpt-4o
    config:
      temperature: 0.7
      max_tokens: 1024

prompts:
  # 内联字符串
  - '翻译以下文本为{{language}}：{{input}}'

  # 外部文件
  - file://prompts/translation.txt

  # 带标签的对象
  - id: file://prompts/v1.txt
    label: 简洁版
  - id: file://prompts/v2.txt
    label: 详细版

prompts:
  - '翻译：{{input}}'
  - '你是一个专业翻译。请将以下{{language}}文本翻译成{{target_language}}，保持原意并符合目标语言习惯：{{input}}'

providers:
  - openai:gpt-4o

tests:
  - vars:
      input: "The quick brown fox"
      language: English
      target_language: 中文
    assert:
      - type: contains
        value: 快速

prompts:
  - '解释量子计算的基本原理。'

providers:
  - openai:gpt-4o
  - anthropic:messages:claude-sonnet-4-20250514
  - google:gemini-2.5-pro

tests:
  - assert:
      - type: contains-any
        value: ["量子", "叠加", "纠缠"]

npx promptfoo@latest eval -r google:gemini-2.5-pro google:gemini-2.5-flash

providers:
  - id: 'http://localhost:3000/api/rag'
    config:
      transformResponse: 'json.data'

prompts:
  - '根据以下上下文回答问题。\n\n上下文：{{context}}\n\n问题：{{question}}'

tests:
  - vars:
      question: 什么是向量数据库？
      context: 向量数据库是一种专门存储和检索向量表示的数据库系统...
    assert:
      - type: contains
        value: 向量
      - type: context-faithfulness
        contextTransform: 'output.sources.map(s => s.content).join("\n")'
        threshold: 0.9

npx promptfoo@latest init --example openai-agents-basic

tests:
  - vars:
      question: 什么是机器学习？
    assert:
      - type: contains
        value: 算法
      - type: not-contains
        value: 我不知道
      - type: similar
        value: 机器学习是人工智能的一个分支，通过数据训练模型
        threshold: 0.7
        provider: openai:gpt-4o-mini
      - type: javascript
        value: output.length >= 50 && output.length <= 500

tests:
  - vars:
      input: 请列出三种编程语言
    assert:
      - type: javascript
        value: |
          const languages = ['python', 'javascript', 'java', 'go', 'rust', 'c++', 'typescript'];
          const output_lower = output.toLowerCase();
          const count = languages.filter(l => output_lower.includes(l)).length;
          return { pass: count >= 3, score: count / 3 };

tests:
  - vars:
      input: 计算斐波那契数列第10项
    assert:
      - type: python
        value: |
          import re
          numbers = re.findall(r'\d+', output)
          pass_test = '55' in numbers
          return {"pass": pass_test, "score": 1.0 if pass_test else 0.0}

defaultTest:
  assert:
    - type: not-contains
      value: "Error"
    - type: not-contains
      value: "我无法"

tests:
  - vars:
      question: 你好
    assert:
      - type: contains
        value: 你好
    threshold: 0.5

# 初始化 Red Team 配置
npx promptfoo@latest redteam

# 运行 Red Team 扫描
npx promptfoo@latest redteam eval

# 查看最近一次评测结果
npx promptfoo@latest view

# 通过 Web UI 配置评测
npx promptfoo@latest eval setup

// custom_provider.js
async function callApi(prompt, context) {
  const response = await fetch('https://your-api.com/generate', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ prompt }),
  });
  const data = await response.json();
  return {
    output: data.text,
    tokenUsage: { total: data.tokens },
  };
}

module.exports = { callApi };

providers:
  - file://custom_provider.js

// extension.js
async function extensionHook(hookName, context) {
  if (hookName === 'beforeAll') {
    console.log('评测开始');
  } else if (hookName === 'afterEach') {
    console.log(`测试完成: ${context.test.description}, 通过: ${context.result.success}`);
  }
}

module.exports = extensionHook;

extensions:
  - file://extension.js:extensionHook

providers:
  - id: 'http://localhost:3000/api'
    config:
      transformResponse: 'json.data.answer'

tests:
  - vars:
      query: 退款政策是什么？
    options:
      transform: 'output.answer'
    assert:
      - type: contains
        value: 30天

# .github/workflows/llm-eval.yml
name: LLM Evaluation

on:
  push:
    paths:
      - 'prompts/**'
      - 'promptfooconfig.yaml'
  pull_request:

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Install Promptfoo
        run: npm install -g promptfoo

      - name: Run Evaluation
        run: promptfoo eval --config promptfooconfig.yaml --output results.json
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

      - name: Upload Results
        uses: actions/upload-artifact@v4
        with:
          name: promptfoo-results
          path: results.json

# 运行评测
promptfoo eval

# 指定配置文件
promptfoo eval -c path/to/config.yaml

# 覆盖 Provider
promptfoo eval -r openai:gpt-4o openai:gpt-4o-mini

# 过滤测试
promptfoo eval --filter-pattern "auth.*"

# 控制并发
promptfoo eval --max-concurrency 10

# 禁用缓存
promptfoo eval --no-cache

# 输出格式
promptfoo eval --output results.json
promptfoo eval --output results.csv
promptfoo eval --output results.html

providers:
  - ollama:chat:llama3

defaultTest:
  assert:
    - type: not-contains
      value: "Error"

tests:
  - vars:
      input: 你好