projects/repobot/v3/eval_agent.py

"""8 道题评估 RepoBot v3。其中 5 道的答案只在源码里，文档没写。

每道题给两个正则表达式：回答里必须出现的内容，和不能出现的内容（用来挡住"提到了关键词，结论却是错的"）。
这仍然是粗略的自动判断，答错的题要人工看一眼。智能体每次的回答都可能不同，建议多跑几次。
    python eval_agent.py
"""
import re
from concurrent.futures import ThreadPoolExecutor

import agent
import tools
from retrieval import Retriever

QUESTIONS = [
    # (问题, 必须出现, 不能出现, 答案在哪)
    ("httpx 默认会自动跟随重定向吗？", r"不会|不跟随|默认.{0,6}False", r"^\W*会", "文档"),
    ("httpx 默认的超时是多少秒？", r"5\s*秒|5\.0|five seconds", None, "文档"),
    ("httpx 的超时分几种？", r"四种|4 种", None, "文档"),
    ("httpx 默认最多跟随几次重定向？", r"20", None, "源码"),
    ("TooManyRedirects 继承自哪个异常类？", r"RequestError", None, "源码"),
    ("response.raise_for_status() 遇到 301 响应会抛异常吗？", r"会(抛|引发)|会。", r"不会(抛|引发)", "源码"),
    ("已经读取过 response.text 之后，再设置 response.encoding 会怎样？", r"ValueError", None, "源码"),
    ("只写 httpx.Limits(max_connections=200)，max_keepalive_connections 是多少？", r"None", r"是\s*\**\s*`?20", "源码"),
]

tools.ensure_source()
tools.retriever = Retriever(tools.DOCS_DIR, tools.HERE / ".cache")


def ask(item):
    question, must, must_not, where = item
    answer, stats = agent.run(question, show=lambda *a: None)
    answer = answer or ""
    ok = bool(re.search(must, answer)) and not (must_not and re.search(must_not, answer))
    return question, where, ok, stats, answer


with ThreadPoolExecutor(4) as pool:
    results = list(pool.map(ask, QUESTIONS))

for where in ("文档", "源码"):
    part = [r for r in results if r[1] == where]
    print(f"答案在{where}里的题：{sum(r[2] for r in part)}/{len(part)} 答对")
print(f"平均每题 {sum(r[3]['steps'] for r in results) / len(results):.1f} 次模型调用，"
      f"{sum(r[3]['tool_calls'] for r in results) / len(results):.1f} 次工具调用，"
      f"共 {sum(r[3]['cost'] for r in results):.4f} 美元")
for question, where, ok, stats, answer in results:
    mark = "对" if ok else "错"
    print(f"\n[{mark}] {question}（{stats['steps']} 步）\n    {answer.replace(chr(10), ' ')[:160]}")