build_dpo_dataset

  1"""Build marola's DPO preference dataset (MIP-0025 §4.3, Layer 3) from `core/llm/Reviewer.scala`'s
  2own historical reject/revise decisions.
  3
  4Real historical decisions are already captured, once, as the DSPy-compiled `review_prompt.json`
  5demos (core/src/main/resources/review_prompt.json) — bootstrap examples where a real Ollama
  6reviewer graded a real draft summary. Wherever a demo's verdict wasn't "approve", the reviewer's
  7own `final_summary` is a real correction of a real flawed draft: that is exactly a (chosen,
  8rejected) preference pair, with no invented text on either side. An "approve" demo carries no
  9preference signal (the draft was fine as-is) and yields no pair — one pair per reject/revise
 10decision, never more, never fabricated when there isn't one.
 11
 12Run:  python build_dpo_dataset.py             (or `just finetune-dpo-dataset`)
 13Self-test:  python build_dpo_dataset.py --self-test   (or `just quality-other`)
 14"""
 15
 16from __future__ import annotations
 17
 18import json
 19import sys
 20from pathlib import Path
 21
 22sys.path.insert(0, str(Path(__file__).resolve().parent))
 23import build_dataset as bd  # reuse INPUT_FIELDS/render_inputs — same conditions shape
 24
 25RESOURCES = bd.RESOURCES
 26OUT = Path(__file__).resolve().parent / "data"
 27
 28DPO_INPUT_FIELDS = bd.INPUT_FIELDS
 29
 30
 31def review_records(path: Path) -> list[dict]:
 32    """Real Reviewer decisions: one dict per review_prompt.json demo, in ReviewResult's own shape
 33    (Reviewer.scala's `ReviewResult(finalSummary, score, verdict)`) plus the draft it reviewed."""
 34    doc = json.loads(path.read_text(encoding="utf-8"))
 35    records = []
 36    for demo in doc.get("demos", []):
 37        if "review_json" not in demo or "summary" not in demo:
 38            continue
 39        review = json.loads(demo["review_json"])
 40        records.append(
 41            {
 42                "conditions": {k: demo[k] for k in DPO_INPUT_FIELDS if k in demo},
 43                "draft_summary": demo["summary"],
 44                "score": review.get("score"),
 45                "verdict": review.get("verdict", "approve"),
 46                "final_summary": review.get("final_summary", demo["summary"]),
 47            }
 48        )
 49    return records
 50
 51
 52def preference_pairs(records: list[dict]) -> list[dict]:
 53    """One (chosen, rejected) pair per real reject/revise record; none for "approve"."""
 54    pairs = []
 55    for r in records:
 56        if r["verdict"] == "approve":
 57            continue
 58        if r["final_summary"] == r["draft_summary"]:
 59            continue  # flagged but the text didn't actually change — no real preference signal
 60        prompt = bd.render_inputs(r["conditions"], DPO_INPUT_FIELDS)
 61        pairs.append(
 62            {"prompt": prompt, "chosen": r["final_summary"], "rejected": r["draft_summary"]}
 63        )
 64    return pairs
 65
 66
 67def main() -> None:
 68    records = review_records(RESOURCES / "review_prompt.json")
 69    pairs = preference_pairs(records)
 70    OUT.mkdir(parents=True, exist_ok=True)
 71    with (OUT / "dpo_pairs.jsonl").open("w", encoding="utf-8") as f:
 72        for p in pairs:
 73            f.write(json.dumps(p, ensure_ascii=False) + "\n")
 74    print(
 75        f"wrote {len(pairs)} DPO preference pairs (from {len(records)} reviewer decisions) to {OUT}"
 76    )
 77
 78
 79# --- self-test: a small fixture of Reviewer decisions, not the real ones ---------------------
 80_FIXTURE_RECORDS = [
 81    {
 82        "conditions": {"beach_name": "Fixture Beach A", "score": "90"},
 83        "draft_summary": "Great conditions, go for it.",
 84        "score": 90,
 85        "verdict": "approve",
 86        "final_summary": "Great conditions, go for it.",
 87    },
 88    {
 89        "conditions": {"beach_name": "Fixture Beach B", "score": "40"},
 90        "draft_summary": "Nice and calm, a pleasant swim.",
 91        "score": 35,
 92        "verdict": "revise",
 93        "final_summary": "Calm but high jellyfish risk — worth a second look before swimming.",
 94    },
 95    {
 96        "conditions": {"beach_name": "Fixture Beach C", "score": "5"},
 97        "draft_summary": "Great day for a swim.",
 98        "score": 5,
 99        "verdict": "reject",
100        "final_summary": "Unsafe: strong riptide and rough seas — do not swim.",
101    },
102]
103
104
105def _self_test() -> None:
106    # Fixture with reject/revise events: exactly one pair per non-approve event, nothing invented.
107    pairs = preference_pairs(_FIXTURE_RECORDS)
108    assert len(pairs) == 2, f"expected 2 pairs (one revise + one reject), got {len(pairs)}"
109    chosen_texts = {p["chosen"] for p in pairs}
110    assert _FIXTURE_RECORDS[1]["final_summary"] in chosen_texts
111    assert _FIXTURE_RECORDS[2]["final_summary"] in chosen_texts
112    for p in pairs:
113        assert p["chosen"] != p["rejected"], "chosen and rejected must differ"
114
115    # A fixture with only "approve" events: no reject/revise → zero pairs, none invented.
116    approve_only = [r for r in _FIXTURE_RECORDS if r["verdict"] == "approve"]
117    assert preference_pairs(approve_only) == [], "an all-approve fixture must yield zero pairs"
118
119    # Real source check: review_prompt.json's own demos really do have reject/revise decisions,
120    # and every generated pair's text matches one of them verbatim — no rewording, no invention.
121    real_records = review_records(RESOURCES / "review_prompt.json")
122    real_pairs = preference_pairs(real_records)
123    assert real_pairs, "no real reject/revise decisions found in review_prompt.json's demos"
124    real_texts = {
125        (r["final_summary"], r["draft_summary"])
126        for r in real_records
127        if r["verdict"] != "approve" and r["final_summary"] != r["draft_summary"]
128    }
129    generated_texts = {(p["chosen"], p["rejected"]) for p in real_pairs}
130    assert generated_texts <= real_texts, "a generated pair's text doesn't match a real decision"
131
132    print(
133        f"self-test OK: {len(pairs)} pairs from the fixture, {len(real_pairs)} from "
134        f"review_prompt.json's real reviewer decisions, all verbatim"
135    )
136
137
138if __name__ == "__main__":
139    if "--self-test" in sys.argv:
140        _self_test()
141    else:
142        main()