added validation
Browse files- .coveragerc +1 -0
- .gitignore +5 -0
- eval/README.md +53 -0
- eval/judge_validation.py +291 -0
- evaluator/LLM_evaluator.py +15 -5
- results/judge_validation.md +3 -0
- tests/test_judge_validation.py +218 -0
.coveragerc
CHANGED
|
@@ -7,6 +7,7 @@ source =
|
|
| 7 |
LegalRag.rag_pipeline
|
| 8 |
LegalRag.citation_check
|
| 9 |
eval.run_answer_eval
|
|
|
|
| 10 |
# The requested parser scope excludes ingestion clients and runnable examples.
|
| 11 |
omit =
|
| 12 |
*/usage.py
|
|
|
|
| 7 |
LegalRag.rag_pipeline
|
| 8 |
LegalRag.citation_check
|
| 9 |
eval.run_answer_eval
|
| 10 |
+
eval.judge_validation
|
| 11 |
# The requested parser scope excludes ingestion clients and runnable examples.
|
| 12 |
omit =
|
| 13 |
*/usage.py
|
.gitignore
CHANGED
|
@@ -117,3 +117,8 @@ pnpm-lock.yaml
|
|
| 117 |
|
| 118 |
# Cached query analyses contain local query text
|
| 119 |
/results/analysis_cache/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
|
| 118 |
# Cached query analyses contain local query text
|
| 119 |
/results/analysis_cache/
|
| 120 |
+
|
| 121 |
+
# Manual judge-validation answers, context and labels
|
| 122 |
+
/results/judge_validation.csv
|
| 123 |
+
/results/judge_validation_samples.jsonl
|
| 124 |
+
/results/judge_consistency.json
|
eval/README.md
CHANGED
|
@@ -365,3 +365,56 @@ The failure stage becomes `CITATION` and uses the existing retry budget. On
|
|
| 365 |
exhaustion, the best attempt retains its own citation report and warnings.
|
| 366 |
The `citation_check` stage is logged, and its duration contributes to API
|
| 367 |
`latency_ms.judge`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 365 |
exhaustion, the best attempt retains its own citation report and warnings.
|
| 366 |
The `citation_check` stage is logged, and its duration contributes to API
|
| 367 |
`latency_ms.judge`.
|
| 368 |
+
|
| 369 |
+
## Validate the LLM judge against manual labels
|
| 370 |
+
|
| 371 |
+
```bash
|
| 372 |
+
# Requires at least 50 reviewed questions and a running full pipeline:
|
| 373 |
+
python -m eval.judge_validation export --seed 42
|
| 374 |
+
# Fill only my_label and my_notes in results/judge_validation.csv.
|
| 375 |
+
# my_label must be correct, partially, or incorrect; leave uncertain rows blank.
|
| 376 |
+
python -m eval.judge_validation consistency
|
| 377 |
+
python -m eval.judge_validation report
|
| 378 |
+
```
|
| 379 |
+
|
| 380 |
+
Export samples 50 reviewed questions without replacement (including reviewed
|
| 381 |
+
out-of-scope items) and runs the full pipeline once per selected question. The
|
| 382 |
+
seed controls question selection, not model generation. `--count` changes the
|
| 383 |
+
sample size. Insufficient reviewed questions stop before model loading; no
|
| 384 |
+
synthetic answers or labels are substituted. Existing export paths are protected;
|
| 385 |
+
use new `--csv` and `--samples` paths for another sample.
|
| 386 |
+
|
| 387 |
+
The CSV columns are `id, question, answer, context_snippet, my_label, my_notes`.
|
| 388 |
+
Judge scores are hidden from this labeling file. Snippets contain up to 4,000
|
| 389 |
+
context characters with an explicit truncation notice. Inspect the full context
|
| 390 |
+
in `results/judge_validation_samples.jsonl` when needed. That snapshot retains
|
| 391 |
+
the final attempt's original question, expanded judge question, answer, formatted
|
| 392 |
+
judge context, raw overall score, correctness score and model. These files contain
|
| 393 |
+
question/answer/source text; they are local evaluation artifacts ignored by Git.
|
| 394 |
+
|
| 395 |
+
Agreement uses the saved original judge, with a fixed mapping declared before
|
| 396 |
+
labeling: correctness 0–1 = incorrect, 2 = partially, 3–4 = correct. It reports
|
| 397 |
+
accuracy, unweighted Cohen's kappa, and a confusion matrix (rows human, columns
|
| 398 |
+
judge). Pearson and Spearman correlations compare original **raw** overall_score
|
| 399 |
+
with manual labels encoded incorrect=0, partially=1, correct=2. Citation penalties
|
| 400 |
+
are excluded from that raw score. Blank labels are excluded and counted; unknown
|
| 401 |
+
labels, changed source columns, duplicate IDs and missing rows are rejected.
|
| 402 |
+
Constant inputs or insufficient observations produce undefined correlations;
|
| 403 |
+
kappa is undefined when chance agreement is one.
|
| 404 |
+
|
| 405 |
+
Consistency makes exactly three fresh judge calls per frozen answer at temperature
|
| 406 |
+
zero, through the production prompt and saved full context. It does not rerun
|
| 407 |
+
retrieval/generation or send human labels to Gemini. This step needs only Gemini
|
| 408 |
+
configuration, not Qdrant/Neo4j/Kanoon credentials. Results are saved in
|
| 409 |
+
`results/judge_consistency.json` and bound to the sample hash. Re-running the
|
| 410 |
+
consistency command replaces those measurements with three new calls per answer.
|
| 411 |
+
The report shows the current judge model, per-answer scores/sample variance
|
| 412 |
+
(ddof=1), mean variance and maximum variance over all sampled answers. Model
|
| 413 |
+
changes may affect comparisons; temperature zero does not guarantee identical
|
| 414 |
+
outputs.
|
| 415 |
+
|
| 416 |
+
`report` runs offline and writes `results/judge_validation.md`. It can report
|
| 417 |
+
agreement before consistency has run, explicitly marking consistency unmeasured.
|
| 418 |
+
Use `--consistency`, `--output`, `--csv`, and `--samples` to select other paths.
|
| 419 |
+
The current gold file has no reviewed questions, so the checked-in report records
|
| 420 |
+
**not run**; manual agreement and live variance have not been measured.
|
eval/judge_validation.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Export blinded manual labels, rerun the production judge, and measure agreement."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import csv
|
| 7 |
+
import hashlib
|
| 8 |
+
import io
|
| 9 |
+
import json
|
| 10 |
+
import math
|
| 11 |
+
import random
|
| 12 |
+
import statistics
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Any, Protocol
|
| 15 |
+
|
| 16 |
+
from pydantic import BaseModel, ConfigDict, Field
|
| 17 |
+
|
| 18 |
+
from eval.models import GoldItem
|
| 19 |
+
from eval.run_answer_eval import load_pipeline
|
| 20 |
+
from eval.storage import atomic_text, load_gold, read_jsonl
|
| 21 |
+
|
| 22 |
+
LABELS = ("incorrect", "partially", "correct")
|
| 23 |
+
COLUMNS = ("id", "question", "answer", "context_snippet", "my_label", "my_notes")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class Sample(BaseModel):
|
| 27 |
+
"""Retain the final attempt's judge inputs and raw outputs separately from labels."""
|
| 28 |
+
|
| 29 |
+
model_config = ConfigDict(extra="forbid")
|
| 30 |
+
id: str
|
| 31 |
+
question: str
|
| 32 |
+
judge_question: str
|
| 33 |
+
answer: str
|
| 34 |
+
context: str
|
| 35 |
+
context_snippet: str
|
| 36 |
+
overall_score: float = Field(ge=0, le=100, allow_inf_nan=False)
|
| 37 |
+
correctness: int = Field(ge=0, le=4, strict=True)
|
| 38 |
+
evaluator_model: str
|
| 39 |
+
seed: int
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class Judge(Protocol):
|
| 43 |
+
"""Production judge boundary; persistence and statistics remain ordinary code."""
|
| 44 |
+
|
| 45 |
+
model: str
|
| 46 |
+
|
| 47 |
+
def evaluate_context(self, question: str, answer: str, context: str) -> dict:
|
| 48 |
+
"""Judge the exact saved context at temperature zero."""
|
| 49 |
+
...
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def sample_questions(gold: list[GoldItem], count: int, seed: int) -> list[GoldItem]:
|
| 53 |
+
"""Select without replacement from reviewed items, independent of input ordering."""
|
| 54 |
+
eligible = sorted((item for item in gold if item.reviewed), key=lambda item: item.id)
|
| 55 |
+
if count < 1 or len(eligible) < count:
|
| 56 |
+
raise ValueError(f"Need {count} reviewed questions; found {len(eligible)}")
|
| 57 |
+
return random.Random(seed).sample(eligible, count)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def export_samples(gold: list[GoldItem], pipeline: Any, count: int, seed: int,
|
| 61 |
+
csv_path: Path, samples_path: Path) -> list[Sample]:
|
| 62 |
+
"""Generate full answers and export blank labels without exposing judge predictions."""
|
| 63 |
+
if csv_path.exists() or samples_path.exists():
|
| 64 |
+
raise ValueError("Export paths already exist; use new paths to preserve manual labels")
|
| 65 |
+
samples = []
|
| 66 |
+
for item in sample_questions(gold, count, seed):
|
| 67 |
+
result = pipeline.search(item.question)
|
| 68 |
+
retrieval, evaluation = result["retrieval"], result["evaluation"]
|
| 69 |
+
# Use the identical source formatter and final retry question used by RAGPipeline.evaluate.
|
| 70 |
+
context = pipeline.evaluator._build_context(
|
| 71 |
+
retrieval.get("statutory_results", []),
|
| 72 |
+
list(retrieval.get("graph_context", {}).values()),
|
| 73 |
+
retrieval.get("judgments", []),
|
| 74 |
+
)
|
| 75 |
+
snippet = "Retrieved context:\n" + context[:4000] + ("\n[TRUNCATED: full context in sample snapshot]" if len(context) > 4000 else "")
|
| 76 |
+
samples.append(Sample(
|
| 77 |
+
id=item.id, question=item.question, judge_question=result["query"],
|
| 78 |
+
answer=result["answer"], context=context, context_snippet=snippet,
|
| 79 |
+
overall_score=evaluation.get("llm_overall_score", evaluation["overall_score"]),
|
| 80 |
+
correctness=evaluation["correctness"]["score"],
|
| 81 |
+
evaluator_model=pipeline.evaluator.model, seed=seed,
|
| 82 |
+
))
|
| 83 |
+
output = io.StringIO(newline="")
|
| 84 |
+
writer = csv.DictWriter(output, fieldnames=COLUMNS)
|
| 85 |
+
writer.writeheader()
|
| 86 |
+
for sample in samples:
|
| 87 |
+
writer.writerow({**{key: getattr(sample, key) for key in COLUMNS[:4]}, "my_label": "", "my_notes": ""})
|
| 88 |
+
atomic_text(samples_path, "".join(sample.model_dump_json() + "\n" for sample in samples))
|
| 89 |
+
atomic_text(csv_path, output.getvalue())
|
| 90 |
+
return samples
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def load_samples(path: Path) -> list[Sample]:
|
| 94 |
+
"""Require a nonempty, unique sample snapshot before scoring or model loading."""
|
| 95 |
+
samples = read_jsonl(path, Sample)
|
| 96 |
+
if not samples or len({sample.id for sample in samples}) != len(samples):
|
| 97 |
+
raise ValueError("Sample snapshot must contain unique, nonempty records")
|
| 98 |
+
return samples
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def load_labels(path: Path, samples: list[Sample]) -> dict[str, str]:
|
| 102 |
+
"""Accept only label/note edits; reject stale, duplicate, missing or unknown rows."""
|
| 103 |
+
expected = {sample.id: sample for sample in samples}
|
| 104 |
+
seen, labels = set(), {}
|
| 105 |
+
with path.open(encoding="utf-8-sig", newline="") as handle:
|
| 106 |
+
reader = csv.DictReader(handle)
|
| 107 |
+
if reader.fieldnames != list(COLUMNS):
|
| 108 |
+
raise ValueError("Manual CSV columns must match the exported columns")
|
| 109 |
+
for row in reader:
|
| 110 |
+
identifier = row["id"]
|
| 111 |
+
if identifier not in expected or identifier in seen or None in row:
|
| 112 |
+
raise ValueError("Manual CSV contains unknown, duplicate or malformed rows")
|
| 113 |
+
seen.add(identifier)
|
| 114 |
+
if any(row[key] != getattr(expected[identifier], key) for key in COLUMNS[:4]):
|
| 115 |
+
raise ValueError("Question, answer or context changed; export a new sample")
|
| 116 |
+
label = (row["my_label"] or "").strip().lower()
|
| 117 |
+
if label and label not in LABELS:
|
| 118 |
+
raise ValueError("my_label must be correct, partially or incorrect")
|
| 119 |
+
if label:
|
| 120 |
+
labels[identifier] = label
|
| 121 |
+
if seen != set(expected):
|
| 122 |
+
raise ValueError("Manual CSV is missing sample rows")
|
| 123 |
+
if not labels:
|
| 124 |
+
raise ValueError("No manual labels yet")
|
| 125 |
+
return labels
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def correlation(left: list[float], right: list[float]) -> float | None:
|
| 129 |
+
"""Pearson correlation, undefined for constant inputs or fewer than two pairs."""
|
| 130 |
+
if len(left) != len(right):
|
| 131 |
+
raise ValueError("Correlation requires paired observations")
|
| 132 |
+
if len(left) < 2:
|
| 133 |
+
return None
|
| 134 |
+
centered_x = [value - statistics.mean(left) for value in left]
|
| 135 |
+
centered_y = [value - statistics.mean(right) for value in right]
|
| 136 |
+
denominator = math.sqrt(sum(value ** 2 for value in centered_x) * sum(value ** 2 for value in centered_y))
|
| 137 |
+
return sum(x * y for x, y in zip(centered_x, centered_y)) / denominator if denominator else None
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def ranks(values: list[float]) -> list[float]:
|
| 141 |
+
"""Assign average one-based ranks to ties for Spearman correlation."""
|
| 142 |
+
return [1 + sum(other < value for other in values) + (values.count(value) - 1) / 2 for value in values]
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def agreement(samples: list[Sample], labels: dict[str, str]) -> dict:
|
| 146 |
+
"""Unweighted Cohen kappa and raw-score Pearson/Spearman on labeled rows only."""
|
| 147 |
+
paired = [sample for sample in samples if sample.id in labels]
|
| 148 |
+
if not paired or set(labels) - {sample.id for sample in samples}:
|
| 149 |
+
raise ValueError("Agreement requires labels belonging to the sample")
|
| 150 |
+
matrix = [[0] * 3 for _ in LABELS]
|
| 151 |
+
human, scores = [], []
|
| 152 |
+
for sample in paired:
|
| 153 |
+
actual = LABELS.index(labels[sample.id])
|
| 154 |
+
predicted = 2 if sample.correctness >= 3 else 1 if sample.correctness == 2 else 0
|
| 155 |
+
matrix[actual][predicted] += 1
|
| 156 |
+
human.append(actual)
|
| 157 |
+
scores.append(sample.overall_score)
|
| 158 |
+
n = len(paired)
|
| 159 |
+
accuracy = sum(matrix[index][index] for index in range(3)) / n
|
| 160 |
+
expected = sum(sum(matrix[index]) * sum(row[index] for row in matrix) for index in range(3)) / n ** 2
|
| 161 |
+
return {
|
| 162 |
+
"sample_count": len(samples), "labeled_count": n, "unlabeled_count": len(samples) - n,
|
| 163 |
+
"accuracy": accuracy, "cohen_kappa": (accuracy - expected) / (1 - expected) if expected < 1 else None,
|
| 164 |
+
"labels": LABELS, "confusion_matrix": matrix,
|
| 165 |
+
"pearson_score_label": correlation(scores, human),
|
| 166 |
+
"spearman_score_label": correlation(ranks(scores), ranks(human)),
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def sample_hash(samples: list[Sample]) -> str:
|
| 171 |
+
"""Bind consistency results to all saved judge inputs and original outputs."""
|
| 172 |
+
serialized = "\n".join(sample.model_dump_json() for sample in sorted(samples, key=lambda sample: sample.id))
|
| 173 |
+
return hashlib.sha256(serialized.encode()).hexdigest()
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def run_consistency(samples: list[Sample], judge: Judge) -> dict:
|
| 177 |
+
"""Make three independent temperature-zero calls per frozen answer/context."""
|
| 178 |
+
details = []
|
| 179 |
+
for sample in samples:
|
| 180 |
+
scores = []
|
| 181 |
+
for _ in range(3):
|
| 182 |
+
result = judge.evaluate_context(sample.judge_question, sample.answer, sample.context)
|
| 183 |
+
score = result["overall_score"]
|
| 184 |
+
if isinstance(score, bool) or not isinstance(score, (int, float)) or not math.isfinite(score) or not 0 <= score <= 100:
|
| 185 |
+
raise ValueError("Judge overall_score must be finite and between 0 and 100")
|
| 186 |
+
scores.append(score)
|
| 187 |
+
details.append({"id": sample.id, "scores": scores, "score_variance": statistics.variance(scores)})
|
| 188 |
+
return {"sample_sha256": sample_hash(samples), "model": judge.model, "temperature": 0,
|
| 189 |
+
"runs_per_answer": 3, "variance_ddof": 1, "details": details}
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def validate_consistency(result: dict, samples: list[Sample]) -> list[dict]:
|
| 193 |
+
"""Reject stale/incomplete reruns and recompute variance from recorded scores."""
|
| 194 |
+
if (result.get("sample_sha256") != sample_hash(samples) or result.get("temperature") != 0
|
| 195 |
+
or result.get("runs_per_answer") != 3 or result.get("variance_ddof") != 1):
|
| 196 |
+
raise ValueError("Consistency results do not match this sample or protocol")
|
| 197 |
+
rows = result["details"]
|
| 198 |
+
if len(rows) != len(samples) or {row["id"] for row in rows} != {sample.id for sample in samples}:
|
| 199 |
+
raise ValueError("Consistency results must cover each sampled answer exactly once")
|
| 200 |
+
for row in rows:
|
| 201 |
+
scores = row["scores"]
|
| 202 |
+
if len(scores) != 3 or any(isinstance(score, bool) or not isinstance(score, (int, float)) or not math.isfinite(score) or not 0 <= score <= 100 for score in scores):
|
| 203 |
+
raise ValueError("Consistency requires three valid scores per answer")
|
| 204 |
+
return [{**row, "score_variance": statistics.variance(row["scores"])} for row in rows]
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def write_report(path: Path, samples: list[Sample], labels: dict[str, str], consistency: dict | None) -> str:
|
| 208 |
+
"""Write measured agreement and optional full-sample consistency as Markdown."""
|
| 209 |
+
metrics = agreement(samples, labels)
|
| 210 |
+
|
| 211 |
+
def display(value: float | None) -> str:
|
| 212 |
+
"""Make undefined statistics explicit rather than substituting zeros."""
|
| 213 |
+
return "Undefined" if value is None else f"{value:.4f}"
|
| 214 |
+
|
| 215 |
+
report = ("# Judge validation\n\n"
|
| 216 |
+
"Human labels: incorrect=0, partially=1, correct=2. Judge correctness: 0–1=incorrect, 2=partially, 3–4=correct. "
|
| 217 |
+
"Correlations use original raw overall_score (before citation penalties).\n\n"
|
| 218 |
+
f"Sampled: {len(samples)}; labeled: {metrics['labeled_count']}; unlabeled excluded: {metrics['unlabeled_count']}.\n\n"
|
| 219 |
+
"| Metric | Value |\n|---|---:|\n")
|
| 220 |
+
for key in ("accuracy", "cohen_kappa", "pearson_score_label", "spearman_score_label"):
|
| 221 |
+
report += f"| {key} | {display(metrics[key])} |\n"
|
| 222 |
+
report += "\nConfusion matrix: rows = human, columns = judge.\n\n| Human \\ Judge | incorrect | partially | correct |\n|---|---:|---:|---:|\n"
|
| 223 |
+
for label, row in zip(LABELS, metrics["confusion_matrix"]):
|
| 224 |
+
report += f"| {label} | " + " | ".join(map(str, row)) + " |\n"
|
| 225 |
+
report += "\nOriginal evaluator models: " + ", ".join(sorted({sample.evaluator_model for sample in samples})) + ".\n"
|
| 226 |
+
report += "\nConsistency uses three fresh calls per answer at temperature 0, with sample variance (ddof=1).\n\n"
|
| 227 |
+
if consistency is None:
|
| 228 |
+
report += "Not run: no saved consistency measurements.\n"
|
| 229 |
+
else:
|
| 230 |
+
rows = validate_consistency(consistency, samples)
|
| 231 |
+
report += f"Model: `{consistency['model']}`; answers: {len(rows)}; mean score variance: {statistics.mean(row['score_variance'] for row in rows):.4f}; max variance: {max(row['score_variance'] for row in rows):.4f}.\n\n"
|
| 232 |
+
report += "| ID | Run 1 | Run 2 | Run 3 | Variance |\n|---|---:|---:|---:|---:|\n"
|
| 233 |
+
for row in rows:
|
| 234 |
+
report += f"| {row['id']} | " + " | ".join(map(str, row["scores"])) + f" | {row['score_variance']:.4f} |\n"
|
| 235 |
+
report += "\nAgreement describes the labeled subset and is not independent ground truth. Temperature zero does not guarantee identical judge outputs. Full context is retained in the sample snapshot; CSV snippets may be truncated.\n"
|
| 236 |
+
atomic_text(path, report)
|
| 237 |
+
return report
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def main() -> None:
|
| 241 |
+
"""Run export, consistency and offline report as separate resumable steps."""
|
| 242 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 243 |
+
parser.add_argument("command", choices=("export", "consistency", "report"))
|
| 244 |
+
parser.add_argument("--gold", type=Path, default=Path("eval/gold.jsonl"))
|
| 245 |
+
parser.add_argument("--csv", type=Path, default=Path("results/judge_validation.csv"))
|
| 246 |
+
parser.add_argument("--samples", type=Path, default=Path("results/judge_validation_samples.jsonl"))
|
| 247 |
+
parser.add_argument("--consistency", type=Path, default=Path("results/judge_consistency.json"))
|
| 248 |
+
parser.add_argument("--output", type=Path, default=Path("results/judge_validation.md"))
|
| 249 |
+
parser.add_argument("--count", type=int, default=50)
|
| 250 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 251 |
+
args = parser.parse_args()
|
| 252 |
+
resource = None
|
| 253 |
+
try:
|
| 254 |
+
if args.command == "export":
|
| 255 |
+
gold = load_gold(args.gold)
|
| 256 |
+
sample_questions(gold, args.count, args.seed) # Gate before loading models/clients.
|
| 257 |
+
if args.csv.exists() or args.samples.exists():
|
| 258 |
+
raise ValueError("Export paths already exist; use new paths")
|
| 259 |
+
resource = load_pipeline()
|
| 260 |
+
export_samples(gold, resource, args.count, args.seed, args.csv, args.samples)
|
| 261 |
+
print(f"Exported {args.count} answers; fill my_label and my_notes in {args.csv}")
|
| 262 |
+
elif args.command == "consistency":
|
| 263 |
+
from evaluator.LLM_evaluator import LegalLLMEvaluator
|
| 264 |
+
from legora.config import EvaluationSettings
|
| 265 |
+
|
| 266 |
+
samples = load_samples(args.samples)
|
| 267 |
+
resource = LegalLLMEvaluator(settings=EvaluationSettings())
|
| 268 |
+
result = run_consistency(samples, resource)
|
| 269 |
+
atomic_text(args.consistency, json.dumps(result, indent=2) + "\n")
|
| 270 |
+
print(f"Saved three-run consistency measurements to {args.consistency}")
|
| 271 |
+
else:
|
| 272 |
+
samples = load_samples(args.samples)
|
| 273 |
+
labels = load_labels(args.csv, samples)
|
| 274 |
+
consistency = json.loads(args.consistency.read_text()) if args.consistency.exists() else None
|
| 275 |
+
print(write_report(args.output, samples, labels, consistency))
|
| 276 |
+
except Exception as error:
|
| 277 |
+
if not args.output.exists():
|
| 278 |
+
atomic_text(args.output, "# Judge validation — not run\n\nNo completed manual agreement/consistency report. Export real pipeline answers and fill manual labels before reporting.\n")
|
| 279 |
+
# Safe validation errors are useful, but upstream failures may contain secrets/prompts.
|
| 280 |
+
message = str(error) if type(error) is ValueError and args.command == "export" and resource is None else type(error).__name__
|
| 281 |
+
parser.exit(1, f"Judge validation failed: {message}. No incomplete run was scored.\n")
|
| 282 |
+
finally:
|
| 283 |
+
if resource is not None:
|
| 284 |
+
if args.command == "export":
|
| 285 |
+
resource.close()
|
| 286 |
+
else:
|
| 287 |
+
resource.client.close()
|
| 288 |
+
|
| 289 |
+
|
| 290 |
+
if __name__ == "__main__":
|
| 291 |
+
main()
|
evaluator/LLM_evaluator.py
CHANGED
|
@@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional
|
|
| 8 |
|
| 9 |
from google import genai
|
| 10 |
from google.genai import types
|
| 11 |
-
from legora.config import get_settings
|
| 12 |
|
| 13 |
logger = logging.getLogger(__name__)
|
| 14 |
|
|
@@ -42,11 +42,14 @@ class LegalLLMEvaluator:
|
|
| 42 |
def __init__(
|
| 43 |
self,
|
| 44 |
api_key: Optional[str] = None,
|
| 45 |
-
model: Optional[str] = None
|
|
|
|
| 46 |
):
|
| 47 |
|
| 48 |
-
settings = get_settings()
|
| 49 |
-
self.api_key = api_key or settings.gemini_api_key.get_secret_value()
|
|
|
|
|
|
|
| 50 |
|
| 51 |
self.client = genai.Client(
|
| 52 |
api_key=self.api_key,
|
|
@@ -79,6 +82,13 @@ class LegalLLMEvaluator:
|
|
| 79 |
kanoon_context
|
| 80 |
)
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
prompt = self._build_prompt(
|
| 83 |
question=question,
|
| 84 |
answer=answer,
|
|
@@ -577,4 +587,4 @@ if __name__ == "__main__":
|
|
| 577 |
kanoon_context=kanoon_results
|
| 578 |
)
|
| 579 |
|
| 580 |
-
logger.debug("%s", json.dumps(result, indent=4, ensure_ascii=False))
|
|
|
|
| 8 |
|
| 9 |
from google import genai
|
| 10 |
from google.genai import types
|
| 11 |
+
from legora.config import EvaluationSettings, Settings, get_settings
|
| 12 |
|
| 13 |
logger = logging.getLogger(__name__)
|
| 14 |
|
|
|
|
| 42 |
def __init__(
|
| 43 |
self,
|
| 44 |
api_key: Optional[str] = None,
|
| 45 |
+
model: Optional[str] = None,
|
| 46 |
+
settings: Settings | EvaluationSettings | None = None,
|
| 47 |
):
|
| 48 |
|
| 49 |
+
settings = settings or get_settings()
|
| 50 |
+
self.api_key = api_key or (settings.gemini_api_key.get_secret_value() if settings.gemini_api_key else None)
|
| 51 |
+
if not self.api_key or not self.api_key.strip():
|
| 52 |
+
raise ValueError("GEMINI_API_KEY is required for judge evaluation")
|
| 53 |
|
| 54 |
self.client = genai.Client(
|
| 55 |
api_key=self.api_key,
|
|
|
|
| 82 |
kanoon_context
|
| 83 |
)
|
| 84 |
|
| 85 |
+
return self.evaluate_context(question, answer, context, reference_answer)
|
| 86 |
+
|
| 87 |
+
def evaluate_context(
|
| 88 |
+
self, question: str, answer: str, context: str,
|
| 89 |
+
reference_answer: str | None = None,
|
| 90 |
+
) -> Dict[str, Any]:
|
| 91 |
+
"""Judge a saved, already formatted context with the production prompt at temperature zero."""
|
| 92 |
prompt = self._build_prompt(
|
| 93 |
question=question,
|
| 94 |
answer=answer,
|
|
|
|
| 587 |
kanoon_context=kanoon_results
|
| 588 |
)
|
| 589 |
|
| 590 |
+
logger.debug("%s", json.dumps(result, indent=4, ensure_ascii=False))
|
results/judge_validation.md
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Judge validation — not run
|
| 2 |
+
|
| 3 |
+
No completed manual agreement/consistency report. Export real pipeline answers and fill manual labels before reporting.
|
tests/test_judge_validation.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Judge-validation workflow exercises real CSV, sampling and statistics offline."""
|
| 2 |
+
|
| 3 |
+
import csv
|
| 4 |
+
import io
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from unittest.mock import Mock
|
| 8 |
+
|
| 9 |
+
import pytest
|
| 10 |
+
|
| 11 |
+
from eval.judge_validation import (
|
| 12 |
+
COLUMNS, Sample, agreement, correlation, export_samples, load_labels,
|
| 13 |
+
load_samples, main, ranks, run_consistency, sample_questions,
|
| 14 |
+
validate_consistency, write_report,
|
| 15 |
+
)
|
| 16 |
+
from tests.test_answer_eval import gold
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def sample(identifier: str = "one", score: float = 90, correctness: int = 4) -> Sample:
|
| 20 |
+
"""Create a frozen judge input only for local tests."""
|
| 21 |
+
return Sample(id=identifier, question="Question?", judge_question="Expanded question?",
|
| 22 |
+
answer='Answer, with "quotes"\nand newline.', context="Full context",
|
| 23 |
+
context_snippet="Snippet", overall_score=score, correctness=correctness,
|
| 24 |
+
evaluator_model="test-model", seed=42)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def label_csv(path: Path, samples: list[Sample], labels: list[str]) -> None:
|
| 28 |
+
"""Write the exact manual format with intentional CSV quoting."""
|
| 29 |
+
output = io.StringIO(newline="")
|
| 30 |
+
writer = csv.DictWriter(output, fieldnames=COLUMNS)
|
| 31 |
+
writer.writeheader()
|
| 32 |
+
for item, label in zip(samples, labels):
|
| 33 |
+
writer.writerow({**{key: getattr(item, key) for key in COLUMNS[:4]}, "my_label": label, "my_notes": "Manual note"})
|
| 34 |
+
path.write_text(output.getvalue())
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def test_sampling_is_reviewed_only_reproducible_and_order_independent() -> None:
|
| 38 |
+
"""Selection has no replacement and excludes unreviewed drafts."""
|
| 39 |
+
questions = [gold(str(index)) for index in range(60)] + [gold("draft", reviewed=False)]
|
| 40 |
+
chosen = sample_questions(questions, 50, 42)
|
| 41 |
+
assert len({item.id for item in chosen}) == 50
|
| 42 |
+
assert chosen == sample_questions(list(reversed(questions)), 50, 42)
|
| 43 |
+
assert chosen != sample_questions(questions, 50, 43)
|
| 44 |
+
assert all(item.reviewed for item in chosen)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@pytest.mark.parametrize("count", [0, -1, 51])
|
| 48 |
+
def test_sampling_requires_enough_reviewed_items(count: int) -> None:
|
| 49 |
+
"""Never silently shrink the requested sample or add fake answers."""
|
| 50 |
+
with pytest.raises(ValueError):
|
| 51 |
+
sample_questions([gold("one")], count, 42)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def test_export_is_blinded_preserves_final_inputs_and_raw_score(tmp_path: Path) -> None:
|
| 55 |
+
"""Human CSV hides judge outputs; sidecar retains exact final retry inputs."""
|
| 56 |
+
pipeline = Mock()
|
| 57 |
+
pipeline.evaluator.model = "test-model"
|
| 58 |
+
pipeline.evaluator._build_context.return_value = "X" * 4100
|
| 59 |
+
pipeline.search.return_value = {
|
| 60 |
+
"query": "Expanded query", "answer": "Answer\nwith comma, and quotes\"",
|
| 61 |
+
"retrieval": {"statutory_results": ["statute"], "graph_context": {"root": ["node"]}, "judgments": ["case"]},
|
| 62 |
+
"evaluation": {"overall_score": 30, "llm_overall_score": 90, "correctness": {"score": 3}},
|
| 63 |
+
}
|
| 64 |
+
csv_path, snapshot = tmp_path / "labels.csv", tmp_path / "samples.jsonl"
|
| 65 |
+
samples = export_samples([gold("one")], pipeline, 1, 42, csv_path, snapshot)
|
| 66 |
+
saved = load_samples(snapshot)
|
| 67 |
+
assert saved == samples
|
| 68 |
+
assert saved[0].overall_score == 90 and saved[0].judge_question == "Expanded query"
|
| 69 |
+
assert len(saved[0].context) == 4100 and "TRUNCATED" in saved[0].context_snippet
|
| 70 |
+
pipeline.evaluator._build_context.assert_called_once_with(["statute"], [["node"]], ["case"])
|
| 71 |
+
with csv_path.open(newline="") as handle:
|
| 72 |
+
rows = list(csv.DictReader(handle))
|
| 73 |
+
assert list(rows[0]) == list(COLUMNS) and rows[0]["my_label"] == ""
|
| 74 |
+
assert rows[0]["answer"] == saved[0].answer
|
| 75 |
+
with pytest.raises(ValueError, match="already exist"):
|
| 76 |
+
export_samples([gold("one")], pipeline, 1, 42, csv_path, snapshot)
|
| 77 |
+
assert pipeline.search.call_count == 1
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def test_labels_allow_blank_rows_and_normalize_manual_case(tmp_path: Path) -> None:
|
| 81 |
+
"""Blank labels are excluded rather than scored as incorrect."""
|
| 82 |
+
path = tmp_path / "labels.csv"
|
| 83 |
+
samples = [sample("a"), sample("b")]
|
| 84 |
+
label_csv(path, samples, [" Correct ", ""])
|
| 85 |
+
assert load_labels(path, samples) == {"a": "correct"}
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@pytest.mark.parametrize("change", ["invalid", "duplicate", "missing", "unknown", "edited", "blank"])
|
| 89 |
+
def test_labels_reject_corrupt_or_unlabeled_exports(tmp_path: Path, change: str) -> None:
|
| 90 |
+
"""Metrics cannot silently join the wrong human answers to judge scores."""
|
| 91 |
+
samples = [sample("a"), sample("b")]
|
| 92 |
+
path = tmp_path / "labels.csv"
|
| 93 |
+
altered = samples.copy()
|
| 94 |
+
labels = ["correct", "partially"]
|
| 95 |
+
if change == "invalid":
|
| 96 |
+
labels[0] = "yes"
|
| 97 |
+
elif change == "duplicate":
|
| 98 |
+
altered[1] = altered[0]
|
| 99 |
+
elif change == "missing":
|
| 100 |
+
altered.pop()
|
| 101 |
+
elif change == "unknown":
|
| 102 |
+
altered[1] = sample("unknown")
|
| 103 |
+
elif change == "edited":
|
| 104 |
+
altered[0] = altered[0].model_copy(update={"answer": "Edited answer"})
|
| 105 |
+
else:
|
| 106 |
+
labels = ["", ""]
|
| 107 |
+
label_csv(path, altered, labels)
|
| 108 |
+
with pytest.raises(ValueError):
|
| 109 |
+
load_labels(path, samples)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def test_agreement_known_matrix_kappa_and_correlations() -> None:
|
| 113 |
+
"""Compare hand-calculated three-class outcomes, including an off-diagonal error."""
|
| 114 |
+
samples = [sample("a", 10, 0), sample("b", 50, 2), sample("c", 90, 4), sample("d", 60, 2), sample("blank")]
|
| 115 |
+
metrics = agreement(samples, {"a": "incorrect", "b": "partially", "c": "correct", "d": "correct"})
|
| 116 |
+
assert metrics["confusion_matrix"] == [[1, 0, 0], [0, 1, 0], [0, 1, 1]]
|
| 117 |
+
assert metrics["accuracy"] == 0.75
|
| 118 |
+
assert metrics["cohen_kappa"] == pytest.approx((.75 - .3125) / (1 - .3125))
|
| 119 |
+
assert metrics["pearson_score_label"] == pytest.approx(correlation([10, 50, 90, 60], [0, 1, 2, 2]))
|
| 120 |
+
assert metrics["spearman_score_label"] == pytest.approx(0.948683298)
|
| 121 |
+
assert metrics["unlabeled_count"] == 1
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
@pytest.mark.parametrize("correctness,predicted", [(0, 0), (1, 0), (2, 1), (3, 2), (4, 2)])
|
| 125 |
+
def test_correctness_mapping(correctness: int, predicted: int) -> None:
|
| 126 |
+
"""The predeclared category boundaries do not depend on human labels."""
|
| 127 |
+
matrix = agreement([sample(correctness=correctness)], {"one": "incorrect"})["confusion_matrix"]
|
| 128 |
+
assert matrix[0][predicted] == 1
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def test_degenerate_statistics_are_undefined() -> None:
|
| 132 |
+
"""Constant/single-row labels cannot produce meaningful kappa or correlation."""
|
| 133 |
+
result = agreement([sample()], {"one": "correct"})
|
| 134 |
+
assert result["accuracy"] == 1 and result["cohen_kappa"] is None
|
| 135 |
+
assert result["pearson_score_label"] is None and result["spearman_score_label"] is None
|
| 136 |
+
assert correlation([1, 1], [0, 2]) is None
|
| 137 |
+
assert ranks([5, 1, 5, 9]) == [2.5, 1, 2.5, 4]
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def test_consistency_three_fresh_calls_and_sample_variance() -> None:
|
| 141 |
+
"""Repeated judgments use exact frozen inputs, not regenerated answers or labels."""
|
| 142 |
+
judge = Mock(model="test-model")
|
| 143 |
+
judge.evaluate_context.side_effect = [{"overall_score": score} for score in [80, 90, 100, 42, 42, 42]]
|
| 144 |
+
samples = [sample("a"), sample("b")]
|
| 145 |
+
result = run_consistency(samples, judge)
|
| 146 |
+
assert judge.evaluate_context.call_count == 6
|
| 147 |
+
assert all(call.args == ("Expanded question?", samples[0].answer, "Full context") for call in judge.evaluate_context.call_args_list)
|
| 148 |
+
assert result["details"][0]["score_variance"] == 100
|
| 149 |
+
assert result["details"][1]["score_variance"] == 0
|
| 150 |
+
assert validate_consistency(result, samples) == result["details"]
|
| 151 |
+
with pytest.raises(ValueError):
|
| 152 |
+
validate_consistency(result, [sample("changed")])
|
| 153 |
+
result["details"].pop()
|
| 154 |
+
with pytest.raises(ValueError):
|
| 155 |
+
validate_consistency(result, samples)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
@pytest.mark.parametrize("score", [float("nan"), float("inf"), -1, 101, True, "90"])
|
| 159 |
+
def test_consistency_rejects_invalid_upstream_scores(score: object) -> None:
|
| 160 |
+
"""Failed or malformed evaluations must not become fabricated variance metrics."""
|
| 161 |
+
judge = Mock(model="test-model")
|
| 162 |
+
judge.evaluate_context.return_value = {"overall_score": score}
|
| 163 |
+
with pytest.raises(ValueError):
|
| 164 |
+
run_consistency([sample()], judge)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def test_report_contains_measured_agreement_and_variance(tmp_path: Path) -> None:
|
| 168 |
+
"""A complete report includes full confusion matrix and per-answer stability."""
|
| 169 |
+
samples = [sample()]
|
| 170 |
+
judge = Mock(model="test-model")
|
| 171 |
+
judge.evaluate_context.side_effect = [{"overall_score": score} for score in [80, 90, 100]]
|
| 172 |
+
output = tmp_path / "report.md"
|
| 173 |
+
report = write_report(output, samples, {"one": "correct"}, run_consistency(samples, judge))
|
| 174 |
+
assert "cohen_kappa | Undefined" in report
|
| 175 |
+
assert "max variance: 100.0000" in report and "| correct | 0 | 0 | 1 |" in report
|
| 176 |
+
assert output.read_text() == report
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def test_export_gate_does_not_load_models_or_fabricate_csv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 180 |
+
"""The present incomplete dataset cannot produce a claimed 50-answer sample."""
|
| 181 |
+
factory = Mock(side_effect=AssertionError("No model startup"))
|
| 182 |
+
monkeypatch.setattr("eval.judge_validation.load_pipeline", factory)
|
| 183 |
+
output, csv_path = tmp_path / "report.md", tmp_path / "manual.csv"
|
| 184 |
+
monkeypatch.setattr("sys.argv", ["judge", "export", "--gold", str(tmp_path / "missing.jsonl"), "--output", str(output), "--csv", str(csv_path)])
|
| 185 |
+
with pytest.raises(SystemExit) as error:
|
| 186 |
+
main()
|
| 187 |
+
assert error.value.code == 1
|
| 188 |
+
factory.assert_not_called()
|
| 189 |
+
assert "not run" in output.read_text() and not csv_path.exists()
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def test_report_command_is_offline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 193 |
+
"""Filled labels can be scored without Gemini, Neo4j or pipeline initialization."""
|
| 194 |
+
csv_path, snapshot, output = (tmp_path / name for name in ("manual.csv", "samples.jsonl", "report.md"))
|
| 195 |
+
samples = [sample()]
|
| 196 |
+
snapshot.write_text(samples[0].model_dump_json() + "\n")
|
| 197 |
+
label_csv(csv_path, samples, ["correct"])
|
| 198 |
+
monkeypatch.setattr("sys.argv", ["judge", "report", "--csv", str(csv_path), "--samples", str(snapshot), "--output", str(output), "--consistency", str(tmp_path / "absent.json")])
|
| 199 |
+
main()
|
| 200 |
+
assert "accuracy | 1.0000" in output.read_text()
|
| 201 |
+
assert "Not run: no saved consistency" in output.read_text()
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def test_production_judge_uses_same_prompt_and_temperature_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 205 |
+
"""Both live evaluation and frozen-context reruns share one real generation path."""
|
| 206 |
+
from evaluator.LLM_evaluator import LegalLLMEvaluator
|
| 207 |
+
from legora.config import EvaluationSettings
|
| 208 |
+
|
| 209 |
+
client = Mock()
|
| 210 |
+
client.models.generate_content.return_value.text = json.dumps({"overall_score": 88})
|
| 211 |
+
monkeypatch.setattr("evaluator.LLM_evaluator.genai.Client", Mock(return_value=client))
|
| 212 |
+
judge = LegalLLMEvaluator(settings=EvaluationSettings(_env_file=None, gemini_api_key="test-key"))
|
| 213 |
+
live = judge.evaluate("Question?", "Answer", qdrant_context=[{"text": "Evidence"}])
|
| 214 |
+
first = client.models.generate_content.call_args
|
| 215 |
+
frozen = judge.evaluate_context("Question?", "Answer", judge._build_context([{"text": "Evidence"}], [], []))
|
| 216 |
+
assert live == frozen == {"overall_score": 88}
|
| 217 |
+
assert first.kwargs["contents"] == client.models.generate_content.call_args.kwargs["contents"]
|
| 218 |
+
assert first.kwargs["config"].temperature == 0
|