updated readme
Browse files- .coveragerc +4 -1
- .dockerignore +23 -0
- .env.example +2 -1
- .github/workflows/ci.yml +57 -0
- .gitignore +6 -0
- Dockerfile +7 -5
- LegalRag/citation_check.py +1 -1
- Makefile +11 -1
- README.md +322 -171
- data/statutes/README.md +7 -0
- docker-compose.yml +103 -0
- eval/generate.py +2 -1
- eval/judge_validation.py +14 -13
- eval/review.py +3 -2
- eval/run_retrieval_eval.py +2 -2
- eval/sample.py +2 -1
- eval/storage.py +2 -2
- legora/logging.py +1 -1
- legora/performance.py +2 -2
- pyproject.toml +18 -0
- requirements-dev.txt +6 -0
- requirements.txt +8 -92
- results/corpus_stats.json +52 -0
- results/corpus_stats.md +18 -0
- retrieval/lexical.py +10 -4
- scripts/__init__.py +1 -0
- scripts/corpus_stats.py +157 -0
- scripts/ingest.py +130 -0
- scripts/ingest.sh +6 -0
- tests/test_corpus_stats.py +104 -0
- tests/test_ingestion.py +137 -0
.coveragerc
CHANGED
|
@@ -10,10 +10,13 @@ source =
|
|
| 10 |
eval.judge_validation
|
| 11 |
eval.profile_pipeline
|
| 12 |
legora.performance
|
|
|
|
|
|
|
| 13 |
# The requested parser scope excludes ingestion clients and runnable examples.
|
| 14 |
omit =
|
| 15 |
*/usage.py
|
| 16 |
-
|
|
|
|
| 17 |
*/search.py
|
| 18 |
*/qdrant_store.py
|
| 19 |
*/embedder.py
|
|
|
|
| 10 |
eval.judge_validation
|
| 11 |
eval.profile_pipeline
|
| 12 |
legora.performance
|
| 13 |
+
scripts.ingest
|
| 14 |
+
scripts.corpus_stats
|
| 15 |
# The requested parser scope excludes ingestion clients and runnable examples.
|
| 16 |
omit =
|
| 17 |
*/usage.py
|
| 18 |
+
db/ingest.py
|
| 19 |
+
db/parsers/*/ingest.py
|
| 20 |
*/search.py
|
| 21 |
*/qdrant_store.py
|
| 22 |
*/embedder.py
|
.dockerignore
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
.env
|
| 3 |
+
.env.*
|
| 4 |
+
!.env.example
|
| 5 |
+
venv
|
| 6 |
+
.venv
|
| 7 |
+
__pycache__
|
| 8 |
+
**/__pycache__
|
| 9 |
+
*.py[cod]
|
| 10 |
+
.cache
|
| 11 |
+
.mypy_cache
|
| 12 |
+
.ruff_cache
|
| 13 |
+
.pytest_cache
|
| 14 |
+
.coverage*
|
| 15 |
+
coverage
|
| 16 |
+
data
|
| 17 |
+
db/pdfs
|
| 18 |
+
**/qdrant_data
|
| 19 |
+
**/*.pdf
|
| 20 |
+
**/*.PDF
|
| 21 |
+
results
|
| 22 |
+
eval/*snapshot*
|
| 23 |
+
eval/gold*.jsonl
|
.env.example
CHANGED
|
@@ -5,7 +5,8 @@ QDRANT_URL=http://localhost:6333
|
|
| 5 |
# Optional for an unauthenticated local instance.
|
| 6 |
QDRANT_API_KEY=
|
| 7 |
NEO4J_URI=bolt://localhost:7687
|
| 8 |
-
|
|
|
|
| 9 |
NEO4J_PASSWORD=replace-with-neo4j-password
|
| 10 |
KANOON_TOKEN=replace-with-kanoon-token
|
| 11 |
TOP_K=30
|
|
|
|
| 5 |
# Optional for an unauthenticated local instance.
|
| 6 |
QDRANT_API_KEY=
|
| 7 |
NEO4J_URI=bolt://localhost:7687
|
| 8 |
+
# Neo4j's official image creates this initial user; use your own strong password.
|
| 9 |
+
NEO4J_USERNAME=neo4j
|
| 10 |
NEO4J_PASSWORD=replace-with-neo4j-password
|
| 11 |
KANOON_TOKEN=replace-with-kanoon-token
|
| 12 |
TOP_K=30
|
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
on:
|
| 3 |
+
push:
|
| 4 |
+
pull_request:
|
| 5 |
+
|
| 6 |
+
permissions:
|
| 7 |
+
contents: read
|
| 8 |
+
|
| 9 |
+
jobs:
|
| 10 |
+
checks:
|
| 11 |
+
runs-on: ubuntu-latest
|
| 12 |
+
timeout-minutes: 30
|
| 13 |
+
env:
|
| 14 |
+
HF_HUB_OFFLINE: "1"
|
| 15 |
+
TRANSFORMERS_OFFLINE: "1"
|
| 16 |
+
steps:
|
| 17 |
+
- uses: actions/checkout@v4
|
| 18 |
+
- uses: actions/setup-python@v5
|
| 19 |
+
with:
|
| 20 |
+
python-version: "3.11"
|
| 21 |
+
cache: pip
|
| 22 |
+
cache-dependency-path: requirements*.txt
|
| 23 |
+
- name: Install pinned dependencies (CPU models)
|
| 24 |
+
run: |
|
| 25 |
+
python -m pip install torch==2.12.0 --index-url https://download.pytorch.org/whl/cpu
|
| 26 |
+
python -m pip install -r requirements-dev.txt
|
| 27 |
+
python -m pip check
|
| 28 |
+
- name: Lint
|
| 29 |
+
run: make lint PYTHON=python
|
| 30 |
+
- name: Type-check new modules
|
| 31 |
+
run: make typecheck PYTHON=python
|
| 32 |
+
- name: Offline tests with coverage
|
| 33 |
+
run: make test PYTHON=python
|
| 34 |
+
- uses: actions/upload-artifact@v4
|
| 35 |
+
if: always()
|
| 36 |
+
with:
|
| 37 |
+
name: coverage
|
| 38 |
+
path: coverage/
|
| 39 |
+
if-no-files-found: ignore
|
| 40 |
+
- name: Validate Compose configuration
|
| 41 |
+
run: |
|
| 42 |
+
cp .env.example .env
|
| 43 |
+
docker compose --env-file .env.example config --quiet
|
| 44 |
+
|
| 45 |
+
docker:
|
| 46 |
+
runs-on: ubuntu-latest
|
| 47 |
+
timeout-minutes: 40
|
| 48 |
+
steps:
|
| 49 |
+
- uses: actions/checkout@v4
|
| 50 |
+
- uses: docker/setup-buildx-action@v3
|
| 51 |
+
- uses: docker/build-push-action@v6
|
| 52 |
+
with:
|
| 53 |
+
context: .
|
| 54 |
+
push: false
|
| 55 |
+
tags: legora:ci
|
| 56 |
+
cache-from: type=gha
|
| 57 |
+
cache-to: type=gha,mode=max
|
.gitignore
CHANGED
|
@@ -122,3 +122,9 @@ pnpm-lock.yaml
|
|
| 122 |
/results/judge_validation.csv
|
| 123 |
/results/judge_validation_samples.jsonl
|
| 124 |
/results/judge_consistency.json
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
/results/judge_validation.csv
|
| 123 |
/results/judge_validation_samples.jsonl
|
| 124 |
/results/judge_consistency.json
|
| 125 |
+
|
| 126 |
+
# User-supplied statute sources and OCR artifacts
|
| 127 |
+
/data/statutes/*
|
| 128 |
+
!/data/statutes/README.md
|
| 129 |
+
.mypy_cache/
|
| 130 |
+
.ruff_cache/
|
Dockerfile
CHANGED
|
@@ -26,6 +26,8 @@ RUN apt-get update && apt-get install -y \
|
|
| 26 |
build-essential \
|
| 27 |
git \
|
| 28 |
curl \
|
|
|
|
|
|
|
| 29 |
libgl1 \
|
| 30 |
libglib2.0-0 \
|
| 31 |
libsm6 \
|
|
@@ -43,9 +45,9 @@ WORKDIR /app
|
|
| 43 |
# ==========================================================
|
| 44 |
COPY requirements.txt .
|
| 45 |
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
|
| 50 |
# ==========================================================
|
| 51 |
# Copy Project
|
|
@@ -55,9 +57,9 @@ COPY . .
|
|
| 55 |
# ==========================================================
|
| 56 |
# Expose Streamlit Port
|
| 57 |
# ==========================================================
|
| 58 |
-
EXPOSE 7860
|
| 59 |
|
| 60 |
# ==========================================================
|
| 61 |
# Launch Streamlit
|
| 62 |
# ==========================================================
|
| 63 |
-
CMD ["
|
|
|
|
| 26 |
build-essential \
|
| 27 |
git \
|
| 28 |
curl \
|
| 29 |
+
poppler-utils \
|
| 30 |
+
tesseract-ocr \
|
| 31 |
libgl1 \
|
| 32 |
libglib2.0-0 \
|
| 33 |
libsm6 \
|
|
|
|
| 45 |
# ==========================================================
|
| 46 |
COPY requirements.txt .
|
| 47 |
|
| 48 |
+
# CPU wheels avoid pulling CUDA runtime libraries into the local deployment.
|
| 49 |
+
RUN pip install --no-cache-dir torch==2.12.0 --index-url https://download.pytorch.org/whl/cpu \
|
| 50 |
+
&& pip install --no-cache-dir -r requirements.txt
|
| 51 |
|
| 52 |
# ==========================================================
|
| 53 |
# Copy Project
|
|
|
|
| 57 |
# ==========================================================
|
| 58 |
# Expose Streamlit Port
|
| 59 |
# ==========================================================
|
| 60 |
+
EXPOSE 8000 7860
|
| 61 |
|
| 62 |
# ==========================================================
|
| 63 |
# Launch Streamlit
|
| 64 |
# ==========================================================
|
| 65 |
+
CMD ["python", "-m", "uvicorn", "api.app:app", "--host", "0.0.0.0", "--port", "8000"]
|
LegalRag/citation_check.py
CHANGED
|
@@ -124,7 +124,7 @@ def extract_citations(answer: str, known_ids: Collection[str] = ()) -> list[Cita
|
|
| 124 |
elif len(codes) == 1:
|
| 125 |
code = next(iter(codes))
|
| 126 |
number_matches = list(re.finditer(NUMBER, reference["numbers"]))
|
| 127 |
-
numbers = []
|
| 128 |
for index, match in enumerate(number_matches):
|
| 129 |
number = normalized_number(match[0])
|
| 130 |
if index:
|
|
|
|
| 124 |
elif len(codes) == 1:
|
| 125 |
code = next(iter(codes))
|
| 126 |
number_matches = list(re.finditer(NUMBER, reference["numbers"]))
|
| 127 |
+
numbers: list[tuple[int, str]] = []
|
| 128 |
for index, match in enumerate(number_matches):
|
| 129 |
number = normalized_number(match[0])
|
| 130 |
if index:
|
Makefile
CHANGED
|
@@ -1,5 +1,15 @@
|
|
| 1 |
PYTHON ?= $(if $(wildcard venv/bin/python),venv/bin/python,python3)
|
| 2 |
|
| 3 |
-
.PHONY: test
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
test:
|
| 5 |
$(PYTHON) -m pytest --cov --cov-branch --cov-report=term-missing --cov-report=html:coverage/html --cov-report=json:coverage/coverage.json
|
|
|
|
| 1 |
PYTHON ?= $(if $(wildcard venv/bin/python),venv/bin/python,python3)
|
| 2 |
|
| 3 |
+
.PHONY: test lint typecheck ingest
|
| 4 |
+
lint:
|
| 5 |
+
$(PYTHON) -m ruff check api legora eval scripts LegalRag/citation_check.py retrieval/lexical.py
|
| 6 |
+
$(PYTHON) -m ruff check --select E9,F63,F7,F82 .
|
| 7 |
+
|
| 8 |
+
typecheck:
|
| 9 |
+
$(PYTHON) -m mypy
|
| 10 |
+
|
| 11 |
+
ingest:
|
| 12 |
+
sh scripts/ingest.sh
|
| 13 |
+
|
| 14 |
test:
|
| 15 |
$(PYTHON) -m pytest --cov --cov-branch --cov-report=term-missing --cov-report=html:coverage/html --cov-report=json:coverage/coverage.json
|
README.md
CHANGED
|
@@ -10,190 +10,341 @@ pinned: false
|
|
| 10 |
|
| 11 |
# Legora
|
| 12 |
|
| 13 |
-
Legora is an
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
-
|
| 20 |
-
-
|
| 21 |
-
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
-
|
| 33 |
-
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
Retrieval defaults are `TOP_K=30`, `RERANK_K=5`, and `JUDGMENT_K=5`.
|
| 51 |
-
`PASS_THRESHOLD=80` uses the evaluator's 0–100 scale; `MAX_ATTEMPTS=3` is the
|
| 52 |
-
maximum total attempts. All candidate limits and the retry budget must be positive.
|
| 53 |
-
|
| 54 |
-
`LOG_LEVEL` defaults to `INFO`. The application configures newline-delimited JSON
|
| 55 |
-
console logs with a fresh `request_id` per `RAGPipeline.search()` call. Retries
|
| 56 |
-
share that ID and carry a 1-based `attempt`; context is restored when the query
|
| 57 |
-
finishes or raises. Stage events include `stage`, `latency_ms`, `candidate_count`,
|
| 58 |
-
`status`, and `error_type`. Counts are unique vector results, retained rerank
|
| 59 |
-
results, expanded graph roots, or fetched judgments, depending on the stage;
|
| 60 |
-
generation and evaluation use the number of supplied statutory/judgment sources.
|
| 61 |
-
Counts are null before output is available or when not applicable (query analysis).
|
| 62 |
-
Query totals report final source counts. Latencies use a monotonic timer. A stage
|
| 63 |
-
status describes execution success, not the evaluator's pass/fail decision.
|
| 64 |
-
|
| 65 |
-
Logs omit questions, prompts, keys, and raw exception messages. Secrets use
|
| 66 |
-
Pydantic `SecretStr`; stage failures log only the exception class. SDK HTTP debug
|
| 67 |
-
logging is suppressed. To correlate a query, filter JSON events by `request_id`.
|
| 68 |
-
|
| 69 |
-
Hierarchy edges use `(child)-[:BELONGS_TO]->(parent)`. Existing BNS graphs built
|
| 70 |
-
with only `HAS_*` edges need the inverse edges added once before BNS hierarchy
|
| 71 |
-
retrieval works. On a backed-up database, run:
|
| 72 |
-
|
| 73 |
-
```cypher
|
| 74 |
-
MATCH (parent)-[r]->(child)
|
| 75 |
-
WHERE type(r) IN ['HAS_PART', 'HAS_CHAPTER', 'HAS_SECTION', 'HAS_CLAUSE',
|
| 76 |
-
'HAS_SUBCLAUSE', 'HAS_ROMANCLAUSE', 'HAS_EXPLANATION',
|
| 77 |
-
'HAS_ILLUSTRATION']
|
| 78 |
-
AND parent.id STARTS WITH 'BNS'
|
| 79 |
-
AND child.id STARTS WITH 'BNS'
|
| 80 |
-
MERGE (child)-[:BELONGS_TO]->(parent);
|
| 81 |
```
|
| 82 |
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
|
| 88 |
```bash
|
| 89 |
-
|
| 90 |
-
|
|
|
|
|
|
|
|
|
|
| 91 |
```
|
| 92 |
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
skips its external calls. For example:
|
| 107 |
|
| 108 |
```bash
|
| 109 |
-
|
| 110 |
-
|
|
|
|
| 111 |
```
|
| 112 |
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
The Gemini probe checks connectivity/authentication, not generation quota.
|
| 128 |
-
|
| 129 |
-
`EXTERNAL_TIMEOUT_SECONDS` (default 30) configures Gemini, Qdrant, Neo4j, and
|
| 130 |
-
Indian Kanoon calls; health probes also enforce a client-side deadline. Gemini
|
| 131 |
-
SDK retries are disabled. Neo4j has connection/acquisition/write and query
|
| 132 |
-
execution timeouts. `QUERY_LOCK_TIMEOUT_SECONDS` (default 180) bounds the wait
|
| 133 |
-
for the shared pipeline. These are per-call/queue deadlines, not a total query
|
| 134 |
-
budget; retries and multiple external calls can take longer in aggregate.
|
| 135 |
-
|
| 136 |
-
Invalid request bodies return 422. Backend failures, upstream timeouts, and
|
| 137 |
-
expired queue waits return 502 with generic messages. No raw SDK errors, API
|
| 138 |
-
keys, graph objects, or stack traces appear in API error responses.
|
| 139 |
-
|
| 140 |
-
Run the offline suite and coverage report with:
|
| 141 |
|
| 142 |
```bash
|
| 143 |
-
|
| 144 |
-
#
|
| 145 |
-
|
|
|
|
| 146 |
```
|
| 147 |
|
| 148 |
-
The
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
query serialization, and Streamlit HTTP behavior.
|
| 163 |
-
|
| 164 |
-
The human-reviewed evaluation workflow is documented in [eval/README.md](eval/README.md):
|
| 165 |
-
balanced stored-provision sampling, Gemini drafts, interactive label review, and
|
| 166 |
-
reviewed-only retrieval/refusal metrics.
|
| 167 |
-
|
| 168 |
-
Optional lexical retrieval is controlled by `USE_BM25=false` (the default), or
|
| 169 |
-
`HybridRetriever(use_bm25=True)`. Enabling it builds one in-memory BM25 index
|
| 170 |
-
per retriever from the same Qdrant chunk payloads and fuses dense/BM25 ranks with
|
| 171 |
-
RRF (k=60) before reranking. The snapshot is read at startup; restart/recreate the
|
| 172 |
-
retriever after corpus changes. BM25 uses case-folded Unicode word/number tokens
|
| 173 |
-
and does not require a second vector collection or new ingestion schema.
|
| 174 |
-
|
| 175 |
-
When BM25 is enabled, explicit section/article references also perform exact
|
| 176 |
-
stored-ID lookup and keep found results at the front after reranking. Override
|
| 177 |
-
this independently with `EXACT_ID_LOOKUP=true/false` or the constructor's
|
| 178 |
-
`exact_id_lookup` argument. Without an explicit act, no statute is guessed.
|
| 179 |
-
Both stages remain off under the default configuration pending reviewed
|
| 180 |
-
Recall@5/MRR evidence; configuration G in the ablation runner evaluates them.
|
| 181 |
-
|
| 182 |
-
Generated answers also pass through `LegalRag/citation_check.py`. It normalizes
|
| 183 |
-
statutory section/article mentions against retained passages and graph nodes.
|
| 184 |
-
Unsupported citations lower the judge score and trigger the existing `CITATION`
|
| 185 |
-
retry path within `MAX_ATTEMPTS`. The API adds `unsupported_citations`,
|
| 186 |
-
`citation_precision`, and `citation_checks` with per-mention support statuses;
|
| 187 |
-
Streamlit displays a warning when unsupported citations remain after retries.
|
| 188 |
-
Precision is null for answers without citations. This verifies source presence,
|
| 189 |
-
not whether a cited provision supports the answer's legal claims.
|
| 190 |
-
|
| 191 |
-
Run `python -m eval.run_answer_eval` for full-pipeline answer metrics on reviewed
|
| 192 |
-
gold items, including out-of-scope questions. See the evaluation guide for metric
|
| 193 |
-
definitions and report paths.
|
| 194 |
-
|
| 195 |
-
Run `python -m eval.profile_pipeline` for paired stage p50/p95 and quality
|
| 196 |
-
comparisons over 50 reviewed questions. Performance switches remain opt-in:
|
| 197 |
-
`VECTOR_SEARCH_WORKERS=4`, `BATCH_GRAPH_EXPANSION=true`, and
|
| 198 |
-
`ENABLE_QUERY_CACHE=true`. Judge skipping remains unavailable pending reviewed
|
| 199 |
-
no-quality-loss evidence. See the evaluation guide for cache and retry semantics.
|
|
|
|
| 10 |
|
| 11 |
# Legora
|
| 12 |
|
| 13 |
+
> **Not legal advice.** Legora is an experimental research system, not a lawyer or a substitute for legal counsel. Answers may contain incorrect, outdated, incomplete, or unsupported legal statements. Verify the current law and consult a qualified lawyer before acting.
|
| 14 |
+
|
| 15 |
+
Finding the provisions relevant to a layperson's legal question requires navigating statute structure, terminology, cross-references and judgments. Legora explores retrieval-augmented answers over the Constitution of India, Bharatiya Nyaya Sanhita (BNS), Bharatiya Nagarik Suraksha Sanhita (BNSS), Bharatiya Sakshya Adhiniyam (BSA), and on-demand Indian Kanoon judgments, combining semantic retrieval, optional lexical retrieval, graph context and source checks. Its intended benefit is easier access to supporting material; the checked-in reports do **not** establish retrieval accuracy, answer quality or latency improvements.
|
| 16 |
+
|
| 17 |
+
Screenshot/GIF placeholders — no demonstration assets are supplied yet:
|
| 18 |
+
|
| 19 |
+
- [ ] Screenshot: a question, answer and retrieved statutory sources in Streamlit.
|
| 20 |
+
- [ ] Screenshot: an unsupported-citation warning and evaluation status.
|
| 21 |
+
- [ ] GIF: retrieval, graph context, retry and final response for a real query.
|
| 22 |
+
|
| 23 |
+
## Architecture
|
| 24 |
+
|
| 25 |
+
Python hosts a FastAPI backend and Streamlit HTTP client. Sentence Transformers
|
| 26 |
+
embeds questions, Qdrant stores statute vectors/payloads, BAAI/bge-reranker-large
|
| 27 |
+
reranks candidates, Neo4j supplies hierarchy/reference context, and Gemini performs
|
| 28 |
+
analysis, generation and judging.
|
| 29 |
+
|
| 30 |
+
```mermaid
|
| 31 |
+
flowchart TD
|
| 32 |
+
UI[Streamlit / API query] --> A[Query analysis]
|
| 33 |
+
A --> M[Multi-query search: embedding + dense retrieval]
|
| 34 |
+
M --> F[Act metadata filter]
|
| 35 |
+
F --> B{BM25 enabled?}
|
| 36 |
+
B -->|yes| RRF[BM25 + dense reciprocal rank fusion]
|
| 37 |
+
B -->|no| R[Rerank]
|
| 38 |
+
RRF --> R
|
| 39 |
+
R --> N[Neo4j expansion when requested]
|
| 40 |
+
N --> G[Generation]
|
| 41 |
+
UI --> K[Indian Kanoon judgments when requested]
|
| 42 |
+
K --> G
|
| 43 |
+
G --> C[Citation presence check]
|
| 44 |
+
C --> J[LLM judge + citation score penalty]
|
| 45 |
+
J --> D{Pass?}
|
| 46 |
+
D -->|yes| O[Answer + sources + citation flags]
|
| 47 |
+
D -->|retry within budget| T[Adjust retrieval or generation]
|
| 48 |
+
T --> A
|
| 49 |
+
D -->|budget exhausted| E[Return best attempt with its own flags]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
```
|
| 51 |
|
| 52 |
+
The filter box is conceptual: the act constraint is passed into Qdrant search,
|
| 53 |
+
not applied after discarding search results. Explicit provision lookup, when
|
| 54 |
+
configured, prepends actual stored hits and preserves them after reranking.
|
| 55 |
+
Graph expansion follows `(child)-[:BELONGS_TO]->(parent)` upward and reverses
|
| 56 |
+
that traversal for descendants. Graph and judgment calls can be disabled per query.
|
| 57 |
+
|
| 58 |
+
Unsupported statutory citations lower the evaluation score and prevent PASS,
|
| 59 |
+
using the existing retry budget. The checker verifies source presence/identity;
|
| 60 |
+
it does not prove that a provision entails the answer's legal claim. The API
|
| 61 |
+
returns `unsupported_citations`, `citation_precision` and per-mention
|
| 62 |
+
`citation_checks`; precision is undefined for answers without citations.
|
| 63 |
+
|
| 64 |
+
## Corpus statistics
|
| 65 |
+
|
| 66 |
+
The source of this table is [results/corpus_stats.md](results/corpus_stats.md),
|
| 67 |
+
with machine-readable provenance in [corpus_stats.json](results/corpus_stats.json).
|
| 68 |
+
The read-only [counting script](scripts/corpus_stats.py) attempted live storage
|
| 69 |
+
reads. Qdrant reported `ResponseHandlingException`; Neo4j reported `ValueError`.
|
| 70 |
+
Both were unavailable. These failures do **not** establish an empty corpus.
|
| 71 |
+
Rows identify the intended acts; their stored coverage is unverified.
|
| 72 |
+
|
| 73 |
+
| Act | Unique stored chunks | Graph nodes | Directed graph edges |
|
| 74 |
+
|---|---:|---:|---:|
|
| 75 |
+
| BNS | Not measured | Not measured | Not measured |
|
| 76 |
+
| BNSS | Not measured | Not measured | Not measured |
|
| 77 |
+
| BSA | Not measured | Not measured | Not measured |
|
| 78 |
+
| Constitution | Not measured | Not measured | Not measured |
|
| 79 |
+
| Unmapped | Not measured | Not measured | Not measured |
|
| 80 |
+
| Total | Not measured | Not measured | Not measured |
|
| 81 |
+
|
| 82 |
+
Chunk counts deduplicate actual stored `chunk_id` values. The JSON also reports
|
| 83 |
+
raw points, duplicate chunk points and points without IDs when storage is readable.
|
| 84 |
+
Graph edges include every directed relationship, including inverse hierarchy
|
| 85 |
+
edges; cross-act edges belong to their source act. Chunk acts use recognized
|
| 86 |
+
metadata with an ID fallback; unresolved identities stay in the Unmapped row.
|
| 87 |
+
Freeze ingestion while collecting counts from both stores.
|
| 88 |
+
|
| 89 |
+
## Results recorded in this repository
|
| 90 |
+
|
| 91 |
+
**No completed benchmark is present.** All tables below reflect only checked-in
|
| 92 |
+
`results/` outputs. “Not evaluated” and “Not measured” are missing measurements,
|
| 93 |
+
not zero scores. There are no demonstrated improvements or confidence intervals.
|
| 94 |
+
Configuration defaults and sample sizes elsewhere in this README are experiment
|
| 95 |
+
settings, not measured outcomes.
|
| 96 |
+
|
| 97 |
+
### Retrieval ablation
|
| 98 |
+
|
| 99 |
+
Source: [retrieval_ablation.md](results/retrieval_ablation.md).
|
| 100 |
+
[retrieval_ablation.csv](results/retrieval_ablation.csv) contains only a header.
|
| 101 |
+
The report states that no reviewed, answerable gold items are available.
|
| 102 |
+
|
| 103 |
+
| Configuration | BM25/RRF | Exact lookup | Recall@5 | MRR |
|
| 104 |
+
|---|---|---|---|---|
|
| 105 |
+
| A | False | False | Not evaluated | Not evaluated |
|
| 106 |
+
| B | False | False | Not evaluated | Not evaluated |
|
| 107 |
+
| C | False | False | Not evaluated | Not evaluated |
|
| 108 |
+
| D | False | False | Not evaluated | Not evaluated |
|
| 109 |
+
| E | False | False | Not evaluated | Not evaluated |
|
| 110 |
+
| F | False | False | Not evaluated | Not evaluated |
|
| 111 |
+
| G | True | True | Not evaluated | Not evaluated |
|
| 112 |
+
|
| 113 |
+
The presets compare dense search, act filtering, multi-query expansion,
|
| 114 |
+
reranking, legal boosts, graph context and BM25 fusion; their precise definitions
|
| 115 |
+
are in [the evaluation guide](eval/README.md#retrieval-ablation-benchmark).
|
| 116 |
+
Root/exact Recall@1/5/10, MRR, nDCG@10, graph context recall, per-intent breakdowns,
|
| 117 |
+
bootstrap intervals and failure lists have not been computed in these outputs.
|
| 118 |
+
|
| 119 |
+
### Answer quality
|
| 120 |
+
|
| 121 |
+
Source: [answer_eval.md](results/answer_eval.md): no reviewed gold items and no
|
| 122 |
+
computed answer metrics.
|
| 123 |
+
|
| 124 |
+
| Measure | Recorded result |
|
| 125 |
+
|---|---|
|
| 126 |
+
| Citation precision | Not measured |
|
| 127 |
+
| Unsupported-citation rate | Not measured |
|
| 128 |
+
| Answer contains expected provision | Not measured |
|
| 129 |
+
| Context-supported expected provision | Not measured |
|
| 130 |
+
| Out-of-scope refusal rate | Not measured |
|
| 131 |
+
|
| 132 |
+
These are citation/coverage/scope measures, not a legal correctness guarantee.
|
| 133 |
+
|
| 134 |
+
### Latency and quality trade-offs
|
| 135 |
+
|
| 136 |
+
Source: [latency.md](results/latency.md): no paired measurements. The planned
|
| 137 |
+
comparison requires restored stores and reviewed gold questions.
|
| 138 |
+
|
| 139 |
+
| Stage | Before p50 / p95 (ms) | After cold p50 / p95 (ms) | After warm p50 / p95 (ms) |
|
| 140 |
+
|---|---|---|---|
|
| 141 |
+
| Analysis | Not measured | Not measured | Not measured |
|
| 142 |
+
| Embedding | Not measured | Not measured | Not measured |
|
| 143 |
+
| Vector search | Not measured | Not measured | Not measured |
|
| 144 |
+
| Rerank | Not measured | Not measured | Not measured |
|
| 145 |
+
| Neo4j expansion | Not measured | Not measured | Not measured |
|
| 146 |
+
| Generation | Not measured | Not measured | Not measured |
|
| 147 |
+
| Judge | Not measured | Not measured | Not measured |
|
| 148 |
+
| Total | Not measured | Not measured | Not measured |
|
| 149 |
+
|
| 150 |
+
The planned before/after Recall@5 and citation-precision comparisons, and the
|
| 151 |
+
quality effect of skipping the judge, are also unmeasured. No speedup is accepted
|
| 152 |
+
as an improvement from these reports.
|
| 153 |
+
|
| 154 |
+
### Judge validation
|
| 155 |
+
|
| 156 |
+
Source: [judge_validation.md](results/judge_validation.md): no completed manual
|
| 157 |
+
agreement or consistency report.
|
| 158 |
+
|
| 159 |
+
| Measure | Recorded result |
|
| 160 |
+
|---|---|
|
| 161 |
+
| Agreement accuracy | Not measured |
|
| 162 |
+
| Cohen's kappa | Not measured |
|
| 163 |
+
| Confusion matrix | Not measured |
|
| 164 |
+
| Overall-score / manual-label Pearson correlation | Not measured |
|
| 165 |
+
| Overall-score / manual-label Spearman correlation | Not measured |
|
| 166 |
+
| Score variance across temperature-zero reruns | Not measured |
|
| 167 |
+
|
| 168 |
+
## Design decisions and unsuccessful experiments
|
| 169 |
+
|
| 170 |
+
- Keep dense retrieval as the default; BM25/RRF, exact lookup and legal boosts
|
| 171 |
+
have explicit evaluation switches. BM25/exact lookup remain off by default
|
| 172 |
+
pending the Recall@5/MRR evidence missing from the ablation report.
|
| 173 |
+
- Use statute hierarchy and references for context, while keeping graph expansion
|
| 174 |
+
optional. More context can include irrelevant material; no quality gain is
|
| 175 |
+
established by the current results.
|
| 176 |
+
- Separate deterministic citation-presence checks from LLM judging. A judge score
|
| 177 |
+
cannot override an unsupported citation, and a high score alone is not proof
|
| 178 |
+
of correctness.
|
| 179 |
+
- Concurrent vector I/O, batched graph expansion and bounded analysis/embedding
|
| 180 |
+
LRUs are opt-in. Input-order merging preserves tie behavior; caches normalize
|
| 181 |
+
whitespace while preserving case and punctuation. Timeouts and bounded transient
|
| 182 |
+
retries limit individual external calls, not total query duration.
|
| 183 |
+
- Evaluate only human-reviewed gold items, retain actual reports and leave
|
| 184 |
+
undefined metrics undefined. Do not promote generated labels to ground truth.
|
| 185 |
+
|
| 186 |
+
**What did not help:** the reports contain no completed comparison from which
|
| 187 |
+
an unsuccessful algorithmic experiment can be identified. Evaluation attempts
|
| 188 |
+
stopped at missing reviewed labels/storage, as their status reports document.
|
| 189 |
+
Those are setup blockers, not evidence that BM25, graph expansion, reranking,
|
| 190 |
+
legal boosts or caching failed. Claiming negative or positive findings would
|
| 191 |
+
invent results. Judge skipping is unavailable: enabling its flag fails fast
|
| 192 |
+
until a reviewed comparison establishes no quality loss.
|
| 193 |
+
|
| 194 |
+
## Known limitations and future work
|
| 195 |
+
|
| 196 |
+
- **Not legal advice; no human lawyer validation is documented.** Obtain qualified
|
| 197 |
+
legal review before considering real-world use.
|
| 198 |
+
- Stored corpus size, completeness, amendment currency and provision coverage
|
| 199 |
+
are unverified. Named acts do not imply that every provision was ingested.
|
| 200 |
+
State laws, other central statutes and the full body of case law are not a
|
| 201 |
+
demonstrated coverage claim. Indian Kanoon access is on-demand.
|
| 202 |
+
- PDF/OCR layout can lose numbering, boundaries, provisos or explanations.
|
| 203 |
+
Citation matching and root-level metrics can miss substantive interpretation
|
| 204 |
+
errors. Out-of-scope refusal recognition uses wording heuristics.
|
| 205 |
+
- Judge reliability, human agreement and temperature-zero consistency have not
|
| 206 |
+
been measured. The generator and judge use the configured Gemini model;
|
| 207 |
+
their agreement is not independent expert validation.
|
| 208 |
+
- External services, model versions and corpus changes affect availability and
|
| 209 |
+
reproducibility. Sequential per-worker queries, aggregate retries and model
|
| 210 |
+
memory use limit deployment scalability.
|
| 211 |
+
|
| 212 |
+
Next steps are to restore and audit the corpus; compute storage counts; collect
|
| 213 |
+
lawyer-reviewed labels; complete the ablations, answer evaluation, judge study
|
| 214 |
+
and latency runs; document regressions as well as improvements; and only then
|
| 215 |
+
consider enabling optional retrieval/performance methods or judge bypass.
|
| 216 |
+
|
| 217 |
+
## Quickstart
|
| 218 |
+
|
| 219 |
+
Use Docker with Compose. No PDFs, source text or credentials are committed.
|
| 220 |
|
| 221 |
```bash
|
| 222 |
+
cp .env.example .env
|
| 223 |
+
# Edit credentials and obtain the statute sources described below.
|
| 224 |
+
make ingest
|
| 225 |
+
# Starts api, ui, qdrant and neo4j, with healthchecks and persistent volumes:
|
| 226 |
+
docker compose up -d --build
|
| 227 |
```
|
| 228 |
|
| 229 |
+
Open the UI at `http://localhost:7860`, API docs at
|
| 230 |
+
`http://localhost:8000/docs`, and Neo4j Browser at `http://localhost:7474`.
|
| 231 |
+
Published ports bind to loopback. API startup loads the embedder/reranker and may
|
| 232 |
+
initially download weights; the UI waits for API dependency health, including
|
| 233 |
+
Gemini reachability. The current result files do not establish that this stack
|
| 234 |
+
has completed a live end-to-end deployment.
|
| 235 |
+
|
| 236 |
+
`make ingest` starts/waits for the databases and loads statutes through
|
| 237 |
+
[scripts/ingest.sh](scripts/ingest.sh). The same parsed documents feed the
|
| 238 |
+
existing chunkers and graph builders; bounded vector batches use stable UUIDs.
|
| 239 |
+
It upserts without clearing stores, is not transactional across both stores,
|
| 240 |
+
and does not prune deleted provisions or convert legacy integer point IDs.
|
| 241 |
+
Use an explicit migration/fresh stores for corpus replacement and restart the
|
| 242 |
+
API after ingestion, especially for a BM25 snapshot. `docker compose down`
|
| 243 |
+
retains named database/model volumes. Sources mount read-only; `.env`, PDFs,
|
| 244 |
+
local data and model caches are excluded from the application image.
|
| 245 |
+
|
| 246 |
+
### Obtain source files
|
| 247 |
+
|
| 248 |
+
Place English text PDFs in `data/statutes/` with these basenames:
|
| 249 |
+
|
| 250 |
+
| File | Official source |
|
| 251 |
+
|---|---|
|
| 252 |
+
| `bns.pdf` | [India Code: BNS](https://www.indiacode.nic.in/indiacode/handle/123456789/20062?col=123456789%2F1362&view_type=search), or the [NCRB New Criminal Laws portal](https://www.ncrb.mha.gov.in/uploads/SankalanPortal/Index.html) |
|
| 253 |
+
| `bnss.pdf` | BNSS gazette from the [NCRB portal](https://www.ncrb.mha.gov.in/uploads/SankalanPortal/Index.html) |
|
| 254 |
+
| `bsa.pdf` | BSA gazette from the [NCRB portal](https://www.ncrb.mha.gov.in/uploads/SankalanPortal/Index.html) |
|
| 255 |
+
| `constitution.pdf` | English edition from the [Legislative Department documents](https://www.legislative.gov.in/documents) |
|
| 256 |
+
|
| 257 |
+
Reviewed UTF-8 `bns.txt`, `bnss.txt`, `bsa.txt` and `constitution.txt` take
|
| 258 |
+
precedence over same-name PDFs. Image-only pages require reviewed OCR text;
|
| 259 |
+
there is no automatic silent OCR. Inspect numbering and contents-page/layout
|
| 260 |
+
artifacts. Record source URL, edition/amendment date and download date locally.
|
| 261 |
+
Successful parsing does not prove complete statute coverage. Do not commit
|
| 262 |
+
copyrighted PDFs or annotated commercial editions.
|
| 263 |
+
|
| 264 |
+
### Environment variables
|
| 265 |
+
|
| 266 |
+
Settings load `.env`; environment variables take precedence. Restart after
|
| 267 |
+
changes. See [.env.example](.env.example) and [legora/config.py](legora/config.py).
|
| 268 |
+
|
| 269 |
+
| Variables | Purpose |
|
| 270 |
+
|---|---|
|
| 271 |
+
| `GEMINI_API_KEY`, `GEMINI_MODEL` | Required API key; configured analysis/generation/judge model |
|
| 272 |
+
| `QDRANT_URL`, `QDRANT_API_KEY` | Required URL; optional local auth key |
|
| 273 |
+
| `NEO4J_URI`, `NEO4J_USERNAME`, `NEO4J_PASSWORD` | Required graph connection settings |
|
| 274 |
+
| `KANOON_TOKEN` | Required backend judgment-service token |
|
| 275 |
+
| `TOP_K`, `RERANK_K`, `JUDGMENT_K` | Retrieval limits |
|
| 276 |
+
| `PASS_THRESHOLD`, `MAX_ATTEMPTS` | Evaluation threshold and total attempt budget |
|
| 277 |
+
| `API_BASE_URL`, `API_TIMEOUT_SECONDS` | Streamlit HTTP settings |
|
| 278 |
+
| `EXTERNAL_TIMEOUT_SECONDS`, `QUERY_LOCK_TIMEOUT_SECONDS` | Per-call and queue deadlines |
|
| 279 |
+
| `USE_BM25`, `EXACT_ID_LOOKUP` | Optional lexical fusion and provision lookup |
|
| 280 |
+
| `VECTOR_SEARCH_WORKERS`, `BATCH_GRAPH_EXPANSION`, `ENABLE_QUERY_CACHE` | Opt-in performance switches |
|
| 281 |
+
| `SKIP_JUDGE_HIGH_CONFIDENCE` | Keep false; bypass is unavailable pending quality evidence |
|
| 282 |
+
| `LOG_LEVEL` | JSON logging verbosity |
|
| 283 |
+
|
| 284 |
+
For Compose's official Neo4j image, the initial username is `neo4j`; choose your
|
| 285 |
+
own strong password. Changing `.env` alone does not reset an existing database
|
| 286 |
+
password. Compose overrides database URLs with internal service addresses; the
|
| 287 |
+
UI receives only its API URL/timeout. Keep host-accessible URLs in `.env` for
|
| 288 |
+
host-side commands. Blank `QDRANT_API_KEY` disables local Qdrant auth.
|
| 289 |
+
|
| 290 |
+
`POST /query` accepts a question and optional retrieval/source controls;
|
| 291 |
+
`GET /health` checks Qdrant, Neo4j and Gemini reachability. The latter does not
|
| 292 |
+
verify generation quota or corpus completeness. Logs correlate stage latency,
|
| 293 |
+
candidate count and attempt by request ID, without logging prompts or keys.
|
| 294 |
+
See [the API models](api/models.py) for the response contract.
|
| 295 |
+
|
| 296 |
+
## Reproduce every table
|
| 297 |
+
|
| 298 |
+
Use Python 3.11 for host commands. Install the pinned developer dependencies
|
| 299 |
+
(or run Python commands through the API container with appropriate volume mounts):
|
| 300 |
|
| 301 |
+
```bash
|
| 302 |
+
python -m pip install -r requirements-dev.txt
|
| 303 |
+
python -m scripts.ingest --check-only --data-dir data/statutes
|
| 304 |
+
```
|
| 305 |
|
| 306 |
+
First restore/ingest the corpus. Generate draft questions from actual stored
|
| 307 |
+
provisions, then review each question and supporting IDs explicitly:
|
|
|
|
| 308 |
|
| 309 |
```bash
|
| 310 |
+
python -m eval.sample
|
| 311 |
+
python -m eval.generate
|
| 312 |
+
python -m eval.review
|
| 313 |
```
|
| 314 |
|
| 315 |
+
Only reviewed items may enter metrics; answerable questions are additionally
|
| 316 |
+
required for retrieval ablations. Drafts are not an evaluation set. Generation,
|
| 317 |
+
review/resume, model/corpus reproducibility and metric definitions are documented
|
| 318 |
+
in [eval/README.md](eval/README.md). The following commands produce each report:
|
| 319 |
+
|
| 320 |
+
| README table | Reproduction command | Authoritative report |
|
| 321 |
+
|---|---|---|
|
| 322 |
+
| Corpus | `python -m scripts.corpus_stats` | `results/corpus_stats.md` and `.json` |
|
| 323 |
+
| Retrieval A–G | `python -m eval.run_retrieval_eval --configs A B C D E F G --seed 42` | `results/retrieval_ablation.md` and `.csv` |
|
| 324 |
+
| Answer quality | `python -m eval.run_answer_eval` | `results/answer_eval.md` and `.json` |
|
| 325 |
+
| Latency / paired quality | `python -m eval.profile_pipeline --count 50 --seed 42 --workers 4` | `results/latency.md` and `.json` |
|
| 326 |
+
| Judge agreement / variance | Commands below | `results/judge_validation.md` |
|
| 327 |
+
|
| 328 |
+
Judge validation requires a real answer export and manual labeling:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
|
| 330 |
```bash
|
| 331 |
+
python -m eval.judge_validation export --count 50 --seed 42
|
| 332 |
+
# Fill my_label (correct/partially/incorrect) and my_notes in results/judge_validation.csv.
|
| 333 |
+
python -m eval.judge_validation consistency
|
| 334 |
+
python -m eval.judge_validation report
|
| 335 |
```
|
| 336 |
|
| 337 |
+
The judge reruns use the same frozen answer/context, temperature zero, and fresh
|
| 338 |
+
calls. Sampling seeds and caches improve reproducibility but do not guarantee
|
| 339 |
+
identical live model outputs. Keep corpus/model versions fixed. The ablation
|
| 340 |
+
runner saves query analyses on disk; `--offline-analysis` requires cache hits.
|
| 341 |
+
The latency runner records baseline, cold-start and warm-cache modes and flags
|
| 342 |
+
observed Recall@5/citation-precision regressions. Missing inputs stop evaluation
|
| 343 |
+
rather than inventing scores. After a completed run, consult the regenerated
|
| 344 |
+
`results/` files and update this README snapshot from those outputs only.
|
| 345 |
+
|
| 346 |
+
Development checks: `make lint`, `make typecheck`, `make test`. The GitHub Actions
|
| 347 |
+
workflow runs these checks, uploads coverage, validates Compose and builds the
|
| 348 |
+
Docker image without publishing. Runtime/developer dependencies are pinned;
|
| 349 |
+
this is not a hash-locked transitive dependency graph. No test or CI pass rate
|
| 350 |
+
is claimed here because those measurements are not recorded in `results/`.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
data/statutes/README.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Local statute sources
|
| 2 |
+
|
| 3 |
+
Place English source files here as `bns.pdf`, `bnss.pdf`, `bsa.pdf`, and
|
| 4 |
+
`constitution.pdf`, or supply reviewed UTF-8 files with the same `.txt` basenames.
|
| 5 |
+
Text takes precedence when both formats exist. See the repository README for
|
| 6 |
+
official download sources, validation and ingestion commands. PDFs, extracted text,
|
| 7 |
+
and OCR intermediates stay local and are excluded from Git and Docker images.
|
docker-compose.yml
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
services:
|
| 2 |
+
api:
|
| 3 |
+
image: legora:local
|
| 4 |
+
build: .
|
| 5 |
+
env_file: .env
|
| 6 |
+
environment:
|
| 7 |
+
QDRANT_URL: http://qdrant:6333
|
| 8 |
+
NEO4J_URI: bolt://neo4j:7687
|
| 9 |
+
command: [python, -m, uvicorn, api.app:app, --host, 0.0.0.0, --port, "8000"]
|
| 10 |
+
ports:
|
| 11 |
+
- "127.0.0.1:8000:8000"
|
| 12 |
+
volumes:
|
| 13 |
+
- model_cache:/app/.cache/huggingface
|
| 14 |
+
- ./data/statutes:/data/statutes:ro
|
| 15 |
+
depends_on:
|
| 16 |
+
qdrant:
|
| 17 |
+
condition: service_healthy
|
| 18 |
+
neo4j:
|
| 19 |
+
condition: service_healthy
|
| 20 |
+
healthcheck:
|
| 21 |
+
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=35).close()"]
|
| 22 |
+
interval: 30s
|
| 23 |
+
timeout: 40s
|
| 24 |
+
retries: 5
|
| 25 |
+
start_period: 600s
|
| 26 |
+
restart: unless-stopped
|
| 27 |
+
|
| 28 |
+
ui:
|
| 29 |
+
image: legora:local
|
| 30 |
+
build: .
|
| 31 |
+
# Only UI settings are passed; backend keys stay in the API container.
|
| 32 |
+
environment:
|
| 33 |
+
API_BASE_URL: http://api:8000
|
| 34 |
+
API_TIMEOUT_SECONDS: ${API_TIMEOUT_SECONDS:-180}
|
| 35 |
+
command: [python, -m, streamlit, run, app.py, --server.address=0.0.0.0, --server.port=7860]
|
| 36 |
+
ports:
|
| 37 |
+
- "127.0.0.1:7860:7860"
|
| 38 |
+
depends_on:
|
| 39 |
+
api:
|
| 40 |
+
condition: service_healthy
|
| 41 |
+
healthcheck:
|
| 42 |
+
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/_stcore/health', timeout=5).close()"]
|
| 43 |
+
interval: 30s
|
| 44 |
+
timeout: 10s
|
| 45 |
+
retries: 3
|
| 46 |
+
start_period: 30s
|
| 47 |
+
restart: unless-stopped
|
| 48 |
+
|
| 49 |
+
qdrant:
|
| 50 |
+
image: qdrant/qdrant:v1.14.1
|
| 51 |
+
environment:
|
| 52 |
+
QDRANT__SERVICE__API_KEY: ${QDRANT_API_KEY:-}
|
| 53 |
+
# An empty API-key setting would enable auth with an empty key upstream.
|
| 54 |
+
command:
|
| 55 |
+
- bash
|
| 56 |
+
- -ec
|
| 57 |
+
- 'if [ -z "$$QDRANT__SERVICE__API_KEY" ]; then unset QDRANT__SERVICE__API_KEY; fi; exec ./entrypoint.sh'
|
| 58 |
+
ports:
|
| 59 |
+
- "127.0.0.1:6333:6333"
|
| 60 |
+
volumes:
|
| 61 |
+
- qdrant_data:/qdrant/storage
|
| 62 |
+
healthcheck:
|
| 63 |
+
# The upstream image has bash; curl/wget are not assumed.
|
| 64 |
+
test:
|
| 65 |
+
- CMD-SHELL
|
| 66 |
+
- >-
|
| 67 |
+
bash -ec 'exec 3<>/dev/tcp/127.0.0.1/6333;
|
| 68 |
+
printf "GET /healthz HTTP/1.0\r\nHost: localhost\r\n\r\n" >&3;
|
| 69 |
+
read -r response <&3; [[ "$$response" == *" 200 "* ]]'
|
| 70 |
+
interval: 10s
|
| 71 |
+
timeout: 5s
|
| 72 |
+
retries: 10
|
| 73 |
+
start_period: 10s
|
| 74 |
+
restart: unless-stopped
|
| 75 |
+
|
| 76 |
+
neo4j:
|
| 77 |
+
image: neo4j:5.26.0-community
|
| 78 |
+
environment:
|
| 79 |
+
NEO4J_AUTH: ${NEO4J_USERNAME:?Set NEO4J_USERNAME in .env}/${NEO4J_PASSWORD:?Set NEO4J_PASSWORD in .env}
|
| 80 |
+
NEO4J_USERNAME: ${NEO4J_USERNAME:?Set NEO4J_USERNAME in .env}
|
| 81 |
+
NEO4J_PASSWORD: ${NEO4J_PASSWORD:?Set NEO4J_PASSWORD in .env}
|
| 82 |
+
NEO4J_server_memory_heap_initial__size: 256m
|
| 83 |
+
NEO4J_server_memory_heap_max__size: 512m
|
| 84 |
+
NEO4J_server_memory_pagecache_size: 256m
|
| 85 |
+
ports:
|
| 86 |
+
- "127.0.0.1:7474:7474"
|
| 87 |
+
- "127.0.0.1:7687:7687"
|
| 88 |
+
volumes:
|
| 89 |
+
- neo4j_data:/data
|
| 90 |
+
- neo4j_logs:/logs
|
| 91 |
+
healthcheck:
|
| 92 |
+
test: [CMD-SHELL, 'cypher-shell -a bolt://127.0.0.1:7687 -u "$$NEO4J_USERNAME" -p "$$NEO4J_PASSWORD" "RETURN 1" > /dev/null']
|
| 93 |
+
interval: 15s
|
| 94 |
+
timeout: 10s
|
| 95 |
+
retries: 10
|
| 96 |
+
start_period: 60s
|
| 97 |
+
restart: unless-stopped
|
| 98 |
+
|
| 99 |
+
volumes:
|
| 100 |
+
qdrant_data:
|
| 101 |
+
neo4j_data:
|
| 102 |
+
neo4j_logs:
|
| 103 |
+
model_cache:
|
eval/generate.py
CHANGED
|
@@ -6,6 +6,7 @@ import json
|
|
| 6 |
import random
|
| 7 |
from collections.abc import Callable
|
| 8 |
from pathlib import Path
|
|
|
|
| 9 |
|
| 10 |
from google import genai
|
| 11 |
from google.genai import types
|
|
@@ -96,7 +97,7 @@ def checked_item(identifier: str, draft: Draft, candidates: list[Provision], mul
|
|
| 96 |
roots = {provision_root(value) for value in draft.expected_chunk_ids}
|
| 97 |
if len(roots) < 2:
|
| 98 |
raise ValueError("A multi-provision question requires distinct supporting provisions")
|
| 99 |
-
act = "MULTI"
|
| 100 |
else:
|
| 101 |
if draft.expected_chunk_ids != [candidates[0].chunk_id]:
|
| 102 |
raise ValueError("Single-provision support must match the sampled provision")
|
|
|
|
| 6 |
import random
|
| 7 |
from collections.abc import Callable
|
| 8 |
from pathlib import Path
|
| 9 |
+
from typing import Literal
|
| 10 |
|
| 11 |
from google import genai
|
| 12 |
from google.genai import types
|
|
|
|
| 97 |
roots = {provision_root(value) for value in draft.expected_chunk_ids}
|
| 98 |
if len(roots) < 2:
|
| 99 |
raise ValueError("A multi-provision question requires distinct supporting provisions")
|
| 100 |
+
act: Literal["BNS", "BNSS", "BSA", "Constitution", "MULTI"] = "MULTI"
|
| 101 |
else:
|
| 102 |
if draft.expected_chunk_ids != [candidates[0].chunk_id]:
|
| 103 |
raise ValueError("Single-provision support must match the sampled provision")
|
eval/judge_validation.py
CHANGED
|
@@ -10,6 +10,7 @@ import json
|
|
| 10 |
import math
|
| 11 |
import random
|
| 12 |
import statistics
|
|
|
|
| 13 |
from pathlib import Path
|
| 14 |
from typing import Any, Protocol
|
| 15 |
|
|
@@ -125,7 +126,7 @@ def load_labels(path: Path, samples: list[Sample]) -> dict[str, str]:
|
|
| 125 |
return labels
|
| 126 |
|
| 127 |
|
| 128 |
-
def correlation(left:
|
| 129 |
"""Pearson correlation, undefined for constant inputs or fewer than two pairs."""
|
| 130 |
if len(left) != len(right):
|
| 131 |
raise ValueError("Correlation requires paired observations")
|
|
@@ -137,7 +138,7 @@ def correlation(left: list[float], right: list[float]) -> float | None:
|
|
| 137 |
return sum(x * y for x, y in zip(centered_x, centered_y)) / denominator if denominator else None
|
| 138 |
|
| 139 |
|
| 140 |
-
def ranks(values:
|
| 141 |
"""Assign average one-based ranks to ties for Spearman correlation."""
|
| 142 |
return [1 + sum(other < value for other in values) + (values.count(value) - 1) / 2 for value in values]
|
| 143 |
|
|
@@ -249,23 +250,24 @@ def main() -> None:
|
|
| 249 |
parser.add_argument("--count", type=int, default=50)
|
| 250 |
parser.add_argument("--seed", type=int, default=42)
|
| 251 |
args = parser.parse_args()
|
| 252 |
-
|
|
|
|
| 253 |
try:
|
| 254 |
if args.command == "export":
|
| 255 |
gold = load_gold(args.gold)
|
| 256 |
sample_questions(gold, args.count, args.seed) # Gate before loading models/clients.
|
| 257 |
if args.csv.exists() or args.samples.exists():
|
| 258 |
raise ValueError("Export paths already exist; use new paths")
|
| 259 |
-
|
| 260 |
-
export_samples(gold,
|
| 261 |
print(f"Exported {args.count} answers; fill my_label and my_notes in {args.csv}")
|
| 262 |
elif args.command == "consistency":
|
| 263 |
from evaluator.LLM_evaluator import LegalLLMEvaluator
|
| 264 |
from legora.config import EvaluationSettings
|
| 265 |
|
| 266 |
samples = load_samples(args.samples)
|
| 267 |
-
|
| 268 |
-
result = run_consistency(samples,
|
| 269 |
atomic_text(args.consistency, json.dumps(result, indent=2) + "\n")
|
| 270 |
print(f"Saved three-run consistency measurements to {args.consistency}")
|
| 271 |
else:
|
|
@@ -277,14 +279,13 @@ def main() -> None:
|
|
| 277 |
if not args.output.exists():
|
| 278 |
atomic_text(args.output, "# Judge validation — not run\n\nNo completed manual agreement/consistency report. Export real pipeline answers and fill manual labels before reporting.\n")
|
| 279 |
# Safe validation errors are useful, but upstream failures may contain secrets/prompts.
|
| 280 |
-
message = str(error) if type(error) is ValueError and args.command == "export" and
|
| 281 |
parser.exit(1, f"Judge validation failed: {message}. No incomplete run was scored.\n")
|
| 282 |
finally:
|
| 283 |
-
if
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
resource.client.close()
|
| 288 |
|
| 289 |
|
| 290 |
if __name__ == "__main__":
|
|
|
|
| 10 |
import math
|
| 11 |
import random
|
| 12 |
import statistics
|
| 13 |
+
from collections.abc import Sequence
|
| 14 |
from pathlib import Path
|
| 15 |
from typing import Any, Protocol
|
| 16 |
|
|
|
|
| 126 |
return labels
|
| 127 |
|
| 128 |
|
| 129 |
+
def correlation(left: Sequence[float], right: Sequence[float]) -> float | None:
|
| 130 |
"""Pearson correlation, undefined for constant inputs or fewer than two pairs."""
|
| 131 |
if len(left) != len(right):
|
| 132 |
raise ValueError("Correlation requires paired observations")
|
|
|
|
| 138 |
return sum(x * y for x, y in zip(centered_x, centered_y)) / denominator if denominator else None
|
| 139 |
|
| 140 |
|
| 141 |
+
def ranks(values: Sequence[float]) -> list[float]:
|
| 142 |
"""Assign average one-based ranks to ties for Spearman correlation."""
|
| 143 |
return [1 + sum(other < value for other in values) + (values.count(value) - 1) / 2 for value in values]
|
| 144 |
|
|
|
|
| 250 |
parser.add_argument("--count", type=int, default=50)
|
| 251 |
parser.add_argument("--seed", type=int, default=42)
|
| 252 |
args = parser.parse_args()
|
| 253 |
+
pipeline = None
|
| 254 |
+
judge = None
|
| 255 |
try:
|
| 256 |
if args.command == "export":
|
| 257 |
gold = load_gold(args.gold)
|
| 258 |
sample_questions(gold, args.count, args.seed) # Gate before loading models/clients.
|
| 259 |
if args.csv.exists() or args.samples.exists():
|
| 260 |
raise ValueError("Export paths already exist; use new paths")
|
| 261 |
+
pipeline = load_pipeline()
|
| 262 |
+
export_samples(gold, pipeline, args.count, args.seed, args.csv, args.samples)
|
| 263 |
print(f"Exported {args.count} answers; fill my_label and my_notes in {args.csv}")
|
| 264 |
elif args.command == "consistency":
|
| 265 |
from evaluator.LLM_evaluator import LegalLLMEvaluator
|
| 266 |
from legora.config import EvaluationSettings
|
| 267 |
|
| 268 |
samples = load_samples(args.samples)
|
| 269 |
+
judge = LegalLLMEvaluator(settings=EvaluationSettings())
|
| 270 |
+
result = run_consistency(samples, judge)
|
| 271 |
atomic_text(args.consistency, json.dumps(result, indent=2) + "\n")
|
| 272 |
print(f"Saved three-run consistency measurements to {args.consistency}")
|
| 273 |
else:
|
|
|
|
| 279 |
if not args.output.exists():
|
| 280 |
atomic_text(args.output, "# Judge validation — not run\n\nNo completed manual agreement/consistency report. Export real pipeline answers and fill manual labels before reporting.\n")
|
| 281 |
# Safe validation errors are useful, but upstream failures may contain secrets/prompts.
|
| 282 |
+
message = str(error) if type(error) is ValueError and args.command == "export" and pipeline is None else type(error).__name__
|
| 283 |
parser.exit(1, f"Judge validation failed: {message}. No incomplete run was scored.\n")
|
| 284 |
finally:
|
| 285 |
+
if pipeline is not None:
|
| 286 |
+
pipeline.close()
|
| 287 |
+
if judge is not None:
|
| 288 |
+
judge.client.close()
|
|
|
|
| 289 |
|
| 290 |
|
| 291 |
if __name__ == "__main__":
|
eval/review.py
CHANGED
|
@@ -93,9 +93,10 @@ def main() -> None:
|
|
| 93 |
print(f"{sum(item.reviewed for item in items)}/{len(items)} reviewed")
|
| 94 |
return
|
| 95 |
if args.id:
|
| 96 |
-
|
| 97 |
-
if
|
| 98 |
raise ValueError("Unknown item ID")
|
|
|
|
| 99 |
changes = {field: value for field, value in {
|
| 100 |
"expected_chunk_ids": args.expected, "acceptable_alternates": args.alternates,
|
| 101 |
"question": args.question, "notes": args.notes, "intent": args.intent,
|
|
|
|
| 93 |
print(f"{sum(item.reviewed for item in items)}/{len(items)} reviewed")
|
| 94 |
return
|
| 95 |
if args.id:
|
| 96 |
+
selected = next((entry for entry in items if entry.id == args.id), None)
|
| 97 |
+
if selected is None:
|
| 98 |
raise ValueError("Unknown item ID")
|
| 99 |
+
item = selected
|
| 100 |
changes = {field: value for field, value in {
|
| 101 |
"expected_chunk_ids": args.expected, "acceptable_alternates": args.alternates,
|
| 102 |
"question": args.question, "notes": args.notes, "intent": args.intent,
|
eval/run_retrieval_eval.py
CHANGED
|
@@ -163,7 +163,7 @@ def run_configuration(
|
|
| 163 |
points.extend(retriever.vector_search_filtered(query, document=document, k=candidates))
|
| 164 |
# Same merged budget for every configuration, and an explicit tie breaker.
|
| 165 |
points = sorted(retriever.dedupe_points(points), key=lambda point: (-float(point.score), point.payload["chunk_id"]))[:candidates]
|
| 166 |
-
exact = []
|
| 167 |
if configuration.bm25 or configuration.exact_id_lookup:
|
| 168 |
points, exact = retriever.fuse_candidates(
|
| 169 |
question, points, document, candidates, configuration.bm25, configuration.exact_id_lookup,
|
|
@@ -208,7 +208,7 @@ def ranking_metrics(item: GoldItem, ranked_ids: list[str], match: str, graph_ids
|
|
| 208 |
positions.append(rank)
|
| 209 |
break
|
| 210 |
count = len(groups)
|
| 211 |
-
values = {f"recall_at_{k}": sum(rank <= k for rank in positions) / count for k in (1, 5, 10)}
|
| 212 |
values["mrr"] = 1 / min(positions) if positions else 0.0
|
| 213 |
ideal = sum(1 / math.log2(rank + 1) for rank in range(1, min(count, 10) + 1))
|
| 214 |
values["ndcg_at_10"] = sum(1 / math.log2(rank + 1) for rank in positions if rank <= 10) / ideal
|
|
|
|
| 163 |
points.extend(retriever.vector_search_filtered(query, document=document, k=candidates))
|
| 164 |
# Same merged budget for every configuration, and an explicit tie breaker.
|
| 165 |
points = sorted(retriever.dedupe_points(points), key=lambda point: (-float(point.score), point.payload["chunk_id"]))[:candidates]
|
| 166 |
+
exact: list[Any] = []
|
| 167 |
if configuration.bm25 or configuration.exact_id_lookup:
|
| 168 |
points, exact = retriever.fuse_candidates(
|
| 169 |
question, points, document, candidates, configuration.bm25, configuration.exact_id_lookup,
|
|
|
|
| 208 |
positions.append(rank)
|
| 209 |
break
|
| 210 |
count = len(groups)
|
| 211 |
+
values: dict[str, float | None] = {f"recall_at_{k}": sum(rank <= k for rank in positions) / count for k in (1, 5, 10)}
|
| 212 |
values["mrr"] = 1 / min(positions) if positions else 0.0
|
| 213 |
ideal = sum(1 / math.log2(rank + 1) for rank in range(1, min(count, 10) + 1))
|
| 214 |
values["ndcg_at_10"] = sum(1 / math.log2(rank + 1) for rank in positions if rank <= 10) / ideal
|
eval/sample.py
CHANGED
|
@@ -6,6 +6,7 @@ import random
|
|
| 6 |
import re
|
| 7 |
from collections.abc import Iterable, Iterator
|
| 8 |
from pathlib import Path
|
|
|
|
| 9 |
|
| 10 |
from qdrant_client import QdrantClient
|
| 11 |
|
|
@@ -107,7 +108,7 @@ def main() -> None:
|
|
| 107 |
if not config.qdrant_url:
|
| 108 |
raise ValueError("QDRANT_URL is required, or supply --chunks")
|
| 109 |
key = config.qdrant_api_key.get_secret_value() if config.qdrant_api_key else None
|
| 110 |
-
client = QdrantClient(url=config.qdrant_url, api_key=key or None, timeout=config.external_timeout_seconds, check_compatibility=False)
|
| 111 |
try:
|
| 112 |
records = stored_provisions(scroll_payloads(client, args.collection or ["legal_rag"]))
|
| 113 |
finally:
|
|
|
|
| 6 |
import re
|
| 7 |
from collections.abc import Iterable, Iterator
|
| 8 |
from pathlib import Path
|
| 9 |
+
from math import ceil
|
| 10 |
|
| 11 |
from qdrant_client import QdrantClient
|
| 12 |
|
|
|
|
| 108 |
if not config.qdrant_url:
|
| 109 |
raise ValueError("QDRANT_URL is required, or supply --chunks")
|
| 110 |
key = config.qdrant_api_key.get_secret_value() if config.qdrant_api_key else None
|
| 111 |
+
client = QdrantClient(url=config.qdrant_url, api_key=key or None, timeout=ceil(config.external_timeout_seconds), check_compatibility=False)
|
| 112 |
try:
|
| 113 |
records = stored_provisions(scroll_payloads(client, args.collection or ["legal_rag"]))
|
| 114 |
finally:
|
eval/storage.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
| 1 |
"""Atomic, validated JSONL storage that preserves human reviews on resume."""
|
| 2 |
|
| 3 |
-
import json
|
| 4 |
import os
|
| 5 |
import tempfile
|
|
|
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import TypeVar
|
| 8 |
|
|
@@ -43,7 +43,7 @@ def atomic_text(path: Path, text: str) -> None:
|
|
| 43 |
temporary.unlink(missing_ok=True)
|
| 44 |
|
| 45 |
|
| 46 |
-
def write_jsonl(path: Path, records:
|
| 47 |
"""Write actual records, with no placeholder or silently invalid rows."""
|
| 48 |
atomic_text(path, "".join(record.model_dump_json() + "\n" for record in records))
|
| 49 |
|
|
|
|
| 1 |
"""Atomic, validated JSONL storage that preserves human reviews on resume."""
|
| 2 |
|
|
|
|
| 3 |
import os
|
| 4 |
import tempfile
|
| 5 |
+
from collections.abc import Sequence
|
| 6 |
from pathlib import Path
|
| 7 |
from typing import TypeVar
|
| 8 |
|
|
|
|
| 43 |
temporary.unlink(missing_ok=True)
|
| 44 |
|
| 45 |
|
| 46 |
+
def write_jsonl(path: Path, records: Sequence[BaseModel]) -> None:
|
| 47 |
"""Write actual records, with no placeholder or silently invalid rows."""
|
| 48 |
atomic_text(path, "".join(record.model_dump_json() + "\n" for record in records))
|
| 49 |
|
legora/logging.py
CHANGED
|
@@ -41,7 +41,7 @@ def setup_logging(level: str = "INFO") -> None:
|
|
| 41 |
if not any(getattr(handler, "_legora_json", False) for handler in root.handlers):
|
| 42 |
handler = logging.StreamHandler()
|
| 43 |
handler.setFormatter(JsonFormatter())
|
| 44 |
-
handler
|
| 45 |
root.handlers[:] = [handler]
|
| 46 |
root.setLevel(level)
|
| 47 |
# HTTP request URLs and SDK diagnostics can contain credentials.
|
|
|
|
| 41 |
if not any(getattr(handler, "_legora_json", False) for handler in root.handlers):
|
| 42 |
handler = logging.StreamHandler()
|
| 43 |
handler.setFormatter(JsonFormatter())
|
| 44 |
+
setattr(handler, "_legora_json", True)
|
| 45 |
root.handlers[:] = [handler]
|
| 46 |
root.setLevel(level)
|
| 47 |
# HTTP request URLs and SDK diagnostics can contain credentials.
|
legora/performance.py
CHANGED
|
@@ -4,7 +4,7 @@ from collections import OrderedDict
|
|
| 4 |
from collections.abc import Callable
|
| 5 |
from copy import deepcopy
|
| 6 |
from threading import RLock
|
| 7 |
-
from typing import ParamSpec, TypeVar
|
| 8 |
|
| 9 |
import httpx
|
| 10 |
from tenacity import Retrying, retry_if_exception, stop_after_attempt, wait_exponential
|
|
@@ -39,7 +39,7 @@ class QueryCache:
|
|
| 39 |
if len(self.values) > self.capacity:
|
| 40 |
self.values.popitem(last=False)
|
| 41 |
self.values.move_to_end(key)
|
| 42 |
-
return deepcopy(self.values[key])
|
| 43 |
|
| 44 |
def clear(self) -> None:
|
| 45 |
"""Remove values before a cold benchmark or a model/prompt change."""
|
|
|
|
| 4 |
from collections.abc import Callable
|
| 5 |
from copy import deepcopy
|
| 6 |
from threading import RLock
|
| 7 |
+
from typing import ParamSpec, TypeVar, cast
|
| 8 |
|
| 9 |
import httpx
|
| 10 |
from tenacity import Retrying, retry_if_exception, stop_after_attempt, wait_exponential
|
|
|
|
| 39 |
if len(self.values) > self.capacity:
|
| 40 |
self.values.popitem(last=False)
|
| 41 |
self.values.move_to_end(key)
|
| 42 |
+
return cast(T, deepcopy(self.values[key]))
|
| 43 |
|
| 44 |
def clear(self) -> None:
|
| 45 |
"""Remove values before a cold benchmark or a model/prompt change."""
|
pyproject.toml
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[tool.ruff]
|
| 2 |
+
target-version = "py311"
|
| 3 |
+
line-length = 120
|
| 4 |
+
extend-exclude = ["venv", ".venv", "coverage"]
|
| 5 |
+
|
| 6 |
+
[tool.ruff.lint]
|
| 7 |
+
select = ["E4", "E7", "E9", "F"]
|
| 8 |
+
|
| 9 |
+
[tool.mypy]
|
| 10 |
+
python_version = "3.11"
|
| 11 |
+
explicit_package_bases = true
|
| 12 |
+
files = ["api", "legora", "eval", "scripts", "LegalRag/citation_check.py", "retrieval/lexical.py"]
|
| 13 |
+
follow_imports = "silent"
|
| 14 |
+
ignore_missing_imports = true
|
| 15 |
+
check_untyped_defs = true
|
| 16 |
+
no_implicit_optional = true
|
| 17 |
+
warn_unused_ignores = true
|
| 18 |
+
plugins = ["pydantic.mypy"]
|
requirements-dev.txt
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r requirements.txt
|
| 2 |
+
pytest==9.2.0
|
| 3 |
+
pytest-cov==7.1.0
|
| 4 |
+
ruff==0.11.13
|
| 5 |
+
mypy==1.15.0
|
| 6 |
+
PyYAML==6.0.3
|
requirements.txt
CHANGED
|
@@ -1,111 +1,27 @@
|
|
| 1 |
-
|
| 2 |
-
aiohttp
|
| 3 |
-
aiosignal
|
| 4 |
-
annotated-doc==0.0.4
|
| 5 |
-
annotated-types==0.7.0
|
| 6 |
anyio==4.13.0
|
| 7 |
-
|
| 8 |
-
certifi==2026.5.20
|
| 9 |
-
cffi==2.0.0
|
| 10 |
-
charset-normalizer==3.4.7
|
| 11 |
-
click==8.4.1
|
| 12 |
-
cryptography==49.0.0
|
| 13 |
-
dotenv==0.9.9
|
| 14 |
filelock==3.29.4
|
| 15 |
-
frozenlist
|
| 16 |
-
fsspec==2026.4.0
|
| 17 |
google-genai==2.8.0
|
| 18 |
-
grpcio==1.81.1
|
| 19 |
-
h11==0.16.0
|
| 20 |
-
h2==4.3.0
|
| 21 |
-
hf-xet==1.5.1
|
| 22 |
-
hpack==4.1.0
|
| 23 |
-
httpcore==1.0.9
|
| 24 |
httpx==0.28.1
|
| 25 |
-
httpx-sse
|
| 26 |
-
huggingface_hub==1.19.0
|
| 27 |
-
hyperframe==6.1.0
|
| 28 |
-
idna==3.18
|
| 29 |
-
Jinja2==3.1.6
|
| 30 |
-
joblib==1.5.3
|
| 31 |
-
jsonpatch
|
| 32 |
-
jsonpointer
|
| 33 |
-
langchain
|
| 34 |
-
langchain-classic
|
| 35 |
-
langchain-community
|
| 36 |
-
langchain-core
|
| 37 |
-
langchain-experimental
|
| 38 |
-
langchain-protocol
|
| 39 |
-
langchain-text-splitters
|
| 40 |
-
langgraph
|
| 41 |
-
langgraph-checkpoint
|
| 42 |
-
langgraph-prebuilt
|
| 43 |
-
langgraph-sdk
|
| 44 |
-
langsmith
|
| 45 |
-
markdown-it-py==4.2.0
|
| 46 |
-
MarkupSafe==3.0.3
|
| 47 |
-
mdurl==0.1.2
|
| 48 |
-
mpmath==1.3.0
|
| 49 |
-
multidict
|
| 50 |
-
narwhals==2.22.1
|
| 51 |
neo4j==6.2.0
|
| 52 |
-
networkx==3.6.1
|
| 53 |
numpy==2.4.6
|
| 54 |
opencv-python==4.13.0.92
|
| 55 |
-
orjson
|
| 56 |
-
ormsgpack
|
| 57 |
-
packaging==26.2
|
| 58 |
pdf2image==1.17.0
|
| 59 |
-
pdfminer.six==20260107
|
| 60 |
pdfplumber==0.11.10
|
| 61 |
pillow==12.2.0
|
| 62 |
-
portalocker==3.2.0
|
| 63 |
-
propcache
|
| 64 |
-
protobuf==7.35.1
|
| 65 |
-
pycparser==3.0
|
| 66 |
-
pycryptodome
|
| 67 |
pydantic==2.13.4
|
| 68 |
-
pydantic-settings
|
| 69 |
-
pydantic_core==2.46.4
|
| 70 |
-
Pygments==2.20.0
|
| 71 |
-
pypdfium2==5.9.0
|
| 72 |
pytesseract==0.3.13
|
| 73 |
-
python-dotenv==1.2.2
|
| 74 |
-
pytz==2026.2
|
| 75 |
-
PyYAML==6.0.3
|
| 76 |
qdrant-client==1.18.0
|
| 77 |
-
|
| 78 |
requests==2.34.2
|
| 79 |
-
requests-toolbelt
|
| 80 |
-
rich==15.0.0
|
| 81 |
-
safetensors==0.8.0
|
| 82 |
-
scikit-learn==1.9.0
|
| 83 |
-
scipy==1.17.1
|
| 84 |
sentence-transformers==5.5.1
|
| 85 |
-
|
| 86 |
-
shellingham==1.5.4
|
| 87 |
-
SQLAlchemy
|
| 88 |
-
sympy==1.14.0
|
| 89 |
symspellpy==6.9.0
|
| 90 |
tenacity==9.1.4
|
| 91 |
-
|
| 92 |
-
tokenizers==0.22.2
|
| 93 |
torch==2.12.0
|
| 94 |
-
torchvision==0.27.0
|
| 95 |
-
tqdm==4.68.2
|
| 96 |
transformers==5.12.0
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
typing-inspection==0.4.2
|
| 100 |
-
urllib3==2.7.0
|
| 101 |
-
uuid_utils
|
| 102 |
-
websockets==16.0
|
| 103 |
-
xxhash
|
| 104 |
-
yarl
|
| 105 |
-
zstandard
|
| 106 |
-
streamlit>=1.40.0
|
| 107 |
-
fastapi>=0.115,<1
|
| 108 |
-
uvicorn>=0.30,<1
|
| 109 |
-
pytest>=8,<10
|
| 110 |
-
pytest-cov>=5,<8
|
| 111 |
-
rank-bm25>=0.2.2,<0.3
|
|
|
|
| 1 |
+
# Runtime dependencies; exact pins match the verified development environment.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
anyio==4.13.0
|
| 3 |
+
fastapi==0.163.1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
filelock==3.29.4
|
|
|
|
|
|
|
| 5 |
google-genai==2.8.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
httpx==0.28.1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
neo4j==6.2.0
|
|
|
|
| 8 |
numpy==2.4.6
|
| 9 |
opencv-python==4.13.0.92
|
|
|
|
|
|
|
|
|
|
| 10 |
pdf2image==1.17.0
|
|
|
|
| 11 |
pdfplumber==0.11.10
|
| 12 |
pillow==12.2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
pydantic==2.13.4
|
| 14 |
+
pydantic-settings==2.16.3
|
|
|
|
|
|
|
|
|
|
| 15 |
pytesseract==0.3.13
|
|
|
|
|
|
|
|
|
|
| 16 |
qdrant-client==1.18.0
|
| 17 |
+
rank-bm25==0.2.2
|
| 18 |
requests==2.34.2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
sentence-transformers==5.5.1
|
| 20 |
+
streamlit==1.58.0
|
|
|
|
|
|
|
|
|
|
| 21 |
symspellpy==6.9.0
|
| 22 |
tenacity==9.1.4
|
| 23 |
+
# Pin the model backend as well as its wrapper for compatibility.
|
|
|
|
| 24 |
torch==2.12.0
|
|
|
|
|
|
|
| 25 |
transformers==5.12.0
|
| 26 |
+
tqdm==4.68.2
|
| 27 |
+
uvicorn==0.49.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
results/corpus_stats.json
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"measured_at_utc": "2026-11-17T03:06:41.993006+00:00",
|
| 3 |
+
"collection": "legal_rag",
|
| 4 |
+
"status": "unavailable",
|
| 5 |
+
"dependencies": {
|
| 6 |
+
"qdrant": {
|
| 7 |
+
"status": "unavailable",
|
| 8 |
+
"error_type": "ResponseHandlingException"
|
| 9 |
+
},
|
| 10 |
+
"neo4j": {
|
| 11 |
+
"status": "unavailable",
|
| 12 |
+
"error_type": "ValueError"
|
| 13 |
+
}
|
| 14 |
+
},
|
| 15 |
+
"acts": [
|
| 16 |
+
{
|
| 17 |
+
"act": "BNS",
|
| 18 |
+
"chunks": null,
|
| 19 |
+
"graph_nodes": null,
|
| 20 |
+
"graph_edges": null
|
| 21 |
+
},
|
| 22 |
+
{
|
| 23 |
+
"act": "BNSS",
|
| 24 |
+
"chunks": null,
|
| 25 |
+
"graph_nodes": null,
|
| 26 |
+
"graph_edges": null
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"act": "BSA",
|
| 30 |
+
"chunks": null,
|
| 31 |
+
"graph_nodes": null,
|
| 32 |
+
"graph_edges": null
|
| 33 |
+
},
|
| 34 |
+
{
|
| 35 |
+
"act": "Constitution",
|
| 36 |
+
"chunks": null,
|
| 37 |
+
"graph_nodes": null,
|
| 38 |
+
"graph_edges": null
|
| 39 |
+
},
|
| 40 |
+
{
|
| 41 |
+
"act": "Unmapped",
|
| 42 |
+
"chunks": null,
|
| 43 |
+
"graph_nodes": null,
|
| 44 |
+
"graph_edges": null
|
| 45 |
+
}
|
| 46 |
+
],
|
| 47 |
+
"totals": {
|
| 48 |
+
"chunks": null,
|
| 49 |
+
"graph_nodes": null,
|
| 50 |
+
"graph_edges": null
|
| 51 |
+
}
|
| 52 |
+
}
|
results/corpus_stats.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Corpus statistics — unavailable
|
| 2 |
+
|
| 3 |
+
Attempted at 2026-11-17T03:06:41.993006+00:00; Qdrant collection `legal_rag`.
|
| 4 |
+
|
| 5 |
+
| Act | Unique stored chunks | Graph nodes | Directed graph edges |
|
| 6 |
+
|---|---:|---:|---:|
|
| 7 |
+
| BNS | Not measured | Not measured | Not measured |
|
| 8 |
+
| BNSS | Not measured | Not measured | Not measured |
|
| 9 |
+
| BSA | Not measured | Not measured | Not measured |
|
| 10 |
+
| Constitution | Not measured | Not measured | Not measured |
|
| 11 |
+
| Unmapped | Not measured | Not measured | Not measured |
|
| 12 |
+
| Total | Not measured | Not measured | Not measured |
|
| 13 |
+
|
| 14 |
+
Unavailable counts are not zeros. Chunk counts use distinct stored chunk_id values; graph edges include every stored relationship, including inverse hierarchy edges, and are grouped by their source node's act. Unmapped metadata is retained. Freeze ingestion while collecting these independent read-only counts.
|
| 15 |
+
|
| 16 |
+
- qdrant: unavailable (ResponseHandlingException).
|
| 17 |
+
|
| 18 |
+
- neo4j: unavailable (ValueError).
|
retrieval/lexical.py
CHANGED
|
@@ -41,7 +41,7 @@ class BM25Retriever:
|
|
| 41 |
|
| 42 |
def __init__(self, hits: Iterable[RetrievalHit]) -> None:
|
| 43 |
"""Index text plus stored IDs, excluding empty or unidentified chunks."""
|
| 44 |
-
unique = {}
|
| 45 |
for hit in hits:
|
| 46 |
identifier = hit.payload.get("chunk_id")
|
| 47 |
text = hit.payload.get("enriched_text") or hit.payload.get("text")
|
|
@@ -75,7 +75,7 @@ def reciprocal_rank_fusion(rankings: Sequence[Sequence[Any]], k: int = 60, limit
|
|
| 75 |
if k < 1:
|
| 76 |
raise ValueError("RRF k must be positive")
|
| 77 |
scores: dict[str, float] = {}
|
| 78 |
-
sources = {}
|
| 79 |
for ranking in rankings:
|
| 80 |
seen = set()
|
| 81 |
for rank, point in enumerate(ranking, 1):
|
|
@@ -108,6 +108,12 @@ def act_code(text: str) -> str:
|
|
| 108 |
return "CONST"
|
| 109 |
|
| 110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
def explicit_provisions(query: str) -> list[list[str]]:
|
| 112 |
"""Parse qualified section/article references; never guess an act from a bare number."""
|
| 113 |
acts = list(re.finditer(rf"\b({ACT_PATTERN})\b", query, re.I))
|
|
@@ -115,7 +121,7 @@ def explicit_provisions(query: str) -> list[list[str]]:
|
|
| 115 |
unique_acts = {act_code(match[1]) for match in acts}
|
| 116 |
qualified = []
|
| 117 |
for reference in references:
|
| 118 |
-
number = reference[1]
|
| 119 |
code = None
|
| 120 |
# In multi-act queries, require an adjacent act with only linking words.
|
| 121 |
for match in acts:
|
|
@@ -128,7 +134,7 @@ def explicit_provisions(query: str) -> list[list[str]]:
|
|
| 128 |
if code is None:
|
| 129 |
continue
|
| 130 |
base = number.split("(", 1)[0].upper()
|
| 131 |
-
suffix = re.sub(r"\(([^)]+)\)",
|
| 132 |
aliases = [f"{code}-{base}{suffix}"]
|
| 133 |
if code == "CONST":
|
| 134 |
parts = re.findall(r"\(([^)]+)\)", suffix)
|
|
|
|
| 41 |
|
| 42 |
def __init__(self, hits: Iterable[RetrievalHit]) -> None:
|
| 43 |
"""Index text plus stored IDs, excluding empty or unidentified chunks."""
|
| 44 |
+
unique: dict[str, RetrievalHit] = {}
|
| 45 |
for hit in hits:
|
| 46 |
identifier = hit.payload.get("chunk_id")
|
| 47 |
text = hit.payload.get("enriched_text") or hit.payload.get("text")
|
|
|
|
| 75 |
if k < 1:
|
| 76 |
raise ValueError("RRF k must be positive")
|
| 77 |
scores: dict[str, float] = {}
|
| 78 |
+
sources: dict[str, Any] = {}
|
| 79 |
for ranking in rankings:
|
| 80 |
seen = set()
|
| 81 |
for rank, point in enumerate(ranking, 1):
|
|
|
|
| 108 |
return "CONST"
|
| 109 |
|
| 110 |
|
| 111 |
+
def _normalized_clause(match: re.Match[str]) -> str:
|
| 112 |
+
"""Preserve numeric clause suffix case while normalizing alphabetic labels."""
|
| 113 |
+
value = match.group(1)
|
| 114 |
+
return "(" + (value.upper() if value[0].isdigit() else value.lower()) + ")"
|
| 115 |
+
|
| 116 |
+
|
| 117 |
def explicit_provisions(query: str) -> list[list[str]]:
|
| 118 |
"""Parse qualified section/article references; never guess an act from a bare number."""
|
| 119 |
acts = list(re.finditer(rf"\b({ACT_PATTERN})\b", query, re.I))
|
|
|
|
| 121 |
unique_acts = {act_code(match[1]) for match in acts}
|
| 122 |
qualified = []
|
| 123 |
for reference in references:
|
| 124 |
+
number: str = reference[1]
|
| 125 |
code = None
|
| 126 |
# In multi-act queries, require an adjacent act with only linking words.
|
| 127 |
for match in acts:
|
|
|
|
| 134 |
if code is None:
|
| 135 |
continue
|
| 136 |
base = number.split("(", 1)[0].upper()
|
| 137 |
+
suffix = re.sub(r"\(([^)]+)\)", _normalized_clause, number[len(base):])
|
| 138 |
aliases = [f"{code}-{base}{suffix}"]
|
| 139 |
if code == "CONST":
|
| 140 |
parts = re.findall(r"\(([^)]+)\)", suffix)
|
scripts/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Operational entry points; importing this package never starts ingestion."""
|
scripts/corpus_stats.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Measure stored corpus counts without loading models or fabricating unavailable totals."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import json
|
| 7 |
+
from collections.abc import Callable
|
| 8 |
+
from datetime import datetime, timezone
|
| 9 |
+
from math import ceil
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Any
|
| 12 |
+
|
| 13 |
+
from eval.storage import atomic_text
|
| 14 |
+
from legora.config import EvaluationSettings
|
| 15 |
+
from legora.performance import call_external
|
| 16 |
+
from retrieval.lexical import document_name
|
| 17 |
+
|
| 18 |
+
ACTS = ("BNS", "BNSS", "BSA", "Constitution", "Unmapped")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def act_name(value: object) -> str:
|
| 22 |
+
"""Map stored act metadata to report rows, preserving unknown data in its own row."""
|
| 23 |
+
return {"BNS": "BNS", "BNSS": "BNSS", "BSA": "BSA", "CONST": "Constitution",
|
| 24 |
+
"CONSTITUTION": "Constitution"}.get(str(value).strip().upper(), "Unmapped")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def vector_counts(client: Any, collection: str) -> dict:
|
| 28 |
+
"""Scroll all stored points; count distinct chunk IDs and expose duplicate/missing IDs."""
|
| 29 |
+
buckets: dict[str, set[str]] = {act: set() for act in ACTS}
|
| 30 |
+
owners: dict[str, str] = {}
|
| 31 |
+
points = missing = 0
|
| 32 |
+
offset = None
|
| 33 |
+
while True:
|
| 34 |
+
records, offset = call_external(
|
| 35 |
+
client.scroll, collection_name=collection, offset=offset, limit=256,
|
| 36 |
+
with_payload=True, with_vectors=False,
|
| 37 |
+
)
|
| 38 |
+
for record in records:
|
| 39 |
+
points += 1
|
| 40 |
+
payload = record.payload or {}
|
| 41 |
+
identifier = payload.get("chunk_id")
|
| 42 |
+
if not isinstance(identifier, str) or not identifier.strip():
|
| 43 |
+
missing += 1
|
| 44 |
+
continue
|
| 45 |
+
act = act_name(document_name(payload))
|
| 46 |
+
if identifier in owners and owners[identifier] != act:
|
| 47 |
+
raise ValueError("Conflicting act metadata for a stored chunk ID")
|
| 48 |
+
owners[identifier] = act
|
| 49 |
+
buckets[act].add(identifier)
|
| 50 |
+
if offset is None:
|
| 51 |
+
break
|
| 52 |
+
return {"chunks_by_act": {act: len(values) for act, values in buckets.items()},
|
| 53 |
+
"stored_points": points, "unique_chunks": len(owners),
|
| 54 |
+
"points_without_chunk_id": missing, "duplicate_chunk_points": points - missing - len(owners)}
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def graph_counts(graph: Any) -> dict:
|
| 58 |
+
"""Count all stored nodes and directed edges, assigning each edge to its source act."""
|
| 59 |
+
results = {}
|
| 60 |
+
for field, query in (
|
| 61 |
+
("nodes_by_act", "MATCH (n) RETURN n.document AS document, count(n) AS count"),
|
| 62 |
+
("edges_by_act", "MATCH (a)-[r]->() RETURN a.document AS document, count(r) AS count"),
|
| 63 |
+
):
|
| 64 |
+
counts = dict.fromkeys(ACTS, 0)
|
| 65 |
+
for row in graph.run_query(query):
|
| 66 |
+
count = row["count"]
|
| 67 |
+
if isinstance(count, bool) or not isinstance(count, int) or count < 0:
|
| 68 |
+
raise ValueError("Invalid graph count")
|
| 69 |
+
counts[act_name(row["document"])] += count
|
| 70 |
+
results[field] = counts
|
| 71 |
+
return results
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def collect_stats(
|
| 75 |
+
vector_factory: Callable[[], Any], graph_factory: Callable[[], Any], collection: str,
|
| 76 |
+
) -> dict:
|
| 77 |
+
"""Measure dependencies independently; failures produce null counts, never zero estimates."""
|
| 78 |
+
dependencies = {}
|
| 79 |
+
for name, factory, measure, close in (
|
| 80 |
+
("qdrant", vector_factory, lambda client: vector_counts(client, collection), lambda client: client.close()),
|
| 81 |
+
("neo4j", graph_factory, graph_counts, lambda graph: graph.close()),
|
| 82 |
+
):
|
| 83 |
+
resource = None
|
| 84 |
+
try:
|
| 85 |
+
resource = factory()
|
| 86 |
+
dependencies[name] = {"status": "ok", **measure(resource)}
|
| 87 |
+
except Exception as error:
|
| 88 |
+
dependencies[name] = {"status": "unavailable", "error_type": type(error).__name__}
|
| 89 |
+
finally:
|
| 90 |
+
if resource is not None:
|
| 91 |
+
try:
|
| 92 |
+
close(resource)
|
| 93 |
+
except Exception as error:
|
| 94 |
+
dependencies[name]["close_error_type"] = type(error).__name__
|
| 95 |
+
available = sum(dependency["status"] == "ok" for dependency in dependencies.values())
|
| 96 |
+
rows = [{"act": act,
|
| 97 |
+
"chunks": dependencies["qdrant"].get("chunks_by_act", {}).get(act),
|
| 98 |
+
"graph_nodes": dependencies["neo4j"].get("nodes_by_act", {}).get(act),
|
| 99 |
+
"graph_edges": dependencies["neo4j"].get("edges_by_act", {}).get(act)} for act in ACTS]
|
| 100 |
+
totals = {key: sum(row[key] for row in rows) if all(row[key] is not None for row in rows) else None
|
| 101 |
+
for key in ("chunks", "graph_nodes", "graph_edges")}
|
| 102 |
+
return {"measured_at_utc": datetime.now(timezone.utc).isoformat(), "collection": collection,
|
| 103 |
+
"status": "completed" if available == 2 else "partial" if available else "unavailable",
|
| 104 |
+
"dependencies": dependencies, "acts": rows, "totals": totals}
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def write_report(directory: Path, result: dict) -> str:
|
| 108 |
+
"""Persist real counts or explicit failure provenance, without endpoint URLs or credentials."""
|
| 109 |
+
report = (f"# Corpus statistics — {result['status']}\n\n"
|
| 110 |
+
f"Attempted at {result['measured_at_utc']}; Qdrant collection `{result['collection']}`.\n\n"
|
| 111 |
+
"| Act | Unique stored chunks | Graph nodes | Directed graph edges |\n|---|---:|---:|---:|\n")
|
| 112 |
+
for row in [*result["acts"], {"act": "Total", **result["totals"]}]:
|
| 113 |
+
values = ["Not measured" if row[key] is None else str(row[key]) for key in ("chunks", "graph_nodes", "graph_edges")]
|
| 114 |
+
report += f"| {row['act']} | " + " | ".join(values) + " |\n"
|
| 115 |
+
report += "\nUnavailable counts are not zeros. Chunk counts use distinct stored chunk_id values; graph edges include every stored relationship, including inverse hierarchy edges, and are grouped by their source node's act. Unmapped metadata is retained. Freeze ingestion while collecting these independent read-only counts.\n"
|
| 116 |
+
for name, dependency in result["dependencies"].items():
|
| 117 |
+
report += f"\n- {name}: {dependency['status']}" + (f" ({dependency['error_type']})" if "error_type" in dependency else "") + ".\n"
|
| 118 |
+
if result["dependencies"]["qdrant"]["status"] == "ok":
|
| 119 |
+
vector = result["dependencies"]["qdrant"]
|
| 120 |
+
report += f"\nStored points: {vector['stored_points']}; unique chunk IDs: {vector['unique_chunks']}; duplicate chunk points: {vector['duplicate_chunk_points']}; points without chunk IDs: {vector['points_without_chunk_id']}.\n"
|
| 121 |
+
atomic_text(directory / "corpus_stats.json", json.dumps(result, indent=2) + "\n")
|
| 122 |
+
atomic_text(directory / "corpus_stats.md", report)
|
| 123 |
+
return report
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def main() -> None:
|
| 127 |
+
"""Load only service clients, measure live storage and exit nonzero on incomplete counts."""
|
| 128 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 129 |
+
parser.add_argument("--collection", default="legal_rag")
|
| 130 |
+
parser.add_argument("--output-dir", type=Path, default=Path("results"))
|
| 131 |
+
args = parser.parse_args()
|
| 132 |
+
settings = EvaluationSettings()
|
| 133 |
+
|
| 134 |
+
def vector_factory() -> Any:
|
| 135 |
+
"""Create a bounded Qdrant connection without requiring model or generation keys."""
|
| 136 |
+
from qdrant_client import QdrantClient
|
| 137 |
+
|
| 138 |
+
if not settings.qdrant_url or not settings.qdrant_url.strip():
|
| 139 |
+
raise ValueError("QDRANT_URL is required")
|
| 140 |
+
return QdrantClient(url=settings.qdrant_url,
|
| 141 |
+
api_key=settings.qdrant_api_key.get_secret_value() if settings.qdrant_api_key else None,
|
| 142 |
+
timeout=ceil(settings.external_timeout_seconds), check_compatibility=False)
|
| 143 |
+
|
| 144 |
+
def graph_factory() -> Any:
|
| 145 |
+
"""Create only the graph client, validating its required connection configuration."""
|
| 146 |
+
from db.neo4j_store import Neo4jStore
|
| 147 |
+
|
| 148 |
+
return Neo4jStore(settings=settings)
|
| 149 |
+
|
| 150 |
+
result = collect_stats(vector_factory, graph_factory, args.collection)
|
| 151 |
+
print(write_report(args.output_dir, result))
|
| 152 |
+
if result["status"] != "completed":
|
| 153 |
+
parser.exit(1, "Corpus statistics incomplete; unavailable counts were left null.\n")
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
if __name__ == "__main__":
|
| 157 |
+
main()
|
scripts/ingest.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Validate and upsert all four statutes using the existing parsers and graph builders."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import argparse
|
| 6 |
+
import logging
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from importlib import import_module
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Any
|
| 11 |
+
from uuid import NAMESPACE_URL, uuid5
|
| 12 |
+
|
| 13 |
+
from legora.logging import setup_logging
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
STATUTES = {"bns": "BNS", "bnss": "BNSS", "bsa": "BSA", "constitution": "Constitution"}
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@dataclass
|
| 20 |
+
class ParsedStatute:
|
| 21 |
+
"""One parsed document shared by vector chunking and graph generation."""
|
| 22 |
+
|
| 23 |
+
name: str
|
| 24 |
+
document: Any
|
| 25 |
+
chunks: list[dict[str, Any]]
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def source_text(directory: Path, name: str) -> str:
|
| 29 |
+
"""Prefer reviewed UTF-8 text; extract text PDFs without silently applying OCR."""
|
| 30 |
+
text_path, pdf_path = directory / f"{name}.txt", directory / f"{name}.pdf"
|
| 31 |
+
if text_path.is_file():
|
| 32 |
+
text = text_path.read_text(encoding="utf-8")
|
| 33 |
+
elif pdf_path.is_file():
|
| 34 |
+
import pdfplumber
|
| 35 |
+
|
| 36 |
+
with pdfplumber.open(pdf_path) as document:
|
| 37 |
+
pages = [page.extract_text() or "" for page in document.pages]
|
| 38 |
+
if any(not page.strip() for page in pages):
|
| 39 |
+
raise ValueError(f"{name}.pdf contains pages without text; supply reviewed OCR text as {name}.txt")
|
| 40 |
+
text = "\n\n".join(pages)
|
| 41 |
+
else:
|
| 42 |
+
raise ValueError(f"Missing {name}.txt or {name}.pdf in the data directory")
|
| 43 |
+
if not text.strip():
|
| 44 |
+
raise ValueError(f"Empty source for {name}")
|
| 45 |
+
return text
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def prepare_corpus(directory: Path) -> list[ParsedStatute]:
|
| 49 |
+
"""Parse all inputs and reject empty/duplicate chunks before opening any service client."""
|
| 50 |
+
texts = {name: source_text(directory, name) for name in STATUTES}
|
| 51 |
+
prepared, seen = [], set()
|
| 52 |
+
for name, prefix in STATUTES.items():
|
| 53 |
+
cleaner = getattr(import_module(f"db.parsers.{name}.cleaner"), f"{prefix}TextCleaner")()
|
| 54 |
+
parser = getattr(import_module(f"db.parsers.{name}.{name}_parser"), f"{prefix}Parser")()
|
| 55 |
+
document = parser.parse(cleaner.clean(texts[name]))
|
| 56 |
+
module = import_module(f"db.parsers.{name}.chunker")
|
| 57 |
+
chunker = module.LegalChunker()
|
| 58 |
+
chunks = module.chunks_to_dicts(chunker.chunk_constitution(document) if name == "constitution" else chunker.chunk_document(document))
|
| 59 |
+
if not chunks:
|
| 60 |
+
raise ValueError(f"No provisions parsed for {name}; inspect PDF layout or supply reviewed text")
|
| 61 |
+
for chunk in chunks:
|
| 62 |
+
identifier = chunk.get("chunk_id")
|
| 63 |
+
if not identifier or identifier in seen or not chunk.get("text", "").strip():
|
| 64 |
+
raise ValueError(f"Invalid or duplicate chunks for {name}")
|
| 65 |
+
seen.add(identifier)
|
| 66 |
+
prepared.append(ParsedStatute(name, document, chunks))
|
| 67 |
+
logger.info("Validated %s: %s chunks", name, len(chunks))
|
| 68 |
+
return prepared
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def ingest_corpus(corpus: list[ParsedStatute], embedder: Any, vectors: Any, graph: Any,
|
| 72 |
+
batch_size: int = 16) -> None:
|
| 73 |
+
"""Upsert bounded embedding batches and hierarchy nodes; never clear existing stores."""
|
| 74 |
+
if batch_size < 1:
|
| 75 |
+
raise ValueError("batch_size must be positive")
|
| 76 |
+
vectors.create_collection(embedder.vector_size())
|
| 77 |
+
graph.create_constraints()
|
| 78 |
+
for statute in corpus:
|
| 79 |
+
builder_name = f"{STATUTES[statute.name]}GraphBuilder"
|
| 80 |
+
builder = getattr(import_module(f"db.graph_builders.{statute.name}_graph_builder"), builder_name)(graph)
|
| 81 |
+
builder.build(statute.document)
|
| 82 |
+
for start in range(0, len(statute.chunks), batch_size):
|
| 83 |
+
batch = statute.chunks[start:start + batch_size]
|
| 84 |
+
embeddings = embedder.embed([chunk.get("enriched_text") or chunk["text"] for chunk in batch])
|
| 85 |
+
if len(embeddings) != len(batch):
|
| 86 |
+
raise ValueError("Embedding count does not match input chunks")
|
| 87 |
+
vectors.upsert_points([
|
| 88 |
+
{"id": str(uuid5(NAMESPACE_URL, "legora:" + chunk["chunk_id"])),
|
| 89 |
+
"vector": embedding.tolist(), "payload": chunk}
|
| 90 |
+
for chunk, embedding in zip(batch, embeddings)
|
| 91 |
+
])
|
| 92 |
+
logger.info("Ingested %s: %s chunks", statute.name, len(statute.chunks))
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def main() -> None:
|
| 96 |
+
"""Offer source validation without models/network and explicit ingestion with client cleanup."""
|
| 97 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 98 |
+
parser.add_argument("--data-dir", type=Path, default=Path("data/statutes"))
|
| 99 |
+
parser.add_argument("--batch-size", type=int, default=16)
|
| 100 |
+
parser.add_argument("--check-only", action="store_true")
|
| 101 |
+
args = parser.parse_args()
|
| 102 |
+
setup_logging()
|
| 103 |
+
vectors = graph = None
|
| 104 |
+
try:
|
| 105 |
+
if args.batch_size < 1:
|
| 106 |
+
raise ValueError("batch_size must be positive")
|
| 107 |
+
corpus = prepare_corpus(args.data_dir)
|
| 108 |
+
if args.check_only:
|
| 109 |
+
return
|
| 110 |
+
from db.embedder import LegalEmbedder
|
| 111 |
+
from db.neo4j_store import Neo4jStore
|
| 112 |
+
from db.vector_store import QdrantStore
|
| 113 |
+
from legora.config import EvaluationSettings
|
| 114 |
+
|
| 115 |
+
settings = EvaluationSettings()
|
| 116 |
+
graph = Neo4jStore(settings=settings)
|
| 117 |
+
vectors = QdrantStore(settings=settings)
|
| 118 |
+
ingest_corpus(corpus, LegalEmbedder(), vectors, graph, args.batch_size)
|
| 119 |
+
except Exception as error:
|
| 120 |
+
message = str(error) if type(error) is ValueError and vectors is None and graph is None else type(error).__name__
|
| 121 |
+
parser.exit(1, f"Ingestion failed: {message}. No successful completion is claimed.\n")
|
| 122 |
+
finally:
|
| 123 |
+
if vectors is not None:
|
| 124 |
+
vectors.client.close()
|
| 125 |
+
if graph is not None:
|
| 126 |
+
graph.close()
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
if __name__ == "__main__":
|
| 130 |
+
main()
|
scripts/ingest.sh
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/sh
|
| 2 |
+
# Run from any directory; Compose loads the project .env without sourcing shell code.
|
| 3 |
+
set -eu
|
| 4 |
+
cd "$(dirname "$0")/.."
|
| 5 |
+
docker compose up -d --wait qdrant neo4j
|
| 6 |
+
docker compose run --rm --build --no-deps api python -m scripts.ingest --data-dir /data/statutes "$@"
|
tests/test_corpus_stats.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Verify measured corpus statistics without network calls or production fixtures."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
from types import SimpleNamespace
|
| 6 |
+
from unittest.mock import Mock
|
| 7 |
+
|
| 8 |
+
import pytest
|
| 9 |
+
|
| 10 |
+
from scripts.corpus_stats import collect_stats, graph_counts, vector_counts, write_report
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def point(identifier: str | None, document: str | None = None) -> SimpleNamespace:
|
| 14 |
+
"""Represent a stored payload, including deliberately malformed rows."""
|
| 15 |
+
return SimpleNamespace(payload={"chunk_id": identifier, "document": document})
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def test_vector_scroll_counts_unique_chunks_duplicates_missing_and_unmapped() -> None:
|
| 19 |
+
"""All pages and legacy IDs count, while duplicate points do not inflate chunk totals."""
|
| 20 |
+
client = Mock()
|
| 21 |
+
client.scroll.side_effect = [
|
| 22 |
+
([point("BNS-303", "bns"), point("BNS-303", "bns"), point(None)], "next"),
|
| 23 |
+
([point("article_21"), point("unknown-id"), point("BNSS-45(2)", "bnss")], None),
|
| 24 |
+
]
|
| 25 |
+
result = vector_counts(client, "legal_rag")
|
| 26 |
+
assert result["stored_points"] == 6
|
| 27 |
+
assert result["unique_chunks"] == 4
|
| 28 |
+
assert result["duplicate_chunk_points"] == 1
|
| 29 |
+
assert result["points_without_chunk_id"] == 1
|
| 30 |
+
assert result["chunks_by_act"] == {"BNS": 1, "BNSS": 1, "BSA": 0, "Constitution": 1, "Unmapped": 1}
|
| 31 |
+
assert client.scroll.call_args.kwargs["offset"] == "next"
|
| 32 |
+
assert client.scroll.call_args.kwargs["with_vectors"] is False
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_conflicting_chunk_metadata_is_not_reported_as_a_valid_total() -> None:
|
| 36 |
+
"""A duplicate ID assigned to two acts is data corruption, not two distinct chunks."""
|
| 37 |
+
client = Mock()
|
| 38 |
+
client.scroll.return_value = ([point("BNS-303", "bns"), point("BNS-303", "bsa")], None)
|
| 39 |
+
with pytest.raises(ValueError):
|
| 40 |
+
vector_counts(client, "legal_rag")
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_graph_counts_merge_metadata_aliases_and_count_edges_by_source() -> None:
|
| 44 |
+
"""Graph counts include unmapped nodes and actual directed edges, without deduping inverses."""
|
| 45 |
+
graph = Mock()
|
| 46 |
+
graph.run_query.side_effect = [
|
| 47 |
+
[{"document": "BNS", "count": 12}, {"document": "CONSTITUTION", "count": 3}, {"document": "CONST", "count": 2}, {"document": None, "count": 1}],
|
| 48 |
+
[{"document": "BNS", "count": 20}, {"document": "CONSTITUTION", "count": 5}],
|
| 49 |
+
]
|
| 50 |
+
result = graph_counts(graph)
|
| 51 |
+
assert result["nodes_by_act"]["Constitution"] == 5
|
| 52 |
+
assert result["nodes_by_act"]["Unmapped"] == 1
|
| 53 |
+
assert result["edges_by_act"]["BNS"] == 20
|
| 54 |
+
assert "a.document AS document" in graph.run_query.call_args.args[0]
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
@pytest.mark.parametrize("invalid", [-1, True, 2.5, "3"])
|
| 58 |
+
def test_invalid_graph_counts_are_rejected(invalid: object) -> None:
|
| 59 |
+
"""Malformed upstream counts cannot appear as measured corpus statistics."""
|
| 60 |
+
graph = Mock()
|
| 61 |
+
graph.run_query.return_value = [{"document": "BNS", "count": invalid}]
|
| 62 |
+
with pytest.raises(ValueError):
|
| 63 |
+
graph_counts(graph)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def test_unavailable_dependencies_produce_nulls_without_secret_messages(tmp_path: Path) -> None:
|
| 67 |
+
"""Failed reads must not imply an empty corpus or leak endpoint credentials."""
|
| 68 |
+
result = collect_stats(Mock(side_effect=ValueError("secret endpoint")), Mock(side_effect=ConnectionError("secret password")), "legal_rag")
|
| 69 |
+
assert result["status"] == "unavailable"
|
| 70 |
+
assert all(value is None for value in result["totals"].values())
|
| 71 |
+
report = write_report(tmp_path, result)
|
| 72 |
+
assert "Not measured" in report and "| Total |" in report
|
| 73 |
+
assert "secret" not in report and "secret" not in (tmp_path / "corpus_stats.json").read_text()
|
| 74 |
+
assert json.loads((tmp_path / "corpus_stats.json").read_text())["totals"]["chunks"] is None
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def test_partial_measurement_preserves_successful_zero_counts_and_closes_client() -> None:
|
| 78 |
+
"""An actually empty store reports zero; an inaccessible other store remains unknown."""
|
| 79 |
+
client = Mock()
|
| 80 |
+
client.scroll.return_value = ([], None)
|
| 81 |
+
result = collect_stats(lambda: client, Mock(side_effect=ValueError()), "legal_rag")
|
| 82 |
+
assert result["status"] == "partial"
|
| 83 |
+
assert result["totals"] == {"chunks": 0, "graph_nodes": None, "graph_edges": None}
|
| 84 |
+
client.close.assert_called_once_with()
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def test_complete_live_shaped_counts_compute_totals_without_model_loading() -> None:
|
| 88 |
+
"""Only successful reads from both stores produce a completed report."""
|
| 89 |
+
client, graph = Mock(), Mock()
|
| 90 |
+
client.scroll.return_value = ([point("BSA-115", "bsa")], None)
|
| 91 |
+
graph.run_query.side_effect = [[{"document": "BSA", "count": 4}], [{"document": "BSA", "count": 6}]]
|
| 92 |
+
result = collect_stats(lambda: client, lambda: graph, "legal_rag")
|
| 93 |
+
assert result["status"] == "completed"
|
| 94 |
+
assert result["totals"] == {"chunks": 1, "graph_nodes": 4, "graph_edges": 6}
|
| 95 |
+
graph.close.assert_called_once_with()
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def test_failed_scroll_closes_client_without_reporting_partial_counts() -> None:
|
| 99 |
+
"""A paginated failure does not publish the first page as the whole corpus."""
|
| 100 |
+
client = Mock()
|
| 101 |
+
client.scroll.side_effect = [([point("BNS-303", "bns")], "next"), ValueError("failed page")]
|
| 102 |
+
result = collect_stats(lambda: client, Mock(side_effect=ValueError()), "legal_rag")
|
| 103 |
+
assert result["totals"]["chunks"] is None
|
| 104 |
+
client.close.assert_called_once_with()
|
tests/test_ingestion.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Exercise real statute parsing, graph building and ingestion without external services."""
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from unittest.mock import Mock
|
| 5 |
+
from uuid import UUID
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
import pytest
|
| 9 |
+
|
| 10 |
+
from scripts.ingest import ingest_corpus, main, prepare_corpus, source_text
|
| 11 |
+
|
| 12 |
+
FIXTURES = Path(__file__).parent / "fixtures"
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def clients() -> tuple[Mock, Mock, Mock]:
|
| 16 |
+
"""Replace only embedding computation and persistence clients."""
|
| 17 |
+
embedder, vectors, graph = Mock(), Mock(), Mock()
|
| 18 |
+
embedder.vector_size.return_value = 2
|
| 19 |
+
embedder.embed.side_effect = lambda texts: np.ones((len(texts), 2))
|
| 20 |
+
return embedder, vectors, graph
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def test_real_corpus_and_graph_builders_use_same_documents_with_bounded_batches() -> None:
|
| 24 |
+
"""All four statutes reach vectors and graph with stable globally unique chunk identities."""
|
| 25 |
+
corpus = prepare_corpus(FIXTURES)
|
| 26 |
+
embedder, vectors, graph = clients()
|
| 27 |
+
ingest_corpus(corpus, embedder, vectors, graph, batch_size=3)
|
| 28 |
+
assert {statute.name for statute in corpus} == {"bns", "bnss", "bsa", "constitution"}
|
| 29 |
+
assert all(len(call.args[0]) <= 3 for call in embedder.embed.call_args_list)
|
| 30 |
+
points = [point for call in vectors.upsert_points.call_args_list for point in call.args[0]]
|
| 31 |
+
assert len(points) == sum(len(statute.chunks) for statute in corpus)
|
| 32 |
+
assert len({point["id"] for point in points}) == len(points)
|
| 33 |
+
assert all(UUID(point["id"]).version == 5 for point in points)
|
| 34 |
+
assert {"BNS-303", "BNSS-35", "BSA-2"} <= {point["payload"]["chunk_id"] for point in points}
|
| 35 |
+
nodes = {call.kwargs.get("node_id", call.args[1] if call.args else "") for call in graph.merge_node.call_args_list}
|
| 36 |
+
assert {"BNS-303", "BNSS-35", "BSA-2", "CONST-19"} <= nodes
|
| 37 |
+
vectors.create_collection.assert_called_once_with(2)
|
| 38 |
+
graph.create_constraints.assert_called_once_with()
|
| 39 |
+
vectors.recreate_collection.assert_not_called()
|
| 40 |
+
graph.clear_graph.assert_not_called()
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def test_repeat_ingestion_keeps_uuid_ids_even_if_source_order_changes() -> None:
|
| 44 |
+
"""Reingestion upserts identical provision IDs rather than renumbering other statutes."""
|
| 45 |
+
corpus = prepare_corpus(FIXTURES)
|
| 46 |
+
embedder, vectors, graph = clients()
|
| 47 |
+
ingest_corpus(corpus, embedder, vectors, graph)
|
| 48 |
+
before = {point["payload"]["chunk_id"]: point["id"] for call in vectors.upsert_points.call_args_list for point in call.args[0]}
|
| 49 |
+
vectors.upsert_points.reset_mock()
|
| 50 |
+
ingest_corpus(list(reversed(corpus)), embedder, vectors, graph)
|
| 51 |
+
after = {point["payload"]["chunk_id"]: point["id"] for call in vectors.upsert_points.call_args_list for point in call.args[0]}
|
| 52 |
+
assert before == after
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def test_embedding_count_mismatch_is_failure_instead_of_silent_zip_truncation() -> None:
|
| 56 |
+
"""No incomplete embedding batch is uploaded as if it succeeded."""
|
| 57 |
+
corpus = prepare_corpus(FIXTURES)
|
| 58 |
+
embedder, vectors, graph = clients()
|
| 59 |
+
embedder.embed.side_effect = lambda texts: np.ones((len(texts) - 1, 2))
|
| 60 |
+
with pytest.raises(ValueError, match="Embedding count"):
|
| 61 |
+
ingest_corpus(corpus, embedder, vectors, graph)
|
| 62 |
+
vectors.upsert_points.assert_not_called()
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
@pytest.mark.parametrize("batch_size", [0, -1])
|
| 66 |
+
def test_invalid_batch_size_never_mutates_stores(batch_size: int) -> None:
|
| 67 |
+
"""Invalid configuration fails before collection/graph creation."""
|
| 68 |
+
embedder, vectors, graph = clients()
|
| 69 |
+
with pytest.raises(ValueError):
|
| 70 |
+
ingest_corpus([], embedder, vectors, graph, batch_size)
|
| 71 |
+
vectors.create_collection.assert_not_called()
|
| 72 |
+
graph.create_constraints.assert_not_called()
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def test_source_text_prefers_reviewed_text_and_rejects_empty_or_missing(tmp_path: Path) -> None:
|
| 76 |
+
"""PDF extraction never replaces a reviewer-supplied UTF-8 source."""
|
| 77 |
+
(tmp_path / "bns.txt").write_text("Reviewed statute text")
|
| 78 |
+
(tmp_path / "bns.pdf").write_bytes(b"test-only-placeholder")
|
| 79 |
+
assert source_text(tmp_path, "bns") == "Reviewed statute text"
|
| 80 |
+
(tmp_path / "bns.txt").write_text(" \n")
|
| 81 |
+
with pytest.raises(ValueError, match="Empty"):
|
| 82 |
+
source_text(tmp_path, "bns")
|
| 83 |
+
with pytest.raises(ValueError, match="Missing"):
|
| 84 |
+
source_text(tmp_path, "bnss")
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def test_pdf_extraction_rejects_scanned_pages(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 88 |
+
"""Image-only pages require reviewed OCR, rather than silently discarding legal text."""
|
| 89 |
+
import pdfplumber
|
| 90 |
+
|
| 91 |
+
(tmp_path / "bns.pdf").write_bytes(b"test-only-placeholder")
|
| 92 |
+
pdf = Mock()
|
| 93 |
+
pdf.__enter__ = Mock(return_value=pdf)
|
| 94 |
+
pdf.__exit__ = Mock(return_value=False)
|
| 95 |
+
page = Mock()
|
| 96 |
+
page.extract_text.return_value = "Section text"
|
| 97 |
+
pdf.pages = [page]
|
| 98 |
+
monkeypatch.setattr(pdfplumber, "open", Mock(return_value=pdf))
|
| 99 |
+
assert source_text(tmp_path, "bns") == "Section text"
|
| 100 |
+
page.extract_text.return_value = None
|
| 101 |
+
with pytest.raises(ValueError, match="without text"):
|
| 102 |
+
source_text(tmp_path, "bns")
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def test_check_only_parses_without_clients(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 106 |
+
"""Source validation can run while containers and model downloads are unavailable."""
|
| 107 |
+
factory = Mock(side_effect=AssertionError("No store startup"))
|
| 108 |
+
monkeypatch.setattr("db.neo4j_store.Neo4jStore", factory)
|
| 109 |
+
monkeypatch.setattr("sys.argv", ["ingest", "--check-only", "--data-dir", str(FIXTURES)])
|
| 110 |
+
main()
|
| 111 |
+
factory.assert_not_called()
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def test_missing_source_fails_before_client_startup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
| 115 |
+
"""Partial local corpora must not start an incomplete database load."""
|
| 116 |
+
factory = Mock(side_effect=AssertionError("No store startup"))
|
| 117 |
+
monkeypatch.setattr("db.neo4j_store.Neo4jStore", factory)
|
| 118 |
+
monkeypatch.setattr("sys.argv", ["ingest", "--data-dir", str(tmp_path)])
|
| 119 |
+
with pytest.raises(SystemExit) as error:
|
| 120 |
+
main()
|
| 121 |
+
assert error.value.code == 1
|
| 122 |
+
factory.assert_not_called()
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def test_ingestion_closes_clients_after_upstream_failure(monkeypatch: pytest.MonkeyPatch) -> None:
|
| 126 |
+
"""A failed write exits unsuccessfully and still releases both clients."""
|
| 127 |
+
embedder, vectors, graph = clients()
|
| 128 |
+
vectors.upsert_points.side_effect = RuntimeError("sensitive upstream detail")
|
| 129 |
+
monkeypatch.setattr("db.embedder.LegalEmbedder", Mock(return_value=embedder))
|
| 130 |
+
monkeypatch.setattr("db.vector_store.QdrantStore", Mock(return_value=vectors))
|
| 131 |
+
monkeypatch.setattr("db.neo4j_store.Neo4jStore", Mock(return_value=graph))
|
| 132 |
+
monkeypatch.setattr("sys.argv", ["ingest", "--data-dir", str(FIXTURES)])
|
| 133 |
+
with pytest.raises(SystemExit) as error:
|
| 134 |
+
main()
|
| 135 |
+
assert error.value.code == 1
|
| 136 |
+
graph.close.assert_called_once_with()
|
| 137 |
+
vectors.client.close.assert_called_once_with()
|