sai-Rohan commited on
Commit
ddc9806
·
1 Parent(s): 1b818bb

updated readme

Browse files
.coveragerc CHANGED
@@ -10,10 +10,13 @@ source =
10
  eval.judge_validation
11
  eval.profile_pipeline
12
  legora.performance
 
 
13
  # The requested parser scope excludes ingestion clients and runnable examples.
14
  omit =
15
  */usage.py
16
- */ingest.py
 
17
  */search.py
18
  */qdrant_store.py
19
  */embedder.py
 
10
  eval.judge_validation
11
  eval.profile_pipeline
12
  legora.performance
13
+ scripts.ingest
14
+ scripts.corpus_stats
15
  # The requested parser scope excludes ingestion clients and runnable examples.
16
  omit =
17
  */usage.py
18
+ db/ingest.py
19
+ db/parsers/*/ingest.py
20
  */search.py
21
  */qdrant_store.py
22
  */embedder.py
.dockerignore ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .git
2
+ .env
3
+ .env.*
4
+ !.env.example
5
+ venv
6
+ .venv
7
+ __pycache__
8
+ **/__pycache__
9
+ *.py[cod]
10
+ .cache
11
+ .mypy_cache
12
+ .ruff_cache
13
+ .pytest_cache
14
+ .coverage*
15
+ coverage
16
+ data
17
+ db/pdfs
18
+ **/qdrant_data
19
+ **/*.pdf
20
+ **/*.PDF
21
+ results
22
+ eval/*snapshot*
23
+ eval/gold*.jsonl
.env.example CHANGED
@@ -5,7 +5,8 @@ QDRANT_URL=http://localhost:6333
5
  # Optional for an unauthenticated local instance.
6
  QDRANT_API_KEY=
7
  NEO4J_URI=bolt://localhost:7687
8
- NEO4J_USERNAME=replace-with-neo4j-username
 
9
  NEO4J_PASSWORD=replace-with-neo4j-password
10
  KANOON_TOKEN=replace-with-kanoon-token
11
  TOP_K=30
 
5
  # Optional for an unauthenticated local instance.
6
  QDRANT_API_KEY=
7
  NEO4J_URI=bolt://localhost:7687
8
+ # Neo4j's official image creates this initial user; use your own strong password.
9
+ NEO4J_USERNAME=neo4j
10
  NEO4J_PASSWORD=replace-with-neo4j-password
11
  KANOON_TOKEN=replace-with-kanoon-token
12
  TOP_K=30
.github/workflows/ci.yml ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI
2
+ on:
3
+ push:
4
+ pull_request:
5
+
6
+ permissions:
7
+ contents: read
8
+
9
+ jobs:
10
+ checks:
11
+ runs-on: ubuntu-latest
12
+ timeout-minutes: 30
13
+ env:
14
+ HF_HUB_OFFLINE: "1"
15
+ TRANSFORMERS_OFFLINE: "1"
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: actions/setup-python@v5
19
+ with:
20
+ python-version: "3.11"
21
+ cache: pip
22
+ cache-dependency-path: requirements*.txt
23
+ - name: Install pinned dependencies (CPU models)
24
+ run: |
25
+ python -m pip install torch==2.12.0 --index-url https://download.pytorch.org/whl/cpu
26
+ python -m pip install -r requirements-dev.txt
27
+ python -m pip check
28
+ - name: Lint
29
+ run: make lint PYTHON=python
30
+ - name: Type-check new modules
31
+ run: make typecheck PYTHON=python
32
+ - name: Offline tests with coverage
33
+ run: make test PYTHON=python
34
+ - uses: actions/upload-artifact@v4
35
+ if: always()
36
+ with:
37
+ name: coverage
38
+ path: coverage/
39
+ if-no-files-found: ignore
40
+ - name: Validate Compose configuration
41
+ run: |
42
+ cp .env.example .env
43
+ docker compose --env-file .env.example config --quiet
44
+
45
+ docker:
46
+ runs-on: ubuntu-latest
47
+ timeout-minutes: 40
48
+ steps:
49
+ - uses: actions/checkout@v4
50
+ - uses: docker/setup-buildx-action@v3
51
+ - uses: docker/build-push-action@v6
52
+ with:
53
+ context: .
54
+ push: false
55
+ tags: legora:ci
56
+ cache-from: type=gha
57
+ cache-to: type=gha,mode=max
.gitignore CHANGED
@@ -122,3 +122,9 @@ pnpm-lock.yaml
122
  /results/judge_validation.csv
123
  /results/judge_validation_samples.jsonl
124
  /results/judge_consistency.json
 
 
 
 
 
 
 
122
  /results/judge_validation.csv
123
  /results/judge_validation_samples.jsonl
124
  /results/judge_consistency.json
125
+
126
+ # User-supplied statute sources and OCR artifacts
127
+ /data/statutes/*
128
+ !/data/statutes/README.md
129
+ .mypy_cache/
130
+ .ruff_cache/
Dockerfile CHANGED
@@ -26,6 +26,8 @@ RUN apt-get update && apt-get install -y \
26
  build-essential \
27
  git \
28
  curl \
 
 
29
  libgl1 \
30
  libglib2.0-0 \
31
  libsm6 \
@@ -43,9 +45,9 @@ WORKDIR /app
43
  # ==========================================================
44
  COPY requirements.txt .
45
 
46
- RUN pip install --upgrade pip
47
-
48
- RUN pip install --no-cache-dir -r requirements.txt
49
 
50
  # ==========================================================
51
  # Copy Project
@@ -55,9 +57,9 @@ COPY . .
55
  # ==========================================================
56
  # Expose Streamlit Port
57
  # ==========================================================
58
- EXPOSE 7860
59
 
60
  # ==========================================================
61
  # Launch Streamlit
62
  # ==========================================================
63
- CMD ["sh", "-c", "python -m uvicorn api.app:app --host 127.0.0.1 --port 8000 & exec streamlit run app.py --server.port=7860 --server.address=0.0.0.0"]
 
26
  build-essential \
27
  git \
28
  curl \
29
+ poppler-utils \
30
+ tesseract-ocr \
31
  libgl1 \
32
  libglib2.0-0 \
33
  libsm6 \
 
45
  # ==========================================================
46
  COPY requirements.txt .
47
 
48
+ # CPU wheels avoid pulling CUDA runtime libraries into the local deployment.
49
+ RUN pip install --no-cache-dir torch==2.12.0 --index-url https://download.pytorch.org/whl/cpu \
50
+ && pip install --no-cache-dir -r requirements.txt
51
 
52
  # ==========================================================
53
  # Copy Project
 
57
  # ==========================================================
58
  # Expose Streamlit Port
59
  # ==========================================================
60
+ EXPOSE 8000 7860
61
 
62
  # ==========================================================
63
  # Launch Streamlit
64
  # ==========================================================
65
+ CMD ["python", "-m", "uvicorn", "api.app:app", "--host", "0.0.0.0", "--port", "8000"]
LegalRag/citation_check.py CHANGED
@@ -124,7 +124,7 @@ def extract_citations(answer: str, known_ids: Collection[str] = ()) -> list[Cita
124
  elif len(codes) == 1:
125
  code = next(iter(codes))
126
  number_matches = list(re.finditer(NUMBER, reference["numbers"]))
127
- numbers = []
128
  for index, match in enumerate(number_matches):
129
  number = normalized_number(match[0])
130
  if index:
 
124
  elif len(codes) == 1:
125
  code = next(iter(codes))
126
  number_matches = list(re.finditer(NUMBER, reference["numbers"]))
127
+ numbers: list[tuple[int, str]] = []
128
  for index, match in enumerate(number_matches):
129
  number = normalized_number(match[0])
130
  if index:
Makefile CHANGED
@@ -1,5 +1,15 @@
1
  PYTHON ?= $(if $(wildcard venv/bin/python),venv/bin/python,python3)
2
 
3
- .PHONY: test
 
 
 
 
 
 
 
 
 
 
4
  test:
5
  $(PYTHON) -m pytest --cov --cov-branch --cov-report=term-missing --cov-report=html:coverage/html --cov-report=json:coverage/coverage.json
 
1
  PYTHON ?= $(if $(wildcard venv/bin/python),venv/bin/python,python3)
2
 
3
+ .PHONY: test lint typecheck ingest
4
+ lint:
5
+ $(PYTHON) -m ruff check api legora eval scripts LegalRag/citation_check.py retrieval/lexical.py
6
+ $(PYTHON) -m ruff check --select E9,F63,F7,F82 .
7
+
8
+ typecheck:
9
+ $(PYTHON) -m mypy
10
+
11
+ ingest:
12
+ sh scripts/ingest.sh
13
+
14
  test:
15
  $(PYTHON) -m pytest --cov --cov-branch --cov-report=term-missing --cov-report=html:coverage/html --cov-report=json:coverage/coverage.json
README.md CHANGED
@@ -10,190 +10,341 @@ pinned: false
10
 
11
  # Legora
12
 
13
- Legora is an AI-powered Legal Retrieval-Augmented Generation (RAG) system designed to answer legal questions using semantic search, graph retrieval, and Large Language Models.
14
-
15
- ## Features
16
-
17
- - Retrieval-Augmented Generation (RAG)
18
- - Semantic Search using Sentence Transformers
19
- - Graph Retrieval using Neo4j
20
- - Vector Search using Qdrant
21
- - Google Gemini Integration
22
- - Streamlit-based User Interface
23
-
24
- ## Tech Stack
25
-
26
- - Python
27
- - Streamlit
28
- - Sentence Transformers
29
- - Transformers
30
- - Qdrant
31
- - Neo4j
32
- - Google Gemini
33
- - Docker
34
-
35
- ## Environment Variables
36
-
37
- Copy `.env.example` to `.env` and replace the placeholder credentials, or set
38
- the same variables in your deployment environment. `legora/config.py` uses
39
- pydantic-settings: environment variables override the project-root `.env` file,
40
- variable names are case-insensitive, and settings are cached for the process.
41
- Restart the app after changing configuration. Imports do not load clients.
42
-
43
- Required: `GEMINI_API_KEY`, `QDRANT_URL`, `NEO4J_URI`, `NEO4J_USERNAME`,
44
- `NEO4J_PASSWORD`, and `KANOON_TOKEN`. `QDRANT_API_KEY` is optional for a local
45
- unauthenticated instance. Neo4j and Qdrant connection configuration comes from
46
- settings; configure local URLs explicitly. Missing or blank required settings
47
- fail validation before retrieval models load. Validation errors hide input values.
48
-
49
- `GEMINI_MODEL` controls all Gemini clients (default `gemini-2.5-flash-lite`).
50
- Retrieval defaults are `TOP_K=30`, `RERANK_K=5`, and `JUDGMENT_K=5`.
51
- `PASS_THRESHOLD=80` uses the evaluator's 0–100 scale; `MAX_ATTEMPTS=3` is the
52
- maximum total attempts. All candidate limits and the retry budget must be positive.
53
-
54
- `LOG_LEVEL` defaults to `INFO`. The application configures newline-delimited JSON
55
- console logs with a fresh `request_id` per `RAGPipeline.search()` call. Retries
56
- share that ID and carry a 1-based `attempt`; context is restored when the query
57
- finishes or raises. Stage events include `stage`, `latency_ms`, `candidate_count`,
58
- `status`, and `error_type`. Counts are unique vector results, retained rerank
59
- results, expanded graph roots, or fetched judgments, depending on the stage;
60
- generation and evaluation use the number of supplied statutory/judgment sources.
61
- Counts are null before output is available or when not applicable (query analysis).
62
- Query totals report final source counts. Latencies use a monotonic timer. A stage
63
- status describes execution success, not the evaluator's pass/fail decision.
64
-
65
- Logs omit questions, prompts, keys, and raw exception messages. Secrets use
66
- Pydantic `SecretStr`; stage failures log only the exception class. SDK HTTP debug
67
- logging is suppressed. To correlate a query, filter JSON events by `request_id`.
68
-
69
- Hierarchy edges use `(child)-[:BELONGS_TO]->(parent)`. Existing BNS graphs built
70
- with only `HAS_*` edges need the inverse edges added once before BNS hierarchy
71
- retrieval works. On a backed-up database, run:
72
-
73
- ```cypher
74
- MATCH (parent)-[r]->(child)
75
- WHERE type(r) IN ['HAS_PART', 'HAS_CHAPTER', 'HAS_SECTION', 'HAS_CLAUSE',
76
- 'HAS_SUBCLAUSE', 'HAS_ROMANCLAUSE', 'HAS_EXPLANATION',
77
- 'HAS_ILLUSTRATION']
78
- AND parent.id STARTS WITH 'BNS'
79
- AND child.id STARTS WITH 'BNS'
80
- MERGE (child)-[:BELONGS_TO]->(parent);
81
  ```
82
 
83
- ## API and UI
84
-
85
- Install dependencies with `python -m pip install -r requirements.txt`. Start the
86
- API and Streamlit in separate terminals:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
 
88
  ```bash
89
- python -m uvicorn api.app:app --host 127.0.0.1 --port 8000
90
- streamlit run app.py
 
 
 
91
  ```
92
 
93
- The API lifespan loads one pipeline (including the embedder and reranker) per
94
- worker and closes its clients on shutdown. Queries are serialized per worker
95
- because the pipeline holds mutable retry state. More Uvicorn workers each load
96
- their own models. The Docker command starts an internal API alongside the UI;
97
- only the Streamlit port is exposed.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
 
99
- Streamlit uses `API_BASE_URL` (default `http://127.0.0.1:8000`) and
100
- `API_TIMEOUT_SECONDS` (default 180). It needs no backend credentials or models.
101
- Set `API_BASE_URL` to the backend's reachable address when running separately.
102
- API startup still requires all backend settings listed above.
103
 
104
- `POST /query` accepts `query` (1–20,000 characters), optional `top_k` (1–100),
105
- `include_graph` and `include_judgments` (both default true). Disabling a source
106
- skips its external calls. For example:
107
 
108
  ```bash
109
- curl -X POST http://127.0.0.1:8000/query -H 'Content-Type: application/json' \
110
- -d '{"query":"What does Article 21 protect?","top_k":10,"include_judgments":false}'
 
111
  ```
112
 
113
- The response contains `answer`, `citations`, `retrieval_debug` (`vector_hits`
114
- and `reranked` counts plus `node_ids` from the returned attempt), `evaluation`
115
- (`score`, `stage`), `attempts`, and `latency_ms` (`analysis`, `retrieval`,
116
- `rerank`, `graph`, `generation`, `judge`). Citations identify retained retrieval
117
- sources. Timings are measured and summed across retries: retrieval covers vector
118
- search plus judgment retrieval; analysis, reranking, graph, generation, and judging
119
- are separate. Skipped stages have zero duration. Exhausted evaluation retries
120
- return the highest-scoring complete attempt with its matching query, sources,
121
- score, and failure stage. Equal scores retain the earlier attempt; `attempts`
122
- still reports the total number executed.
123
-
124
- `GET /health` probes Qdrant collections, a Neo4j `RETURN 1` query, and Gemini
125
- model metadata using the startup clients. It returns status and measured latency
126
- for each dependency, with HTTP 200 when all are reachable and 502 otherwise.
127
- The Gemini probe checks connectivity/authentication, not generation quota.
128
-
129
- `EXTERNAL_TIMEOUT_SECONDS` (default 30) configures Gemini, Qdrant, Neo4j, and
130
- Indian Kanoon calls; health probes also enforce a client-side deadline. Gemini
131
- SDK retries are disabled. Neo4j has connection/acquisition/write and query
132
- execution timeouts. `QUERY_LOCK_TIMEOUT_SECONDS` (default 180) bounds the wait
133
- for the shared pipeline. These are per-call/queue deadlines, not a total query
134
- budget; retries and multiple external calls can take longer in aggregate.
135
-
136
- Invalid request bodies return 422. Backend failures, upstream timeouts, and
137
- expired queue waits return 502 with generic messages. No raw SDK errors, API
138
- keys, graph objects, or stack traces appear in API error responses.
139
-
140
- Run the offline suite and coverage report with:
141
 
142
  ```bash
143
- make test
144
- # Or choose an interpreter explicitly:
145
- make test PYTHON=python3
 
146
  ```
147
 
148
- The target uses `venv/bin/python` when present, runs pytest-cov with branch
149
- coverage, and writes `coverage/html/index.html` and `coverage/coverage.json`.
150
- `python -m pytest` runs the same suite without generating coverage reports.
151
- Coverage is scoped to the four parser packages, hybrid retrieval, reranking,
152
- the RAG pipeline, citation verification, and the evaluation/performance runners;
153
- ingestion clients and executable examples are excluded.
154
- There is no artificially imposed coverage threshold.
155
-
156
- The suite blocks socket connections and DNS lookups and forces Hugging Face
157
- into offline mode. Models and service clients are replaced only at their
158
- boundaries; parser, chunking, root extraction, deduplication, boosting, and retry
159
- logic run unchanged by mocks. Structural statute fixtures are abridged test text,
160
- with explicit test-only connector sentences, rather than authoritative statutes.
161
- Tests also cover FastAPI TestClient lifecycle/validation, dependency timeouts,
162
- query serialization, and Streamlit HTTP behavior.
163
-
164
- The human-reviewed evaluation workflow is documented in [eval/README.md](eval/README.md):
165
- balanced stored-provision sampling, Gemini drafts, interactive label review, and
166
- reviewed-only retrieval/refusal metrics.
167
-
168
- Optional lexical retrieval is controlled by `USE_BM25=false` (the default), or
169
- `HybridRetriever(use_bm25=True)`. Enabling it builds one in-memory BM25 index
170
- per retriever from the same Qdrant chunk payloads and fuses dense/BM25 ranks with
171
- RRF (k=60) before reranking. The snapshot is read at startup; restart/recreate the
172
- retriever after corpus changes. BM25 uses case-folded Unicode word/number tokens
173
- and does not require a second vector collection or new ingestion schema.
174
-
175
- When BM25 is enabled, explicit section/article references also perform exact
176
- stored-ID lookup and keep found results at the front after reranking. Override
177
- this independently with `EXACT_ID_LOOKUP=true/false` or the constructor's
178
- `exact_id_lookup` argument. Without an explicit act, no statute is guessed.
179
- Both stages remain off under the default configuration pending reviewed
180
- Recall@5/MRR evidence; configuration G in the ablation runner evaluates them.
181
-
182
- Generated answers also pass through `LegalRag/citation_check.py`. It normalizes
183
- statutory section/article mentions against retained passages and graph nodes.
184
- Unsupported citations lower the judge score and trigger the existing `CITATION`
185
- retry path within `MAX_ATTEMPTS`. The API adds `unsupported_citations`,
186
- `citation_precision`, and `citation_checks` with per-mention support statuses;
187
- Streamlit displays a warning when unsupported citations remain after retries.
188
- Precision is null for answers without citations. This verifies source presence,
189
- not whether a cited provision supports the answer's legal claims.
190
-
191
- Run `python -m eval.run_answer_eval` for full-pipeline answer metrics on reviewed
192
- gold items, including out-of-scope questions. See the evaluation guide for metric
193
- definitions and report paths.
194
-
195
- Run `python -m eval.profile_pipeline` for paired stage p50/p95 and quality
196
- comparisons over 50 reviewed questions. Performance switches remain opt-in:
197
- `VECTOR_SEARCH_WORKERS=4`, `BATCH_GRAPH_EXPANSION=true`, and
198
- `ENABLE_QUERY_CACHE=true`. Judge skipping remains unavailable pending reviewed
199
- no-quality-loss evidence. See the evaluation guide for cache and retry semantics.
 
10
 
11
  # Legora
12
 
13
+ > **Not legal advice.** Legora is an experimental research system, not a lawyer or a substitute for legal counsel. Answers may contain incorrect, outdated, incomplete, or unsupported legal statements. Verify the current law and consult a qualified lawyer before acting.
14
+
15
+ Finding the provisions relevant to a layperson's legal question requires navigating statute structure, terminology, cross-references and judgments. Legora explores retrieval-augmented answers over the Constitution of India, Bharatiya Nyaya Sanhita (BNS), Bharatiya Nagarik Suraksha Sanhita (BNSS), Bharatiya Sakshya Adhiniyam (BSA), and on-demand Indian Kanoon judgments, combining semantic retrieval, optional lexical retrieval, graph context and source checks. Its intended benefit is easier access to supporting material; the checked-in reports do **not** establish retrieval accuracy, answer quality or latency improvements.
16
+
17
+ Screenshot/GIF placeholders — no demonstration assets are supplied yet:
18
+
19
+ - [ ] Screenshot: a question, answer and retrieved statutory sources in Streamlit.
20
+ - [ ] Screenshot: an unsupported-citation warning and evaluation status.
21
+ - [ ] GIF: retrieval, graph context, retry and final response for a real query.
22
+
23
+ ## Architecture
24
+
25
+ Python hosts a FastAPI backend and Streamlit HTTP client. Sentence Transformers
26
+ embeds questions, Qdrant stores statute vectors/payloads, BAAI/bge-reranker-large
27
+ reranks candidates, Neo4j supplies hierarchy/reference context, and Gemini performs
28
+ analysis, generation and judging.
29
+
30
+ ```mermaid
31
+ flowchart TD
32
+ UI[Streamlit / API query] --> A[Query analysis]
33
+ A --> M[Multi-query search: embedding + dense retrieval]
34
+ M --> F[Act metadata filter]
35
+ F --> B{BM25 enabled?}
36
+ B -->|yes| RRF[BM25 + dense reciprocal rank fusion]
37
+ B -->|no| R[Rerank]
38
+ RRF --> R
39
+ R --> N[Neo4j expansion when requested]
40
+ N --> G[Generation]
41
+ UI --> K[Indian Kanoon judgments when requested]
42
+ K --> G
43
+ G --> C[Citation presence check]
44
+ C --> J[LLM judge + citation score penalty]
45
+ J --> D{Pass?}
46
+ D -->|yes| O[Answer + sources + citation flags]
47
+ D -->|retry within budget| T[Adjust retrieval or generation]
48
+ T --> A
49
+ D -->|budget exhausted| E[Return best attempt with its own flags]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  ```
51
 
52
+ The filter box is conceptual: the act constraint is passed into Qdrant search,
53
+ not applied after discarding search results. Explicit provision lookup, when
54
+ configured, prepends actual stored hits and preserves them after reranking.
55
+ Graph expansion follows `(child)-[:BELONGS_TO]->(parent)` upward and reverses
56
+ that traversal for descendants. Graph and judgment calls can be disabled per query.
57
+
58
+ Unsupported statutory citations lower the evaluation score and prevent PASS,
59
+ using the existing retry budget. The checker verifies source presence/identity;
60
+ it does not prove that a provision entails the answer's legal claim. The API
61
+ returns `unsupported_citations`, `citation_precision` and per-mention
62
+ `citation_checks`; precision is undefined for answers without citations.
63
+
64
+ ## Corpus statistics
65
+
66
+ The source of this table is [results/corpus_stats.md](results/corpus_stats.md),
67
+ with machine-readable provenance in [corpus_stats.json](results/corpus_stats.json).
68
+ The read-only [counting script](scripts/corpus_stats.py) attempted live storage
69
+ reads. Qdrant reported `ResponseHandlingException`; Neo4j reported `ValueError`.
70
+ Both were unavailable. These failures do **not** establish an empty corpus.
71
+ Rows identify the intended acts; their stored coverage is unverified.
72
+
73
+ | Act | Unique stored chunks | Graph nodes | Directed graph edges |
74
+ |---|---:|---:|---:|
75
+ | BNS | Not measured | Not measured | Not measured |
76
+ | BNSS | Not measured | Not measured | Not measured |
77
+ | BSA | Not measured | Not measured | Not measured |
78
+ | Constitution | Not measured | Not measured | Not measured |
79
+ | Unmapped | Not measured | Not measured | Not measured |
80
+ | Total | Not measured | Not measured | Not measured |
81
+
82
+ Chunk counts deduplicate actual stored `chunk_id` values. The JSON also reports
83
+ raw points, duplicate chunk points and points without IDs when storage is readable.
84
+ Graph edges include every directed relationship, including inverse hierarchy
85
+ edges; cross-act edges belong to their source act. Chunk acts use recognized
86
+ metadata with an ID fallback; unresolved identities stay in the Unmapped row.
87
+ Freeze ingestion while collecting counts from both stores.
88
+
89
+ ## Results recorded in this repository
90
+
91
+ **No completed benchmark is present.** All tables below reflect only checked-in
92
+ `results/` outputs. “Not evaluated” and “Not measured” are missing measurements,
93
+ not zero scores. There are no demonstrated improvements or confidence intervals.
94
+ Configuration defaults and sample sizes elsewhere in this README are experiment
95
+ settings, not measured outcomes.
96
+
97
+ ### Retrieval ablation
98
+
99
+ Source: [retrieval_ablation.md](results/retrieval_ablation.md).
100
+ [retrieval_ablation.csv](results/retrieval_ablation.csv) contains only a header.
101
+ The report states that no reviewed, answerable gold items are available.
102
+
103
+ | Configuration | BM25/RRF | Exact lookup | Recall@5 | MRR |
104
+ |---|---|---|---|---|
105
+ | A | False | False | Not evaluated | Not evaluated |
106
+ | B | False | False | Not evaluated | Not evaluated |
107
+ | C | False | False | Not evaluated | Not evaluated |
108
+ | D | False | False | Not evaluated | Not evaluated |
109
+ | E | False | False | Not evaluated | Not evaluated |
110
+ | F | False | False | Not evaluated | Not evaluated |
111
+ | G | True | True | Not evaluated | Not evaluated |
112
+
113
+ The presets compare dense search, act filtering, multi-query expansion,
114
+ reranking, legal boosts, graph context and BM25 fusion; their precise definitions
115
+ are in [the evaluation guide](eval/README.md#retrieval-ablation-benchmark).
116
+ Root/exact Recall@1/5/10, MRR, nDCG@10, graph context recall, per-intent breakdowns,
117
+ bootstrap intervals and failure lists have not been computed in these outputs.
118
+
119
+ ### Answer quality
120
+
121
+ Source: [answer_eval.md](results/answer_eval.md): no reviewed gold items and no
122
+ computed answer metrics.
123
+
124
+ | Measure | Recorded result |
125
+ |---|---|
126
+ | Citation precision | Not measured |
127
+ | Unsupported-citation rate | Not measured |
128
+ | Answer contains expected provision | Not measured |
129
+ | Context-supported expected provision | Not measured |
130
+ | Out-of-scope refusal rate | Not measured |
131
+
132
+ These are citation/coverage/scope measures, not a legal correctness guarantee.
133
+
134
+ ### Latency and quality trade-offs
135
+
136
+ Source: [latency.md](results/latency.md): no paired measurements. The planned
137
+ comparison requires restored stores and reviewed gold questions.
138
+
139
+ | Stage | Before p50 / p95 (ms) | After cold p50 / p95 (ms) | After warm p50 / p95 (ms) |
140
+ |---|---|---|---|
141
+ | Analysis | Not measured | Not measured | Not measured |
142
+ | Embedding | Not measured | Not measured | Not measured |
143
+ | Vector search | Not measured | Not measured | Not measured |
144
+ | Rerank | Not measured | Not measured | Not measured |
145
+ | Neo4j expansion | Not measured | Not measured | Not measured |
146
+ | Generation | Not measured | Not measured | Not measured |
147
+ | Judge | Not measured | Not measured | Not measured |
148
+ | Total | Not measured | Not measured | Not measured |
149
+
150
+ The planned before/after Recall@5 and citation-precision comparisons, and the
151
+ quality effect of skipping the judge, are also unmeasured. No speedup is accepted
152
+ as an improvement from these reports.
153
+
154
+ ### Judge validation
155
+
156
+ Source: [judge_validation.md](results/judge_validation.md): no completed manual
157
+ agreement or consistency report.
158
+
159
+ | Measure | Recorded result |
160
+ |---|---|
161
+ | Agreement accuracy | Not measured |
162
+ | Cohen's kappa | Not measured |
163
+ | Confusion matrix | Not measured |
164
+ | Overall-score / manual-label Pearson correlation | Not measured |
165
+ | Overall-score / manual-label Spearman correlation | Not measured |
166
+ | Score variance across temperature-zero reruns | Not measured |
167
+
168
+ ## Design decisions and unsuccessful experiments
169
+
170
+ - Keep dense retrieval as the default; BM25/RRF, exact lookup and legal boosts
171
+ have explicit evaluation switches. BM25/exact lookup remain off by default
172
+ pending the Recall@5/MRR evidence missing from the ablation report.
173
+ - Use statute hierarchy and references for context, while keeping graph expansion
174
+ optional. More context can include irrelevant material; no quality gain is
175
+ established by the current results.
176
+ - Separate deterministic citation-presence checks from LLM judging. A judge score
177
+ cannot override an unsupported citation, and a high score alone is not proof
178
+ of correctness.
179
+ - Concurrent vector I/O, batched graph expansion and bounded analysis/embedding
180
+ LRUs are opt-in. Input-order merging preserves tie behavior; caches normalize
181
+ whitespace while preserving case and punctuation. Timeouts and bounded transient
182
+ retries limit individual external calls, not total query duration.
183
+ - Evaluate only human-reviewed gold items, retain actual reports and leave
184
+ undefined metrics undefined. Do not promote generated labels to ground truth.
185
+
186
+ **What did not help:** the reports contain no completed comparison from which
187
+ an unsuccessful algorithmic experiment can be identified. Evaluation attempts
188
+ stopped at missing reviewed labels/storage, as their status reports document.
189
+ Those are setup blockers, not evidence that BM25, graph expansion, reranking,
190
+ legal boosts or caching failed. Claiming negative or positive findings would
191
+ invent results. Judge skipping is unavailable: enabling its flag fails fast
192
+ until a reviewed comparison establishes no quality loss.
193
+
194
+ ## Known limitations and future work
195
+
196
+ - **Not legal advice; no human lawyer validation is documented.** Obtain qualified
197
+ legal review before considering real-world use.
198
+ - Stored corpus size, completeness, amendment currency and provision coverage
199
+ are unverified. Named acts do not imply that every provision was ingested.
200
+ State laws, other central statutes and the full body of case law are not a
201
+ demonstrated coverage claim. Indian Kanoon access is on-demand.
202
+ - PDF/OCR layout can lose numbering, boundaries, provisos or explanations.
203
+ Citation matching and root-level metrics can miss substantive interpretation
204
+ errors. Out-of-scope refusal recognition uses wording heuristics.
205
+ - Judge reliability, human agreement and temperature-zero consistency have not
206
+ been measured. The generator and judge use the configured Gemini model;
207
+ their agreement is not independent expert validation.
208
+ - External services, model versions and corpus changes affect availability and
209
+ reproducibility. Sequential per-worker queries, aggregate retries and model
210
+ memory use limit deployment scalability.
211
+
212
+ Next steps are to restore and audit the corpus; compute storage counts; collect
213
+ lawyer-reviewed labels; complete the ablations, answer evaluation, judge study
214
+ and latency runs; document regressions as well as improvements; and only then
215
+ consider enabling optional retrieval/performance methods or judge bypass.
216
+
217
+ ## Quickstart
218
+
219
+ Use Docker with Compose. No PDFs, source text or credentials are committed.
220
 
221
  ```bash
222
+ cp .env.example .env
223
+ # Edit credentials and obtain the statute sources described below.
224
+ make ingest
225
+ # Starts api, ui, qdrant and neo4j, with healthchecks and persistent volumes:
226
+ docker compose up -d --build
227
  ```
228
 
229
+ Open the UI at `http://localhost:7860`, API docs at
230
+ `http://localhost:8000/docs`, and Neo4j Browser at `http://localhost:7474`.
231
+ Published ports bind to loopback. API startup loads the embedder/reranker and may
232
+ initially download weights; the UI waits for API dependency health, including
233
+ Gemini reachability. The current result files do not establish that this stack
234
+ has completed a live end-to-end deployment.
235
+
236
+ `make ingest` starts/waits for the databases and loads statutes through
237
+ [scripts/ingest.sh](scripts/ingest.sh). The same parsed documents feed the
238
+ existing chunkers and graph builders; bounded vector batches use stable UUIDs.
239
+ It upserts without clearing stores, is not transactional across both stores,
240
+ and does not prune deleted provisions or convert legacy integer point IDs.
241
+ Use an explicit migration/fresh stores for corpus replacement and restart the
242
+ API after ingestion, especially for a BM25 snapshot. `docker compose down`
243
+ retains named database/model volumes. Sources mount read-only; `.env`, PDFs,
244
+ local data and model caches are excluded from the application image.
245
+
246
+ ### Obtain source files
247
+
248
+ Place English text PDFs in `data/statutes/` with these basenames:
249
+
250
+ | File | Official source |
251
+ |---|---|
252
+ | `bns.pdf` | [India Code: BNS](https://www.indiacode.nic.in/indiacode/handle/123456789/20062?col=123456789%2F1362&view_type=search), or the [NCRB New Criminal Laws portal](https://www.ncrb.mha.gov.in/uploads/SankalanPortal/Index.html) |
253
+ | `bnss.pdf` | BNSS gazette from the [NCRB portal](https://www.ncrb.mha.gov.in/uploads/SankalanPortal/Index.html) |
254
+ | `bsa.pdf` | BSA gazette from the [NCRB portal](https://www.ncrb.mha.gov.in/uploads/SankalanPortal/Index.html) |
255
+ | `constitution.pdf` | English edition from the [Legislative Department documents](https://www.legislative.gov.in/documents) |
256
+
257
+ Reviewed UTF-8 `bns.txt`, `bnss.txt`, `bsa.txt` and `constitution.txt` take
258
+ precedence over same-name PDFs. Image-only pages require reviewed OCR text;
259
+ there is no automatic silent OCR. Inspect numbering and contents-page/layout
260
+ artifacts. Record source URL, edition/amendment date and download date locally.
261
+ Successful parsing does not prove complete statute coverage. Do not commit
262
+ copyrighted PDFs or annotated commercial editions.
263
+
264
+ ### Environment variables
265
+
266
+ Settings load `.env`; environment variables take precedence. Restart after
267
+ changes. See [.env.example](.env.example) and [legora/config.py](legora/config.py).
268
+
269
+ | Variables | Purpose |
270
+ |---|---|
271
+ | `GEMINI_API_KEY`, `GEMINI_MODEL` | Required API key; configured analysis/generation/judge model |
272
+ | `QDRANT_URL`, `QDRANT_API_KEY` | Required URL; optional local auth key |
273
+ | `NEO4J_URI`, `NEO4J_USERNAME`, `NEO4J_PASSWORD` | Required graph connection settings |
274
+ | `KANOON_TOKEN` | Required backend judgment-service token |
275
+ | `TOP_K`, `RERANK_K`, `JUDGMENT_K` | Retrieval limits |
276
+ | `PASS_THRESHOLD`, `MAX_ATTEMPTS` | Evaluation threshold and total attempt budget |
277
+ | `API_BASE_URL`, `API_TIMEOUT_SECONDS` | Streamlit HTTP settings |
278
+ | `EXTERNAL_TIMEOUT_SECONDS`, `QUERY_LOCK_TIMEOUT_SECONDS` | Per-call and queue deadlines |
279
+ | `USE_BM25`, `EXACT_ID_LOOKUP` | Optional lexical fusion and provision lookup |
280
+ | `VECTOR_SEARCH_WORKERS`, `BATCH_GRAPH_EXPANSION`, `ENABLE_QUERY_CACHE` | Opt-in performance switches |
281
+ | `SKIP_JUDGE_HIGH_CONFIDENCE` | Keep false; bypass is unavailable pending quality evidence |
282
+ | `LOG_LEVEL` | JSON logging verbosity |
283
+
284
+ For Compose's official Neo4j image, the initial username is `neo4j`; choose your
285
+ own strong password. Changing `.env` alone does not reset an existing database
286
+ password. Compose overrides database URLs with internal service addresses; the
287
+ UI receives only its API URL/timeout. Keep host-accessible URLs in `.env` for
288
+ host-side commands. Blank `QDRANT_API_KEY` disables local Qdrant auth.
289
+
290
+ `POST /query` accepts a question and optional retrieval/source controls;
291
+ `GET /health` checks Qdrant, Neo4j and Gemini reachability. The latter does not
292
+ verify generation quota or corpus completeness. Logs correlate stage latency,
293
+ candidate count and attempt by request ID, without logging prompts or keys.
294
+ See [the API models](api/models.py) for the response contract.
295
+
296
+ ## Reproduce every table
297
+
298
+ Use Python 3.11 for host commands. Install the pinned developer dependencies
299
+ (or run Python commands through the API container with appropriate volume mounts):
300
 
301
+ ```bash
302
+ python -m pip install -r requirements-dev.txt
303
+ python -m scripts.ingest --check-only --data-dir data/statutes
304
+ ```
305
 
306
+ First restore/ingest the corpus. Generate draft questions from actual stored
307
+ provisions, then review each question and supporting IDs explicitly:
 
308
 
309
  ```bash
310
+ python -m eval.sample
311
+ python -m eval.generate
312
+ python -m eval.review
313
  ```
314
 
315
+ Only reviewed items may enter metrics; answerable questions are additionally
316
+ required for retrieval ablations. Drafts are not an evaluation set. Generation,
317
+ review/resume, model/corpus reproducibility and metric definitions are documented
318
+ in [eval/README.md](eval/README.md). The following commands produce each report:
319
+
320
+ | README table | Reproduction command | Authoritative report |
321
+ |---|---|---|
322
+ | Corpus | `python -m scripts.corpus_stats` | `results/corpus_stats.md` and `.json` |
323
+ | Retrieval A–G | `python -m eval.run_retrieval_eval --configs A B C D E F G --seed 42` | `results/retrieval_ablation.md` and `.csv` |
324
+ | Answer quality | `python -m eval.run_answer_eval` | `results/answer_eval.md` and `.json` |
325
+ | Latency / paired quality | `python -m eval.profile_pipeline --count 50 --seed 42 --workers 4` | `results/latency.md` and `.json` |
326
+ | Judge agreement / variance | Commands below | `results/judge_validation.md` |
327
+
328
+ Judge validation requires a real answer export and manual labeling:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
329
 
330
  ```bash
331
+ python -m eval.judge_validation export --count 50 --seed 42
332
+ # Fill my_label (correct/partially/incorrect) and my_notes in results/judge_validation.csv.
333
+ python -m eval.judge_validation consistency
334
+ python -m eval.judge_validation report
335
  ```
336
 
337
+ The judge reruns use the same frozen answer/context, temperature zero, and fresh
338
+ calls. Sampling seeds and caches improve reproducibility but do not guarantee
339
+ identical live model outputs. Keep corpus/model versions fixed. The ablation
340
+ runner saves query analyses on disk; `--offline-analysis` requires cache hits.
341
+ The latency runner records baseline, cold-start and warm-cache modes and flags
342
+ observed Recall@5/citation-precision regressions. Missing inputs stop evaluation
343
+ rather than inventing scores. After a completed run, consult the regenerated
344
+ `results/` files and update this README snapshot from those outputs only.
345
+
346
+ Development checks: `make lint`, `make typecheck`, `make test`. The GitHub Actions
347
+ workflow runs these checks, uploads coverage, validates Compose and builds the
348
+ Docker image without publishing. Runtime/developer dependencies are pinned;
349
+ this is not a hash-locked transitive dependency graph. No test or CI pass rate
350
+ is claimed here because those measurements are not recorded in `results/`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
data/statutes/README.md ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # Local statute sources
2
+
3
+ Place English source files here as `bns.pdf`, `bnss.pdf`, `bsa.pdf`, and
4
+ `constitution.pdf`, or supply reviewed UTF-8 files with the same `.txt` basenames.
5
+ Text takes precedence when both formats exist. See the repository README for
6
+ official download sources, validation and ingestion commands. PDFs, extracted text,
7
+ and OCR intermediates stay local and are excluded from Git and Docker images.
docker-compose.yml ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ services:
2
+ api:
3
+ image: legora:local
4
+ build: .
5
+ env_file: .env
6
+ environment:
7
+ QDRANT_URL: http://qdrant:6333
8
+ NEO4J_URI: bolt://neo4j:7687
9
+ command: [python, -m, uvicorn, api.app:app, --host, 0.0.0.0, --port, "8000"]
10
+ ports:
11
+ - "127.0.0.1:8000:8000"
12
+ volumes:
13
+ - model_cache:/app/.cache/huggingface
14
+ - ./data/statutes:/data/statutes:ro
15
+ depends_on:
16
+ qdrant:
17
+ condition: service_healthy
18
+ neo4j:
19
+ condition: service_healthy
20
+ healthcheck:
21
+ test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=35).close()"]
22
+ interval: 30s
23
+ timeout: 40s
24
+ retries: 5
25
+ start_period: 600s
26
+ restart: unless-stopped
27
+
28
+ ui:
29
+ image: legora:local
30
+ build: .
31
+ # Only UI settings are passed; backend keys stay in the API container.
32
+ environment:
33
+ API_BASE_URL: http://api:8000
34
+ API_TIMEOUT_SECONDS: ${API_TIMEOUT_SECONDS:-180}
35
+ command: [python, -m, streamlit, run, app.py, --server.address=0.0.0.0, --server.port=7860]
36
+ ports:
37
+ - "127.0.0.1:7860:7860"
38
+ depends_on:
39
+ api:
40
+ condition: service_healthy
41
+ healthcheck:
42
+ test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/_stcore/health', timeout=5).close()"]
43
+ interval: 30s
44
+ timeout: 10s
45
+ retries: 3
46
+ start_period: 30s
47
+ restart: unless-stopped
48
+
49
+ qdrant:
50
+ image: qdrant/qdrant:v1.14.1
51
+ environment:
52
+ QDRANT__SERVICE__API_KEY: ${QDRANT_API_KEY:-}
53
+ # An empty API-key setting would enable auth with an empty key upstream.
54
+ command:
55
+ - bash
56
+ - -ec
57
+ - 'if [ -z "$$QDRANT__SERVICE__API_KEY" ]; then unset QDRANT__SERVICE__API_KEY; fi; exec ./entrypoint.sh'
58
+ ports:
59
+ - "127.0.0.1:6333:6333"
60
+ volumes:
61
+ - qdrant_data:/qdrant/storage
62
+ healthcheck:
63
+ # The upstream image has bash; curl/wget are not assumed.
64
+ test:
65
+ - CMD-SHELL
66
+ - >-
67
+ bash -ec 'exec 3<>/dev/tcp/127.0.0.1/6333;
68
+ printf "GET /healthz HTTP/1.0\r\nHost: localhost\r\n\r\n" >&3;
69
+ read -r response <&3; [[ "$$response" == *" 200 "* ]]'
70
+ interval: 10s
71
+ timeout: 5s
72
+ retries: 10
73
+ start_period: 10s
74
+ restart: unless-stopped
75
+
76
+ neo4j:
77
+ image: neo4j:5.26.0-community
78
+ environment:
79
+ NEO4J_AUTH: ${NEO4J_USERNAME:?Set NEO4J_USERNAME in .env}/${NEO4J_PASSWORD:?Set NEO4J_PASSWORD in .env}
80
+ NEO4J_USERNAME: ${NEO4J_USERNAME:?Set NEO4J_USERNAME in .env}
81
+ NEO4J_PASSWORD: ${NEO4J_PASSWORD:?Set NEO4J_PASSWORD in .env}
82
+ NEO4J_server_memory_heap_initial__size: 256m
83
+ NEO4J_server_memory_heap_max__size: 512m
84
+ NEO4J_server_memory_pagecache_size: 256m
85
+ ports:
86
+ - "127.0.0.1:7474:7474"
87
+ - "127.0.0.1:7687:7687"
88
+ volumes:
89
+ - neo4j_data:/data
90
+ - neo4j_logs:/logs
91
+ healthcheck:
92
+ test: [CMD-SHELL, 'cypher-shell -a bolt://127.0.0.1:7687 -u "$$NEO4J_USERNAME" -p "$$NEO4J_PASSWORD" "RETURN 1" > /dev/null']
93
+ interval: 15s
94
+ timeout: 10s
95
+ retries: 10
96
+ start_period: 60s
97
+ restart: unless-stopped
98
+
99
+ volumes:
100
+ qdrant_data:
101
+ neo4j_data:
102
+ neo4j_logs:
103
+ model_cache:
eval/generate.py CHANGED
@@ -6,6 +6,7 @@ import json
6
  import random
7
  from collections.abc import Callable
8
  from pathlib import Path
 
9
 
10
  from google import genai
11
  from google.genai import types
@@ -96,7 +97,7 @@ def checked_item(identifier: str, draft: Draft, candidates: list[Provision], mul
96
  roots = {provision_root(value) for value in draft.expected_chunk_ids}
97
  if len(roots) < 2:
98
  raise ValueError("A multi-provision question requires distinct supporting provisions")
99
- act = "MULTI"
100
  else:
101
  if draft.expected_chunk_ids != [candidates[0].chunk_id]:
102
  raise ValueError("Single-provision support must match the sampled provision")
 
6
  import random
7
  from collections.abc import Callable
8
  from pathlib import Path
9
+ from typing import Literal
10
 
11
  from google import genai
12
  from google.genai import types
 
97
  roots = {provision_root(value) for value in draft.expected_chunk_ids}
98
  if len(roots) < 2:
99
  raise ValueError("A multi-provision question requires distinct supporting provisions")
100
+ act: Literal["BNS", "BNSS", "BSA", "Constitution", "MULTI"] = "MULTI"
101
  else:
102
  if draft.expected_chunk_ids != [candidates[0].chunk_id]:
103
  raise ValueError("Single-provision support must match the sampled provision")
eval/judge_validation.py CHANGED
@@ -10,6 +10,7 @@ import json
10
  import math
11
  import random
12
  import statistics
 
13
  from pathlib import Path
14
  from typing import Any, Protocol
15
 
@@ -125,7 +126,7 @@ def load_labels(path: Path, samples: list[Sample]) -> dict[str, str]:
125
  return labels
126
 
127
 
128
- def correlation(left: list[float], right: list[float]) -> float | None:
129
  """Pearson correlation, undefined for constant inputs or fewer than two pairs."""
130
  if len(left) != len(right):
131
  raise ValueError("Correlation requires paired observations")
@@ -137,7 +138,7 @@ def correlation(left: list[float], right: list[float]) -> float | None:
137
  return sum(x * y for x, y in zip(centered_x, centered_y)) / denominator if denominator else None
138
 
139
 
140
- def ranks(values: list[float]) -> list[float]:
141
  """Assign average one-based ranks to ties for Spearman correlation."""
142
  return [1 + sum(other < value for other in values) + (values.count(value) - 1) / 2 for value in values]
143
 
@@ -249,23 +250,24 @@ def main() -> None:
249
  parser.add_argument("--count", type=int, default=50)
250
  parser.add_argument("--seed", type=int, default=42)
251
  args = parser.parse_args()
252
- resource = None
 
253
  try:
254
  if args.command == "export":
255
  gold = load_gold(args.gold)
256
  sample_questions(gold, args.count, args.seed) # Gate before loading models/clients.
257
  if args.csv.exists() or args.samples.exists():
258
  raise ValueError("Export paths already exist; use new paths")
259
- resource = load_pipeline()
260
- export_samples(gold, resource, args.count, args.seed, args.csv, args.samples)
261
  print(f"Exported {args.count} answers; fill my_label and my_notes in {args.csv}")
262
  elif args.command == "consistency":
263
  from evaluator.LLM_evaluator import LegalLLMEvaluator
264
  from legora.config import EvaluationSettings
265
 
266
  samples = load_samples(args.samples)
267
- resource = LegalLLMEvaluator(settings=EvaluationSettings())
268
- result = run_consistency(samples, resource)
269
  atomic_text(args.consistency, json.dumps(result, indent=2) + "\n")
270
  print(f"Saved three-run consistency measurements to {args.consistency}")
271
  else:
@@ -277,14 +279,13 @@ def main() -> None:
277
  if not args.output.exists():
278
  atomic_text(args.output, "# Judge validation — not run\n\nNo completed manual agreement/consistency report. Export real pipeline answers and fill manual labels before reporting.\n")
279
  # Safe validation errors are useful, but upstream failures may contain secrets/prompts.
280
- message = str(error) if type(error) is ValueError and args.command == "export" and resource is None else type(error).__name__
281
  parser.exit(1, f"Judge validation failed: {message}. No incomplete run was scored.\n")
282
  finally:
283
- if resource is not None:
284
- if args.command == "export":
285
- resource.close()
286
- else:
287
- resource.client.close()
288
 
289
 
290
  if __name__ == "__main__":
 
10
  import math
11
  import random
12
  import statistics
13
+ from collections.abc import Sequence
14
  from pathlib import Path
15
  from typing import Any, Protocol
16
 
 
126
  return labels
127
 
128
 
129
+ def correlation(left: Sequence[float], right: Sequence[float]) -> float | None:
130
  """Pearson correlation, undefined for constant inputs or fewer than two pairs."""
131
  if len(left) != len(right):
132
  raise ValueError("Correlation requires paired observations")
 
138
  return sum(x * y for x, y in zip(centered_x, centered_y)) / denominator if denominator else None
139
 
140
 
141
+ def ranks(values: Sequence[float]) -> list[float]:
142
  """Assign average one-based ranks to ties for Spearman correlation."""
143
  return [1 + sum(other < value for other in values) + (values.count(value) - 1) / 2 for value in values]
144
 
 
250
  parser.add_argument("--count", type=int, default=50)
251
  parser.add_argument("--seed", type=int, default=42)
252
  args = parser.parse_args()
253
+ pipeline = None
254
+ judge = None
255
  try:
256
  if args.command == "export":
257
  gold = load_gold(args.gold)
258
  sample_questions(gold, args.count, args.seed) # Gate before loading models/clients.
259
  if args.csv.exists() or args.samples.exists():
260
  raise ValueError("Export paths already exist; use new paths")
261
+ pipeline = load_pipeline()
262
+ export_samples(gold, pipeline, args.count, args.seed, args.csv, args.samples)
263
  print(f"Exported {args.count} answers; fill my_label and my_notes in {args.csv}")
264
  elif args.command == "consistency":
265
  from evaluator.LLM_evaluator import LegalLLMEvaluator
266
  from legora.config import EvaluationSettings
267
 
268
  samples = load_samples(args.samples)
269
+ judge = LegalLLMEvaluator(settings=EvaluationSettings())
270
+ result = run_consistency(samples, judge)
271
  atomic_text(args.consistency, json.dumps(result, indent=2) + "\n")
272
  print(f"Saved three-run consistency measurements to {args.consistency}")
273
  else:
 
279
  if not args.output.exists():
280
  atomic_text(args.output, "# Judge validation — not run\n\nNo completed manual agreement/consistency report. Export real pipeline answers and fill manual labels before reporting.\n")
281
  # Safe validation errors are useful, but upstream failures may contain secrets/prompts.
282
+ message = str(error) if type(error) is ValueError and args.command == "export" and pipeline is None else type(error).__name__
283
  parser.exit(1, f"Judge validation failed: {message}. No incomplete run was scored.\n")
284
  finally:
285
+ if pipeline is not None:
286
+ pipeline.close()
287
+ if judge is not None:
288
+ judge.client.close()
 
289
 
290
 
291
  if __name__ == "__main__":
eval/review.py CHANGED
@@ -93,9 +93,10 @@ def main() -> None:
93
  print(f"{sum(item.reviewed for item in items)}/{len(items)} reviewed")
94
  return
95
  if args.id:
96
- item = next((entry for entry in items if entry.id == args.id), None)
97
- if item is None:
98
  raise ValueError("Unknown item ID")
 
99
  changes = {field: value for field, value in {
100
  "expected_chunk_ids": args.expected, "acceptable_alternates": args.alternates,
101
  "question": args.question, "notes": args.notes, "intent": args.intent,
 
93
  print(f"{sum(item.reviewed for item in items)}/{len(items)} reviewed")
94
  return
95
  if args.id:
96
+ selected = next((entry for entry in items if entry.id == args.id), None)
97
+ if selected is None:
98
  raise ValueError("Unknown item ID")
99
+ item = selected
100
  changes = {field: value for field, value in {
101
  "expected_chunk_ids": args.expected, "acceptable_alternates": args.alternates,
102
  "question": args.question, "notes": args.notes, "intent": args.intent,
eval/run_retrieval_eval.py CHANGED
@@ -163,7 +163,7 @@ def run_configuration(
163
  points.extend(retriever.vector_search_filtered(query, document=document, k=candidates))
164
  # Same merged budget for every configuration, and an explicit tie breaker.
165
  points = sorted(retriever.dedupe_points(points), key=lambda point: (-float(point.score), point.payload["chunk_id"]))[:candidates]
166
- exact = []
167
  if configuration.bm25 or configuration.exact_id_lookup:
168
  points, exact = retriever.fuse_candidates(
169
  question, points, document, candidates, configuration.bm25, configuration.exact_id_lookup,
@@ -208,7 +208,7 @@ def ranking_metrics(item: GoldItem, ranked_ids: list[str], match: str, graph_ids
208
  positions.append(rank)
209
  break
210
  count = len(groups)
211
- values = {f"recall_at_{k}": sum(rank <= k for rank in positions) / count for k in (1, 5, 10)}
212
  values["mrr"] = 1 / min(positions) if positions else 0.0
213
  ideal = sum(1 / math.log2(rank + 1) for rank in range(1, min(count, 10) + 1))
214
  values["ndcg_at_10"] = sum(1 / math.log2(rank + 1) for rank in positions if rank <= 10) / ideal
 
163
  points.extend(retriever.vector_search_filtered(query, document=document, k=candidates))
164
  # Same merged budget for every configuration, and an explicit tie breaker.
165
  points = sorted(retriever.dedupe_points(points), key=lambda point: (-float(point.score), point.payload["chunk_id"]))[:candidates]
166
+ exact: list[Any] = []
167
  if configuration.bm25 or configuration.exact_id_lookup:
168
  points, exact = retriever.fuse_candidates(
169
  question, points, document, candidates, configuration.bm25, configuration.exact_id_lookup,
 
208
  positions.append(rank)
209
  break
210
  count = len(groups)
211
+ values: dict[str, float | None] = {f"recall_at_{k}": sum(rank <= k for rank in positions) / count for k in (1, 5, 10)}
212
  values["mrr"] = 1 / min(positions) if positions else 0.0
213
  ideal = sum(1 / math.log2(rank + 1) for rank in range(1, min(count, 10) + 1))
214
  values["ndcg_at_10"] = sum(1 / math.log2(rank + 1) for rank in positions if rank <= 10) / ideal
eval/sample.py CHANGED
@@ -6,6 +6,7 @@ import random
6
  import re
7
  from collections.abc import Iterable, Iterator
8
  from pathlib import Path
 
9
 
10
  from qdrant_client import QdrantClient
11
 
@@ -107,7 +108,7 @@ def main() -> None:
107
  if not config.qdrant_url:
108
  raise ValueError("QDRANT_URL is required, or supply --chunks")
109
  key = config.qdrant_api_key.get_secret_value() if config.qdrant_api_key else None
110
- client = QdrantClient(url=config.qdrant_url, api_key=key or None, timeout=config.external_timeout_seconds, check_compatibility=False)
111
  try:
112
  records = stored_provisions(scroll_payloads(client, args.collection or ["legal_rag"]))
113
  finally:
 
6
  import re
7
  from collections.abc import Iterable, Iterator
8
  from pathlib import Path
9
+ from math import ceil
10
 
11
  from qdrant_client import QdrantClient
12
 
 
108
  if not config.qdrant_url:
109
  raise ValueError("QDRANT_URL is required, or supply --chunks")
110
  key = config.qdrant_api_key.get_secret_value() if config.qdrant_api_key else None
111
+ client = QdrantClient(url=config.qdrant_url, api_key=key or None, timeout=ceil(config.external_timeout_seconds), check_compatibility=False)
112
  try:
113
  records = stored_provisions(scroll_payloads(client, args.collection or ["legal_rag"]))
114
  finally:
eval/storage.py CHANGED
@@ -1,8 +1,8 @@
1
  """Atomic, validated JSONL storage that preserves human reviews on resume."""
2
 
3
- import json
4
  import os
5
  import tempfile
 
6
  from pathlib import Path
7
  from typing import TypeVar
8
 
@@ -43,7 +43,7 @@ def atomic_text(path: Path, text: str) -> None:
43
  temporary.unlink(missing_ok=True)
44
 
45
 
46
- def write_jsonl(path: Path, records: list[BaseModel]) -> None:
47
  """Write actual records, with no placeholder or silently invalid rows."""
48
  atomic_text(path, "".join(record.model_dump_json() + "\n" for record in records))
49
 
 
1
  """Atomic, validated JSONL storage that preserves human reviews on resume."""
2
 
 
3
  import os
4
  import tempfile
5
+ from collections.abc import Sequence
6
  from pathlib import Path
7
  from typing import TypeVar
8
 
 
43
  temporary.unlink(missing_ok=True)
44
 
45
 
46
+ def write_jsonl(path: Path, records: Sequence[BaseModel]) -> None:
47
  """Write actual records, with no placeholder or silently invalid rows."""
48
  atomic_text(path, "".join(record.model_dump_json() + "\n" for record in records))
49
 
legora/logging.py CHANGED
@@ -41,7 +41,7 @@ def setup_logging(level: str = "INFO") -> None:
41
  if not any(getattr(handler, "_legora_json", False) for handler in root.handlers):
42
  handler = logging.StreamHandler()
43
  handler.setFormatter(JsonFormatter())
44
- handler._legora_json = True
45
  root.handlers[:] = [handler]
46
  root.setLevel(level)
47
  # HTTP request URLs and SDK diagnostics can contain credentials.
 
41
  if not any(getattr(handler, "_legora_json", False) for handler in root.handlers):
42
  handler = logging.StreamHandler()
43
  handler.setFormatter(JsonFormatter())
44
+ setattr(handler, "_legora_json", True)
45
  root.handlers[:] = [handler]
46
  root.setLevel(level)
47
  # HTTP request URLs and SDK diagnostics can contain credentials.
legora/performance.py CHANGED
@@ -4,7 +4,7 @@ from collections import OrderedDict
4
  from collections.abc import Callable
5
  from copy import deepcopy
6
  from threading import RLock
7
- from typing import ParamSpec, TypeVar
8
 
9
  import httpx
10
  from tenacity import Retrying, retry_if_exception, stop_after_attempt, wait_exponential
@@ -39,7 +39,7 @@ class QueryCache:
39
  if len(self.values) > self.capacity:
40
  self.values.popitem(last=False)
41
  self.values.move_to_end(key)
42
- return deepcopy(self.values[key])
43
 
44
  def clear(self) -> None:
45
  """Remove values before a cold benchmark or a model/prompt change."""
 
4
  from collections.abc import Callable
5
  from copy import deepcopy
6
  from threading import RLock
7
+ from typing import ParamSpec, TypeVar, cast
8
 
9
  import httpx
10
  from tenacity import Retrying, retry_if_exception, stop_after_attempt, wait_exponential
 
39
  if len(self.values) > self.capacity:
40
  self.values.popitem(last=False)
41
  self.values.move_to_end(key)
42
+ return cast(T, deepcopy(self.values[key]))
43
 
44
  def clear(self) -> None:
45
  """Remove values before a cold benchmark or a model/prompt change."""
pyproject.toml ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [tool.ruff]
2
+ target-version = "py311"
3
+ line-length = 120
4
+ extend-exclude = ["venv", ".venv", "coverage"]
5
+
6
+ [tool.ruff.lint]
7
+ select = ["E4", "E7", "E9", "F"]
8
+
9
+ [tool.mypy]
10
+ python_version = "3.11"
11
+ explicit_package_bases = true
12
+ files = ["api", "legora", "eval", "scripts", "LegalRag/citation_check.py", "retrieval/lexical.py"]
13
+ follow_imports = "silent"
14
+ ignore_missing_imports = true
15
+ check_untyped_defs = true
16
+ no_implicit_optional = true
17
+ warn_unused_ignores = true
18
+ plugins = ["pydantic.mypy"]
requirements-dev.txt ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ -r requirements.txt
2
+ pytest==9.2.0
3
+ pytest-cov==7.1.0
4
+ ruff==0.11.13
5
+ mypy==1.15.0
6
+ PyYAML==6.0.3
requirements.txt CHANGED
@@ -1,111 +1,27 @@
1
- aiohappyeyeballs
2
- aiohttp
3
- aiosignal
4
- annotated-doc==0.0.4
5
- annotated-types==0.7.0
6
  anyio==4.13.0
7
- attrs==26.1.0
8
- certifi==2026.5.20
9
- cffi==2.0.0
10
- charset-normalizer==3.4.7
11
- click==8.4.1
12
- cryptography==49.0.0
13
- dotenv==0.9.9
14
  filelock==3.29.4
15
- frozenlist
16
- fsspec==2026.4.0
17
  google-genai==2.8.0
18
- grpcio==1.81.1
19
- h11==0.16.0
20
- h2==4.3.0
21
- hf-xet==1.5.1
22
- hpack==4.1.0
23
- httpcore==1.0.9
24
  httpx==0.28.1
25
- httpx-sse
26
- huggingface_hub==1.19.0
27
- hyperframe==6.1.0
28
- idna==3.18
29
- Jinja2==3.1.6
30
- joblib==1.5.3
31
- jsonpatch
32
- jsonpointer
33
- langchain
34
- langchain-classic
35
- langchain-community
36
- langchain-core
37
- langchain-experimental
38
- langchain-protocol
39
- langchain-text-splitters
40
- langgraph
41
- langgraph-checkpoint
42
- langgraph-prebuilt
43
- langgraph-sdk
44
- langsmith
45
- markdown-it-py==4.2.0
46
- MarkupSafe==3.0.3
47
- mdurl==0.1.2
48
- mpmath==1.3.0
49
- multidict
50
- narwhals==2.22.1
51
  neo4j==6.2.0
52
- networkx==3.6.1
53
  numpy==2.4.6
54
  opencv-python==4.13.0.92
55
- orjson
56
- ormsgpack
57
- packaging==26.2
58
  pdf2image==1.17.0
59
- pdfminer.six==20260107
60
  pdfplumber==0.11.10
61
  pillow==12.2.0
62
- portalocker==3.2.0
63
- propcache
64
- protobuf==7.35.1
65
- pycparser==3.0
66
- pycryptodome
67
  pydantic==2.13.4
68
- pydantic-settings>=2,<3
69
- pydantic_core==2.46.4
70
- Pygments==2.20.0
71
- pypdfium2==5.9.0
72
  pytesseract==0.3.13
73
- python-dotenv==1.2.2
74
- pytz==2026.2
75
- PyYAML==6.0.3
76
  qdrant-client==1.18.0
77
- regex==2026.5.9
78
  requests==2.34.2
79
- requests-toolbelt
80
- rich==15.0.0
81
- safetensors==0.8.0
82
- scikit-learn==1.9.0
83
- scipy==1.17.1
84
  sentence-transformers==5.5.1
85
- shellescape
86
- shellingham==1.5.4
87
- SQLAlchemy
88
- sympy==1.14.0
89
  symspellpy==6.9.0
90
  tenacity==9.1.4
91
- threadpoolctl==3.6.0
92
- tokenizers==0.22.2
93
  torch==2.12.0
94
- torchvision==0.27.0
95
- tqdm==4.68.2
96
  transformers==5.12.0
97
- typer==0.25.1
98
- typing_extensions==4.15.0
99
- typing-inspection==0.4.2
100
- urllib3==2.7.0
101
- uuid_utils
102
- websockets==16.0
103
- xxhash
104
- yarl
105
- zstandard
106
- streamlit>=1.40.0
107
- fastapi>=0.115,<1
108
- uvicorn>=0.30,<1
109
- pytest>=8,<10
110
- pytest-cov>=5,<8
111
- rank-bm25>=0.2.2,<0.3
 
1
+ # Runtime dependencies; exact pins match the verified development environment.
 
 
 
 
2
  anyio==4.13.0
3
+ fastapi==0.163.1
 
 
 
 
 
 
4
  filelock==3.29.4
 
 
5
  google-genai==2.8.0
 
 
 
 
 
 
6
  httpx==0.28.1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  neo4j==6.2.0
 
8
  numpy==2.4.6
9
  opencv-python==4.13.0.92
 
 
 
10
  pdf2image==1.17.0
 
11
  pdfplumber==0.11.10
12
  pillow==12.2.0
 
 
 
 
 
13
  pydantic==2.13.4
14
+ pydantic-settings==2.16.3
 
 
 
15
  pytesseract==0.3.13
 
 
 
16
  qdrant-client==1.18.0
17
+ rank-bm25==0.2.2
18
  requests==2.34.2
 
 
 
 
 
19
  sentence-transformers==5.5.1
20
+ streamlit==1.58.0
 
 
 
21
  symspellpy==6.9.0
22
  tenacity==9.1.4
23
+ # Pin the model backend as well as its wrapper for compatibility.
 
24
  torch==2.12.0
 
 
25
  transformers==5.12.0
26
+ tqdm==4.68.2
27
+ uvicorn==0.49.0
 
 
 
 
 
 
 
 
 
 
 
 
 
results/corpus_stats.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "measured_at_utc": "2026-11-17T03:06:41.993006+00:00",
3
+ "collection": "legal_rag",
4
+ "status": "unavailable",
5
+ "dependencies": {
6
+ "qdrant": {
7
+ "status": "unavailable",
8
+ "error_type": "ResponseHandlingException"
9
+ },
10
+ "neo4j": {
11
+ "status": "unavailable",
12
+ "error_type": "ValueError"
13
+ }
14
+ },
15
+ "acts": [
16
+ {
17
+ "act": "BNS",
18
+ "chunks": null,
19
+ "graph_nodes": null,
20
+ "graph_edges": null
21
+ },
22
+ {
23
+ "act": "BNSS",
24
+ "chunks": null,
25
+ "graph_nodes": null,
26
+ "graph_edges": null
27
+ },
28
+ {
29
+ "act": "BSA",
30
+ "chunks": null,
31
+ "graph_nodes": null,
32
+ "graph_edges": null
33
+ },
34
+ {
35
+ "act": "Constitution",
36
+ "chunks": null,
37
+ "graph_nodes": null,
38
+ "graph_edges": null
39
+ },
40
+ {
41
+ "act": "Unmapped",
42
+ "chunks": null,
43
+ "graph_nodes": null,
44
+ "graph_edges": null
45
+ }
46
+ ],
47
+ "totals": {
48
+ "chunks": null,
49
+ "graph_nodes": null,
50
+ "graph_edges": null
51
+ }
52
+ }
results/corpus_stats.md ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Corpus statistics — unavailable
2
+
3
+ Attempted at 2026-11-17T03:06:41.993006+00:00; Qdrant collection `legal_rag`.
4
+
5
+ | Act | Unique stored chunks | Graph nodes | Directed graph edges |
6
+ |---|---:|---:|---:|
7
+ | BNS | Not measured | Not measured | Not measured |
8
+ | BNSS | Not measured | Not measured | Not measured |
9
+ | BSA | Not measured | Not measured | Not measured |
10
+ | Constitution | Not measured | Not measured | Not measured |
11
+ | Unmapped | Not measured | Not measured | Not measured |
12
+ | Total | Not measured | Not measured | Not measured |
13
+
14
+ Unavailable counts are not zeros. Chunk counts use distinct stored chunk_id values; graph edges include every stored relationship, including inverse hierarchy edges, and are grouped by their source node's act. Unmapped metadata is retained. Freeze ingestion while collecting these independent read-only counts.
15
+
16
+ - qdrant: unavailable (ResponseHandlingException).
17
+
18
+ - neo4j: unavailable (ValueError).
retrieval/lexical.py CHANGED
@@ -41,7 +41,7 @@ class BM25Retriever:
41
 
42
  def __init__(self, hits: Iterable[RetrievalHit]) -> None:
43
  """Index text plus stored IDs, excluding empty or unidentified chunks."""
44
- unique = {}
45
  for hit in hits:
46
  identifier = hit.payload.get("chunk_id")
47
  text = hit.payload.get("enriched_text") or hit.payload.get("text")
@@ -75,7 +75,7 @@ def reciprocal_rank_fusion(rankings: Sequence[Sequence[Any]], k: int = 60, limit
75
  if k < 1:
76
  raise ValueError("RRF k must be positive")
77
  scores: dict[str, float] = {}
78
- sources = {}
79
  for ranking in rankings:
80
  seen = set()
81
  for rank, point in enumerate(ranking, 1):
@@ -108,6 +108,12 @@ def act_code(text: str) -> str:
108
  return "CONST"
109
 
110
 
 
 
 
 
 
 
111
  def explicit_provisions(query: str) -> list[list[str]]:
112
  """Parse qualified section/article references; never guess an act from a bare number."""
113
  acts = list(re.finditer(rf"\b({ACT_PATTERN})\b", query, re.I))
@@ -115,7 +121,7 @@ def explicit_provisions(query: str) -> list[list[str]]:
115
  unique_acts = {act_code(match[1]) for match in acts}
116
  qualified = []
117
  for reference in references:
118
- number = reference[1]
119
  code = None
120
  # In multi-act queries, require an adjacent act with only linking words.
121
  for match in acts:
@@ -128,7 +134,7 @@ def explicit_provisions(query: str) -> list[list[str]]:
128
  if code is None:
129
  continue
130
  base = number.split("(", 1)[0].upper()
131
- suffix = re.sub(r"\(([^)]+)\)", lambda match: "(" + (match[1].upper() if match[1][0].isdigit() else match[1].lower()) + ")", number[len(base):])
132
  aliases = [f"{code}-{base}{suffix}"]
133
  if code == "CONST":
134
  parts = re.findall(r"\(([^)]+)\)", suffix)
 
41
 
42
  def __init__(self, hits: Iterable[RetrievalHit]) -> None:
43
  """Index text plus stored IDs, excluding empty or unidentified chunks."""
44
+ unique: dict[str, RetrievalHit] = {}
45
  for hit in hits:
46
  identifier = hit.payload.get("chunk_id")
47
  text = hit.payload.get("enriched_text") or hit.payload.get("text")
 
75
  if k < 1:
76
  raise ValueError("RRF k must be positive")
77
  scores: dict[str, float] = {}
78
+ sources: dict[str, Any] = {}
79
  for ranking in rankings:
80
  seen = set()
81
  for rank, point in enumerate(ranking, 1):
 
108
  return "CONST"
109
 
110
 
111
+ def _normalized_clause(match: re.Match[str]) -> str:
112
+ """Preserve numeric clause suffix case while normalizing alphabetic labels."""
113
+ value = match.group(1)
114
+ return "(" + (value.upper() if value[0].isdigit() else value.lower()) + ")"
115
+
116
+
117
  def explicit_provisions(query: str) -> list[list[str]]:
118
  """Parse qualified section/article references; never guess an act from a bare number."""
119
  acts = list(re.finditer(rf"\b({ACT_PATTERN})\b", query, re.I))
 
121
  unique_acts = {act_code(match[1]) for match in acts}
122
  qualified = []
123
  for reference in references:
124
+ number: str = reference[1]
125
  code = None
126
  # In multi-act queries, require an adjacent act with only linking words.
127
  for match in acts:
 
134
  if code is None:
135
  continue
136
  base = number.split("(", 1)[0].upper()
137
+ suffix = re.sub(r"\(([^)]+)\)", _normalized_clause, number[len(base):])
138
  aliases = [f"{code}-{base}{suffix}"]
139
  if code == "CONST":
140
  parts = re.findall(r"\(([^)]+)\)", suffix)
scripts/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Operational entry points; importing this package never starts ingestion."""
scripts/corpus_stats.py ADDED
@@ -0,0 +1,157 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Measure stored corpus counts without loading models or fabricating unavailable totals."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ from collections.abc import Callable
8
+ from datetime import datetime, timezone
9
+ from math import ceil
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ from eval.storage import atomic_text
14
+ from legora.config import EvaluationSettings
15
+ from legora.performance import call_external
16
+ from retrieval.lexical import document_name
17
+
18
+ ACTS = ("BNS", "BNSS", "BSA", "Constitution", "Unmapped")
19
+
20
+
21
+ def act_name(value: object) -> str:
22
+ """Map stored act metadata to report rows, preserving unknown data in its own row."""
23
+ return {"BNS": "BNS", "BNSS": "BNSS", "BSA": "BSA", "CONST": "Constitution",
24
+ "CONSTITUTION": "Constitution"}.get(str(value).strip().upper(), "Unmapped")
25
+
26
+
27
+ def vector_counts(client: Any, collection: str) -> dict:
28
+ """Scroll all stored points; count distinct chunk IDs and expose duplicate/missing IDs."""
29
+ buckets: dict[str, set[str]] = {act: set() for act in ACTS}
30
+ owners: dict[str, str] = {}
31
+ points = missing = 0
32
+ offset = None
33
+ while True:
34
+ records, offset = call_external(
35
+ client.scroll, collection_name=collection, offset=offset, limit=256,
36
+ with_payload=True, with_vectors=False,
37
+ )
38
+ for record in records:
39
+ points += 1
40
+ payload = record.payload or {}
41
+ identifier = payload.get("chunk_id")
42
+ if not isinstance(identifier, str) or not identifier.strip():
43
+ missing += 1
44
+ continue
45
+ act = act_name(document_name(payload))
46
+ if identifier in owners and owners[identifier] != act:
47
+ raise ValueError("Conflicting act metadata for a stored chunk ID")
48
+ owners[identifier] = act
49
+ buckets[act].add(identifier)
50
+ if offset is None:
51
+ break
52
+ return {"chunks_by_act": {act: len(values) for act, values in buckets.items()},
53
+ "stored_points": points, "unique_chunks": len(owners),
54
+ "points_without_chunk_id": missing, "duplicate_chunk_points": points - missing - len(owners)}
55
+
56
+
57
+ def graph_counts(graph: Any) -> dict:
58
+ """Count all stored nodes and directed edges, assigning each edge to its source act."""
59
+ results = {}
60
+ for field, query in (
61
+ ("nodes_by_act", "MATCH (n) RETURN n.document AS document, count(n) AS count"),
62
+ ("edges_by_act", "MATCH (a)-[r]->() RETURN a.document AS document, count(r) AS count"),
63
+ ):
64
+ counts = dict.fromkeys(ACTS, 0)
65
+ for row in graph.run_query(query):
66
+ count = row["count"]
67
+ if isinstance(count, bool) or not isinstance(count, int) or count < 0:
68
+ raise ValueError("Invalid graph count")
69
+ counts[act_name(row["document"])] += count
70
+ results[field] = counts
71
+ return results
72
+
73
+
74
+ def collect_stats(
75
+ vector_factory: Callable[[], Any], graph_factory: Callable[[], Any], collection: str,
76
+ ) -> dict:
77
+ """Measure dependencies independently; failures produce null counts, never zero estimates."""
78
+ dependencies = {}
79
+ for name, factory, measure, close in (
80
+ ("qdrant", vector_factory, lambda client: vector_counts(client, collection), lambda client: client.close()),
81
+ ("neo4j", graph_factory, graph_counts, lambda graph: graph.close()),
82
+ ):
83
+ resource = None
84
+ try:
85
+ resource = factory()
86
+ dependencies[name] = {"status": "ok", **measure(resource)}
87
+ except Exception as error:
88
+ dependencies[name] = {"status": "unavailable", "error_type": type(error).__name__}
89
+ finally:
90
+ if resource is not None:
91
+ try:
92
+ close(resource)
93
+ except Exception as error:
94
+ dependencies[name]["close_error_type"] = type(error).__name__
95
+ available = sum(dependency["status"] == "ok" for dependency in dependencies.values())
96
+ rows = [{"act": act,
97
+ "chunks": dependencies["qdrant"].get("chunks_by_act", {}).get(act),
98
+ "graph_nodes": dependencies["neo4j"].get("nodes_by_act", {}).get(act),
99
+ "graph_edges": dependencies["neo4j"].get("edges_by_act", {}).get(act)} for act in ACTS]
100
+ totals = {key: sum(row[key] for row in rows) if all(row[key] is not None for row in rows) else None
101
+ for key in ("chunks", "graph_nodes", "graph_edges")}
102
+ return {"measured_at_utc": datetime.now(timezone.utc).isoformat(), "collection": collection,
103
+ "status": "completed" if available == 2 else "partial" if available else "unavailable",
104
+ "dependencies": dependencies, "acts": rows, "totals": totals}
105
+
106
+
107
+ def write_report(directory: Path, result: dict) -> str:
108
+ """Persist real counts or explicit failure provenance, without endpoint URLs or credentials."""
109
+ report = (f"# Corpus statistics — {result['status']}\n\n"
110
+ f"Attempted at {result['measured_at_utc']}; Qdrant collection `{result['collection']}`.\n\n"
111
+ "| Act | Unique stored chunks | Graph nodes | Directed graph edges |\n|---|---:|---:|---:|\n")
112
+ for row in [*result["acts"], {"act": "Total", **result["totals"]}]:
113
+ values = ["Not measured" if row[key] is None else str(row[key]) for key in ("chunks", "graph_nodes", "graph_edges")]
114
+ report += f"| {row['act']} | " + " | ".join(values) + " |\n"
115
+ report += "\nUnavailable counts are not zeros. Chunk counts use distinct stored chunk_id values; graph edges include every stored relationship, including inverse hierarchy edges, and are grouped by their source node's act. Unmapped metadata is retained. Freeze ingestion while collecting these independent read-only counts.\n"
116
+ for name, dependency in result["dependencies"].items():
117
+ report += f"\n- {name}: {dependency['status']}" + (f" ({dependency['error_type']})" if "error_type" in dependency else "") + ".\n"
118
+ if result["dependencies"]["qdrant"]["status"] == "ok":
119
+ vector = result["dependencies"]["qdrant"]
120
+ report += f"\nStored points: {vector['stored_points']}; unique chunk IDs: {vector['unique_chunks']}; duplicate chunk points: {vector['duplicate_chunk_points']}; points without chunk IDs: {vector['points_without_chunk_id']}.\n"
121
+ atomic_text(directory / "corpus_stats.json", json.dumps(result, indent=2) + "\n")
122
+ atomic_text(directory / "corpus_stats.md", report)
123
+ return report
124
+
125
+
126
+ def main() -> None:
127
+ """Load only service clients, measure live storage and exit nonzero on incomplete counts."""
128
+ parser = argparse.ArgumentParser(description=__doc__)
129
+ parser.add_argument("--collection", default="legal_rag")
130
+ parser.add_argument("--output-dir", type=Path, default=Path("results"))
131
+ args = parser.parse_args()
132
+ settings = EvaluationSettings()
133
+
134
+ def vector_factory() -> Any:
135
+ """Create a bounded Qdrant connection without requiring model or generation keys."""
136
+ from qdrant_client import QdrantClient
137
+
138
+ if not settings.qdrant_url or not settings.qdrant_url.strip():
139
+ raise ValueError("QDRANT_URL is required")
140
+ return QdrantClient(url=settings.qdrant_url,
141
+ api_key=settings.qdrant_api_key.get_secret_value() if settings.qdrant_api_key else None,
142
+ timeout=ceil(settings.external_timeout_seconds), check_compatibility=False)
143
+
144
+ def graph_factory() -> Any:
145
+ """Create only the graph client, validating its required connection configuration."""
146
+ from db.neo4j_store import Neo4jStore
147
+
148
+ return Neo4jStore(settings=settings)
149
+
150
+ result = collect_stats(vector_factory, graph_factory, args.collection)
151
+ print(write_report(args.output_dir, result))
152
+ if result["status"] != "completed":
153
+ parser.exit(1, "Corpus statistics incomplete; unavailable counts were left null.\n")
154
+
155
+
156
+ if __name__ == "__main__":
157
+ main()
scripts/ingest.py ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Validate and upsert all four statutes using the existing parsers and graph builders."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import logging
7
+ from dataclasses import dataclass
8
+ from importlib import import_module
9
+ from pathlib import Path
10
+ from typing import Any
11
+ from uuid import NAMESPACE_URL, uuid5
12
+
13
+ from legora.logging import setup_logging
14
+
15
+ logger = logging.getLogger(__name__)
16
+ STATUTES = {"bns": "BNS", "bnss": "BNSS", "bsa": "BSA", "constitution": "Constitution"}
17
+
18
+
19
+ @dataclass
20
+ class ParsedStatute:
21
+ """One parsed document shared by vector chunking and graph generation."""
22
+
23
+ name: str
24
+ document: Any
25
+ chunks: list[dict[str, Any]]
26
+
27
+
28
+ def source_text(directory: Path, name: str) -> str:
29
+ """Prefer reviewed UTF-8 text; extract text PDFs without silently applying OCR."""
30
+ text_path, pdf_path = directory / f"{name}.txt", directory / f"{name}.pdf"
31
+ if text_path.is_file():
32
+ text = text_path.read_text(encoding="utf-8")
33
+ elif pdf_path.is_file():
34
+ import pdfplumber
35
+
36
+ with pdfplumber.open(pdf_path) as document:
37
+ pages = [page.extract_text() or "" for page in document.pages]
38
+ if any(not page.strip() for page in pages):
39
+ raise ValueError(f"{name}.pdf contains pages without text; supply reviewed OCR text as {name}.txt")
40
+ text = "\n\n".join(pages)
41
+ else:
42
+ raise ValueError(f"Missing {name}.txt or {name}.pdf in the data directory")
43
+ if not text.strip():
44
+ raise ValueError(f"Empty source for {name}")
45
+ return text
46
+
47
+
48
+ def prepare_corpus(directory: Path) -> list[ParsedStatute]:
49
+ """Parse all inputs and reject empty/duplicate chunks before opening any service client."""
50
+ texts = {name: source_text(directory, name) for name in STATUTES}
51
+ prepared, seen = [], set()
52
+ for name, prefix in STATUTES.items():
53
+ cleaner = getattr(import_module(f"db.parsers.{name}.cleaner"), f"{prefix}TextCleaner")()
54
+ parser = getattr(import_module(f"db.parsers.{name}.{name}_parser"), f"{prefix}Parser")()
55
+ document = parser.parse(cleaner.clean(texts[name]))
56
+ module = import_module(f"db.parsers.{name}.chunker")
57
+ chunker = module.LegalChunker()
58
+ chunks = module.chunks_to_dicts(chunker.chunk_constitution(document) if name == "constitution" else chunker.chunk_document(document))
59
+ if not chunks:
60
+ raise ValueError(f"No provisions parsed for {name}; inspect PDF layout or supply reviewed text")
61
+ for chunk in chunks:
62
+ identifier = chunk.get("chunk_id")
63
+ if not identifier or identifier in seen or not chunk.get("text", "").strip():
64
+ raise ValueError(f"Invalid or duplicate chunks for {name}")
65
+ seen.add(identifier)
66
+ prepared.append(ParsedStatute(name, document, chunks))
67
+ logger.info("Validated %s: %s chunks", name, len(chunks))
68
+ return prepared
69
+
70
+
71
+ def ingest_corpus(corpus: list[ParsedStatute], embedder: Any, vectors: Any, graph: Any,
72
+ batch_size: int = 16) -> None:
73
+ """Upsert bounded embedding batches and hierarchy nodes; never clear existing stores."""
74
+ if batch_size < 1:
75
+ raise ValueError("batch_size must be positive")
76
+ vectors.create_collection(embedder.vector_size())
77
+ graph.create_constraints()
78
+ for statute in corpus:
79
+ builder_name = f"{STATUTES[statute.name]}GraphBuilder"
80
+ builder = getattr(import_module(f"db.graph_builders.{statute.name}_graph_builder"), builder_name)(graph)
81
+ builder.build(statute.document)
82
+ for start in range(0, len(statute.chunks), batch_size):
83
+ batch = statute.chunks[start:start + batch_size]
84
+ embeddings = embedder.embed([chunk.get("enriched_text") or chunk["text"] for chunk in batch])
85
+ if len(embeddings) != len(batch):
86
+ raise ValueError("Embedding count does not match input chunks")
87
+ vectors.upsert_points([
88
+ {"id": str(uuid5(NAMESPACE_URL, "legora:" + chunk["chunk_id"])),
89
+ "vector": embedding.tolist(), "payload": chunk}
90
+ for chunk, embedding in zip(batch, embeddings)
91
+ ])
92
+ logger.info("Ingested %s: %s chunks", statute.name, len(statute.chunks))
93
+
94
+
95
+ def main() -> None:
96
+ """Offer source validation without models/network and explicit ingestion with client cleanup."""
97
+ parser = argparse.ArgumentParser(description=__doc__)
98
+ parser.add_argument("--data-dir", type=Path, default=Path("data/statutes"))
99
+ parser.add_argument("--batch-size", type=int, default=16)
100
+ parser.add_argument("--check-only", action="store_true")
101
+ args = parser.parse_args()
102
+ setup_logging()
103
+ vectors = graph = None
104
+ try:
105
+ if args.batch_size < 1:
106
+ raise ValueError("batch_size must be positive")
107
+ corpus = prepare_corpus(args.data_dir)
108
+ if args.check_only:
109
+ return
110
+ from db.embedder import LegalEmbedder
111
+ from db.neo4j_store import Neo4jStore
112
+ from db.vector_store import QdrantStore
113
+ from legora.config import EvaluationSettings
114
+
115
+ settings = EvaluationSettings()
116
+ graph = Neo4jStore(settings=settings)
117
+ vectors = QdrantStore(settings=settings)
118
+ ingest_corpus(corpus, LegalEmbedder(), vectors, graph, args.batch_size)
119
+ except Exception as error:
120
+ message = str(error) if type(error) is ValueError and vectors is None and graph is None else type(error).__name__
121
+ parser.exit(1, f"Ingestion failed: {message}. No successful completion is claimed.\n")
122
+ finally:
123
+ if vectors is not None:
124
+ vectors.client.close()
125
+ if graph is not None:
126
+ graph.close()
127
+
128
+
129
+ if __name__ == "__main__":
130
+ main()
scripts/ingest.sh ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ #!/bin/sh
2
+ # Run from any directory; Compose loads the project .env without sourcing shell code.
3
+ set -eu
4
+ cd "$(dirname "$0")/.."
5
+ docker compose up -d --wait qdrant neo4j
6
+ docker compose run --rm --build --no-deps api python -m scripts.ingest --data-dir /data/statutes "$@"
tests/test_corpus_stats.py ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Verify measured corpus statistics without network calls or production fixtures."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from types import SimpleNamespace
6
+ from unittest.mock import Mock
7
+
8
+ import pytest
9
+
10
+ from scripts.corpus_stats import collect_stats, graph_counts, vector_counts, write_report
11
+
12
+
13
+ def point(identifier: str | None, document: str | None = None) -> SimpleNamespace:
14
+ """Represent a stored payload, including deliberately malformed rows."""
15
+ return SimpleNamespace(payload={"chunk_id": identifier, "document": document})
16
+
17
+
18
+ def test_vector_scroll_counts_unique_chunks_duplicates_missing_and_unmapped() -> None:
19
+ """All pages and legacy IDs count, while duplicate points do not inflate chunk totals."""
20
+ client = Mock()
21
+ client.scroll.side_effect = [
22
+ ([point("BNS-303", "bns"), point("BNS-303", "bns"), point(None)], "next"),
23
+ ([point("article_21"), point("unknown-id"), point("BNSS-45(2)", "bnss")], None),
24
+ ]
25
+ result = vector_counts(client, "legal_rag")
26
+ assert result["stored_points"] == 6
27
+ assert result["unique_chunks"] == 4
28
+ assert result["duplicate_chunk_points"] == 1
29
+ assert result["points_without_chunk_id"] == 1
30
+ assert result["chunks_by_act"] == {"BNS": 1, "BNSS": 1, "BSA": 0, "Constitution": 1, "Unmapped": 1}
31
+ assert client.scroll.call_args.kwargs["offset"] == "next"
32
+ assert client.scroll.call_args.kwargs["with_vectors"] is False
33
+
34
+
35
+ def test_conflicting_chunk_metadata_is_not_reported_as_a_valid_total() -> None:
36
+ """A duplicate ID assigned to two acts is data corruption, not two distinct chunks."""
37
+ client = Mock()
38
+ client.scroll.return_value = ([point("BNS-303", "bns"), point("BNS-303", "bsa")], None)
39
+ with pytest.raises(ValueError):
40
+ vector_counts(client, "legal_rag")
41
+
42
+
43
+ def test_graph_counts_merge_metadata_aliases_and_count_edges_by_source() -> None:
44
+ """Graph counts include unmapped nodes and actual directed edges, without deduping inverses."""
45
+ graph = Mock()
46
+ graph.run_query.side_effect = [
47
+ [{"document": "BNS", "count": 12}, {"document": "CONSTITUTION", "count": 3}, {"document": "CONST", "count": 2}, {"document": None, "count": 1}],
48
+ [{"document": "BNS", "count": 20}, {"document": "CONSTITUTION", "count": 5}],
49
+ ]
50
+ result = graph_counts(graph)
51
+ assert result["nodes_by_act"]["Constitution"] == 5
52
+ assert result["nodes_by_act"]["Unmapped"] == 1
53
+ assert result["edges_by_act"]["BNS"] == 20
54
+ assert "a.document AS document" in graph.run_query.call_args.args[0]
55
+
56
+
57
+ @pytest.mark.parametrize("invalid", [-1, True, 2.5, "3"])
58
+ def test_invalid_graph_counts_are_rejected(invalid: object) -> None:
59
+ """Malformed upstream counts cannot appear as measured corpus statistics."""
60
+ graph = Mock()
61
+ graph.run_query.return_value = [{"document": "BNS", "count": invalid}]
62
+ with pytest.raises(ValueError):
63
+ graph_counts(graph)
64
+
65
+
66
+ def test_unavailable_dependencies_produce_nulls_without_secret_messages(tmp_path: Path) -> None:
67
+ """Failed reads must not imply an empty corpus or leak endpoint credentials."""
68
+ result = collect_stats(Mock(side_effect=ValueError("secret endpoint")), Mock(side_effect=ConnectionError("secret password")), "legal_rag")
69
+ assert result["status"] == "unavailable"
70
+ assert all(value is None for value in result["totals"].values())
71
+ report = write_report(tmp_path, result)
72
+ assert "Not measured" in report and "| Total |" in report
73
+ assert "secret" not in report and "secret" not in (tmp_path / "corpus_stats.json").read_text()
74
+ assert json.loads((tmp_path / "corpus_stats.json").read_text())["totals"]["chunks"] is None
75
+
76
+
77
+ def test_partial_measurement_preserves_successful_zero_counts_and_closes_client() -> None:
78
+ """An actually empty store reports zero; an inaccessible other store remains unknown."""
79
+ client = Mock()
80
+ client.scroll.return_value = ([], None)
81
+ result = collect_stats(lambda: client, Mock(side_effect=ValueError()), "legal_rag")
82
+ assert result["status"] == "partial"
83
+ assert result["totals"] == {"chunks": 0, "graph_nodes": None, "graph_edges": None}
84
+ client.close.assert_called_once_with()
85
+
86
+
87
+ def test_complete_live_shaped_counts_compute_totals_without_model_loading() -> None:
88
+ """Only successful reads from both stores produce a completed report."""
89
+ client, graph = Mock(), Mock()
90
+ client.scroll.return_value = ([point("BSA-115", "bsa")], None)
91
+ graph.run_query.side_effect = [[{"document": "BSA", "count": 4}], [{"document": "BSA", "count": 6}]]
92
+ result = collect_stats(lambda: client, lambda: graph, "legal_rag")
93
+ assert result["status"] == "completed"
94
+ assert result["totals"] == {"chunks": 1, "graph_nodes": 4, "graph_edges": 6}
95
+ graph.close.assert_called_once_with()
96
+
97
+
98
+ def test_failed_scroll_closes_client_without_reporting_partial_counts() -> None:
99
+ """A paginated failure does not publish the first page as the whole corpus."""
100
+ client = Mock()
101
+ client.scroll.side_effect = [([point("BNS-303", "bns")], "next"), ValueError("failed page")]
102
+ result = collect_stats(lambda: client, Mock(side_effect=ValueError()), "legal_rag")
103
+ assert result["totals"]["chunks"] is None
104
+ client.close.assert_called_once_with()
tests/test_ingestion.py ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Exercise real statute parsing, graph building and ingestion without external services."""
2
+
3
+ from pathlib import Path
4
+ from unittest.mock import Mock
5
+ from uuid import UUID
6
+
7
+ import numpy as np
8
+ import pytest
9
+
10
+ from scripts.ingest import ingest_corpus, main, prepare_corpus, source_text
11
+
12
+ FIXTURES = Path(__file__).parent / "fixtures"
13
+
14
+
15
+ def clients() -> tuple[Mock, Mock, Mock]:
16
+ """Replace only embedding computation and persistence clients."""
17
+ embedder, vectors, graph = Mock(), Mock(), Mock()
18
+ embedder.vector_size.return_value = 2
19
+ embedder.embed.side_effect = lambda texts: np.ones((len(texts), 2))
20
+ return embedder, vectors, graph
21
+
22
+
23
+ def test_real_corpus_and_graph_builders_use_same_documents_with_bounded_batches() -> None:
24
+ """All four statutes reach vectors and graph with stable globally unique chunk identities."""
25
+ corpus = prepare_corpus(FIXTURES)
26
+ embedder, vectors, graph = clients()
27
+ ingest_corpus(corpus, embedder, vectors, graph, batch_size=3)
28
+ assert {statute.name for statute in corpus} == {"bns", "bnss", "bsa", "constitution"}
29
+ assert all(len(call.args[0]) <= 3 for call in embedder.embed.call_args_list)
30
+ points = [point for call in vectors.upsert_points.call_args_list for point in call.args[0]]
31
+ assert len(points) == sum(len(statute.chunks) for statute in corpus)
32
+ assert len({point["id"] for point in points}) == len(points)
33
+ assert all(UUID(point["id"]).version == 5 for point in points)
34
+ assert {"BNS-303", "BNSS-35", "BSA-2"} <= {point["payload"]["chunk_id"] for point in points}
35
+ nodes = {call.kwargs.get("node_id", call.args[1] if call.args else "") for call in graph.merge_node.call_args_list}
36
+ assert {"BNS-303", "BNSS-35", "BSA-2", "CONST-19"} <= nodes
37
+ vectors.create_collection.assert_called_once_with(2)
38
+ graph.create_constraints.assert_called_once_with()
39
+ vectors.recreate_collection.assert_not_called()
40
+ graph.clear_graph.assert_not_called()
41
+
42
+
43
+ def test_repeat_ingestion_keeps_uuid_ids_even_if_source_order_changes() -> None:
44
+ """Reingestion upserts identical provision IDs rather than renumbering other statutes."""
45
+ corpus = prepare_corpus(FIXTURES)
46
+ embedder, vectors, graph = clients()
47
+ ingest_corpus(corpus, embedder, vectors, graph)
48
+ before = {point["payload"]["chunk_id"]: point["id"] for call in vectors.upsert_points.call_args_list for point in call.args[0]}
49
+ vectors.upsert_points.reset_mock()
50
+ ingest_corpus(list(reversed(corpus)), embedder, vectors, graph)
51
+ after = {point["payload"]["chunk_id"]: point["id"] for call in vectors.upsert_points.call_args_list for point in call.args[0]}
52
+ assert before == after
53
+
54
+
55
+ def test_embedding_count_mismatch_is_failure_instead_of_silent_zip_truncation() -> None:
56
+ """No incomplete embedding batch is uploaded as if it succeeded."""
57
+ corpus = prepare_corpus(FIXTURES)
58
+ embedder, vectors, graph = clients()
59
+ embedder.embed.side_effect = lambda texts: np.ones((len(texts) - 1, 2))
60
+ with pytest.raises(ValueError, match="Embedding count"):
61
+ ingest_corpus(corpus, embedder, vectors, graph)
62
+ vectors.upsert_points.assert_not_called()
63
+
64
+
65
+ @pytest.mark.parametrize("batch_size", [0, -1])
66
+ def test_invalid_batch_size_never_mutates_stores(batch_size: int) -> None:
67
+ """Invalid configuration fails before collection/graph creation."""
68
+ embedder, vectors, graph = clients()
69
+ with pytest.raises(ValueError):
70
+ ingest_corpus([], embedder, vectors, graph, batch_size)
71
+ vectors.create_collection.assert_not_called()
72
+ graph.create_constraints.assert_not_called()
73
+
74
+
75
+ def test_source_text_prefers_reviewed_text_and_rejects_empty_or_missing(tmp_path: Path) -> None:
76
+ """PDF extraction never replaces a reviewer-supplied UTF-8 source."""
77
+ (tmp_path / "bns.txt").write_text("Reviewed statute text")
78
+ (tmp_path / "bns.pdf").write_bytes(b"test-only-placeholder")
79
+ assert source_text(tmp_path, "bns") == "Reviewed statute text"
80
+ (tmp_path / "bns.txt").write_text(" \n")
81
+ with pytest.raises(ValueError, match="Empty"):
82
+ source_text(tmp_path, "bns")
83
+ with pytest.raises(ValueError, match="Missing"):
84
+ source_text(tmp_path, "bnss")
85
+
86
+
87
+ def test_pdf_extraction_rejects_scanned_pages(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
88
+ """Image-only pages require reviewed OCR, rather than silently discarding legal text."""
89
+ import pdfplumber
90
+
91
+ (tmp_path / "bns.pdf").write_bytes(b"test-only-placeholder")
92
+ pdf = Mock()
93
+ pdf.__enter__ = Mock(return_value=pdf)
94
+ pdf.__exit__ = Mock(return_value=False)
95
+ page = Mock()
96
+ page.extract_text.return_value = "Section text"
97
+ pdf.pages = [page]
98
+ monkeypatch.setattr(pdfplumber, "open", Mock(return_value=pdf))
99
+ assert source_text(tmp_path, "bns") == "Section text"
100
+ page.extract_text.return_value = None
101
+ with pytest.raises(ValueError, match="without text"):
102
+ source_text(tmp_path, "bns")
103
+
104
+
105
+ def test_check_only_parses_without_clients(monkeypatch: pytest.MonkeyPatch) -> None:
106
+ """Source validation can run while containers and model downloads are unavailable."""
107
+ factory = Mock(side_effect=AssertionError("No store startup"))
108
+ monkeypatch.setattr("db.neo4j_store.Neo4jStore", factory)
109
+ monkeypatch.setattr("sys.argv", ["ingest", "--check-only", "--data-dir", str(FIXTURES)])
110
+ main()
111
+ factory.assert_not_called()
112
+
113
+
114
+ def test_missing_source_fails_before_client_startup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
115
+ """Partial local corpora must not start an incomplete database load."""
116
+ factory = Mock(side_effect=AssertionError("No store startup"))
117
+ monkeypatch.setattr("db.neo4j_store.Neo4jStore", factory)
118
+ monkeypatch.setattr("sys.argv", ["ingest", "--data-dir", str(tmp_path)])
119
+ with pytest.raises(SystemExit) as error:
120
+ main()
121
+ assert error.value.code == 1
122
+ factory.assert_not_called()
123
+
124
+
125
+ def test_ingestion_closes_clients_after_upstream_failure(monkeypatch: pytest.MonkeyPatch) -> None:
126
+ """A failed write exits unsuccessfully and still releases both clients."""
127
+ embedder, vectors, graph = clients()
128
+ vectors.upsert_points.side_effect = RuntimeError("sensitive upstream detail")
129
+ monkeypatch.setattr("db.embedder.LegalEmbedder", Mock(return_value=embedder))
130
+ monkeypatch.setattr("db.vector_store.QdrantStore", Mock(return_value=vectors))
131
+ monkeypatch.setattr("db.neo4j_store.Neo4jStore", Mock(return_value=graph))
132
+ monkeypatch.setattr("sys.argv", ["ingest", "--data-dir", str(FIXTURES)])
133
+ with pytest.raises(SystemExit) as error:
134
+ main()
135
+ assert error.value.code == 1
136
+ graph.close.assert_called_once_with()
137
+ vectors.client.close.assert_called_once_with()