Download tests/test_ingestion.py from sai-Rohan/Legora: direct link, hf CLI and curl.
- Browser
- Download file 6.68 kB
-
https://huggingface.co/spaces/sai-Rohan/Legora/resolve/main/tests/test_ingestion.py
- Command line
-
hf download hf://spaces/sai-Rohan/Legora/tests/test_ingestion.py
-
curl -L -o test_ingestion.py https://huggingface.co/spaces/sai-Rohan/Legora/resolve/main/tests/test_ingestion.py
6.68 kB
| """Exercise real statute parsing, graph building and ingestion without external services.""" | |
| from pathlib import Path | |
| from unittest.mock import Mock | |
| from uuid import UUID | |
| import numpy as np | |
| import pytest | |
| from scripts.ingest import ingest_corpus, main, prepare_corpus, source_text | |
| FIXTURES = Path(__file__).parent / "fixtures" | |
| def clients() -> tuple[Mock, Mock, Mock]: | |
| """Replace only embedding computation and persistence clients.""" | |
| embedder, vectors, graph = Mock(), Mock(), Mock() | |
| embedder.vector_size.return_value = 2 | |
| embedder.embed.side_effect = lambda texts: np.ones((len(texts), 2)) | |
| return embedder, vectors, graph | |
| def test_real_corpus_and_graph_builders_use_same_documents_with_bounded_batches() -> None: | |
| """All four statutes reach vectors and graph with stable globally unique chunk identities.""" | |
| corpus = prepare_corpus(FIXTURES) | |
| embedder, vectors, graph = clients() | |
| ingest_corpus(corpus, embedder, vectors, graph, batch_size=3) | |
| assert {statute.name for statute in corpus} == {"bns", "bnss", "bsa", "constitution"} | |
| assert all(len(call.args[0]) <= 3 for call in embedder.embed.call_args_list) | |
| points = [point for call in vectors.upsert_points.call_args_list for point in call.args[0]] | |
| assert len(points) == sum(len(statute.chunks) for statute in corpus) | |
| assert len({point["id"] for point in points}) == len(points) | |
| assert all(UUID(point["id"]).version == 5 for point in points) | |
| assert {"BNS-303", "BNSS-35", "BSA-2"} <= {point["payload"]["chunk_id"] for point in points} | |
| nodes = {call.kwargs.get("node_id", call.args[1] if call.args else "") for call in graph.merge_node.call_args_list} | |
| assert {"BNS-303", "BNSS-35", "BSA-2", "CONST-19"} <= nodes | |
| vectors.create_collection.assert_called_once_with(2) | |
| graph.create_constraints.assert_called_once_with() | |
| vectors.recreate_collection.assert_not_called() | |
| graph.clear_graph.assert_not_called() | |
| def test_repeat_ingestion_keeps_uuid_ids_even_if_source_order_changes() -> None: | |
| """Reingestion upserts identical provision IDs rather than renumbering other statutes.""" | |
| corpus = prepare_corpus(FIXTURES) | |
| embedder, vectors, graph = clients() | |
| ingest_corpus(corpus, embedder, vectors, graph) | |
| before = {point["payload"]["chunk_id"]: point["id"] for call in vectors.upsert_points.call_args_list for point in call.args[0]} | |
| vectors.upsert_points.reset_mock() | |
| ingest_corpus(list(reversed(corpus)), embedder, vectors, graph) | |
| after = {point["payload"]["chunk_id"]: point["id"] for call in vectors.upsert_points.call_args_list for point in call.args[0]} | |
| assert before == after | |
| def test_embedding_count_mismatch_is_failure_instead_of_silent_zip_truncation() -> None: | |
| """No incomplete embedding batch is uploaded as if it succeeded.""" | |
| corpus = prepare_corpus(FIXTURES) | |
| embedder, vectors, graph = clients() | |
| embedder.embed.side_effect = lambda texts: np.ones((len(texts) - 1, 2)) | |
| with pytest.raises(ValueError, match="Embedding count"): | |
| ingest_corpus(corpus, embedder, vectors, graph) | |
| vectors.upsert_points.assert_not_called() | |
| def test_invalid_batch_size_never_mutates_stores(batch_size: int) -> None: | |
| """Invalid configuration fails before collection/graph creation.""" | |
| embedder, vectors, graph = clients() | |
| with pytest.raises(ValueError): | |
| ingest_corpus([], embedder, vectors, graph, batch_size) | |
| vectors.create_collection.assert_not_called() | |
| graph.create_constraints.assert_not_called() | |
| def test_source_text_prefers_reviewed_text_and_rejects_empty_or_missing(tmp_path: Path) -> None: | |
| """PDF extraction never replaces a reviewer-supplied UTF-8 source.""" | |
| (tmp_path / "bns.txt").write_text("Reviewed statute text") | |
| (tmp_path / "bns.pdf").write_bytes(b"test-only-placeholder") | |
| assert source_text(tmp_path, "bns") == "Reviewed statute text" | |
| (tmp_path / "bns.txt").write_text(" \n") | |
| with pytest.raises(ValueError, match="Empty"): | |
| source_text(tmp_path, "bns") | |
| with pytest.raises(ValueError, match="Missing"): | |
| source_text(tmp_path, "bnss") | |
| def test_pdf_extraction_rejects_scanned_pages(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| """Image-only pages require reviewed OCR, rather than silently discarding legal text.""" | |
| import pdfplumber | |
| (tmp_path / "bns.pdf").write_bytes(b"test-only-placeholder") | |
| pdf = Mock() | |
| pdf.__enter__ = Mock(return_value=pdf) | |
| pdf.__exit__ = Mock(return_value=False) | |
| page = Mock() | |
| page.extract_text.return_value = "Section text" | |
| pdf.pages = [page] | |
| monkeypatch.setattr(pdfplumber, "open", Mock(return_value=pdf)) | |
| assert source_text(tmp_path, "bns") == "Section text" | |
| page.extract_text.return_value = None | |
| with pytest.raises(ValueError, match="without text"): | |
| source_text(tmp_path, "bns") | |
| def test_check_only_parses_without_clients(monkeypatch: pytest.MonkeyPatch) -> None: | |
| """Source validation can run while containers and model downloads are unavailable.""" | |
| factory = Mock(side_effect=AssertionError("No store startup")) | |
| monkeypatch.setattr("db.neo4j_store.Neo4jStore", factory) | |
| monkeypatch.setattr("sys.argv", ["ingest", "--check-only", "--data-dir", str(FIXTURES)]) | |
| main() | |
| factory.assert_not_called() | |
| def test_missing_source_fails_before_client_startup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: | |
| """Partial local corpora must not start an incomplete database load.""" | |
| factory = Mock(side_effect=AssertionError("No store startup")) | |
| monkeypatch.setattr("db.neo4j_store.Neo4jStore", factory) | |
| monkeypatch.setattr("sys.argv", ["ingest", "--data-dir", str(tmp_path)]) | |
| with pytest.raises(SystemExit) as error: | |
| main() | |
| assert error.value.code == 1 | |
| factory.assert_not_called() | |
| def test_ingestion_closes_clients_after_upstream_failure(monkeypatch: pytest.MonkeyPatch) -> None: | |
| """A failed write exits unsuccessfully and still releases both clients.""" | |
| embedder, vectors, graph = clients() | |
| vectors.upsert_points.side_effect = RuntimeError("sensitive upstream detail") | |
| monkeypatch.setattr("db.embedder.LegalEmbedder", Mock(return_value=embedder)) | |
| monkeypatch.setattr("db.vector_store.QdrantStore", Mock(return_value=vectors)) | |
| monkeypatch.setattr("db.neo4j_store.Neo4jStore", Mock(return_value=graph)) | |
| monkeypatch.setattr("sys.argv", ["ingest", "--data-dir", str(FIXTURES)]) | |
| with pytest.raises(SystemExit) as error: | |
| main() | |
| assert error.value.code == 1 | |
| graph.close.assert_called_once_with() | |
| vectors.client.close.assert_called_once_with() | |