tests/test_scrap.py

import httpx
import pytest
import respx

from helpers import load_fixture
from scrap import parse_patent_html, scrap_patent_async, scrap_patent_bulk_async

FULL_PATENT_HTML = load_fixture("patents", "full_patent.html")
MISSING_TITLE_HTML = load_fixture("patents", "missing_title.html")


def test_parse_patent_html_extracts_all_fields_with_no_network_involved():
    """`scrap_patent_async` is now a thin fetch that delegates to this - a
    plain, synchronous function of the HTML string, so the parsing logic
    (including the regex-based section splitting, which is the part most
    likely to break when Google Patents changes its markup) can be tested
    directly against a saved page with no client/respx/event loop needed.
    """
    result = parse_patent_html(FULL_PATENT_HTML, "https://patents.google.com/patent/US11930446B2/en")

    assert result.title == "Widget with improved gadget mechanism"
    codes = {c.code: c.description for c in result.classifications}
    assert codes == {
        "G06F17/30": "Database structures therefor",
        "G06F17/50": "Other database related",
        "H04L9/00": "Cryptographic mechanisms",
    }


def test_parse_patent_html_raises_when_page_has_no_title():
    with pytest.raises(ValueError):
        parse_patent_html(MISSING_TITLE_HTML, "https://patents.google.com/patent/BOGUS/en")


async def test_scrap_patent_async_extracts_all_fields():
    url = "https://patents.google.com/patent/US11930446B2/en"
    with respx.mock:
        respx.get(url).mock(return_value=httpx.Response(200, text=FULL_PATENT_HTML))
        async with httpx.AsyncClient() as client:
            result = await scrap_patent_async(client, url)

    assert result.title == "Widget with improved gadget mechanism"
    assert result.abstract == "A widget comprising a gadget and a mechanism for improving widget performance."
    assert result.field_of_invention == "This invention relates to widgets and gadgets for testing purposes."
    assert result.background == "Prior art widgets suffered from several problems described herein."
    assert "1. A widget comprising a gadget." in result.claims
    assert "2. The widget of claim 1, further comprising a mechanism." in result.claims

    codes = {c.code: c.description for c in result.classifications}
    assert codes == {
        "G06F17/30": "Database structures therefor",
        "G06F17/50": "Other database related",
        "H04L9/00": "Cryptographic mechanisms",
    }


async def test_scrap_patent_async_raises_when_page_has_no_title():
    """Pins the documented behavior at scrap.py: an unexpected page layout
    (interstitial, non-patent page, markup change) raises ValueError rather
    than crashing on `None.get(...)`, so callers can treat it as a failed
    scrape.
    """
    url = "https://patents.google.com/patent/BOGUS/en"
    with respx.mock:
        respx.get(url).mock(return_value=httpx.Response(200, text=MISSING_TITLE_HTML))
        async with httpx.AsyncClient() as client:
            with pytest.raises(ValueError):
                await scrap_patent_async(client, url)


async def test_scrap_patent_async_raises_on_http_error():
    url = "https://patents.google.com/patent/US00000000/en"
    with respx.mock:
        respx.get(url).mock(return_value=httpx.Response(404))
        async with httpx.AsyncClient() as client:
            with pytest.raises(httpx.HTTPStatusError):
                await scrap_patent_async(client, url)


async def test_scrap_patent_bulk_async_separates_successes_from_failures():
    ok_url = "https://patents.google.com/patent/US_OK/en"
    fail_url = "https://patents.google.com/patent/US_FAIL/en"
    with respx.mock:
        respx.get(ok_url).mock(return_value=httpx.Response(200, text=FULL_PATENT_HTML))
        respx.get(fail_url).mock(return_value=httpx.Response(404))
        async with httpx.AsyncClient() as client:
            result = await scrap_patent_bulk_async(client, ["US_OK", "US_FAIL"])

    assert len(result.patents) == 1
    assert result.patents[0].title == "Widget with improved gadget mechanism"
    assert result.failed_ids == ["US_FAIL"]


def test_parse_patent_html_strips_whitespace_around_the_title():
    """Google Patents' DC.title meta content is frequently padded with
    newlines and indentation from the surrounding markup, which would
    otherwise end up in the API response and in every downstream citation.
    """
    html = '<html><head><meta name="DC.title" content="  Widget apparatus\n  "></head><body></body></html>'

    result = parse_patent_html(html, "https://patents.google.com/patent/US1/en")

    assert result.title == "Widget apparatus"