Merge branch 'dev'

.env.example

@@ -8,8 +8,16 @@ OPENAI_API_KEY=sk-your-key-here
 ANTHROPIC_API_KEY=sk-ant-your-key-here
 
 # Model names (optional - sensible defaults)
-OPENAI_MODEL=gpt-5.1
-ANTHROPIC_MODEL=claude-sonnet-4-5-20250929
+ANTHROPIC_MODEL=claude-3-5-sonnet-20240620
+OPENAI_MODEL=gpt-4-turbo
+
+# ============== EMBEDDINGS ==============
+
+# OpenAI Embedding Model (used if LLM_PROVIDER is openai and performing RAG/Embeddings)
+OPENAI_EMBEDDING_MODEL=text-embedding-3-small
+
+# Local Embedding Model (used for local/offline embeddings)
+LOCAL_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
 
 # ============== HUGGINGFACE (FREE TIER) ==============
 
@@ -20,7 +28,7 @@ ANTHROPIC_MODEL=claude-sonnet-4-5-20250929
 # WITH HF_TOKEN: Uses Llama 3.1 8B Instruct (requires accepting license)
 #
 # For HuggingFace Spaces deployment:
-#   Set this as a "Secret" in Space Settings → Variables and secrets
+#   Set this as a "Secret" in Space Settings -> Variables and secrets
 #   Users/judges don't need their own token - the Space secret is used
 #
 HF_TOKEN=hf_your-token-here
@@ -36,9 +44,5 @@ LOG_LEVEL=INFO
 # PubMed (optional - higher rate limits)
 NCBI_API_KEY=your-ncbi-key-here
 
-# Modal Sandbox (optional - for secure code execution)
-MODAL_TOKEN_ID=ak-your-modal-token-id-here
-MODAL_TOKEN_SECRET=your-modal-token-secret-here
-
 # Vector Database (optional - for LlamaIndex RAG)
 CHROMA_DB_PATH=./chroma_db

docs/brainstorming/00_ROADMAP_SUMMARY.md

@@ -0,0 +1,194 @@
+# DeepCritical Data Sources: Roadmap Summary
+
+**Created**: 2024-11-27
+**Purpose**: Future maintainability and hackathon continuation
+
+---
+
+## Current State
+
+### Working Tools
+
+| Tool | Status | Data Quality |
+|------|--------|--------------|
+| PubMed | ✅ Works | Good (abstracts only) |
+| ClinicalTrials.gov | ✅ Works | Good (filtered for interventional) |
+| Europe PMC | ✅ Works | Good (includes preprints) |
+
+### Removed Tools
+
+| Tool | Status | Reason |
+|------|--------|--------|
+| bioRxiv | ❌ Removed | No search API - only date/DOI lookup |
+
+---
+
+## Priority Improvements
+
+### P0: Critical (Do First)
+
+1. **Add Rate Limiting to PubMed**
+   - NCBI will block us without it
+   - Use `limits` library (see reference repo)
+   - 3/sec without key, 10/sec with key
+
+### P1: High Value, Medium Effort
+
+2. **Add OpenAlex as 4th Source**
+   - Citation network (huge for drug repurposing)
+   - Concept tagging (semantic discovery)
+   - Already implemented in reference repo
+   - Free, no API key
+
+3. **PubMed Full-Text via BioC**
+   - Get full paper text for PMC papers
+   - Already in reference repo
+
+### P2: Nice to Have
+
+4. **ClinicalTrials.gov Results**
+   - Get efficacy data from completed trials
+   - Requires more complex API calls
+
+5. **Europe PMC Annotations**
+   - Text-mined entities (genes, drugs, diseases)
+   - Automatic entity extraction
+
+---
+
+## Effort Estimates
+
+| Improvement | Effort | Impact | Priority |
+|-------------|--------|--------|----------|
+| PubMed rate limiting | 1 hour | Stability | P0 |
+| OpenAlex basic search | 2 hours | High | P1 |
+| OpenAlex citations | 2 hours | Very High | P1 |
+| PubMed full-text | 3 hours | Medium | P1 |
+| CT.gov results | 4 hours | Medium | P2 |
+| Europe PMC annotations | 3 hours | Medium | P2 |
+
+---
+
+## Architecture Decision
+
+### Option A: Keep Current + Add OpenAlex
+
+```
+                    User Query
+                        ↓
+    ┌───────────────────┼───────────────────┐
+    ↓                   ↓                   ↓
+ PubMed          ClinicalTrials        Europe PMC
+ (abstracts)     (trials only)         (preprints)
+    ↓                   ↓                   ↓
+    └───────────────────┼───────────────────┘
+                        ↓
+                   OpenAlex              ← NEW
+               (citations, concepts)
+                        ↓
+                  Orchestrator
+                        ↓
+                     Report
+```
+
+**Pros**: Low risk, additive
+**Cons**: More complexity, some overlap
+
+### Option B: OpenAlex as Primary
+
+```
+                    User Query
+                        ↓
+    ┌───────────────────┼───────────────────┐
+    ↓                   ↓                   ↓
+ OpenAlex          ClinicalTrials      Europe PMC
+ (primary          (trials only)       (full-text
+  search)                               fallback)
+    ↓                   ↓                   ↓
+    └───────────────────┼───────────────────┘
+                        ↓
+                  Orchestrator
+                        ↓
+                     Report
+```
+
+**Pros**: Simpler, citation network built-in
+**Cons**: Lose some PubMed-specific features
+
+### Recommendation: Option A
+
+Keep current architecture working, add OpenAlex incrementally.
+
+---
+
+## Quick Wins (Can Do Today)
+
+1. **Add `limits` to `pyproject.toml`**
+   ```toml
+   dependencies = [
+       "limits>=3.0",
+   ]
+   ```
+
+2. **Copy OpenAlex tool from reference repo**
+   - File: `reference_repos/DeepCritical/DeepResearch/src/tools/openalex_tools.py`
+   - Adapt to our `SearchTool` base class
+
+3. **Enable NCBI API Key**
+   - Add to `.env`: `NCBI_API_KEY=your_key`
+   - 10x rate limit improvement
+
+---
+
+## External Resources Worth Exploring
+
+### Python Libraries
+
+| Library | For | Notes |
+|---------|-----|-------|
+| `limits` | Rate limiting | Used by reference repo |
+| `pyalex` | OpenAlex wrapper | [GitHub](https://github.com/J535D165/pyalex) |
+| `metapub` | PubMed | Full-featured |
+| `sentence-transformers` | Semantic search | For embeddings |
+
+### APIs Not Yet Used
+
+| API | Provides | Effort |
+|-----|----------|--------|
+| RxNorm | Drug name normalization | Low |
+| DrugBank | Drug targets/mechanisms | Medium (license) |
+| UniProt | Protein data | Medium |
+| ChEMBL | Bioactivity data | Medium |
+
+### RAG Tools (Future)
+
+| Tool | Purpose |
+|------|---------|
+| [PaperQA](https://github.com/Future-House/paper-qa) | RAG for scientific papers |
+| [txtai](https://github.com/neuml/txtai) | Embeddings + search |
+| [PubMedBERT](https://huggingface.co/NeuML/pubmedbert-base-embeddings) | Biomedical embeddings |
+
+---
+
+## Files in This Directory
+
+| File | Contents |
+|------|----------|
+| `00_ROADMAP_SUMMARY.md` | This file |
+| `01_PUBMED_IMPROVEMENTS.md` | PubMed enhancement details |
+| `02_CLINICALTRIALS_IMPROVEMENTS.md` | ClinicalTrials.gov details |
+| `03_EUROPEPMC_IMPROVEMENTS.md` | Europe PMC details |
+| `04_OPENALEX_INTEGRATION.md` | OpenAlex integration plan |
+
+---
+
+## For Future Maintainers
+
+If you're picking this up after the hackathon:
+
+1. **Start with OpenAlex** - biggest bang for buck
+2. **Add rate limiting** - prevents API blocks
+3. **Don't bother with bioRxiv** - use Europe PMC instead
+4. **Reference repo is gold** - `reference_repos/DeepCritical/` has working implementations
+
+Good luck! 🚀

docs/brainstorming/01_PUBMED_IMPROVEMENTS.md

@@ -0,0 +1,125 @@
+# PubMed Tool: Current State & Future Improvements
+
+**Status**: Currently Implemented
+**Priority**: High (Core Data Source)
+
+---
+
+## Current Implementation
+
+### What We Have (`src/tools/pubmed.py`)
+
+- Basic E-utilities search via `esearch.fcgi` and `efetch.fcgi`
+- Query preprocessing (strips question words, expands synonyms)
+- Returns: title, abstract, authors, journal, PMID
+- Rate limiting: None implemented (relying on NCBI defaults)
+
+### Current Limitations
+
+1. **No Full-Text Access**: Only retrieves abstracts, not full paper text
+2. **No Rate Limiting**: Risk of being blocked by NCBI
+3. **No BioC Format**: Missing structured full-text extraction
+4. **No Figure Retrieval**: No supplementary materials access
+5. **No PMC Integration**: Missing open-access full-text via PMC
+
+---
+
+## Reference Implementation (DeepCritical Reference Repo)
+
+The reference repo at `reference_repos/DeepCritical/DeepResearch/src/tools/bioinformatics_tools.py` has a more sophisticated implementation:
+
+### Features We're Missing
+
+```python
+# Rate limiting (lines 47-50)
+from limits import parse
+from limits.storage import MemoryStorage
+from limits.strategies import MovingWindowRateLimiter
+
+storage = MemoryStorage()
+limiter = MovingWindowRateLimiter(storage)
+rate_limit = parse("3/second")  # NCBI allows 3/sec without API key, 10/sec with
+
+# Full-text via BioC format (lines 108-120)
+def _get_fulltext(pmid: int) -> dict[str, Any] | None:
+    pmid_url = f"https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/{pmid}/unicode"
+    # Returns structured JSON with full text for open-access papers
+
+# Figure retrieval via Europe PMC (lines 123-149)
+def _get_figures(pmcid: str) -> dict[str, str]:
+    suppl_url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/supplementaryFiles"
+    # Returns base64-encoded images from supplementary materials
+```
+
+---
+
+## Recommended Improvements
+
+### Phase 1: Rate Limiting (Critical)
+
+```python
+# Add to src/tools/pubmed.py
+from limits import parse
+from limits.storage import MemoryStorage
+from limits.strategies import MovingWindowRateLimiter
+
+storage = MemoryStorage()
+limiter = MovingWindowRateLimiter(storage)
+
+# With NCBI_API_KEY: 10/sec, without: 3/sec
+def get_rate_limit():
+    if settings.ncbi_api_key:
+        return parse("10/second")
+    return parse("3/second")
+```
+
+**Dependencies**: `pip install limits`
+
+### Phase 2: Full-Text Retrieval
+
+```python
+async def get_fulltext(pmid: str) -> str | None:
+    """Get full text for open-access papers via BioC API."""
+    url = f"https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/{pmid}/unicode"
+    # Only works for PMC papers (open access)
+```
+
+### Phase 3: PMC ID Resolution
+
+```python
+async def get_pmc_id(pmid: str) -> str | None:
+    """Convert PMID to PMCID for full-text access."""
+    url = f"https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids={pmid}&format=json"
+```
+
+---
+
+## Python Libraries to Consider
+
+| Library | Purpose | Notes |
+|---------|---------|-------|
+| [Biopython](https://biopython.org/) | `Bio.Entrez` module | Official, well-maintained |
+| [PyMed](https://pypi.org/project/pymed/) | PubMed wrapper | Simpler API, less control |
+| [metapub](https://pypi.org/project/metapub/) | Full-featured | Tested on 1/3 of PubMed |
+| [limits](https://pypi.org/project/limits/) | Rate limiting | Used by reference repo |
+
+---
+
+## API Endpoints Reference
+
+| Endpoint | Purpose | Rate Limit |
+|----------|---------|------------|
+| `esearch.fcgi` | Search for PMIDs | 3/sec (10 with key) |
+| `efetch.fcgi` | Fetch metadata | 3/sec (10 with key) |
+| `esummary.fcgi` | Quick metadata | 3/sec (10 with key) |
+| `pmcoa.cgi/BioC_json` | Full text (PMC only) | Unknown |
+| `idconv/v1.0` | PMID ↔ PMCID | Unknown |
+
+---
+
+## Sources
+
+- [PubMed E-utilities Documentation](https://www.ncbi.nlm.nih.gov/books/NBK25501/)
+- [NCBI BioC API](https://www.ncbi.nlm.nih.gov/research/bionlp/APIs/)
+- [Searching PubMed with Python](https://marcobonzanini.com/2015/01/12/searching-pubmed-with-python/)
+- [PyMed on PyPI](https://pypi.org/project/pymed/)

docs/brainstorming/02_CLINICALTRIALS_IMPROVEMENTS.md

@@ -0,0 +1,193 @@
+# ClinicalTrials.gov Tool: Current State & Future Improvements
+
+**Status**: Currently Implemented
+**Priority**: High (Core Data Source for Drug Repurposing)
+
+---
+
+## Current Implementation
+
+### What We Have (`src/tools/clinicaltrials.py`)
+
+- V2 API search via `clinicaltrials.gov/api/v2/studies`
+- Filters: `INTERVENTIONAL` study type, `RECRUITING` status
+- Returns: NCT ID, title, conditions, interventions, phase, status
+- Query preprocessing via shared `query_utils.py`
+
+### Current Strengths
+
+1. **Good Filtering**: Already filtering for interventional + recruiting
+2. **V2 API**: Using the modern API (v1 deprecated)
+3. **Phase Info**: Extracting trial phases for drug development context
+
+### Current Limitations
+
+1. **No Outcome Data**: Missing primary/secondary outcomes
+2. **No Eligibility Criteria**: Missing inclusion/exclusion details
+3. **No Sponsor Info**: Missing who's running the trial
+4. **No Result Data**: For completed trials, no efficacy data
+5. **Limited Drug Mapping**: No integration with drug databases
+
+---
+
+## API Capabilities We're Not Using
+
+### Fields We Could Request
+
+```python
+# Current fields
+fields = ["NCTId", "BriefTitle", "Condition", "InterventionName", "Phase", "OverallStatus"]
+
+# Additional valuable fields
+additional_fields = [
+    "PrimaryOutcomeMeasure",      # What are they measuring?
+    "SecondaryOutcomeMeasure",    # Secondary endpoints
+    "EligibilityCriteria",        # Who can participate?
+    "LeadSponsorName",            # Who's funding?
+    "ResultsFirstPostDate",       # Has results?
+    "StudyFirstPostDate",         # When started?
+    "CompletionDate",             # When finished?
+    "EnrollmentCount",            # Sample size
+    "InterventionDescription",    # Drug details
+    "ArmGroupLabel",              # Treatment arms
+    "InterventionOtherName",      # Drug aliases
+]
+```
+
+### Filter Enhancements
+
+```python
+# Current
+aggFilters = "studyType:INTERVENTIONAL,status:RECRUITING"
+
+# Could add
+"status:RECRUITING,ACTIVE_NOT_RECRUITING,COMPLETED"  # Include completed for results
+"phase:PHASE2,PHASE3"  # Only later-stage trials
+"resultsFirstPostDateRange:2020-01-01_"  # Trials with posted results
+```
+
+---
+
+## Recommended Improvements
+
+### Phase 1: Richer Metadata
+
+```python
+EXTENDED_FIELDS = [
+    "NCTId",
+    "BriefTitle",
+    "OfficialTitle",
+    "Condition",
+    "InterventionName",
+    "InterventionDescription",
+    "InterventionOtherName",  # Drug synonyms!
+    "Phase",
+    "OverallStatus",
+    "PrimaryOutcomeMeasure",
+    "EnrollmentCount",
+    "LeadSponsorName",
+    "StudyFirstPostDate",
+]
+```
+
+### Phase 2: Results Retrieval
+
+For completed trials, we can get actual efficacy data:
+
+```python
+async def get_trial_results(nct_id: str) -> dict | None:
+    """Fetch results for completed trials."""
+    url = f"https://clinicaltrials.gov/api/v2/studies/{nct_id}"
+    params = {
+        "fields": "ResultsSection",
+    }
+    # Returns outcome measures and statistics
+```
+
+### Phase 3: Drug Name Normalization
+
+Map intervention names to standard identifiers:
+
+```python
+# Problem: "Metformin", "Metformin HCl", "Glucophage" are the same drug
+# Solution: Use RxNorm or DrugBank for normalization
+
+async def normalize_drug_name(intervention: str) -> str:
+    """Normalize drug name via RxNorm API."""
+    url = f"https://rxnav.nlm.nih.gov/REST/rxcui.json?name={intervention}"
+    # Returns standardized RxCUI
+```
+
+---
+
+## Integration Opportunities
+
+### With PubMed
+
+Cross-reference trials with publications:
+```python
+# ClinicalTrials.gov provides PMID links
+# Can correlate trial results with published papers
+```
+
+### With DrugBank/ChEMBL
+
+Map interventions to:
+- Mechanism of action
+- Known targets
+- Adverse effects
+- Drug-drug interactions
+
+---
+
+## Python Libraries to Consider
+
+| Library | Purpose | Notes |
+|---------|---------|-------|
+| [pytrials](https://pypi.org/project/pytrials/) | CT.gov wrapper | V2 API support unclear |
+| [clinicaltrials](https://github.com/ebmdatalab/clinicaltrials-act-tracker) | Data tracking | More for analysis |
+| [drugbank-downloader](https://pypi.org/project/drugbank-downloader/) | Drug mapping | Requires license |
+
+---
+
+## API Quirks & Gotchas
+
+1. **Rate Limiting**: Undocumented, be conservative
+2. **Pagination**: Max 1000 results per request
+3. **Field Names**: Case-sensitive, camelCase
+4. **Empty Results**: Some fields may be null even if requested
+5. **Status Changes**: Trials change status frequently
+
+---
+
+## Example Enhanced Query
+
+```python
+async def search_drug_repurposing_trials(
+    drug_name: str,
+    condition: str,
+    include_completed: bool = True,
+) -> list[Evidence]:
+    """Search for trials repurposing a drug for a new condition."""
+
+    statuses = ["RECRUITING", "ACTIVE_NOT_RECRUITING"]
+    if include_completed:
+        statuses.append("COMPLETED")
+
+    params = {
+        "query.intr": drug_name,
+        "query.cond": condition,
+        "filter.overallStatus": ",".join(statuses),
+        "filter.studyType": "INTERVENTIONAL",
+        "fields": ",".join(EXTENDED_FIELDS),
+        "pageSize": 50,
+    }
+```
+
+---
+
+## Sources
+
+- [ClinicalTrials.gov API Documentation](https://clinicaltrials.gov/data-api/api)
+- [CT.gov Field Definitions](https://clinicaltrials.gov/data-api/about-api/study-data-structure)
+- [RxNorm API](https://lhncbc.nlm.nih.gov/RxNav/APIs/api-RxNorm.findRxcuiByString.html)

docs/brainstorming/03_EUROPEPMC_IMPROVEMENTS.md

@@ -0,0 +1,211 @@
+# Europe PMC Tool: Current State & Future Improvements
+
+**Status**: Currently Implemented (Replaced bioRxiv)
+**Priority**: High (Preprint + Open Access Source)
+
+---
+
+## Why Europe PMC Over bioRxiv?
+
+### bioRxiv API Limitations (Why We Abandoned It)
+
+1. **No Search API**: Only returns papers by date range or DOI
+2. **No Query Capability**: Cannot search for "metformin cancer"
+3. **Workaround Required**: Would need to download ALL preprints and build local search
+4. **Known Issue**: [Gradio Issue #8861](https://github.com/gradio-app/gradio/issues/8861) documents the limitation
+
+### Europe PMC Advantages
+
+1. **Full Search API**: Boolean queries, filters, facets
+2. **Aggregates bioRxiv**: Includes bioRxiv, medRxiv content anyway
+3. **Includes PubMed**: Also has MEDLINE content
+4. **34 Preprint Servers**: Not just bioRxiv
+5. **Open Access Focus**: Full-text when available
+
+---
+
+## Current Implementation
+
+### What We Have (`src/tools/europepmc.py`)
+
+- REST API search via `europepmc.org/webservices/rest/search`
+- Preprint flagging via `firstPublicationDate` heuristics
+- Returns: title, abstract, authors, DOI, source
+- Marks preprints for transparency
+
+### Current Limitations
+
+1. **No Full-Text Retrieval**: Only metadata/abstracts
+2. **No Citation Network**: Missing references/citations
+3. **No Supplementary Files**: Not fetching figures/data
+4. **Basic Preprint Detection**: Heuristic, not explicit flag
+
+---
+
+## Europe PMC API Capabilities
+
+### Endpoints We Could Use
+
+| Endpoint | Purpose | Currently Using |
+|----------|---------|-----------------|
+| `/search` | Query papers | Yes |
+| `/fulltext/{ID}` | Full text (XML/JSON) | No |
+| `/{PMCID}/supplementaryFiles` | Figures, data | No |
+| `/citations/{ID}` | Who cited this | No |
+| `/references/{ID}` | What this cites | No |
+| `/annotations` | Text-mined entities | No |
+
+### Rich Query Syntax
+
+```python
+# Current simple query
+query = "metformin cancer"
+
+# Could use advanced syntax
+query = "(TITLE:metformin OR ABSTRACT:metformin) AND (cancer OR oncology)"
+query += " AND (SRC:PPR)"  # Only preprints
+query += " AND (FIRST_PDATE:[2023-01-01 TO 2024-12-31])"  # Date range
+query += " AND (OPEN_ACCESS:y)"  # Only open access
+```
+
+### Source Filters
+
+```python
+# Filter by source
+"SRC:MED"     # MEDLINE
+"SRC:PMC"     # PubMed Central
+"SRC:PPR"     # Preprints (bioRxiv, medRxiv, etc.)
+"SRC:AGR"     # Agricola
+"SRC:CBA"     # Chinese Biological Abstracts
+```
+
+---
+
+## Recommended Improvements
+
+### Phase 1: Rich Metadata
+
+```python
+# Add to search results
+additional_fields = [
+    "citedByCount",           # Impact indicator
+    "source",                 # Explicit source (MED, PMC, PPR)
+    "isOpenAccess",           # Boolean flag
+    "fullTextUrlList",        # URLs for full text
+    "authorAffiliations",     # Institution info
+    "grantsList",             # Funding info
+]
+```
+
+### Phase 2: Full-Text Retrieval
+
+```python
+async def get_fulltext(pmcid: str) -> str | None:
+    """Get full text for open access papers."""
+    # XML format
+    url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextXML"
+    # Or JSON
+    url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextJSON"
+```
+
+### Phase 3: Citation Network
+
+```python
+async def get_citations(pmcid: str) -> list[str]:
+    """Get papers that cite this one."""
+    url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/citations"
+
+async def get_references(pmcid: str) -> list[str]:
+    """Get papers this one cites."""
+    url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/references"
+```
+
+### Phase 4: Text-Mined Annotations
+
+Europe PMC extracts entities automatically:
+
+```python
+async def get_annotations(pmcid: str) -> dict:
+    """Get text-mined entities (genes, diseases, drugs)."""
+    url = f"https://www.ebi.ac.uk/europepmc/annotations_api/annotationsByArticleIds"
+    params = {
+        "articleIds": f"PMC:{pmcid}",
+        "type": "Gene_Proteins,Diseases,Chemicals",
+        "format": "JSON",
+    }
+    # Returns structured entity mentions with positions
+```
+
+---
+
+## Supplementary File Retrieval
+
+From reference repo (`bioinformatics_tools.py` lines 123-149):
+
+```python
+def get_figures(pmcid: str) -> dict[str, str]:
+    """Download figures and supplementary files."""
+    url = f"https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/supplementaryFiles?includeInlineImage=true"
+    # Returns ZIP with images, returns base64-encoded
+```
+
+---
+
+## Preprint-Specific Features
+
+### Identify Preprint Servers
+
+```python
+PREPRINT_SOURCES = {
+    "PPR": "General preprints",
+    "bioRxiv": "Biology preprints",
+    "medRxiv": "Medical preprints",
+    "chemRxiv": "Chemistry preprints",
+    "Research Square": "Multi-disciplinary",
+    "Preprints.org": "MDPI preprints",
+}
+
+# Check if published version exists
+async def check_published_version(preprint_doi: str) -> str | None:
+    """Check if preprint has been peer-reviewed and published."""
+    # Europe PMC links preprints to final versions
+```
+
+---
+
+## Rate Limiting
+
+Europe PMC is more generous than NCBI:
+
+```python
+# No documented hard limit, but be respectful
+# Recommend: 10-20 requests/second max
+# Use email in User-Agent for polite pool
+headers = {
+    "User-Agent": "DeepCritical/1.0 (mailto:your@email.com)"
+}
+```
+
+---
+
+## vs. The Lens & OpenAlex
+
+| Feature | Europe PMC | The Lens | OpenAlex |
+|---------|------------|----------|----------|
+| Biomedical Focus | Yes | Partial | Partial |
+| Preprints | Yes (34 servers) | Yes | Yes |
+| Full Text | PMC papers | Links | No |
+| Citations | Yes | Yes | Yes |
+| Annotations | Yes (text-mined) | No | No |
+| Rate Limits | Generous | Moderate | Very generous |
+| API Key | Optional | Required | Optional |
+
+---
+
+## Sources
+
+- [Europe PMC REST API](https://europepmc.org/RestfulWebService)
+- [Europe PMC Annotations API](https://europepmc.org/AnnotationsApi)
+- [Europe PMC Articles API](https://europepmc.org/ArticlesApi)
+- [rOpenSci medrxivr](https://docs.ropensci.org/medrxivr/)
+- [bioRxiv TDM Resources](https://www.biorxiv.org/tdm)

docs/brainstorming/04_OPENALEX_INTEGRATION.md

@@ -0,0 +1,303 @@
+# OpenAlex Integration: The Missing Piece?
+
+**Status**: NOT Implemented (Candidate for Addition)
+**Priority**: HIGH - Could Replace Multiple Tools
+**Reference**: Already implemented in `reference_repos/DeepCritical`
+
+---
+
+## What is OpenAlex?
+
+OpenAlex is a **fully open** index of the global research system:
+
+- **209M+ works** (papers, books, datasets)
+- **2B+ author records** (disambiguated)
+- **124K+ venues** (journals, repositories)
+- **109K+ institutions**
+- **65K+ concepts** (hierarchical, linked to Wikidata)
+
+**Free. Open. No API key required.**
+
+---
+
+## Why OpenAlex for DeepCritical?
+
+### Current Architecture
+
+```
+User Query
+    ↓
+┌──────────────────────────────────────┐
+│  PubMed    ClinicalTrials  Europe PMC │  ← 3 separate APIs
+└──────────────────────────────────────┘
+    ↓
+Orchestrator (deduplicate, judge, synthesize)
+```
+
+### With OpenAlex
+
+```
+User Query
+    ↓
+┌──────────────────────────────────────┐
+│              OpenAlex                 │  ← Single API
+│  (includes PubMed + preprints +       │
+│   citations + concepts + authors)     │
+└──────────────────────────────────────┘
+    ↓
+Orchestrator (enrich with CT.gov for trials)
+```
+
+**OpenAlex already aggregates**:
+- PubMed/MEDLINE
+- Crossref
+- ORCID
+- Unpaywall (open access links)
+- Microsoft Academic Graph (legacy)
+- Preprint servers
+
+---
+
+## Reference Implementation
+
+From `reference_repos/DeepCritical/DeepResearch/src/tools/openalex_tools.py`:
+
+```python
+class OpenAlexFetchTool(ToolRunner):
+    def __init__(self):
+        super().__init__(
+            ToolSpec(
+                name="openalex_fetch",
+                description="Fetch OpenAlex work or author",
+                inputs={"entity": "TEXT", "identifier": "TEXT"},
+                outputs={"result": "JSON"},
+            )
+        )
+
+    def run(self, params: dict[str, Any]) -> ExecutionResult:
+        entity = params["entity"]      # "works", "authors", "venues"
+        identifier = params["identifier"]
+        base = "https://api.openalex.org"
+        url = f"{base}/{entity}/{identifier}"
+        resp = requests.get(url, timeout=30)
+        return ExecutionResult(success=True, data={"result": resp.json()})
+```
+
+---
+
+## OpenAlex API Features
+
+### Search Works (Papers)
+
+```python
+# Search for metformin + cancer papers
+url = "https://api.openalex.org/works"
+params = {
+    "search": "metformin cancer drug repurposing",
+    "filter": "publication_year:>2020,type:article",
+    "sort": "cited_by_count:desc",
+    "per_page": 50,
+}
+```
+
+### Rich Filtering
+
+```python
+# Filter examples
+"publication_year:2023"
+"type:article"                      # vs preprint, book, etc.
+"is_oa:true"                        # Open access only
+"concepts.id:C71924100"             # Papers about "Medicine"
+"authorships.institutions.id:I27837315"  # From Harvard
+"cited_by_count:>100"               # Highly cited
+"has_fulltext:true"                 # Full text available
+```
+
+### What You Get Back
+
+```json
+{
+    "id": "W2741809807",
+    "title": "Metformin: A candidate drug for...",
+    "publication_year": 2023,
+    "type": "article",
+    "cited_by_count": 45,
+    "is_oa": true,
+    "primary_location": {
+        "source": {"display_name": "Nature Medicine"},
+        "pdf_url": "https://...",
+        "landing_page_url": "https://..."
+    },
+    "concepts": [
+        {"id": "C71924100", "display_name": "Medicine", "score": 0.95},
+        {"id": "C54355233", "display_name": "Pharmacology", "score": 0.88}
+    ],
+    "authorships": [
+        {
+            "author": {"id": "A123", "display_name": "John Smith"},
+            "institutions": [{"display_name": "Harvard Medical School"}]
+        }
+    ],
+    "referenced_works": ["W123", "W456"],  # Citations
+    "related_works": ["W789", "W012"]       # Similar papers
+}
+```
+
+---
+
+## Key Advantages Over Current Tools
+
+### 1. Citation Network (We Don't Have This!)
+
+```python
+# Get papers that cite a work
+url = f"https://api.openalex.org/works?filter=cites:{work_id}"
+
+# Get papers cited by a work
+# Already in `referenced_works` field
+```
+
+### 2. Concept Tagging (We Don't Have This!)
+
+OpenAlex auto-tags papers with hierarchical concepts:
+- "Medicine" → "Pharmacology" → "Drug Repurposing"
+- Can search by concept, not just keywords
+
+### 3. Author Disambiguation (We Don't Have This!)
+
+```python
+# Find all works by an author
+url = f"https://api.openalex.org/works?filter=authorships.author.id:{author_id}"
+```
+
+### 4. Institution Tracking
+
+```python
+# Find drug repurposing papers from top institutions
+url = "https://api.openalex.org/works"
+params = {
+    "search": "drug repurposing",
+    "filter": "authorships.institutions.id:I27837315",  # Harvard
+}
+```
+
+### 5. Related Works
+
+Each paper comes with `related_works` - semantically similar papers discovered by OpenAlex's ML.
+
+---
+
+## Proposed Implementation
+
+### New Tool: `src/tools/openalex.py`
+
+```python
+"""OpenAlex search tool for comprehensive scholarly data."""
+
+import httpx
+from src.tools.base import SearchTool
+from src.utils.models import Evidence
+
+class OpenAlexTool(SearchTool):
+    """Search OpenAlex for scholarly works with rich metadata."""
+
+    name = "openalex"
+
+    async def search(self, query: str, max_results: int = 10) -> list[Evidence]:
+        async with httpx.AsyncClient() as client:
+            resp = await client.get(
+                "https://api.openalex.org/works",
+                params={
+                    "search": query,
+                    "filter": "type:article,is_oa:true",
+                    "sort": "cited_by_count:desc",
+                    "per_page": max_results,
+                    "mailto": "deepcritical@example.com",  # Polite pool
+                },
+            )
+            data = resp.json()
+
+        return [
+            Evidence(
+                source="openalex",
+                title=work["title"],
+                abstract=work.get("abstract", ""),
+                url=work["primary_location"]["landing_page_url"],
+                metadata={
+                    "cited_by_count": work["cited_by_count"],
+                    "concepts": [c["display_name"] for c in work["concepts"][:5]],
+                    "is_open_access": work["is_oa"],
+                    "pdf_url": work["primary_location"].get("pdf_url"),
+                },
+            )
+            for work in data["results"]
+        ]
+```
+
+---
+
+## Rate Limits
+
+OpenAlex is **extremely generous**:
+
+- No hard rate limit documented
+- Recommended: <100,000 requests/day
+- **Polite pool**: Add `mailto=your@email.com` param for faster responses
+- No API key required (optional for priority support)
+
+---
+
+## Should We Add OpenAlex?
+
+### Arguments FOR
+
+1. **Already in reference repo** - proven pattern
+2. **Richer data** - citations, concepts, authors
+3. **Single source** - reduces API complexity
+4. **Free & open** - no keys, no limits
+5. **Institution adoption** - Leiden, Sorbonne switched to it
+
+### Arguments AGAINST
+
+1. **Adds complexity** - another data source
+2. **Overlap** - duplicates some PubMed data
+3. **Not biomedical-focused** - covers all disciplines
+4. **No full text** - still need PMC/Europe PMC for that
+
+### Recommendation
+
+**Add OpenAlex as a 4th source**, don't replace existing tools.
+
+Use it for:
+- Citation network analysis
+- Concept-based discovery
+- High-impact paper finding
+- Author/institution tracking
+
+Keep PubMed, ClinicalTrials, Europe PMC for:
+- Authoritative biomedical search
+- Clinical trial data
+- Full-text access
+- Preprint tracking
+
+---
+
+## Implementation Priority
+
+| Task | Effort | Value |
+|------|--------|-------|
+| Basic search | Low | High |
+| Citation network | Medium | Very High |
+| Concept filtering | Low | High |
+| Related works | Low | High |
+| Author tracking | Medium | Medium |
+
+---
+
+## Sources
+
+- [OpenAlex Documentation](https://docs.openalex.org)
+- [OpenAlex API Overview](https://docs.openalex.org/api)
+- [OpenAlex Wikipedia](https://en.wikipedia.org/wiki/OpenAlex)
+- [Leiden University Announcement](https://www.leidenranking.com/information/openalex)
+- [OpenAlex: A fully-open index (Paper)](https://arxiv.org/abs/2205.01833)

docs/brainstorming/implementation/15_PHASE_OPENALEX.md

@@ -0,0 +1,603 @@
+# Phase 15: OpenAlex Integration
+
+**Priority**: HIGH - Biggest bang for buck
+**Effort**: ~2-3 hours
+**Dependencies**: None (existing codebase patterns sufficient)
+
+---
+
+## Prerequisites (COMPLETED)
+
+The following model changes have been implemented to support this integration:
+
+1. **`SourceName` Literal Updated** (`src/utils/models.py:9`)
+   ```python
+   SourceName = Literal["pubmed", "clinicaltrials", "europepmc", "preprint", "openalex"]
+   ```
+   - Without this, `source="openalex"` would fail Pydantic validation
+
+2. **`Evidence.metadata` Field Added** (`src/utils/models.py:39-42`)
+   ```python
+   metadata: dict[str, Any] = Field(
+       default_factory=dict,
+       description="Additional metadata (e.g., cited_by_count, concepts, is_open_access)",
+   )
+   ```
+   - Required for storing `cited_by_count`, `concepts`, etc.
+   - Model is still frozen - metadata must be passed at construction time
+
+3. **`__init__.py` Exports Updated** (`src/tools/__init__.py`)
+   - All tools are now exported: `ClinicalTrialsTool`, `EuropePMCTool`, `PubMedTool`
+   - OpenAlexTool should be added here after implementation
+
+---
+
+## Overview
+
+Add OpenAlex as a 4th data source for comprehensive scholarly data including:
+- Citation networks (who cites whom)
+- Concept tagging (hierarchical topic classification)
+- Author disambiguation
+- 209M+ works indexed
+
+**Why OpenAlex?**
+- Free, no API key required
+- Already implemented in reference repo
+- Provides citation data we don't have
+- Aggregates PubMed + preprints + more
+
+---
+
+## TDD Implementation Plan
+
+### Step 1: Write the Tests First
+
+**File**: `tests/unit/tools/test_openalex.py`
+
+```python
+"""Tests for OpenAlex search tool."""
+
+import pytest
+import respx
+from httpx import Response
+
+from src.tools.openalex import OpenAlexTool
+from src.utils.models import Evidence
+
+
+class TestOpenAlexTool:
+    """Test suite for OpenAlex search functionality."""
+
+    @pytest.fixture
+    def tool(self) -> OpenAlexTool:
+        return OpenAlexTool()
+
+    def test_name_property(self, tool: OpenAlexTool) -> None:
+        """Tool should identify itself as 'openalex'."""
+        assert tool.name == "openalex"
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_search_returns_evidence(self, tool: OpenAlexTool) -> None:
+        """Search should return list of Evidence objects."""
+        mock_response = {
+            "results": [
+                {
+                    "id": "W2741809807",
+                    "title": "Metformin and cancer: A systematic review",
+                    "publication_year": 2023,
+                    "cited_by_count": 45,
+                    "type": "article",
+                    "is_oa": True,
+                    "primary_location": {
+                        "source": {"display_name": "Nature Medicine"},
+                        "landing_page_url": "https://doi.org/10.1038/example",
+                        "pdf_url": None,
+                    },
+                    "abstract_inverted_index": {
+                        "Metformin": [0],
+                        "shows": [1],
+                        "anticancer": [2],
+                        "effects": [3],
+                    },
+                    "concepts": [
+                        {"display_name": "Medicine", "score": 0.95},
+                        {"display_name": "Oncology", "score": 0.88},
+                    ],
+                    "authorships": [
+                        {
+                            "author": {"display_name": "John Smith"},
+                            "institutions": [{"display_name": "Harvard"}],
+                        }
+                    ],
+                }
+            ]
+        }
+
+        respx.get("https://api.openalex.org/works").mock(
+            return_value=Response(200, json=mock_response)
+        )
+
+        results = await tool.search("metformin cancer", max_results=10)
+
+        assert len(results) == 1
+        assert isinstance(results[0], Evidence)
+        assert "Metformin and cancer" in results[0].citation.title
+        assert results[0].citation.source == "openalex"
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_search_empty_results(self, tool: OpenAlexTool) -> None:
+        """Search with no results should return empty list."""
+        respx.get("https://api.openalex.org/works").mock(
+            return_value=Response(200, json={"results": []})
+        )
+
+        results = await tool.search("xyznonexistentquery123")
+        assert results == []
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_search_handles_missing_abstract(self, tool: OpenAlexTool) -> None:
+        """Tool should handle papers without abstracts."""
+        mock_response = {
+            "results": [
+                {
+                    "id": "W123",
+                    "title": "Paper without abstract",
+                    "publication_year": 2023,
+                    "cited_by_count": 10,
+                    "type": "article",
+                    "is_oa": False,
+                    "primary_location": {
+                        "source": {"display_name": "Journal"},
+                        "landing_page_url": "https://example.com",
+                    },
+                    "abstract_inverted_index": None,
+                    "concepts": [],
+                    "authorships": [],
+                }
+            ]
+        }
+
+        respx.get("https://api.openalex.org/works").mock(
+            return_value=Response(200, json=mock_response)
+        )
+
+        results = await tool.search("test query")
+        assert len(results) == 1
+        assert results[0].content == ""  # No abstract
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_search_extracts_citation_count(self, tool: OpenAlexTool) -> None:
+        """Citation count should be in metadata."""
+        mock_response = {
+            "results": [
+                {
+                    "id": "W456",
+                    "title": "Highly cited paper",
+                    "publication_year": 2020,
+                    "cited_by_count": 500,
+                    "type": "article",
+                    "is_oa": True,
+                    "primary_location": {
+                        "source": {"display_name": "Science"},
+                        "landing_page_url": "https://example.com",
+                    },
+                    "abstract_inverted_index": {"Test": [0]},
+                    "concepts": [],
+                    "authorships": [],
+                }
+            ]
+        }
+
+        respx.get("https://api.openalex.org/works").mock(
+            return_value=Response(200, json=mock_response)
+        )
+
+        results = await tool.search("highly cited")
+        assert results[0].metadata["cited_by_count"] == 500
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_search_extracts_concepts(self, tool: OpenAlexTool) -> None:
+        """Concepts should be extracted for semantic discovery."""
+        mock_response = {
+            "results": [
+                {
+                    "id": "W789",
+                    "title": "Drug repurposing study",
+                    "publication_year": 2023,
+                    "cited_by_count": 25,
+                    "type": "article",
+                    "is_oa": True,
+                    "primary_location": {
+                        "source": {"display_name": "PLOS ONE"},
+                        "landing_page_url": "https://example.com",
+                    },
+                    "abstract_inverted_index": {"Drug": [0], "repurposing": [1]},
+                    "concepts": [
+                        {"display_name": "Pharmacology", "score": 0.92},
+                        {"display_name": "Drug Discovery", "score": 0.85},
+                        {"display_name": "Medicine", "score": 0.80},
+                    ],
+                    "authorships": [],
+                }
+            ]
+        }
+
+        respx.get("https://api.openalex.org/works").mock(
+            return_value=Response(200, json=mock_response)
+        )
+
+        results = await tool.search("drug repurposing")
+        assert "Pharmacology" in results[0].metadata["concepts"]
+        assert "Drug Discovery" in results[0].metadata["concepts"]
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_search_api_error_raises_search_error(
+        self, tool: OpenAlexTool
+    ) -> None:
+        """API errors should raise SearchError."""
+        from src.utils.exceptions import SearchError
+
+        respx.get("https://api.openalex.org/works").mock(
+            return_value=Response(500, text="Internal Server Error")
+        )
+
+        with pytest.raises(SearchError):
+            await tool.search("test query")
+
+    def test_reconstruct_abstract(self, tool: OpenAlexTool) -> None:
+        """Test abstract reconstruction from inverted index."""
+        inverted_index = {
+            "Metformin": [0, 5],
+            "is": [1],
+            "a": [2],
+            "diabetes": [3],
+            "drug": [4],
+            "effective": [6],
+        }
+        abstract = tool._reconstruct_abstract(inverted_index)
+        assert abstract == "Metformin is a diabetes drug Metformin effective"
+```
+
+---
+
+### Step 2: Create the Implementation
+
+**File**: `src/tools/openalex.py`
+
+```python
+"""OpenAlex search tool for comprehensive scholarly data."""
+
+from typing import Any
+
+import httpx
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+from src.utils.exceptions import SearchError
+from src.utils.models import Citation, Evidence
+
+
+class OpenAlexTool:
+    """
+    Search OpenAlex for scholarly works with rich metadata.
+
+    OpenAlex provides:
+    - 209M+ scholarly works
+    - Citation counts and networks
+    - Concept tagging (hierarchical)
+    - Author disambiguation
+    - Open access links
+
+    API Docs: https://docs.openalex.org/
+    """
+
+    BASE_URL = "https://api.openalex.org/works"
+
+    def __init__(self, email: str | None = None) -> None:
+        """
+        Initialize OpenAlex tool.
+
+        Args:
+            email: Optional email for polite pool (faster responses)
+        """
+        self.email = email or "deepcritical@example.com"
+
+    @property
+    def name(self) -> str:
+        return "openalex"
+
+    @retry(
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=1, max=10),
+        reraise=True,
+    )
+    async def search(self, query: str, max_results: int = 10) -> list[Evidence]:
+        """
+        Search OpenAlex for scholarly works.
+
+        Args:
+            query: Search terms
+            max_results: Maximum results to return (max 200 per request)
+
+        Returns:
+            List of Evidence objects with citation metadata
+
+        Raises:
+            SearchError: If API request fails
+        """
+        params = {
+            "search": query,
+            "filter": "type:article",  # Only peer-reviewed articles
+            "sort": "cited_by_count:desc",  # Most cited first
+            "per_page": min(max_results, 200),
+            "mailto": self.email,  # Polite pool for faster responses
+        }
+
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            try:
+                response = await client.get(self.BASE_URL, params=params)
+                response.raise_for_status()
+
+                data = response.json()
+                results = data.get("results", [])
+
+                return [self._to_evidence(work) for work in results[:max_results]]
+
+            except httpx.HTTPStatusError as e:
+                raise SearchError(f"OpenAlex API error: {e}") from e
+            except httpx.RequestError as e:
+                raise SearchError(f"OpenAlex connection failed: {e}") from e
+
+    def _to_evidence(self, work: dict[str, Any]) -> Evidence:
+        """Convert OpenAlex work to Evidence object."""
+        title = work.get("title", "Untitled")
+        pub_year = work.get("publication_year", "Unknown")
+        cited_by = work.get("cited_by_count", 0)
+        is_oa = work.get("is_oa", False)
+
+        # Reconstruct abstract from inverted index
+        abstract_index = work.get("abstract_inverted_index")
+        abstract = self._reconstruct_abstract(abstract_index) if abstract_index else ""
+
+        # Extract concepts (top 5)
+        concepts = [
+            c.get("display_name", "")
+            for c in work.get("concepts", [])[:5]
+            if c.get("display_name")
+        ]
+
+        # Extract authors (top 5)
+        authorships = work.get("authorships", [])
+        authors = [
+            a.get("author", {}).get("display_name", "")
+            for a in authorships[:5]
+            if a.get("author", {}).get("display_name")
+        ]
+
+        # Get URL
+        primary_loc = work.get("primary_location") or {}
+        url = primary_loc.get("landing_page_url", "")
+        if not url:
+            # Fallback to OpenAlex page
+            work_id = work.get("id", "").replace("https://openalex.org/", "")
+            url = f"https://openalex.org/{work_id}"
+
+        return Evidence(
+            content=abstract[:2000],
+            citation=Citation(
+                source="openalex",
+                title=title[:500],
+                url=url,
+                date=str(pub_year),
+                authors=authors,
+            ),
+            relevance=min(0.9, 0.5 + (cited_by / 1000)),  # Boost by citations
+            metadata={
+                "cited_by_count": cited_by,
+                "is_open_access": is_oa,
+                "concepts": concepts,
+                "pdf_url": primary_loc.get("pdf_url"),
+            },
+        )
+
+    def _reconstruct_abstract(
+        self, inverted_index: dict[str, list[int]]
+    ) -> str:
+        """
+        Reconstruct abstract from OpenAlex inverted index format.
+
+        OpenAlex stores abstracts as {"word": [position1, position2, ...]}.
+        This rebuilds the original text.
+        """
+        if not inverted_index:
+            return ""
+
+        # Build position -> word mapping
+        position_word: dict[int, str] = {}
+        for word, positions in inverted_index.items():
+            for pos in positions:
+                position_word[pos] = word
+
+        # Reconstruct in order
+        if not position_word:
+            return ""
+
+        max_pos = max(position_word.keys())
+        words = [position_word.get(i, "") for i in range(max_pos + 1)]
+        return " ".join(w for w in words if w)
+```
+
+---
+
+### Step 3: Register in Search Handler
+
+**File**: `src/tools/search_handler.py` (add to imports and tool list)
+
+```python
+# Add import
+from src.tools.openalex import OpenAlexTool
+
+# Add to _create_tools method
+def _create_tools(self) -> list[SearchTool]:
+    return [
+        PubMedTool(),
+        ClinicalTrialsTool(),
+        EuropePMCTool(),
+        OpenAlexTool(),  # NEW
+    ]
+```
+
+---
+
+### Step 4: Update `__init__.py`
+
+**File**: `src/tools/__init__.py`
+
+```python
+from src.tools.openalex import OpenAlexTool
+
+__all__ = [
+    "PubMedTool",
+    "ClinicalTrialsTool",
+    "EuropePMCTool",
+    "OpenAlexTool",  # NEW
+    # ...
+]
+```
+
+---
+
+## Demo Script
+
+**File**: `examples/openalex_demo.py`
+
+```python
+#!/usr/bin/env python3
+"""Demo script to verify OpenAlex integration."""
+
+import asyncio
+from src.tools.openalex import OpenAlexTool
+
+
+async def main():
+    """Run OpenAlex search demo."""
+    tool = OpenAlexTool()
+
+    print("=" * 60)
+    print("OpenAlex Integration Demo")
+    print("=" * 60)
+
+    # Test 1: Basic drug repurposing search
+    print("\n[Test 1] Searching for 'metformin cancer drug repurposing'...")
+    results = await tool.search("metformin cancer drug repurposing", max_results=5)
+
+    for i, evidence in enumerate(results, 1):
+        print(f"\n--- Result {i} ---")
+        print(f"Title: {evidence.citation.title}")
+        print(f"Year: {evidence.citation.date}")
+        print(f"Citations: {evidence.metadata.get('cited_by_count', 'N/A')}")
+        print(f"Concepts: {', '.join(evidence.metadata.get('concepts', []))}")
+        print(f"Open Access: {evidence.metadata.get('is_open_access', False)}")
+        print(f"URL: {evidence.citation.url}")
+        if evidence.content:
+            print(f"Abstract: {evidence.content[:200]}...")
+
+    # Test 2: High-impact papers
+    print("\n" + "=" * 60)
+    print("[Test 2] Finding highly-cited papers on 'long COVID treatment'...")
+    results = await tool.search("long COVID treatment", max_results=3)
+
+    for evidence in results:
+        print(f"\n- {evidence.citation.title}")
+        print(f"  Citations: {evidence.metadata.get('cited_by_count', 0)}")
+
+    print("\n" + "=" * 60)
+    print("Demo complete!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## Verification Checklist
+
+### Unit Tests
+```bash
+# Run just OpenAlex tests
+uv run pytest tests/unit/tools/test_openalex.py -v
+
+# Expected: All tests pass
+```
+
+### Integration Test (Manual)
+```bash
+# Run demo script with real API
+uv run python examples/openalex_demo.py
+
+# Expected: Real results from OpenAlex API
+```
+
+### Full Test Suite
+```bash
+# Ensure nothing broke
+make check
+
+# Expected: All 110+ tests pass, mypy clean
+```
+
+---
+
+## Success Criteria
+
+1. **Unit tests pass**: All mocked tests in `test_openalex.py` pass
+2. **Integration works**: Demo script returns real results
+3. **No regressions**: `make check` passes completely
+4. **SearchHandler integration**: OpenAlex appears in search results alongside other sources
+5. **Citation metadata**: Results include `cited_by_count`, `concepts`, `is_open_access`
+
+---
+
+## Future Enhancements (P2)
+
+Once basic integration works:
+
+1. **Citation Network Queries**
+   ```python
+   # Get papers citing a specific work
+   async def get_citing_works(self, work_id: str) -> list[Evidence]:
+       params = {"filter": f"cites:{work_id}"}
+       ...
+   ```
+
+2. **Concept-Based Search**
+   ```python
+   # Search by OpenAlex concept ID
+   async def search_by_concept(self, concept_id: str) -> list[Evidence]:
+       params = {"filter": f"concepts.id:{concept_id}"}
+       ...
+   ```
+
+3. **Author Tracking**
+   ```python
+   # Find all works by an author
+   async def search_by_author(self, author_id: str) -> list[Evidence]:
+       params = {"filter": f"authorships.author.id:{author_id}"}
+       ...
+   ```
+
+---
+
+## Notes
+
+- OpenAlex is **very generous** with rate limits (no documented hard limit)
+- Adding `mailto` parameter gives priority access (polite pool)
+- Abstract is stored as inverted index - must reconstruct
+- Citation count is a good proxy for paper quality/impact
+- Consider caching responses for repeated queries

docs/brainstorming/implementation/16_PHASE_PUBMED_FULLTEXT.md

@@ -0,0 +1,586 @@
+# Phase 16: PubMed Full-Text Retrieval
+
+**Priority**: MEDIUM - Enhances evidence quality
+**Effort**: ~3 hours
+**Dependencies**: None (existing PubMed tool sufficient)
+
+---
+
+## Prerequisites (COMPLETED)
+
+The `Evidence.metadata` field has been added to `src/utils/models.py` to support:
+```python
+metadata={"has_fulltext": True}
+```
+
+---
+
+## Architecture Decision: Constructor Parameter vs Method Parameter
+
+**IMPORTANT**: The original spec proposed `include_fulltext` as a method parameter:
+```python
+# WRONG - SearchHandler won't pass this parameter
+async def search(self, query: str, max_results: int = 10, include_fulltext: bool = False):
+```
+
+**Problem**: `SearchHandler` calls `tool.search(query, max_results)` uniformly across all tools.
+It has no mechanism to pass tool-specific parameters like `include_fulltext`.
+
+**Solution**: Use constructor parameter instead:
+```python
+# CORRECT - Configured at instantiation time
+class PubMedTool:
+    def __init__(self, api_key: str | None = None, include_fulltext: bool = False):
+        self.include_fulltext = include_fulltext
+        ...
+```
+
+This way, you can create a full-text-enabled PubMed tool:
+```python
+# In orchestrator or wherever tools are created
+tools = [
+    PubMedTool(include_fulltext=True),  # Full-text enabled
+    ClinicalTrialsTool(),
+    EuropePMCTool(),
+]
+```
+
+---
+
+## Overview
+
+Add full-text retrieval for PubMed papers via the BioC API, enabling:
+- Complete paper text for open-access PMC papers
+- Structured sections (intro, methods, results, discussion)
+- Better evidence for LLM synthesis
+
+**Why Full-Text?**
+- Abstracts only give ~200-300 words
+- Full text provides detailed methods, results, figures
+- Reference repo already has this implemented
+- Makes LLM judgments more accurate
+
+---
+
+## TDD Implementation Plan
+
+### Step 1: Write the Tests First
+
+**File**: `tests/unit/tools/test_pubmed_fulltext.py`
+
+```python
+"""Tests for PubMed full-text retrieval."""
+
+import pytest
+import respx
+from httpx import Response
+
+from src.tools.pubmed import PubMedTool
+
+
+class TestPubMedFullText:
+    """Test suite for PubMed full-text functionality."""
+
+    @pytest.fixture
+    def tool(self) -> PubMedTool:
+        return PubMedTool()
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_get_pmc_id_success(self, tool: PubMedTool) -> None:
+        """Should convert PMID to PMCID for full-text access."""
+        mock_response = {
+            "records": [
+                {
+                    "pmid": "12345678",
+                    "pmcid": "PMC1234567",
+                }
+            ]
+        }
+
+        respx.get("https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/").mock(
+            return_value=Response(200, json=mock_response)
+        )
+
+        pmcid = await tool.get_pmc_id("12345678")
+        assert pmcid == "PMC1234567"
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_get_pmc_id_not_in_pmc(self, tool: PubMedTool) -> None:
+        """Should return None if paper not in PMC."""
+        mock_response = {
+            "records": [
+                {
+                    "pmid": "12345678",
+                    # No pmcid means not in PMC
+                }
+            ]
+        }
+
+        respx.get("https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/").mock(
+            return_value=Response(200, json=mock_response)
+        )
+
+        pmcid = await tool.get_pmc_id("12345678")
+        assert pmcid is None
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_get_fulltext_success(self, tool: PubMedTool) -> None:
+        """Should retrieve full text for PMC papers."""
+        # Mock BioC API response
+        mock_bioc = {
+            "documents": [
+                {
+                    "passages": [
+                        {
+                            "infons": {"section_type": "INTRO"},
+                            "text": "Introduction text here.",
+                        },
+                        {
+                            "infons": {"section_type": "METHODS"},
+                            "text": "Methods description here.",
+                        },
+                        {
+                            "infons": {"section_type": "RESULTS"},
+                            "text": "Results summary here.",
+                        },
+                        {
+                            "infons": {"section_type": "DISCUSS"},
+                            "text": "Discussion and conclusions.",
+                        },
+                    ]
+                }
+            ]
+        }
+
+        respx.get(
+            "https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/12345678/unicode"
+        ).mock(return_value=Response(200, json=mock_bioc))
+
+        fulltext = await tool.get_fulltext("12345678")
+
+        assert fulltext is not None
+        assert "Introduction text here" in fulltext
+        assert "Methods description here" in fulltext
+        assert "Results summary here" in fulltext
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_get_fulltext_not_available(self, tool: PubMedTool) -> None:
+        """Should return None if full text not available."""
+        respx.get(
+            "https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/99999999/unicode"
+        ).mock(return_value=Response(404))
+
+        fulltext = await tool.get_fulltext("99999999")
+        assert fulltext is None
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_get_fulltext_structured(self, tool: PubMedTool) -> None:
+        """Should return structured sections dict."""
+        mock_bioc = {
+            "documents": [
+                {
+                    "passages": [
+                        {"infons": {"section_type": "INTRO"}, "text": "Intro..."},
+                        {"infons": {"section_type": "METHODS"}, "text": "Methods..."},
+                        {"infons": {"section_type": "RESULTS"}, "text": "Results..."},
+                        {"infons": {"section_type": "DISCUSS"}, "text": "Discussion..."},
+                    ]
+                }
+            ]
+        }
+
+        respx.get(
+            "https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/12345678/unicode"
+        ).mock(return_value=Response(200, json=mock_bioc))
+
+        sections = await tool.get_fulltext_structured("12345678")
+
+        assert sections is not None
+        assert "introduction" in sections
+        assert "methods" in sections
+        assert "results" in sections
+        assert "discussion" in sections
+
+    @respx.mock
+    @pytest.mark.asyncio
+    async def test_search_with_fulltext_enabled(self) -> None:
+        """Search should include full text when tool is configured for it."""
+        # Create tool WITH full-text enabled via constructor
+        tool = PubMedTool(include_fulltext=True)
+
+        # Mock esearch
+        respx.get("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi").mock(
+            return_value=Response(
+                200, json={"esearchresult": {"idlist": ["12345678"]}}
+            )
+        )
+
+        # Mock efetch (abstract)
+        mock_xml = """
+        <PubmedArticleSet>
+          <PubmedArticle>
+            <MedlineCitation>
+              <PMID>12345678</PMID>
+              <Article>
+                <ArticleTitle>Test Paper</ArticleTitle>
+                <Abstract><AbstractText>Short abstract.</AbstractText></Abstract>
+                <AuthorList><Author><LastName>Smith</LastName></Author></AuthorList>
+              </Article>
+            </MedlineCitation>
+          </PubmedArticle>
+        </PubmedArticleSet>
+        """
+        respx.get("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi").mock(
+            return_value=Response(200, text=mock_xml)
+        )
+
+        # Mock ID converter
+        respx.get("https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/").mock(
+            return_value=Response(
+                200, json={"records": [{"pmid": "12345678", "pmcid": "PMC1234567"}]}
+            )
+        )
+
+        # Mock BioC full text
+        mock_bioc = {
+            "documents": [
+                {
+                    "passages": [
+                        {"infons": {"section_type": "INTRO"}, "text": "Full intro..."},
+                    ]
+                }
+            ]
+        }
+        respx.get(
+            "https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/12345678/unicode"
+        ).mock(return_value=Response(200, json=mock_bioc))
+
+        # NOTE: No include_fulltext param - it's set via constructor
+        results = await tool.search("test", max_results=1)
+
+        assert len(results) == 1
+        # Full text should be appended or replace abstract
+        assert "Full intro" in results[0].content or "Short abstract" in results[0].content
+```
+
+---
+
+### Step 2: Implement Full-Text Methods
+
+**File**: `src/tools/pubmed.py` (additions to existing class)
+
+```python
+# Add these methods to PubMedTool class
+
+async def get_pmc_id(self, pmid: str) -> str | None:
+    """
+    Convert PMID to PMCID for full-text access.
+
+    Args:
+        pmid: PubMed ID
+
+    Returns:
+        PMCID if paper is in PMC, None otherwise
+    """
+    url = "https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/"
+    params = {"ids": pmid, "format": "json"}
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        try:
+            response = await client.get(url, params=params)
+            response.raise_for_status()
+            data = response.json()
+
+            records = data.get("records", [])
+            if records and records[0].get("pmcid"):
+                return records[0]["pmcid"]
+            return None
+
+        except httpx.HTTPError:
+            return None
+
+
+async def get_fulltext(self, pmid: str) -> str | None:
+    """
+    Get full text for a PubMed paper via BioC API.
+
+    Only works for open-access papers in PubMed Central.
+
+    Args:
+        pmid: PubMed ID
+
+    Returns:
+        Full text as string, or None if not available
+    """
+    url = f"https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/{pmid}/unicode"
+
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        try:
+            response = await client.get(url)
+            if response.status_code == 404:
+                return None
+            response.raise_for_status()
+            data = response.json()
+
+            # Extract text from all passages
+            documents = data.get("documents", [])
+            if not documents:
+                return None
+
+            passages = documents[0].get("passages", [])
+            text_parts = [p.get("text", "") for p in passages if p.get("text")]
+
+            return "\n\n".join(text_parts) if text_parts else None
+
+        except httpx.HTTPError:
+            return None
+
+
+async def get_fulltext_structured(self, pmid: str) -> dict[str, str] | None:
+    """
+    Get structured full text with sections.
+
+    Args:
+        pmid: PubMed ID
+
+    Returns:
+        Dict mapping section names to text, or None if not available
+    """
+    url = f"https://www.ncbi.nlm.nih.gov/research/bionlp/RESTful/pmcoa.cgi/BioC_json/{pmid}/unicode"
+
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        try:
+            response = await client.get(url)
+            if response.status_code == 404:
+                return None
+            response.raise_for_status()
+            data = response.json()
+
+            documents = data.get("documents", [])
+            if not documents:
+                return None
+
+            # Map section types to readable names
+            section_map = {
+                "INTRO": "introduction",
+                "METHODS": "methods",
+                "RESULTS": "results",
+                "DISCUSS": "discussion",
+                "CONCL": "conclusion",
+                "ABSTRACT": "abstract",
+            }
+
+            sections: dict[str, list[str]] = {}
+            for passage in documents[0].get("passages", []):
+                section_type = passage.get("infons", {}).get("section_type", "other")
+                section_name = section_map.get(section_type, "other")
+                text = passage.get("text", "")
+
+                if text:
+                    if section_name not in sections:
+                        sections[section_name] = []
+                    sections[section_name].append(text)
+
+            # Join multiple passages per section
+            return {k: "\n\n".join(v) for k, v in sections.items()}
+
+        except httpx.HTTPError:
+            return None
+```
+
+---
+
+### Step 3: Update Constructor and Search Method
+
+Add full-text flag to constructor and update search to use it:
+
+```python
+class PubMedTool:
+    """Search tool for PubMed/NCBI."""
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        include_fulltext: bool = False,  # NEW CONSTRUCTOR PARAM
+    ) -> None:
+        self.api_key = api_key or settings.ncbi_api_key
+        if self.api_key == "your-ncbi-key-here":
+            self.api_key = None
+        self._last_request_time = 0.0
+        self.include_fulltext = include_fulltext  # Store for use in search()
+
+    async def search(self, query: str, max_results: int = 10) -> list[Evidence]:
+        """
+        Search PubMed and return evidence.
+
+        Note: Full-text enrichment is controlled by constructor parameter,
+        not method parameter, because SearchHandler doesn't pass extra args.
+        """
+        # ... existing search logic ...
+
+        evidence_list = self._parse_pubmed_xml(fetch_resp.text)
+
+        # Optionally enrich with full text (if configured at construction)
+        if self.include_fulltext:
+            evidence_list = await self._enrich_with_fulltext(evidence_list)
+
+        return evidence_list
+
+
+async def _enrich_with_fulltext(
+    self, evidence_list: list[Evidence]
+) -> list[Evidence]:
+    """Attempt to add full text to evidence items."""
+    enriched = []
+
+    for evidence in evidence_list:
+        # Extract PMID from URL
+        url = evidence.citation.url
+        pmid = url.rstrip("/").split("/")[-1] if url else None
+
+        if pmid:
+            fulltext = await self.get_fulltext(pmid)
+            if fulltext:
+                # Replace abstract with full text (truncated)
+                evidence = Evidence(
+                    content=fulltext[:8000],  # Larger limit for full text
+                    citation=evidence.citation,
+                    relevance=evidence.relevance,
+                    metadata={
+                        **evidence.metadata,
+                        "has_fulltext": True,
+                    },
+                )
+
+        enriched.append(evidence)
+
+    return enriched
+```
+
+---
+
+## Demo Script
+
+**File**: `examples/pubmed_fulltext_demo.py`
+
+```python
+#!/usr/bin/env python3
+"""Demo script to verify PubMed full-text retrieval."""
+
+import asyncio
+from src.tools.pubmed import PubMedTool
+
+
+async def main():
+    """Run PubMed full-text demo."""
+    tool = PubMedTool()
+
+    print("=" * 60)
+    print("PubMed Full-Text Demo")
+    print("=" * 60)
+
+    # Test 1: Convert PMID to PMCID
+    print("\n[Test 1] Converting PMID to PMCID...")
+    # Use a known open-access paper
+    test_pmid = "34450029"  # Example: COVID-related open-access paper
+    pmcid = await tool.get_pmc_id(test_pmid)
+    print(f"PMID {test_pmid} -> PMCID: {pmcid or 'Not in PMC'}")
+
+    # Test 2: Get full text
+    print("\n[Test 2] Fetching full text...")
+    if pmcid:
+        fulltext = await tool.get_fulltext(test_pmid)
+        if fulltext:
+            print(f"Full text length: {len(fulltext)} characters")
+            print(f"Preview: {fulltext[:500]}...")
+        else:
+            print("Full text not available")
+
+    # Test 3: Get structured sections
+    print("\n[Test 3] Fetching structured sections...")
+    if pmcid:
+        sections = await tool.get_fulltext_structured(test_pmid)
+        if sections:
+            print("Available sections:")
+            for section, text in sections.items():
+                print(f"  - {section}: {len(text)} chars")
+        else:
+            print("Structured text not available")
+
+    # Test 4: Search with full text
+    print("\n[Test 4] Search with full-text enrichment...")
+    results = await tool.search(
+        "metformin cancer open access",
+        max_results=3,
+        include_fulltext=True
+    )
+
+    for i, evidence in enumerate(results, 1):
+        has_ft = evidence.metadata.get("has_fulltext", False)
+        print(f"\n--- Result {i} ---")
+        print(f"Title: {evidence.citation.title}")
+        print(f"Has Full Text: {has_ft}")
+        print(f"Content Length: {len(evidence.content)} chars")
+
+    print("\n" + "=" * 60)
+    print("Demo complete!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## Verification Checklist
+
+### Unit Tests
+```bash
+# Run full-text tests
+uv run pytest tests/unit/tools/test_pubmed_fulltext.py -v
+
+# Run all PubMed tests
+uv run pytest tests/unit/tools/test_pubmed.py -v
+
+# Expected: All tests pass
+```
+
+### Integration Test (Manual)
+```bash
+# Run demo with real API
+uv run python examples/pubmed_fulltext_demo.py
+
+# Expected: Real full text from PMC papers
+```
+
+### Full Test Suite
+```bash
+make check
+# Expected: All tests pass, mypy clean
+```
+
+---
+
+## Success Criteria
+
+1. **ID Conversion works**: PMID -> PMCID conversion successful
+2. **Full text retrieval works**: BioC API returns paper text
+3. **Structured sections work**: Can get intro/methods/results/discussion separately
+4. **Search integration works**: `include_fulltext=True` enriches results
+5. **No regressions**: Existing tests still pass
+6. **Graceful degradation**: Non-PMC papers still return abstracts
+
+---
+
+## Notes
+
+- Only ~30% of PubMed papers have full text in PMC
+- BioC API has no documented rate limit, but be respectful
+- Full text can be very long - truncate appropriately
+- Consider caching full text responses (they don't change)
+- Timeout should be longer for full text (60s vs 30s)

docs/brainstorming/implementation/17_PHASE_RATE_LIMITING.md

@@ -0,0 +1,540 @@
+# Phase 17: Rate Limiting with `limits` Library
+
+**Priority**: P0 CRITICAL - Prevents API blocks
+**Effort**: ~1 hour
+**Dependencies**: None
+
+---
+
+## CRITICAL: Async Safety Requirements
+
+**WARNING**: The rate limiter MUST be async-safe. Blocking the event loop will freeze:
+- The Gradio UI
+- All parallel searches
+- The orchestrator
+
+**Rules**:
+1. **NEVER use `time.sleep()`** - Always use `await asyncio.sleep()`
+2. **NEVER use blocking while loops** - Use async-aware polling
+3. **The `limits` library check is synchronous** - Wrap it carefully
+
+The implementation below uses a polling pattern that:
+- Checks the limit (synchronous, fast)
+- If exceeded, `await asyncio.sleep()` (non-blocking)
+- Retry the check
+
+**Alternative**: If `limits` proves problematic, use `aiolimiter` which is pure-async.
+
+---
+
+## Overview
+
+Replace naive `asyncio.sleep` rate limiting with proper rate limiter using the `limits` library, which provides:
+- Moving window rate limiting
+- Per-API configurable limits
+- Thread-safe storage
+- Already used in reference repo
+
+**Why This Matters?**
+- NCBI will block us without proper rate limiting (3/sec without key, 10/sec with)
+- Current implementation only has simple sleep delay
+- Need coordinated limits across all PubMed calls
+- Professional-grade rate limiting prevents production issues
+
+---
+
+## Current State
+
+### What We Have (`src/tools/pubmed.py:20-21, 34-41`)
+
+```python
+RATE_LIMIT_DELAY = 0.34  # ~3 requests/sec without API key
+
+async def _rate_limit(self) -> None:
+    """Enforce NCBI rate limiting."""
+    loop = asyncio.get_running_loop()
+    now = loop.time()
+    elapsed = now - self._last_request_time
+    if elapsed < self.RATE_LIMIT_DELAY:
+        await asyncio.sleep(self.RATE_LIMIT_DELAY - elapsed)
+    self._last_request_time = loop.time()
+```
+
+### Problems
+
+1. **Not shared across instances**: Each `PubMedTool()` has its own counter
+2. **Simple delay vs moving window**: Doesn't handle bursts properly
+3. **Hardcoded rate**: Doesn't adapt to API key presence
+4. **No backoff on 429**: Just retries blindly
+
+---
+
+## TDD Implementation Plan
+
+### Step 1: Add Dependency
+
+**File**: `pyproject.toml`
+
+```toml
+dependencies = [
+    # ... existing deps ...
+    "limits>=3.0",
+]
+```
+
+Then run:
+```bash
+uv sync
+```
+
+---
+
+### Step 2: Write the Tests First
+
+**File**: `tests/unit/tools/test_rate_limiting.py`
+
+```python
+"""Tests for rate limiting functionality."""
+
+import asyncio
+import time
+
+import pytest
+
+from src.tools.rate_limiter import RateLimiter, get_pubmed_limiter
+
+
+class TestRateLimiter:
+    """Test suite for rate limiter."""
+
+    def test_create_limiter_without_api_key(self) -> None:
+        """Should create 3/sec limiter without API key."""
+        limiter = RateLimiter(rate="3/second")
+        assert limiter.rate == "3/second"
+
+    def test_create_limiter_with_api_key(self) -> None:
+        """Should create 10/sec limiter with API key."""
+        limiter = RateLimiter(rate="10/second")
+        assert limiter.rate == "10/second"
+
+    @pytest.mark.asyncio
+    async def test_limiter_allows_requests_under_limit(self) -> None:
+        """Should allow requests under the rate limit."""
+        limiter = RateLimiter(rate="10/second")
+
+        # 3 requests should all succeed immediately
+        for _ in range(3):
+            allowed = await limiter.acquire()
+            assert allowed is True
+
+    @pytest.mark.asyncio
+    async def test_limiter_blocks_when_exceeded(self) -> None:
+        """Should wait when rate limit exceeded."""
+        limiter = RateLimiter(rate="2/second")
+
+        # First 2 should be instant
+        await limiter.acquire()
+        await limiter.acquire()
+
+        # Third should block briefly
+        start = time.monotonic()
+        await limiter.acquire()
+        elapsed = time.monotonic() - start
+
+        # Should have waited ~0.5 seconds (half second window for 2/sec)
+        assert elapsed >= 0.3
+
+    @pytest.mark.asyncio
+    async def test_limiter_resets_after_window(self) -> None:
+        """Rate limit should reset after time window."""
+        limiter = RateLimiter(rate="5/second")
+
+        # Use up the limit
+        for _ in range(5):
+            await limiter.acquire()
+
+        # Wait for window to pass
+        await asyncio.sleep(1.1)
+
+        # Should be allowed again
+        start = time.monotonic()
+        await limiter.acquire()
+        elapsed = time.monotonic() - start
+
+        assert elapsed < 0.1  # Should be nearly instant
+
+
+class TestGetPubmedLimiter:
+    """Test PubMed-specific limiter factory."""
+
+    def test_limiter_without_api_key(self) -> None:
+        """Should return 3/sec limiter without key."""
+        limiter = get_pubmed_limiter(api_key=None)
+        assert "3" in limiter.rate
+
+    def test_limiter_with_api_key(self) -> None:
+        """Should return 10/sec limiter with key."""
+        limiter = get_pubmed_limiter(api_key="my-api-key")
+        assert "10" in limiter.rate
+
+    def test_limiter_is_singleton(self) -> None:
+        """Same API key should return same limiter instance."""
+        limiter1 = get_pubmed_limiter(api_key="key1")
+        limiter2 = get_pubmed_limiter(api_key="key1")
+        assert limiter1 is limiter2
+
+    def test_different_keys_different_limiters(self) -> None:
+        """Different API keys should return different limiters."""
+        limiter1 = get_pubmed_limiter(api_key="key1")
+        limiter2 = get_pubmed_limiter(api_key="key2")
+        # Clear cache for clean test
+        # Actually, different keys SHOULD share the same limiter
+        # since we're limiting against the same API
+        assert limiter1 is limiter2  # Shared NCBI rate limit
+```
+
+---
+
+### Step 3: Create Rate Limiter Module
+
+**File**: `src/tools/rate_limiter.py`
+
+```python
+"""Rate limiting utilities using the limits library."""
+
+import asyncio
+from typing import ClassVar
+
+from limits import RateLimitItem, parse
+from limits.storage import MemoryStorage
+from limits.strategies import MovingWindowRateLimiter
+
+
+class RateLimiter:
+    """
+    Async-compatible rate limiter using limits library.
+
+    Uses moving window algorithm for smooth rate limiting.
+    """
+
+    def __init__(self, rate: str) -> None:
+        """
+        Initialize rate limiter.
+
+        Args:
+            rate: Rate string like "3/second" or "10/second"
+        """
+        self.rate = rate
+        self._storage = MemoryStorage()
+        self._limiter = MovingWindowRateLimiter(self._storage)
+        self._rate_limit: RateLimitItem = parse(rate)
+        self._identity = "default"  # Single identity for shared limiting
+
+    async def acquire(self, wait: bool = True) -> bool:
+        """
+        Acquire permission to make a request.
+
+        ASYNC-SAFE: Uses asyncio.sleep(), never time.sleep().
+        The polling pattern allows other coroutines to run while waiting.
+
+        Args:
+            wait: If True, wait until allowed. If False, return immediately.
+
+        Returns:
+            True if allowed, False if not (only when wait=False)
+        """
+        while True:
+            # Check if we can proceed (synchronous, fast - ~microseconds)
+            if self._limiter.hit(self._rate_limit, self._identity):
+                return True
+
+            if not wait:
+                return False
+
+            # CRITICAL: Use asyncio.sleep(), NOT time.sleep()
+            # This yields control to the event loop, allowing other
+            # coroutines (UI, parallel searches) to run
+            await asyncio.sleep(0.1)
+
+    def reset(self) -> None:
+        """Reset the rate limiter (for testing)."""
+        self._storage.reset()
+
+
+# Singleton limiter for PubMed/NCBI
+_pubmed_limiter: RateLimiter | None = None
+
+
+def get_pubmed_limiter(api_key: str | None = None) -> RateLimiter:
+    """
+    Get the shared PubMed rate limiter.
+
+    Rate depends on whether API key is provided:
+    - Without key: 3 requests/second
+    - With key: 10 requests/second
+
+    Args:
+        api_key: NCBI API key (optional)
+
+    Returns:
+        Shared RateLimiter instance
+    """
+    global _pubmed_limiter
+
+    if _pubmed_limiter is None:
+        rate = "10/second" if api_key else "3/second"
+        _pubmed_limiter = RateLimiter(rate)
+
+    return _pubmed_limiter
+
+
+def reset_pubmed_limiter() -> None:
+    """Reset the PubMed limiter (for testing)."""
+    global _pubmed_limiter
+    _pubmed_limiter = None
+
+
+# Factory for other APIs
+class RateLimiterFactory:
+    """Factory for creating/getting rate limiters for different APIs."""
+
+    _limiters: ClassVar[dict[str, RateLimiter]] = {}
+
+    @classmethod
+    def get(cls, api_name: str, rate: str) -> RateLimiter:
+        """
+        Get or create a rate limiter for an API.
+
+        Args:
+            api_name: Unique identifier for the API
+            rate: Rate limit string (e.g., "10/second")
+
+        Returns:
+            RateLimiter instance (shared for same api_name)
+        """
+        if api_name not in cls._limiters:
+            cls._limiters[api_name] = RateLimiter(rate)
+        return cls._limiters[api_name]
+
+    @classmethod
+    def reset_all(cls) -> None:
+        """Reset all limiters (for testing)."""
+        cls._limiters.clear()
+```
+
+---
+
+### Step 4: Update PubMed Tool
+
+**File**: `src/tools/pubmed.py` (replace rate limiting code)
+
+```python
+# Replace imports and rate limiting
+
+from src.tools.rate_limiter import get_pubmed_limiter
+
+
+class PubMedTool:
+    """Search tool for PubMed/NCBI."""
+
+    BASE_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
+    HTTP_TOO_MANY_REQUESTS = 429
+
+    def __init__(self, api_key: str | None = None) -> None:
+        self.api_key = api_key or settings.ncbi_api_key
+        if self.api_key == "your-ncbi-key-here":
+            self.api_key = None
+        # Use shared rate limiter
+        self._limiter = get_pubmed_limiter(self.api_key)
+
+    async def _rate_limit(self) -> None:
+        """Enforce NCBI rate limiting using shared limiter."""
+        await self._limiter.acquire()
+
+    # ... rest of class unchanged ...
+```
+
+---
+
+### Step 5: Add Rate Limiters for Other APIs
+
+**File**: `src/tools/clinicaltrials.py` (optional)
+
+```python
+from src.tools.rate_limiter import RateLimiterFactory
+
+
+class ClinicalTrialsTool:
+    def __init__(self) -> None:
+        # ClinicalTrials.gov doesn't document limits, but be conservative
+        self._limiter = RateLimiterFactory.get("clinicaltrials", "5/second")
+
+    async def search(self, query: str, max_results: int = 10) -> list[Evidence]:
+        await self._limiter.acquire()
+        # ... rest of method ...
+```
+
+**File**: `src/tools/europepmc.py` (optional)
+
+```python
+from src.tools.rate_limiter import RateLimiterFactory
+
+
+class EuropePMCTool:
+    def __init__(self) -> None:
+        # Europe PMC is generous, but still be respectful
+        self._limiter = RateLimiterFactory.get("europepmc", "10/second")
+
+    async def search(self, query: str, max_results: int = 10) -> list[Evidence]:
+        await self._limiter.acquire()
+        # ... rest of method ...
+```
+
+---
+
+## Demo Script
+
+**File**: `examples/rate_limiting_demo.py`
+
+```python
+#!/usr/bin/env python3
+"""Demo script to verify rate limiting works correctly."""
+
+import asyncio
+import time
+
+from src.tools.rate_limiter import RateLimiter, get_pubmed_limiter, reset_pubmed_limiter
+from src.tools.pubmed import PubMedTool
+
+
+async def test_basic_limiter():
+    """Test basic rate limiter behavior."""
+    print("=" * 60)
+    print("Rate Limiting Demo")
+    print("=" * 60)
+
+    # Test 1: Basic limiter
+    print("\n[Test 1] Testing 3/second limiter...")
+    limiter = RateLimiter("3/second")
+
+    start = time.monotonic()
+    for i in range(6):
+        await limiter.acquire()
+        elapsed = time.monotonic() - start
+        print(f"  Request {i+1} at {elapsed:.2f}s")
+
+    total = time.monotonic() - start
+    print(f"  Total time for 6 requests: {total:.2f}s (expected ~2s)")
+
+
+async def test_pubmed_limiter():
+    """Test PubMed-specific limiter."""
+    print("\n[Test 2] Testing PubMed limiter (shared)...")
+
+    reset_pubmed_limiter()  # Clean state
+
+    # Without API key: 3/sec
+    limiter = get_pubmed_limiter(api_key=None)
+    print(f"  Rate without key: {limiter.rate}")
+
+    # Multiple tools should share the same limiter
+    tool1 = PubMedTool()
+    tool2 = PubMedTool()
+
+    # Verify they share the limiter
+    print(f"  Tools share limiter: {tool1._limiter is tool2._limiter}")
+
+
+async def test_concurrent_requests():
+    """Test rate limiting under concurrent load."""
+    print("\n[Test 3] Testing concurrent request limiting...")
+
+    limiter = RateLimiter("5/second")
+
+    async def make_request(i: int):
+        await limiter.acquire()
+        return time.monotonic()
+
+    start = time.monotonic()
+    # Launch 10 concurrent requests
+    tasks = [make_request(i) for i in range(10)]
+    times = await asyncio.gather(*tasks)
+
+    # Calculate distribution
+    relative_times = [t - start for t in times]
+    print(f"  Request times: {[f'{t:.2f}s' for t in sorted(relative_times)]}")
+
+    total = max(relative_times)
+    print(f"  All 10 requests completed in {total:.2f}s (expected ~2s)")
+
+
+async def main():
+    await test_basic_limiter()
+    await test_pubmed_limiter()
+    await test_concurrent_requests()
+
+    print("\n" + "=" * 60)
+    print("Demo complete!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+---
+
+## Verification Checklist
+
+### Unit Tests
+```bash
+# Run rate limiting tests
+uv run pytest tests/unit/tools/test_rate_limiting.py -v
+
+# Expected: All tests pass
+```
+
+### Integration Test (Manual)
+```bash
+# Run demo
+uv run python examples/rate_limiting_demo.py
+
+# Expected: Requests properly spaced
+```
+
+### Full Test Suite
+```bash
+make check
+# Expected: All tests pass, mypy clean
+```
+
+---
+
+## Success Criteria
+
+1. **`limits` library installed**: Dependency added to pyproject.toml
+2. **RateLimiter class works**: Can create and use limiters
+3. **PubMed uses new limiter**: Shared limiter across instances
+4. **Rate adapts to API key**: 3/sec without, 10/sec with
+5. **Concurrent requests handled**: Multiple async requests properly queued
+6. **No regressions**: All existing tests pass
+
+---
+
+## API Rate Limit Reference
+
+| API | Without Key | With Key |
+|-----|-------------|----------|
+| PubMed/NCBI | 3/sec | 10/sec |
+| ClinicalTrials.gov | Undocumented (~5/sec safe) | N/A |
+| Europe PMC | ~10-20/sec (generous) | N/A |
+| OpenAlex | ~100k/day (no per-sec limit) | Faster with `mailto` |
+
+---
+
+## Notes
+
+- `limits` library uses moving window algorithm (fairer than fixed window)
+- Singleton pattern ensures all PubMed calls share the limit
+- The factory pattern allows easy extension to other APIs
+- Consider adding 429 response detection + exponential backoff
+- In production, consider Redis storage for distributed rate limiting

docs/brainstorming/implementation/README.md

@@ -0,0 +1,143 @@
+# Implementation Plans
+
+TDD implementation plans based on the brainstorming documents. Each phase is a self-contained vertical slice with tests, implementation, and demo scripts.
+
+---
+
+## Prerequisites (COMPLETED)
+
+The following foundational changes have been implemented to support all three phases:
+
+| Change | File | Status |
+|--------|------|--------|
+| Add `"openalex"` to `SourceName` | `src/utils/models.py:9` | ✅ Done |
+| Add `metadata` field to `Evidence` | `src/utils/models.py:39-42` | ✅ Done |
+| Export all tools from `__init__.py` | `src/tools/__init__.py` | ✅ Done |
+
+All 110 tests pass after these changes.
+
+---
+
+## Priority Order
+
+| Phase | Name | Priority | Effort | Value |
+|-------|------|----------|--------|-------|
+| **17** | Rate Limiting | P0 CRITICAL | 1 hour | Stability |
+| **15** | OpenAlex | HIGH | 2-3 hours | Very High |
+| **16** | PubMed Full-Text | MEDIUM | 3 hours | High |
+
+**Recommended implementation order**: 17 → 15 → 16
+
+---
+
+## Phase 15: OpenAlex Integration
+
+**File**: [15_PHASE_OPENALEX.md](./15_PHASE_OPENALEX.md)
+
+Add OpenAlex as 4th data source for:
+- Citation networks (who cites whom)
+- Concept tagging (semantic discovery)
+- 209M+ scholarly works
+- Free, no API key required
+
+**Quick Start**:
+```bash
+# Create the tool
+touch src/tools/openalex.py
+touch tests/unit/tools/test_openalex.py
+
+# Run tests first (TDD)
+uv run pytest tests/unit/tools/test_openalex.py -v
+
+# Demo
+uv run python examples/openalex_demo.py
+```
+
+---
+
+## Phase 16: PubMed Full-Text
+
+**File**: [16_PHASE_PUBMED_FULLTEXT.md](./16_PHASE_PUBMED_FULLTEXT.md)
+
+Add full-text retrieval via BioC API for:
+- Complete paper text (not just abstracts)
+- Structured sections (intro, methods, results)
+- Better evidence for LLM synthesis
+
+**Quick Start**:
+```bash
+# Add methods to existing pubmed.py
+# Tests in test_pubmed_fulltext.py
+
+# Run tests
+uv run pytest tests/unit/tools/test_pubmed_fulltext.py -v
+
+# Demo
+uv run python examples/pubmed_fulltext_demo.py
+```
+
+---
+
+## Phase 17: Rate Limiting
+
+**File**: [17_PHASE_RATE_LIMITING.md](./17_PHASE_RATE_LIMITING.md)
+
+Replace naive sleep-based rate limiting with `limits` library for:
+- Moving window algorithm
+- Shared limits across instances
+- Configurable per-API rates
+- Production-grade stability
+
+**Quick Start**:
+```bash
+# Add dependency
+uv add limits
+
+# Create module
+touch src/tools/rate_limiter.py
+touch tests/unit/tools/test_rate_limiting.py
+
+# Run tests
+uv run pytest tests/unit/tools/test_rate_limiting.py -v
+
+# Demo
+uv run python examples/rate_limiting_demo.py
+```
+
+---
+
+## TDD Workflow
+
+Each implementation doc follows this pattern:
+
+1. **Write tests first** - Define expected behavior
+2. **Run tests** - Verify they fail (red)
+3. **Implement** - Write minimal code to pass
+4. **Run tests** - Verify they pass (green)
+5. **Refactor** - Clean up if needed
+6. **Demo** - Verify end-to-end with real APIs
+7. **`make check`** - Ensure no regressions
+
+---
+
+## Related Brainstorming Docs
+
+These implementation plans are derived from:
+
+- [00_ROADMAP_SUMMARY.md](../00_ROADMAP_SUMMARY.md) - Priority overview
+- [01_PUBMED_IMPROVEMENTS.md](../01_PUBMED_IMPROVEMENTS.md) - PubMed details
+- [02_CLINICALTRIALS_IMPROVEMENTS.md](../02_CLINICALTRIALS_IMPROVEMENTS.md) - CT.gov details
+- [03_EUROPEPMC_IMPROVEMENTS.md](../03_EUROPEPMC_IMPROVEMENTS.md) - Europe PMC details
+- [04_OPENALEX_INTEGRATION.md](../04_OPENALEX_INTEGRATION.md) - OpenAlex integration
+
+---
+
+## Future Phases (Not Yet Documented)
+
+Based on brainstorming, these could be added later:
+
+- **Phase 18**: ClinicalTrials.gov Results Retrieval
+- **Phase 19**: Europe PMC Annotations API
+- **Phase 20**: Drug Name Normalization (RxNorm)
+- **Phase 21**: Citation Network Queries (OpenAlex)
+- **Phase 22**: Semantic Search with Embeddings

docs/bugs/P1_GRADIO_SETTINGS_CLEANUP.md

@@ -3,131 +3,79 @@
 **Priority**: P1 (UX Bug)
 **Status**: OPEN
 **Date**: 2025-11-27
+**Target Component**: `src/app.py`
 
 ---
 
-## Bug Description
+## 1. Problem Description
 
-The "Settings" accordion in the Gradio UI does not collapse/hide its content. Even when the accordion arrow shows "collapsed" state, all settings (Orchestrator Mode, API Key, API Provider) remain visible.
+The "Settings" accordion in the Gradio UI (containing Orchestrator Mode, API Key, Provider) fails to collapse, even when configured with `open=False`. It remains permanently expanded, cluttering the interface and obscuring the chat history.
 
----
-
-## Root Cause
+### Symptoms
+- Accordion arrow toggles visually, but content remains visible.
+- Occurs in both local development (`uv run src/app.py`) and HuggingFace Spaces.
 
-**Known Gradio Bug**: `additional_inputs_accordion` does not work correctly when `ChatInterface` is used inside `gr.Blocks()`.
+---
 
-**GitHub Issue**: [gradio-app/gradio#8861](https://github.com/gradio-app/gradio/issues/8861)
-> "Is there any subsequent plan to support gr.ChatInterface inheritance under gr.Block()? Currently using accordion is not working well."
+## 2. Root Cause Analysis
 
-**Our Code** (`src/app.py` lines 196-250):
-```python
-with gr.Blocks(...) as demo:  # <-- Using gr.Blocks wrapper
-    gr.ChatInterface(
-        ...
-        additional_inputs_accordion=gr.Accordion(label="⚙️ Settings", open=False),
-        additional_inputs=[...],
-    )
-```
+**Definitive Cause**: Nested `Blocks` Context Bug.
+`gr.ChatInterface` is itself a high-level abstraction that creates a `gr.Blocks` context. Wrapping `gr.ChatInterface` inside an external `with gr.Blocks():` context causes event listener conflicts, specifically breaking the JavaScript state management for `additional_inputs_accordion`.
 
-The `additional_inputs_accordion` parameter is designed for standalone `ChatInterface`, but breaks when wrapped in `gr.Blocks()`.
+**Reference**: [Gradio Issue #8861](https://github.com/gradio-app/gradio/issues/8861) confirms that `additional_inputs_accordion` malfunctions when `ChatInterface` is not the top-level block.
 
 ---
 
-## Evidence
+## 3. Solution Strategy: "The Unwrap Fix"
 
-- Accordion arrow toggles (visual feedback works)
-- Content does NOT hide when collapsed
-- Same behavior in local dev and HuggingFace Spaces
-
----
+We will remove the redundant `gr.Blocks` wrapper. This restores the native behavior of `ChatInterface`, ensuring the accordion respects `open=False`.
 
-## Possible Fixes
+### Implementation Plan
 
-### Option 1: Remove gr.Blocks Wrapper (Recommended)
+**Refactor `src/app.py` / `create_demo()`**:
 
-If we don't need the header/footer markdown, use standalone `ChatInterface`:
+1.  **Remove** the `with gr.Blocks() as demo:` context manager.
+2.  **Instantiate** `gr.ChatInterface` directly as the `demo` object.
+3.  **Migrate UI Elements**:
+    *   **Header**: Move the H1/Title text into the `title` parameter of `ChatInterface`.
+    *   **Footer**: Move the footer text ("MCP Server Active...") into the `description` parameter. `ChatInterface` supports Markdown in `description`, making it the ideal place for static info below the title but above the chat.
 
+### Before (Buggy)
 ```python
-# Instead of gr.Blocks wrapper
-demo = gr.ChatInterface(
-    fn=research_agent,
-    title="🧬 DeepCritical",
-    description="AI-Powered Drug Repurposing Agent",
-    additional_inputs_accordion=gr.Accordion(label="⚙️ Settings", open=False),
-    additional_inputs=[...],
-)
+def create_demo():
+    with gr.Blocks() as demo:  # <--- CAUSE OF BUG
+        gr.Markdown("# Title")
+        gr.ChatInterface(..., additional_inputs_accordion=gr.Accordion(open=False))
+        gr.Markdown("Footer")
+    return demo
 ```
 
-**Pros**: Accordion should work correctly
-**Cons**: Less control over layout, no custom header/footer
-
-### Option 2: Manual Accordion Outside ChatInterface
-
-Move settings outside `ChatInterface` into a proper `gr.Accordion`:
-
+### After (Correct)
 ```python
-with gr.Blocks() as demo:
-    gr.Markdown("# 🧬 DeepCritical")
-
-    with gr.Accordion("⚙️ Settings", open=False):
-        mode = gr.Radio(choices=["simple", "magentic"], value="simple", label="Mode")
-        api_key = gr.Textbox(label="API Key", type="password")
-        provider = gr.Radio(choices=["openai", "anthropic"], value="openai", label="Provider")
-
-    chatbot = gr.Chatbot()
-    msg = gr.Textbox(label="Ask a research question")
-
-    msg.submit(research_agent, [msg, chatbot, mode, api_key, provider], chatbot)
-```
-
-**Pros**: Full control, accordion works
-**Cons**: More code, lose ChatInterface conveniences (examples, etc.)
-
-### Option 3: Wait for Gradio Fix
-
-Gradio added `.expand()` and `.collapse()` events in recent versions. Upgrading might help.
-
-**Check current version**:
-```bash
-pip show gradio | grep Version
-```
-
-**Upgrade**:
-```bash
-pip install --upgrade gradio
+def create_demo():
+    return gr.ChatInterface(   # <--- FIX: Top-level component
+        ...,
+        title="🧬 DeepCritical",
+        description="*AI-Powered Drug Repurposing Agent...*\n\n---\n**MCP Server Active**...",
+        additional_inputs_accordion=gr.Accordion(label="⚙️ Settings", open=False)
+    )
 ```
 
 ---
 
-## Recommendation
-
-**Option 1** (Remove gr.Blocks) is cleanest if we can live without custom header/footer.
-
-If header/footer needed, **Option 2** gives working accordion at cost of more code.
-
----
-
-## Files to Modify
-
-| File | Change |
-|------|--------|
-| `src/app.py` | Restructure UI per chosen option |
-| `pyproject.toml` | Possibly upgrade Gradio version |
-
----
-
-## Test Plan
+## 4. Validation
 
-1. Run locally: `uv run python -m src.app`
-2. Click Settings accordion to collapse
-3. Verify content hides when collapsed
-4. Verify content shows when expanded
-5. Test on HuggingFace Spaces after deploy
+1.  **Run**: `uv run python src/app.py`
+2.  **Check**: Open `http://localhost:7860`
+3.  **Verify**:
+    *   Settings accordion starts **COLLAPSED**.
+    *   Header title ("DeepCritical") is visible.
+    *   Footer text ("MCP Server Active") is visible in the description area.
+    *   Chat functionality works (Magentic/Simple modes).
 
 ---
 
-## Sources
+## 5. Constraints & Notes
 
-- [Gradio Issue #8861 - Accordion not working in Blocks](https://github.com/gradio-app/gradio/issues/8861)
-- [Gradio ChatInterface Docs](https://www.gradio.app/docs/gradio/chatinterface)
-- [Gradio Accordion Docs](https://www.gradio.app/docs/gradio/accordion)
+- **Layout**: We lose the ability to place arbitrary elements *below* the chat box (footer will move to top, under title), but this is an acceptable trade-off for a working UI.
+- **CSS**: `ChatInterface` handles its own CSS; any custom class styling from the previous footer will be standardized to the description text style.

examples/rate_limiting_demo.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Demo script to verify rate limiting works correctly."""
+
+import asyncio
+import time
+
+from src.tools.pubmed import PubMedTool
+from src.tools.rate_limiter import RateLimiter, get_pubmed_limiter, reset_pubmed_limiter
+
+
+async def test_basic_limiter():
+    """Test basic rate limiter behavior."""
+    print("=" * 60)
+    print("Rate Limiting Demo")
+    print("=" * 60)
+
+    # Test 1: Basic limiter
+    print("\n[Test 1] Testing 3/second limiter...")
+    limiter = RateLimiter("3/second")
+
+    start = time.monotonic()
+    for i in range(6):
+        await limiter.acquire()
+        elapsed = time.monotonic() - start
+        print(f"  Request {i+1} at {elapsed:.2f}s")
+
+    total = time.monotonic() - start
+    print(f"  Total time for 6 requests: {total:.2f}s (expected ~2s)")
+
+
+async def test_pubmed_limiter():
+    """Test PubMed-specific limiter."""
+    print("\n[Test 2] Testing PubMed limiter (shared)...")
+
+    reset_pubmed_limiter()  # Clean state
+
+    # Without API key: 3/sec
+    limiter = get_pubmed_limiter(api_key=None)
+    print(f"  Rate without key: {limiter.rate}")
+
+    # Multiple tools should share the same limiter
+    tool1 = PubMedTool()
+    tool2 = PubMedTool()
+
+    # Verify they share the limiter
+    print(f"  Tools share limiter: {tool1._limiter is tool2._limiter}")
+
+
+async def test_concurrent_requests():
+    """Test rate limiting under concurrent load."""
+    print("\n[Test 3] Testing concurrent request limiting...")
+
+    limiter = RateLimiter("5/second")
+
+    async def make_request(i: int):
+        await limiter.acquire()
+        return time.monotonic()
+
+    start = time.monotonic()
+    # Launch 10 concurrent requests
+    tasks = [make_request(i) for i in range(10)]
+    times = await asyncio.gather(*tasks)
+
+    # Calculate distribution
+    relative_times = [t - start for t in times]
+    print(f"  Request times: {[f'{t:.2f}s' for t in sorted(relative_times)]}")
+
+    total = max(relative_times)
+    print(f"  All 10 requests completed in {total:.2f}s (expected ~2s)")
+
+
+async def main():
+    await test_basic_limiter()
+    await test_pubmed_limiter()
+    await test_concurrent_requests()
+
+    print("\n" + "=" * 60)
+    print("Demo complete!")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "tenacity>=8.2", # Retry logic
     "structlog>=24.1", # Structured logging
     "requests>=2.32.5", # ClinicalTrials.gov (httpx blocked by WAF)
+    "limits>=3.0", # Rate limiting
 ]
 
 [project.optional-dependencies]

src/app.py

@@ -186,78 +186,66 @@ async def research_agent(
         yield f"❌ **Error**: {e!s}"
 
 
-def create_demo() -> Any:
+def create_demo() -> gr.ChatInterface:
     """
     Create the Gradio demo interface with MCP support.
 
     Returns:
         Configured Gradio Blocks interface with MCP server enabled
     """
-    with gr.Blocks(
-        title="DeepCritical - Drug Repurposing Research Agent",
-    ) as demo:
-        # 1. Minimal Header (Option A: 2 lines max)
-        gr.Markdown(
-            "# 🧬 DeepCritical\n"
-            "*AI-Powered Drug Repurposing Agent — searches PubMed, ClinicalTrials.gov & Europe PMC*"
-        )
-
-        # 2. Main Chat Interface
-        # Config inputs will be in a collapsed accordion below the chat input
-        gr.ChatInterface(
-            fn=research_agent,
-            examples=[
-                [
-                    "What drugs could be repurposed for Alzheimer's disease?",
-                    "simple",
-                    "",
-                    "openai",
-                ],
-                [
-                    "Is metformin effective for treating cancer?",
-                    "simple",
-                    "",
-                    "openai",
-                ],
-                [
-                    "What medications show promise for Long COVID treatment?",
-                    "simple",
-                    "",
-                    "openai",
-                ],
+    # 1. Unwrapped ChatInterface (Fixes Accordion Bug)
+    demo = gr.ChatInterface(
+        fn=research_agent,
+        title="🧬 DeepCritical",
+        description=(
+            "*AI-Powered Drug Repurposing Agent — searches PubMed, "
+            "ClinicalTrials.gov & Europe PMC*\n\n"
+            "---\n"
+            "*Research tool only — not for medical advice.*  \n"
+            "**MCP Server Active**: Connect Claude Desktop to `/gradio_api/mcp/`"
+        ),
+        examples=[
+            [
+                "What drugs could be repurposed for Alzheimer's disease?",
+                "simple",
+                "",
+                "openai",
             ],
-            additional_inputs_accordion=gr.Accordion(label="⚙️ Settings", open=False),
-            additional_inputs=[
-                gr.Radio(
-                    choices=["simple", "magentic"],
-                    value="simple",
-                    label="Orchestrator Mode",
-                    info="Simple: Linear | Magentic: Multi-Agent (OpenAI)",
-                ),
-                gr.Textbox(
-                    label="🔑 API Key (Optional - BYOK)",
-                    placeholder="sk-... or sk-ant-...",
-                    type="password",
-                    info="Enter your own API key. Never stored.",
-                ),
-                gr.Radio(
-                    choices=["openai", "anthropic"],
-                    value="openai",
-                    label="API Provider",
-                    info="Select the provider for your API key",
-                ),
+            [
+                "Is metformin effective for treating cancer?",
+                "simple",
+                "",
+                "openai",
             ],
-        )
-
-        # 3. Minimal Footer (Option C: Remove MCP Tabs, keep info)
-        gr.Markdown(
-            """
-            ---
-            *Research tool only — not for medical advice.*
-            **MCP Server Active**: Connect Claude Desktop to `/gradio_api/mcp/`
-            """,
-            elem_classes=["footer"],
-        )
+            [
+                "What medications show promise for Long COVID treatment?",
+                "simple",
+                "",
+                "openai",
+            ],
+        ],
+        additional_inputs_accordion=gr.Accordion(label="⚙️ Settings", open=False),
+        additional_inputs=[
+            gr.Radio(
+                choices=["simple", "magentic"],
+                value="simple",
+                label="Orchestrator Mode",
+                info="Simple: Linear | Magentic: Multi-Agent (OpenAI)",
+            ),
+            gr.Textbox(
+                label="🔑 API Key (Optional - BYOK)",
+                placeholder="sk-... or sk-ant-...",
+                type="password",
+                info="Enter your own API key. Never stored.",
+            ),
+            gr.Radio(
+                choices=["openai", "anthropic"],
+                value="openai",
+                label="API Provider",
+                info="Select the provider for your API key",
+            ),
+        ],
+    )
 
     return demo
 

src/tools/__init__.py

@@ -1,8 +1,16 @@
 """Search tools package."""
 
 from src.tools.base import SearchTool
+from src.tools.clinicaltrials import ClinicalTrialsTool
+from src.tools.europepmc import EuropePMCTool
 from src.tools.pubmed import PubMedTool
 from src.tools.search_handler import SearchHandler
 
-# Re-export
-__all__ = ["PubMedTool", "SearchHandler", "SearchTool"]
+# Re-export all search tools
+__all__ = [
+    "ClinicalTrialsTool",
+    "EuropePMCTool",
+    "PubMedTool",
+    "SearchHandler",
+    "SearchTool",
+]

src/tools/pubmed.py

@@ -1,6 +1,5 @@
 """PubMed search tool using NCBI E-utilities."""
 
-import asyncio
 from typing import Any
 
 import httpx
@@ -8,6 +7,7 @@ import xmltodict
 from tenacity import retry, stop_after_attempt, wait_exponential
 
 from src.tools.query_utils import preprocess_query
+from src.tools.rate_limiter import get_pubmed_limiter
 from src.utils.config import settings
 from src.utils.exceptions import RateLimitError, SearchError
 from src.utils.models import Citation, Evidence
@@ -17,7 +17,6 @@ class PubMedTool:
     """Search tool for PubMed/NCBI."""
 
     BASE_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
-    RATE_LIMIT_DELAY = 0.34  # ~3 requests/sec without API key
     HTTP_TOO_MANY_REQUESTS = 429
 
     def __init__(self, api_key: str | None = None) -> None:
@@ -25,7 +24,9 @@ class PubMedTool:
         # Ignore placeholder values from .env.example
         if self.api_key == "your-ncbi-key-here":
             self.api_key = None
-        self._last_request_time = 0.0
+
+        # Use shared rate limiter
+        self._limiter = get_pubmed_limiter(self.api_key)
 
     @property
     def name(self) -> str:
@@ -33,12 +34,7 @@ class PubMedTool:
 
     async def _rate_limit(self) -> None:
         """Enforce NCBI rate limiting."""
-        loop = asyncio.get_running_loop()
-        now = loop.time()
-        elapsed = now - self._last_request_time
-        if elapsed < self.RATE_LIMIT_DELAY:
-            await asyncio.sleep(self.RATE_LIMIT_DELAY - elapsed)
-        self._last_request_time = loop.time()
+        await self._limiter.acquire()
 
     def _build_params(self, **kwargs: Any) -> dict[str, Any]:
         """Build request params with optional API key."""

src/tools/rate_limiter.py

@@ -0,0 +1,121 @@
+"""Rate limiting utilities using the limits library."""
+
+import asyncio
+from typing import ClassVar
+
+from limits import RateLimitItem, parse
+from limits.storage import MemoryStorage
+from limits.strategies import MovingWindowRateLimiter
+
+
+class RateLimiter:
+    """
+    Async-compatible rate limiter using limits library.
+
+    Uses moving window algorithm for smooth rate limiting.
+    """
+
+    def __init__(self, rate: str) -> None:
+        """
+        Initialize rate limiter.
+
+        Args:
+            rate: Rate string like "3/second" or "10/second"
+        """
+        self.rate = rate
+        self._storage = MemoryStorage()
+        self._limiter = MovingWindowRateLimiter(self._storage)
+        self._rate_limit: RateLimitItem = parse(rate)
+        self._identity = "default"  # Single identity for shared limiting
+
+    async def acquire(self, wait: bool = True) -> bool:
+        """
+        Acquire permission to make a request.
+
+        ASYNC-SAFE: Uses asyncio.sleep(), never time.sleep().
+        The polling pattern allows other coroutines to run while waiting.
+
+        Args:
+            wait: If True, wait until allowed. If False, return immediately.
+
+        Returns:
+            True if allowed, False if not (only when wait=False)
+        """
+        while True:
+            # Check if we can proceed (synchronous, fast - ~microseconds)
+            if self._limiter.hit(self._rate_limit, self._identity):
+                return True
+
+            if not wait:
+                return False
+
+            # CRITICAL: Use asyncio.sleep(), NOT time.sleep()
+            # This yields control to the event loop, allowing other
+            # coroutines (UI, parallel searches) to run.
+            # Using 0.01s for fine-grained responsiveness.
+            await asyncio.sleep(0.01)
+
+    def reset(self) -> None:
+        """Reset the rate limiter (for testing)."""
+        self._storage.reset()
+
+
+# Singleton limiter for PubMed/NCBI
+_pubmed_limiter: RateLimiter | None = None
+
+
+def get_pubmed_limiter(api_key: str | None = None) -> RateLimiter:
+    """
+    Get the shared PubMed rate limiter.
+
+    Rate depends on whether API key is provided:
+    - Without key: 3 requests/second
+    - With key: 10 requests/second
+
+    Args:
+        api_key: NCBI API key (optional)
+
+    Returns:
+        Shared RateLimiter instance
+    """
+    global _pubmed_limiter
+
+    if _pubmed_limiter is None:
+        rate = "10/second" if api_key else "3/second"
+        _pubmed_limiter = RateLimiter(rate)
+
+    return _pubmed_limiter
+
+
+def reset_pubmed_limiter() -> None:
+    """Reset the PubMed limiter (for testing)."""
+    global _pubmed_limiter
+    _pubmed_limiter = None
+
+
+# Factory for other APIs
+class RateLimiterFactory:
+    """Factory for creating/getting rate limiters for different APIs."""
+
+    _limiters: ClassVar[dict[str, RateLimiter]] = {}
+
+    @classmethod
+    def get(cls, api_name: str, rate: str) -> RateLimiter:
+        """
+        Get or create a rate limiter for an API.
+
+        Args:
+            api_name: Unique identifier for the API
+            rate: Rate limit string (e.g., "10/second")
+
+        Returns:
+            RateLimiter instance (shared for same api_name)
+        """
+        if api_name not in cls._limiters:
+            cls._limiters[api_name] = RateLimiter(rate)
+        return cls._limiters[api_name]
+
+    @classmethod
+    def reset_all(cls) -> None:
+        """Reset all limiters (for testing)."""
+        cls._limiters.clear()

src/utils/models.py

@@ -6,7 +6,7 @@ from typing import Any, ClassVar, Literal
 from pydantic import BaseModel, Field
 
 # Centralized source type - add new sources here (e.g., new databases)
-SourceName = Literal["pubmed", "clinicaltrials", "europepmc", "preprint"]
+SourceName = Literal["pubmed", "clinicaltrials", "europepmc", "preprint", "openalex"]
 
 
 class Citation(BaseModel):
@@ -36,6 +36,10 @@ class Evidence(BaseModel):
     content: str = Field(min_length=1, description="The actual text content")
     citation: Citation
     relevance: float = Field(default=0.0, ge=0.0, le=1.0, description="Relevance score 0-1")
+    metadata: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata (e.g., cited_by_count, concepts, is_open_access)",
+    )
 
     model_config = {"frozen": True}
 

tests/unit/tools/test_rate_limiting.py

@@ -0,0 +1,104 @@
+"""Tests for rate limiting functionality."""
+
+import asyncio
+import time
+
+import pytest
+
+from src.tools.rate_limiter import RateLimiter, get_pubmed_limiter, reset_pubmed_limiter
+
+
+class TestRateLimiter:
+    """Test suite for rate limiter."""
+
+    def test_create_limiter_without_api_key(self) -> None:
+        """Should create 3/sec limiter without API key."""
+        limiter = RateLimiter(rate="3/second")
+        assert limiter.rate == "3/second"
+
+    def test_create_limiter_with_api_key(self) -> None:
+        """Should create 10/sec limiter with API key."""
+        limiter = RateLimiter(rate="10/second")
+        assert limiter.rate == "10/second"
+
+    @pytest.mark.asyncio
+    async def test_limiter_allows_requests_under_limit(self) -> None:
+        """Should allow requests under the rate limit."""
+        limiter = RateLimiter(rate="10/second")
+
+        # 3 requests should all succeed immediately
+        for _ in range(3):
+            allowed = await limiter.acquire()
+            assert allowed is True
+
+    @pytest.mark.asyncio
+    async def test_limiter_blocks_when_exceeded(self) -> None:
+        """Should wait when rate limit exceeded."""
+        limiter = RateLimiter(rate="2/second")
+
+        # First 2 should be instant
+        await limiter.acquire()
+        await limiter.acquire()
+
+        # Third should block briefly
+        start = time.monotonic()
+        await limiter.acquire()
+        elapsed = time.monotonic() - start
+
+        # Should have waited ~0.5 seconds (half second window for 2/sec)
+        assert elapsed >= 0.3
+
+    @pytest.mark.asyncio
+    async def test_limiter_resets_after_window(self) -> None:
+        """Rate limit should reset after time window."""
+        limiter = RateLimiter(rate="5/second")
+
+        # Use up the limit
+        for _ in range(5):
+            await limiter.acquire()
+
+        # Wait for window to pass
+        await asyncio.sleep(1.1)
+
+        # Should be allowed again
+        start = time.monotonic()
+        await limiter.acquire()
+        elapsed = time.monotonic() - start
+
+        assert elapsed < 0.1  # Should be nearly instant
+
+
+class TestGetPubmedLimiter:
+    """Test PubMed-specific limiter factory."""
+
+    @pytest.fixture(autouse=True)
+    def setup_teardown(self):
+        """Reset limiter before and after each test."""
+        reset_pubmed_limiter()
+        yield
+        reset_pubmed_limiter()
+
+    def test_limiter_without_api_key(self) -> None:
+        """Should return 3/sec limiter without key."""
+        limiter = get_pubmed_limiter(api_key=None)
+        assert "3" in limiter.rate
+
+    def test_limiter_with_api_key(self) -> None:
+        """Should return 10/sec limiter with key."""
+        limiter = get_pubmed_limiter(api_key="my-api-key")
+        assert "10" in limiter.rate
+
+    def test_limiter_is_singleton(self) -> None:
+        """Same API key should return same limiter instance."""
+        limiter1 = get_pubmed_limiter(api_key="key1")
+        limiter2 = get_pubmed_limiter(api_key="key1")
+        assert limiter1 is limiter2
+
+    def test_different_keys_different_limiters(self) -> None:
+        """Different API keys should return different limiters."""
+        limiter1 = get_pubmed_limiter(api_key="key1")
+        limiter2 = get_pubmed_limiter(api_key="key2")
+        # Clear cache for clean test
+        # Actually, different keys SHOULD share the same limiter
+        # since we're limiting against the same API
+        assert limiter1 is limiter2  # Shared NCBI rate limit

uv.lock

@@ -1066,6 +1066,7 @@ dependencies = [
     { name = "gradio", extra = ["mcp"] },
     { name = "httpx" },
     { name = "huggingface-hub" },
+    { name = "limits" },
     { name = "openai" },
     { name = "pydantic" },
     { name = "pydantic-ai" },
@@ -1116,6 +1117,7 @@ requires-dist = [
     { name = "gradio", extras = ["mcp"], specifier = ">=6.0.0" },
     { name = "httpx", specifier = ">=0.27" },
     { name = "huggingface-hub", specifier = ">=0.20.0" },
+    { name = "limits", specifier = ">=3.0" },
     { name = "llama-index", marker = "extra == 'modal'", specifier = ">=0.11.0" },
     { name = "llama-index-embeddings-openai", marker = "extra == 'modal'" },
     { name = "llama-index-llms-openai", marker = "extra == 'modal'" },
@@ -2259,6 +2261,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/ca/ec/65f7d563aa4a62dd58777e8f6aa882f15db53b14eb29aba0c28a20f7eb26/kubernetes-34.1.0-py2.py3-none-any.whl", hash = "sha256:bffba2272534e224e6a7a74d582deb0b545b7c9879d2cd9e4aae9481d1f2cc2a", size = 2008380 },
 ]
 
+[[package]]
+name = "limits"
+version = "5.6.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "deprecated" },
+    { name = "packaging" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/bb/e5/c968d43a65128cd54fb685f257aafb90cd5e4e1c67d084a58f0e4cbed557/limits-5.6.0.tar.gz", hash = "sha256:807fac75755e73912e894fdd61e2838de574c5721876a19f7ab454ae1fffb4b5", size = 182984 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/40/96/4fcd44aed47b8fcc457653b12915fcad192cd646510ef3f29fd216f4b0ab/limits-5.6.0-py3-none-any.whl", hash = "sha256:b585c2104274528536a5b68864ec3835602b3c4a802cd6aa0b07419798394021", size = 60604 },
+]
+
 [[package]]
 name = "llama-cloud"
 version = "0.1.35"