| import os |
| from scrapingbee import ScrapingBeeClient |
| from logger import setup_logger |
| import json |
|
|
| logger = setup_logger("scraper") |
|
|
| |
| client = ScrapingBeeClient(api_key=os.getenv('SCRAPINGBEE_API_KEY', '')) |
|
|
| def scrape_url(url: str) -> str: |
| """ |
| Scrape content from URL using ScrapingBee with AI extraction |
| |
| Args: |
| url: The URL to scrape |
| |
| Returns: |
| str: Extracted text content or error message |
| """ |
| try: |
| logger.info(f"Scraping URL: {url}") |
| response = client.get( |
| url, |
| params={ |
| 'stealth_proxy': True, |
| 'country_code': 'us', |
| 'ai_query': 'Extract the main text content from this page' |
| } |
| ) |
| |
| if response.status_code == 200: |
| logger.info(f"Successfully scraped URL: {url}") |
| return response.text if response.text else "No content could be extracted from the URL" |
| else: |
| logger.error(f"Failed to scrape URL: {url}, Status: {response.status_code}") |
| return f"Failed to download the URL. Status code: {response.status_code}" |
| |
| except Exception as e: |
| logger.error(f"Error scraping URL: {url}", exc_info=True) |
| return f"Error scraping the URL: {str(e)}" |
