From 78a627f1e9e4fe2903ffb5d3131ea2b879455adc Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sat, 13 Sep 2025 23:26:19 -0500 Subject: [PATCH 1/6] rewrite this extension by migrating ruby code to python --- .gitignore | 3 + LICENSE | 21 +++ README.md | 213 ++++++++++++++++++++++ examples/async_batch_search.py | 99 +++++++++++ examples/basic_usage.py | 70 ++++++++ examples/multiple_engines.py | 112 ++++++++++++ pyproject.toml | 98 +++++++++++ requirements.txt | 10 ++ serpapi/__init__.py | 18 ++ serpapi/client.py | 310 +++++++++++++++++++++++++++++++++ serpapi/error.py | 19 ++ serpapi/version.py | 5 + tests/__init__.py | 1 + tests/test_client.py | 265 ++++++++++++++++++++++++++++ tests/test_error.py | 36 ++++ 15 files changed, 1280 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 examples/async_batch_search.py create mode 100644 examples/basic_usage.py create mode 100644 examples/multiple_engines.py create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 serpapi/__init__.py create mode 100644 serpapi/client.py create mode 100644 serpapi/error.py create mode 100644 serpapi/version.py create mode 100644 tests/__init__.py create mode 100644 tests/test_client.py create mode 100644 tests/test_error.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9a4391f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +## PROJECT::GENERAL +__pycache__ + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c7affab --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 SerpApi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b1a575b --- /dev/null +++ b/README.md @@ -0,0 +1,213 @@ +# SerpApi Python Library + +Official Python client for SerpApi.com - Search Engine Results API. + +## Features + +- Async/await support for non-blocking HTTP requests +- Persistent connections for 2x faster response times +- Multiple search engines (Google, Bing, Yahoo, Baidu, Yandex, etc.) +- Comprehensive API coverage (Search, Location, Account, Search Archive) +- Type hints and extensive documentation + +## Installation + +### Using pip + +```bash +pip install serpapi +``` + +### Using uv (recommended) + +```bash +# Install with uv +uv add serpapi + +# Or clone and install locally +git clone https://github.com/serpapi/serpapi-python.git +cd serpapi-python +uv sync +``` + +### Development Setup + +```bash +# Clone the repository +git clone https://github.com/serpapi/serpapi-python.git +cd serpapi-python + +# Install with uv (recommended) +uv sync --dev + +# Or with pip +pip install -e ".[dev]" +``` + +## Quick Start + +```python +import asyncio +from serpapi import Client + +async def main(): + client = Client(api_key="your_api_key", engine="google") + results = await client.search({"q": "coffee"}) + + for result in results.get("organic_results", []): + print(f"Title: {result.get('title')}") + print(f"Link: {result.get('link')}") + + await client.close() + +asyncio.run(main()) +``` + +## API Key + +Get your API key from [serpapi.com/signup](https://serpapi.com/users/sign_up?plan=free). + +Set environment variable: +```bash +export SERPAPI_KEY="your_secret_key" +``` + +## Usage Examples + +### Basic Search +```python +results = await client.search({"q": "coffee"}) +``` + +### HTML Search +```python +html_content = await client.html({"q": "coffee"}) +``` + +### Location API +```python +locations = await client.location({"q": "Austin", "limit": 3}) +``` + +### Search Archive +```python +archived = await client.search_archive(search_id) +``` + +### Account Info +```python +account = await client.account() +``` + +## Async Batch Processing + +```python +import asyncio + +async def search_company(client, company): + results = await client.search({"q": company}) + return {"company": company, "count": len(results.get("organic_results", []))} + +async def main(): + client = Client(api_key="your_api_key", persistent=True) + companies = ["meta", "amazon", "apple", "netflix", "google"] + + tasks = [search_company(client, company) for company in companies] + results = await asyncio.gather(*tasks) + + for result in results: + print(f"{result['company']}: {result['count']} results") + + await client.close() + +asyncio.run(main()) +``` + +## Context Manager + +```python +async with Client(api_key="your_api_key") as client: + results = await client.search({"q": "coffee"}) + # Client automatically closed +``` + +## Error Handling + +```python +from serpapi import SerpApiError + +try: + results = await client.search({"q": "coffee"}) +except SerpApiError as e: + print(f"SerpApi error: {e}") +``` + +## Development + +```bash +# Install dependencies +pip install -e ".[dev]" + +# Run tests +pytest + +# Type checking +mypy serpapi/ +``` + +## UV Package Manager + +This project is fully configured for [uv](https://docs.astral.sh/uv/), a fast Python package manager. + +### UV Commands + +```bash +# Install all dependencies +uv sync + +# Install with development dependencies +uv sync --dev + +# Run Python scripts +uv run python script.py + +# Run tests +uv run pytest + +# Add new dependency +uv add package-name + +# Add development dependency +uv add --dev package-name + +# Show installed packages +uv pip list + +# Update dependencies +uv sync --upgrade +``` + +### UV Benefits + +- **Fast**: 10-100x faster than pip +- **Reliable**: Lock file ensures reproducible builds +- **Simple**: Single command for most operations +- **Modern**: Built for Python 3.11+ with async support + +### Project Structure with UV + +``` +serpapi-python/ +├── .python-version # Python version (3.11) +├── uv.lock # Dependency lock file +├── .venv/ # Virtual environment (auto-created) +├── pyproject.toml # Project configuration +├── serpapi/ # Package source code +├── tests/ # Test suite +├── examples/ # Usage examples +└── README.md # This file +``` + +## License + +MIT License - see LICENSE file for details. diff --git a/examples/async_batch_search.py b/examples/async_batch_search.py new file mode 100644 index 0000000..548db6d --- /dev/null +++ b/examples/async_batch_search.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +""" +Async batch search example for SerpApi Python client. + +This example demonstrates how to perform multiple searches concurrently +using async/await for better performance. +""" + +import asyncio +import os +from serpapi import Client, SerpApiError + + +async def search_company(client: Client, company: str) -> dict: + """Search for a specific company.""" + try: + results = await client.search({"q": company}) + return { + "company": company, + "status": results.get("search_metadata", {}).get("status", "Unknown"), + "search_id": results.get("search_metadata", {}).get("id", "N/A"), + "organic_count": len(results.get("organic_results", [])), + "success": True + } + except SerpApiError as e: + return { + "company": company, + "error": str(e), + "success": False + } + + +async def main(): + """Main example function.""" + # Get API key from environment variable + api_key = os.getenv('SERPAPI_KEY') + if not api_key: + print("Please set SERPAPI_KEY environment variable") + return + + # Create client with persistent connections for better performance + client = Client(api_key=api_key, engine="google", persistent=True) + + # List of companies to search + companies = ["meta", "amazon", "apple", "netflix", "google", "microsoft", "tesla"] + + try: + print("=== Async Batch Search ===") + print(f"Searching for {len(companies)} companies concurrently...") + print() + + # Create tasks for concurrent execution + tasks = [search_company(client, company) for company in companies] + + # Execute all searches concurrently + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Process results + successful_searches = [] + failed_searches = [] + + for result in results: + if isinstance(result, Exception): + print(f"Unexpected error: {result}") + continue + + if result["success"]: + successful_searches.append(result) + print(f"✓ {result['company']}: {result['organic_count']} results " + f"(Status: {result['status']}, ID: {result['search_id'][:8]}...)") + else: + failed_searches.append(result) + print(f"✗ {result['company']}: {result['error']}") + + print() + print(f"Summary: {len(successful_searches)} successful, {len(failed_searches)} failed") + + # Demonstrate search archive functionality + if successful_searches: + print("\n=== Search Archive Example ===") + first_result = successful_searches[0] + search_id = first_result["search_id"] + + try: + archived_result = await client.search_archive(search_id) + print(f"Retrieved archived result for {first_result['company']}") + print(f"Archive status: {archived_result.get('search_metadata', {}).get('status', 'Unknown')}") + except SerpApiError as e: + print(f"Failed to retrieve archive: {e}") + + except Exception as e: + print(f"Unexpected error: {e}") + finally: + # Close the client + await client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/basic_usage.py b/examples/basic_usage.py new file mode 100644 index 0000000..08a50a3 --- /dev/null +++ b/examples/basic_usage.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +""" +Basic usage example for SerpApi Python client. + +This example demonstrates how to use the SerpApi client for basic search operations. +""" + +import asyncio +import os +from serpapi import Client, SerpApiError + + +async def main(): + """Main example function.""" + # Get API key from environment variable + api_key = os.getenv('SERPAPI_KEY') + if not api_key: + print("Please set SERPAPI_KEY environment variable") + return + + # Create client + client = Client(api_key=api_key, engine="google") + + try: + # Basic search + print("=== Basic Google Search ===") + results = await client.search({"q": "coffee"}) + + if "organic_results" in results: + print(f"Found {len(results['organic_results'])} organic results:") + for i, result in enumerate(results["organic_results"][:3], 1): + print(f"{i}. {result.get('title', 'No title')}") + print(f" {result.get('link', 'No link')}") + print() + else: + print("No organic results found") + + # HTML search + print("=== HTML Search ===") + html_content = await client.html({"q": "python programming"}) + print(f"HTML content length: {len(html_content)} characters") + print(f"First 200 characters: {html_content[:200]}...") + print() + + # Location search + print("=== Location Search ===") + locations = await client.location({"q": "Austin", "limit": 3}) + print(f"Found {len(locations)} locations:") + for location in locations: + print(f"- {location.get('name', 'No name')} ({location.get('country_code', 'No country')})") + print() + + # Account information + print("=== Account Information ===") + account = await client.account() + print(f"Account ID: {account.get('account_id', 'N/A')}") + print(f"Plan: {account.get('plan_name', 'N/A')}") + print(f"Searches left: {account.get('total_searches_left', 'N/A')}") + + except SerpApiError as e: + print(f"SerpApi Error: {e}") + except Exception as e: + print(f"Unexpected error: {e}") + finally: + # Close the client + await client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/multiple_engines.py b/examples/multiple_engines.py new file mode 100644 index 0000000..a0265fc --- /dev/null +++ b/examples/multiple_engines.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Multiple search engines example for SerpApi Python client. + +This example demonstrates how to use different search engines +and compare their results. +""" + +import asyncio +import os +from serpapi import Client, SerpApiError + + +async def search_with_engine(client: Client, engine: str, query: str) -> dict: + """Search using a specific engine.""" + try: + results = await client.search({"engine": engine, "q": query}) + return { + "engine": engine, + "query": query, + "status": results.get("search_metadata", {}).get("status", "Unknown"), + "organic_count": len(results.get("organic_results", [])), + "total_results": results.get("search_information", {}).get("total_results", "N/A"), + "success": True, + "results": results.get("organic_results", [])[:3] # First 3 results + } + except SerpApiError as e: + return { + "engine": engine, + "query": query, + "error": str(e), + "success": False + } + + +async def main(): + """Main example function.""" + # Get API key from environment variable + api_key = os.getenv('SERPAPI_KEY') + if not api_key: + print("Please set SERPAPI_KEY environment variable") + return + + # Create client + client = Client(api_key=api_key, persistent=True) + + # Search query + query = "artificial intelligence" + + # Different search engines to try + engines = [ + "google", + "bing", + "yahoo", + "duckduckgo", + "baidu" + ] + + try: + print(f"=== Multi-Engine Search: '{query}' ===") + print(f"Searching with {len(engines)} different engines...") + print() + + # Create tasks for concurrent execution + tasks = [search_with_engine(client, engine, query) for engine in engines] + + # Execute all searches concurrently + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Process and display results + for result in results: + if isinstance(result, Exception): + print(f"Unexpected error: {result}") + continue + + print(f"--- {result['engine'].upper()} ---") + + if result["success"]: + print(f"Status: {result['status']}") + print(f"Organic results: {result['organic_count']}") + print(f"Total results: {result['total_results']}") + + if result["results"]: + print("Top results:") + for i, res in enumerate(result["results"], 1): + title = res.get("title", "No title") + link = res.get("link", "No link") + print(f" {i}. {title}") + print(f" {link}") + else: + print("No organic results found") + else: + print(f"Error: {result['error']}") + + print() + + # Compare results across engines + successful_results = [r for r in results if isinstance(r, dict) and r.get("success")] + if len(successful_results) > 1: + print("=== Comparison Summary ===") + for result in successful_results: + print(f"{result['engine']}: {result['organic_count']} organic results") + + except Exception as e: + print(f"Unexpected error: {e}") + finally: + # Close the client + await client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..870a742 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,98 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "serpapi" +version = "1.0.1" +description = "Official Python library for SerpApi.com - Search Engine Results API" +readme = "README.md" +requires-python = ">=3.11" +license = {text = "MIT"} +authors = [ + {name = "victor benarbia", email = "victor@serpapi.com"}, + {name = "Julien Khaleghy"} +] +keywords = ["serpapi", "search", "google", "bing", "yahoo", "scraping", "api"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "aiohttp>=3.9.0,<4", + "yarl>=1.9", + "multidict>=6", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7", + "pytest-asyncio>=0.23", + "mypy>=1.8", + "pytest-cov>=4.1", +] + +[project.urls] +Homepage = "https://github.com/serpapi/serpapi-python" +Documentation = "https://serpapi.com" +Repository = "https://github.com/serpapi/serpapi-python" +Issues = "https://github.com/serpapi/serpapi-python/issues" + +[tool.setuptools.packages.find] +where = ["."] +include = ["serpapi*"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +disallow_incomplete_defs = true +check_untyped_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +strict_equality = true + +[tool.coverage.run] +source = ["serpapi"] +omit = ["tests/*", "*/tests/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", + "class .*\\bProtocol\\):", + "@(abc\\.)?abstractmethod", +] + +[dependency-groups] +dev = [ + "mypy>=1.18.1", + "pytest>=8.4.2", + "pytest-asyncio>=1.2.0", + "pytest-cov>=7.0.0", +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5f1a014 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# Core dependencies +aiohttp>=3.9.0,<4 +yarl>=1.9 +multidict>=6 + +# Development dependencies (optional) +pytest>=7 +pytest-asyncio>=0.23 +mypy>=1.8 +pytest-cov>=4.1 diff --git a/serpapi/__init__.py b/serpapi/__init__.py new file mode 100644 index 0000000..2faa2cf --- /dev/null +++ b/serpapi/__init__.py @@ -0,0 +1,18 @@ +""" +SerpApi Python Client Library + +Official Python client for SerpApi.com - Search Engine Results API. +Integrate search data into your AI workflow, RAG / fine-tuning, or Python application. + +Supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and more. +""" + +from .error import SerpApiError +from .version import __version__ + +# Import Client only when aiohttp is available +try: + from .client import Client + __all__ = ['Client', 'SerpApiError', '__version__'] +except ImportError: + __all__ = ['SerpApiError', '__version__'] diff --git a/serpapi/client.py b/serpapi/client.py new file mode 100644 index 0000000..3cbeea9 --- /dev/null +++ b/serpapi/client.py @@ -0,0 +1,310 @@ +""" +Client implementation for SerpApi.com + +Powered by aiohttp for async HTTP requests and persistent connections. +""" + +import asyncio +import json +import os +from typing import Dict, Any, Optional, Union, List +from urllib.parse import urlencode + +import aiohttp +from aiohttp import ClientSession, ClientTimeout + +from .error import SerpApiError +from .version import __version__ + + +class Client: + """ + Client for SerpApi.com + + Features: + - Async non-blocking search + - Persistent HTTP connections + - Search API + - Location API + - Account API + - Search Archive API + """ + + # Backend service URL + BACKEND = 'serpapi.com' + + def __init__( + self, + api_key: Optional[str] = None, + engine: str = 'google', + persistent: bool = True, + async_mode: bool = False, + timeout: int = 120, + symbolize_names: bool = True, + **kwargs + ): + """ + Initialize SerpApi client. + + Args: + api_key: User secret API key. If None, will try to get from SERPAPI_KEY env var. + engine: Default search engine selection. + persistent: Keep socket connection open for faster response times (2x faster). + async_mode: Enable async mode for non-blocking operations. + timeout: HTTP request timeout in seconds. + symbolize_names: Convert JSON keys to symbols (not applicable in Python, kept for compatibility). + **kwargs: Additional parameters to store as default parameters. + + Raises: + SerpApiError: If parameters are invalid. + """ + if api_key is None: + api_key = os.getenv('SERPAPI_KEY') + if api_key is None: + raise SerpApiError('API key is required. Set api_key parameter or SERPAPI_KEY environment variable.') + + # Store configuration + self._timeout = timeout + self._persistent = persistent + self._async_mode = async_mode + self._symbolize_names = symbolize_names + + # Set default query parameters + self._params = { + 'api_key': api_key, + 'engine': engine, + 'source': f'serpapi-python:{__version__}', + **kwargs + } + + # HTTP client session (will be created when needed) + self._session: Optional[ClientSession] = None + self._session_lock = asyncio.Lock() + + @property + def timeout(self) -> int: + """Get HTTP timeout in seconds.""" + return self._timeout + + @property + def persistent(self) -> bool: + """Check if persistent connections are enabled.""" + return self._persistent + + @property + def engine(self) -> str: + """Get default search engine.""" + return self._params.get('engine', 'google') + + @property + def api_key(self) -> str: + """Get API key.""" + return self._params.get('api_key', '') + + async def _get_session(self) -> ClientSession: + """Get or create HTTP session.""" + if not self._persistent or self._session is None or self._session.closed: + async with self._session_lock: + if not self._persistent or self._session is None or self._session.closed: + timeout = ClientTimeout(total=self._timeout) + connector = aiohttp.TCPConnector(limit=100, limit_per_host=30) + self._session = ClientSession( + timeout=timeout, + connector=connector, + base_url=f'https://{self.BACKEND}' + ) + return self._session + + def _merge_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """ + Merge runtime parameters with default parameters. + + Args: + params: Runtime parameters to merge. + + Returns: + Merged parameters after cleanup. + + Raises: + SerpApiError: If params is not a dictionary. + """ + if not isinstance(params, dict): + raise SerpApiError(f"params must be dict, not: {type(params)}") + + # Merge default params with custom params + merged = self._params.copy() + merged.update(params) + + # Remove client-specific configuration + merged.pop('symbolize_names', None) + + # Remove None values + return {k: v for k, v in merged.items() if v is not None} + + async def _make_request( + self, + endpoint: str, + params: Dict[str, Any], + response_format: str = 'json' + ) -> Union[Dict[str, Any], str]: + """ + Make HTTP request to SerpApi backend. + + Args: + endpoint: API endpoint path. + params: Request parameters. + response_format: Response format ('json' or 'html'). + + Returns: + Response data as dict (JSON) or str (HTML). + + Raises: + SerpApiError: If request fails or response is invalid. + """ + session = await self._get_session() + query_params = self._merge_params(params) + + try: + async with session.get(endpoint, params=query_params) as response: + if response_format == 'json': + try: + data = await response.json() + if isinstance(data, dict) and 'error' in data: + raise SerpApiError( + f"HTTP request failed with error: {data['error']} " + f"from url: https://{self.BACKEND}{endpoint}, " + f"params: {params}, response status: {response.status}" + ) + elif response.status != 200: + raise SerpApiError( + f"HTTP request failed with response status: {response.status} " + f"response: {data} on get url: https://{self.BACKEND}{endpoint}, " + f"params: {params}" + ) + return data + except json.JSONDecodeError as e: + text = await response.text() + raise SerpApiError( + f"JSON parse error: {text} on get url: https://{self.BACKEND}{endpoint}, " + f"params: {params}, response status: {response.status}" + ) + elif response_format == 'html': + if response.status != 200: + text = await response.text() + raise SerpApiError( + f"HTTP request failed with response status: {response.status} " + f"response: {text} on get url: https://{self.BACKEND}{endpoint}, " + f"params: {params}" + ) + return await response.text() + else: + raise SerpApiError(f"Unsupported response format: {response_format}") + except aiohttp.ClientError as e: + raise SerpApiError(f"HTTP client error: {str(e)}") + + async def search(self, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + """ + Perform a search using SerpApi.com + + Args: + params: Search parameters including engine, query, etc. + + Returns: + Search results as a dictionary. + """ + if params is None: + params = {} + return await self._make_request('/search', params, 'json') + + async def html(self, params: Optional[Dict[str, Any]] = None) -> str: + """ + Perform a search and return raw HTML. + + Useful for training AI models, RAG, debugging, or custom parsing. + + Args: + params: Search parameters. + + Returns: + Raw HTML search results directly from the search engine. + """ + if params is None: + params = {} + return await self._make_request('/search', params, 'html') + + async def location(self, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: + """ + Get location suggestions using Location API. + + Args: + params: Location parameters including 'q' (query) and 'limit'. + + Returns: + List of matching locations. + """ + if params is None: + params = {} + return await self._make_request('/locations.json', params, 'json') + + async def search_archive( + self, + search_id: Union[str, int], + format_type: str = 'json' + ) -> Union[Dict[str, Any], str]: + """ + Retrieve search result from the Search Archive API. + + Args: + search_id: Search ID from original search results. + format_type: Response format ('json' or 'html'). + + Returns: + Archived search results as dict (JSON) or str (HTML). + + Raises: + SerpApiError: If format_type is invalid. + """ + if format_type not in ['json', 'html']: + raise SerpApiError('format_type must be json or html') + + return await self._make_request(f'/searches/{search_id}.{format_type}', {}, format_type) + + async def account(self, api_key: Optional[str] = None) -> Dict[str, Any]: + """ + Get account information using Account API. + + Args: + api_key: API key (optional if already provided to constructor). + + Returns: + Account information dictionary. + """ + params = {'api_key': api_key} if api_key else {} + return await self._make_request('/account', params, 'json') + + async def close(self): + """Close HTTP session if persistent connections are enabled.""" + if self._session and not self._session.closed: + await self._session.close() + + async def __aenter__(self): + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Async context manager exit.""" + await self.close() + + def __del__(self): + """Destructor to ensure session is closed.""" + if hasattr(self, '_session') and self._session and not self._session.closed: + # Schedule the session close in the event loop + try: + loop = asyncio.get_event_loop() + if loop.is_running(): + loop.create_task(self._session.close()) + else: + loop.run_until_complete(self._session.close()) + except RuntimeError: + # No event loop running, can't close session + pass diff --git a/serpapi/error.py b/serpapi/error.py new file mode 100644 index 0000000..e2de00b --- /dev/null +++ b/serpapi/error.py @@ -0,0 +1,19 @@ +""" +SerpApi error handling module. +""" + + +class SerpApiError(Exception): + """ + SerpApiError wraps any errors related to the SerpApi client. + + Handles the following types of errors: + - HTTP response errors from SerpApi.com + - Missing API key + - Credit limit exceeded + - Incorrect query parameters + - JSON parsing errors + - Network timeouts + - And more... + """ + pass diff --git a/serpapi/version.py b/serpapi/version.py new file mode 100644 index 0000000..fd48a0b --- /dev/null +++ b/serpapi/version.py @@ -0,0 +1,5 @@ +""" +Version information for SerpApi Python client. +""" + +__version__ = '0.2.0' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8c4ba8d --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# Test package for SerpApi Python client diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..dff315d --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,265 @@ +""" +Test suite for SerpApi Client. +""" + +import asyncio +import os +import pytest +from unittest.mock import AsyncMock, patch, MagicMock + +from serpapi import Client, SerpApiError + + +class TestClient: + """Test cases for SerpApi Client.""" + + def test_client_initialization_with_api_key(self): + """Test client initialization with API key.""" + client = Client(api_key="test_key", engine="google") + assert client.api_key == "test_key" + assert client.engine == "google" + assert client.persistent is True + assert client.timeout == 120 + + def test_client_initialization_with_env_var(self): + """Test client initialization with environment variable.""" + with patch.dict(os.environ, {'SERPAPI_KEY': 'env_key'}): + client = Client(engine="bing") + assert client.api_key == "env_key" + assert client.engine == "bing" + + def test_client_initialization_no_api_key(self): + """Test client initialization without API key raises error.""" + with patch.dict(os.environ, {}, clear=True): + with pytest.raises(SerpApiError, match="API key is required"): + Client() + + def test_client_initialization_custom_params(self): + """Test client initialization with custom parameters.""" + client = Client( + api_key="test_key", + engine="yahoo", + persistent=False, + timeout=60, + custom_param="value" + ) + assert client.api_key == "test_key" + assert client.engine == "yahoo" + assert client.persistent is False + assert client.timeout == 60 + assert client._params["custom_param"] == "value" + + def test_merge_params(self): + """Test parameter merging.""" + client = Client(api_key="test_key", engine="google", param1="value1") + + # Test with valid params + merged = client._merge_params({"param2": "value2", "q": "coffee"}) + expected = { + "api_key": "test_key", + "engine": "google", + "source": "serpapi-python:1.0.1", + "param1": "value1", + "param2": "value2", + "q": "coffee" + } + assert merged == expected + + # Test with invalid params + with pytest.raises(SerpApiError, match="params must be dict"): + client._merge_params("invalid") + + @pytest.mark.asyncio + async def test_search_success(self): + """Test successful search request.""" + client = Client(api_key="test_key") + + mock_response = { + "search_metadata": {"id": "test_id", "status": "Success"}, + "organic_results": [{"title": "Test Result", "link": "https://example.com"}] + } + + with patch.object(client, '_make_request', return_value=mock_response) as mock_request: + result = await client.search({"q": "coffee"}) + + mock_request.assert_called_once_with('/search', {"q": "coffee"}, 'json') + assert result == mock_response + + @pytest.mark.asyncio + async def test_html_success(self): + """Test successful HTML request.""" + client = Client(api_key="test_key") + + mock_html = "Test HTML" + + with patch.object(client, '_make_request', return_value=mock_html) as mock_request: + result = await client.html({"q": "coffee"}) + + mock_request.assert_called_once_with('/search', {"q": "coffee"}, 'html') + assert result == mock_html + + @pytest.mark.asyncio + async def test_location_success(self): + """Test successful location request.""" + client = Client(api_key="test_key") + + mock_locations = [ + {"id": "1", "name": "Austin, TX", "country_code": "US"} + ] + + with patch.object(client, '_make_request', return_value=mock_locations) as mock_request: + result = await client.location({"q": "Austin", "limit": 3}) + + mock_request.assert_called_once_with('/locations.json', {"q": "Austin", "limit": 3}, 'json') + assert result == mock_locations + + @pytest.mark.asyncio + async def test_search_archive_success(self): + """Test successful search archive request.""" + client = Client(api_key="test_key") + + mock_archive = {"search_metadata": {"id": "test_id"}, "organic_results": []} + + with patch.object(client, '_make_request', return_value=mock_archive) as mock_request: + result = await client.search_archive("test_id", "json") + + mock_request.assert_called_once_with('/searches/test_id.json', {}, 'json') + assert result == mock_archive + + @pytest.mark.asyncio + async def test_search_archive_invalid_format(self): + """Test search archive with invalid format.""" + client = Client(api_key="test_key") + + with pytest.raises(SerpApiError, match="format_type must be json or html"): + await client.search_archive("test_id", "invalid") + + @pytest.mark.asyncio + async def test_account_success(self): + """Test successful account request.""" + client = Client(api_key="test_key") + + mock_account = { + "account_id": "123456", + "account_email": "test@example.com", + "plan_name": "Free Plan" + } + + with patch.object(client, '_make_request', return_value=mock_account) as mock_request: + result = await client.account() + + mock_request.assert_called_once_with('/account', {}, 'json') + assert result == mock_account + + @pytest.mark.asyncio + async def test_account_with_api_key(self): + """Test account request with specific API key.""" + client = Client(api_key="test_key") + + mock_account = {"account_id": "123456"} + + with patch.object(client, '_make_request', return_value=mock_account) as mock_request: + result = await client.account("custom_key") + + mock_request.assert_called_once_with('/account', {"api_key": "custom_key"}, 'json') + assert result == mock_account + + @pytest.mark.asyncio + async def test_make_request_json_success(self): + """Test successful JSON request.""" + client = Client(api_key="test_key") + + mock_response = MagicMock() + mock_response.status = 200 + mock_response.json = AsyncMock(return_value={"result": "success"}) + + with patch.object(client, '_get_session') as mock_session: + mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + + result = await client._make_request('/test', {"param": "value"}, 'json') + + assert result == {"result": "success"} + + @pytest.mark.asyncio + async def test_make_request_json_error_response(self): + """Test JSON request with error response.""" + client = Client(api_key="test_key") + + mock_response = MagicMock() + mock_response.status = 200 + mock_response.json = AsyncMock(return_value={"error": "API Error"}) + + with patch.object(client, '_get_session') as mock_session: + mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + + with pytest.raises(SerpApiError, match="HTTP request failed with error: API Error"): + await client._make_request('/test', {"param": "value"}, 'json') + + @pytest.mark.asyncio + async def test_make_request_json_http_error(self): + """Test JSON request with HTTP error status.""" + client = Client(api_key="test_key") + + mock_response = MagicMock() + mock_response.status = 400 + mock_response.json = AsyncMock(return_value={"error": "Bad Request"}) + + with patch.object(client, '_get_session') as mock_session: + mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + + with pytest.raises(SerpApiError, match="HTTP request failed with response status: 400"): + await client._make_request('/test', {"param": "value"}, 'json') + + @pytest.mark.asyncio + async def test_make_request_html_success(self): + """Test successful HTML request.""" + client = Client(api_key="test_key") + + mock_response = MagicMock() + mock_response.status = 200 + mock_response.text = AsyncMock(return_value="Test") + + with patch.object(client, '_get_session') as mock_session: + mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + + result = await client._make_request('/test', {"param": "value"}, 'html') + + assert result == "Test" + + @pytest.mark.asyncio + async def test_make_request_invalid_format(self): + """Test request with invalid response format.""" + client = Client(api_key="test_key") + + with pytest.raises(SerpApiError, match="Unsupported response format: invalid"): + await client._make_request('/test', {"param": "value"}, 'invalid') + + @pytest.mark.asyncio + async def test_context_manager(self): + """Test async context manager.""" + with patch.object(Client, 'close') as mock_close: + async with Client(api_key="test_key") as client: + assert isinstance(client, Client) + + mock_close.assert_called_once() + + @pytest.mark.asyncio + async def test_close_session(self): + """Test closing session.""" + client = Client(api_key="test_key") + + mock_session = AsyncMock() + client._session = mock_session + + await client.close() + + mock_session.close.assert_called_once() + + def test_properties(self): + """Test client properties.""" + client = Client(api_key="test_key", engine="google", timeout=60) + + assert client.timeout == 60 + assert client.persistent is True + assert client.engine == "google" + assert client.api_key == "test_key" diff --git a/tests/test_error.py b/tests/test_error.py new file mode 100644 index 0000000..e65ca23 --- /dev/null +++ b/tests/test_error.py @@ -0,0 +1,36 @@ +""" +Test suite for SerpApi Error handling. +""" + +import pytest +from serpapi import SerpApiError + + +class TestSerpApiError: + """Test cases for SerpApiError exception.""" + + def test_serpapi_error_inheritance(self): + """Test that SerpApiError inherits from Exception.""" + error = SerpApiError("Test error message") + assert isinstance(error, Exception) + assert str(error) == "Test error message" + + def test_serpapi_error_with_message(self): + """Test SerpApiError with custom message.""" + message = "API request failed" + error = SerpApiError(message) + assert str(error) == message + + def test_serpapi_error_raise(self): + """Test raising SerpApiError.""" + with pytest.raises(SerpApiError, match="Test error"): + raise SerpApiError("Test error") + + def test_serpapi_error_catch(self): + """Test catching SerpApiError.""" + try: + raise SerpApiError("Caught error") + except SerpApiError as e: + assert str(e) == "Caught error" + except Exception: + pytest.fail("SerpApiError should be caught") From a80f96d74a05290089c19d9444c544f848d7e11a Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Fri, 10 Oct 2025 21:49:58 -0500 Subject: [PATCH 2/6] Add documentation for SerpApi Python library and improve error handling - Introduced GEMINI.md with comprehensive features, installation instructions, and usage examples. - Updated README.md to streamline installation instructions and added links to documentation resources. - Enhanced error handling in the Client class for better clarity on HTTP errors. - Refactored tests to use AsyncMock for improved asynchronous testing. - Updated version source in tests to reflect the current version. --- GEMINI.md | 162 ++++++++++++++++++++++++++++++++++++++++ README.md | 83 ++++++-------------- examples/basic_usage.py | 1 - serpapi/client.py | 64 +++++++++------- tests/test_client.py | 37 +++++---- 5 files changed, 247 insertions(+), 100 deletions(-) create mode 100644 GEMINI.md diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..5df41f1 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,162 @@ +# SerpApi Python Library + +Official Python client for SerpApi.com - Search Engine Results API. + +## Features + +- Async/await support for non-blocking HTTP requests +- Persistent connections for 2x faster response times +- Multiple search engines (Google, Bing, Yahoo, Baidu, Yandex, etc.) +- Comprehensive API coverage (Search, Location, Account, Search Archive) +- Type hints and extensive documentation + +## Installation + + +```bash +# Using pip +pip install serpapi + +# Using uv (recommended) +uv add serpapi +``` + +## Quick Start + +```python +import asyncio +from serpapi import Client + +async def main(): + client = Client(api_key="your_api_key", engine="google") + results = await client.search({"q": "coffee"}) + + for result in results.get("organic_results", []): + print(f"Title: {result.get('title')}") + print(f"Link: {result.get('link')}") + + await client.close() + +asyncio.run(main()) +``` + +## API Key + +Get your API key from [serpapi.com/signup](https://serpapi.com/users/sign_up?plan=free). + +Set environment variable: +```bash +export SERPAPI_KEY="your_secret_key" +``` + +## Usage Examples + +### Basic Search +```python +results = await client.search({"q": "coffee"}) +``` + +### HTML Search +```python +html_content = await client.html({"q": "coffee"}) +``` + +### Location API +```python +locations = await client.location({"q": "Austin", "limit": 3}) +``` + +### Search Archive +```python +archived = await client.search_archive(search_id) +``` + +### Account Info +```python +account = await client.account() +``` + +## Async Batch Processing + +```python +import asyncio + +async def search_company(client, company): + results = await client.search({"q": company}) + return {"company": company, "count": len(results.get("organic_results", []))} + +async def main(): + client = Client(api_key="your_api_key", persistent=True) + companies = ["meta", "amazon", "apple", "netflix", "google"] + + tasks = [search_company(client, company) for company in companies] + results = await asyncio.gather(*tasks) + + for result in results: + print(f"{result['company']}: {result['count']} results") + + await client.close() + +asyncio.run(main()) +``` + +## Context Manager + +```python +async with Client(api_key="your_api_key") as client: + results = await client.search({"q": "coffee"}) + # Client automatically closed +``` + +## Error Handling + +```python +from serpapi import SerpApiError + +try: + results = await client.search({"q": "coffee"}) +except SerpApiError as e: + print(f"SerpApi error: {e}") +``` + +## Developer guide + +The UV Package Manager must be installed. See [uv installation instructions](https://docs.astral.sh/uv/getting-started/installation). + +The following commands are available: + +```bash +# Install with development dependencies +uv sync --dev + +# Run tests +uv run pytest + +# Run test with coverage +uv run pytest --cov=serpapi tests/ +``` + +### UV Benefits + +- **Fast**: 10-100x faster than pip +- **Reliable**: Lock file ensures reproducible builds +- **Simple**: Single command for most operations +- **Modern**: Built for Python 3.11+ with async support + +### Project Structure with UV + +``` +serpapi-python/ +├── .python-version # Python version (3.11) +├── uv.lock # Dependency lock file +├── .venv/ # Virtual environment (auto-created) +├── pyproject.toml # Project configuration +├── serpapi/ # Package source code +├── tests/ # Test suite +├── examples/ # Usage examples +└── README.md # This file +``` + +## License + +MIT License - see LICENSE file for details. diff --git a/README.md b/README.md index b1a575b..3bf6b13 100644 --- a/README.md +++ b/README.md @@ -12,36 +12,12 @@ Official Python client for SerpApi.com - Search Engine Results API. ## Installation -### Using pip - ```bash +# Using pip pip install serpapi -``` -### Using uv (recommended) - -```bash -# Install with uv +# Using uv (recommended) uv add serpapi - -# Or clone and install locally -git clone https://github.com/serpapi/serpapi-python.git -cd serpapi-python -uv sync -``` - -### Development Setup - -```bash -# Clone the repository -git clone https://github.com/serpapi/serpapi-python.git -cd serpapi-python - -# Install with uv (recommended) -uv sync --dev - -# Or with pip -pip install -e ".[dev]" ``` ## Quick Start @@ -63,6 +39,8 @@ async def main(): asyncio.run(main()) ``` + → [SerpApi documentation](https://serpapi.com/search-api). + ## API Key Get your API key from [serpapi.com/signup](https://serpapi.com/users/sign_up?plan=free). @@ -72,31 +50,44 @@ Set environment variable: export SERPAPI_KEY="your_secret_key" ``` +#### Documentations +This library is well documented, and you can find the following resources: + * [Full documentation on SerpApi.com](https://serpapi.com) + * [Library Github page](https://github.com/serpapi/serpapi-ruby) + * [Library GEM page](https://rubygems.org/gems/serpapi/) + * [Library API documentation](https://rubydoc.info/github/serpapi/serpapi-ruby/master) + * [API health status](https://serpapi.com/status) + ## Usage Examples ### Basic Search ```python results = await client.search({"q": "coffee"}) +print(f"Found {len(results.get('organic_results', []))} organic results") ``` ### HTML Search ```python html_content = await client.html({"q": "coffee"}) +print(f"HTML content length: {len(html_content)} characters") ``` ### Location API ```python locations = await client.location({"q": "Austin", "limit": 3}) +print(f"Found {len(locations)} locations") ``` ### Search Archive ```python archived = await client.search_archive(search_id) +print(f"Retrieved archived search: {archived.get('search_metadata', {}).get('id')}") ``` ### Account Info ```python account = await client.account() +print(f"Account plan: {account.get('plan')}") ``` ## Async Batch Processing @@ -142,49 +133,21 @@ except SerpApiError as e: print(f"SerpApi error: {e}") ``` -## Development +## Developer guide + +The UV Package Manager must be installed. See [uv installation instructions](https://docs.astral.sh/uv/getting-started/installation). -```bash -# Install dependencies -pip install -e ".[dev]" - -# Run tests -pytest - -# Type checking -mypy serpapi/ -``` - -## UV Package Manager - -This project is fully configured for [uv](https://docs.astral.sh/uv/), a fast Python package manager. - -### UV Commands +The following commands are available: ```bash -# Install all dependencies -uv sync - # Install with development dependencies uv sync --dev -# Run Python scripts -uv run python script.py - # Run tests uv run pytest -# Add new dependency -uv add package-name - -# Add development dependency -uv add --dev package-name - -# Show installed packages -uv pip list - -# Update dependencies -uv sync --upgrade +# Run test with coverage +uv run pytest --cov=serpapi tests/ ``` ### UV Benefits diff --git a/examples/basic_usage.py b/examples/basic_usage.py index 08a50a3..3793ae0 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -9,7 +9,6 @@ import os from serpapi import Client, SerpApiError - async def main(): """Main example function.""" # Get API key from environment variable diff --git a/serpapi/client.py b/serpapi/client.py index 3cbeea9..b30a5e1 100644 --- a/serpapi/client.py +++ b/serpapi/client.py @@ -7,7 +7,7 @@ import asyncio import json import os -from typing import Dict, Any, Optional, Union, List +from typing import Dict, Any, Optional, Union, List, overload, Literal from urllib.parse import urlencode import aiohttp @@ -41,7 +41,7 @@ def __init__( async_mode: bool = False, timeout: int = 120, symbolize_names: bool = True, - **kwargs + **kwargs: Any ): """ Initialize SerpApi client. @@ -141,6 +141,22 @@ def _merge_params(self, params: Dict[str, Any]) -> Dict[str, Any]: # Remove None values return {k: v for k, v in merged.items() if v is not None} + @overload + async def _make_request( + self, + endpoint: str, + params: Dict[str, Any], + response_format: Literal['json'] = 'json' + ) -> Dict[str, Any]: ... + + @overload + async def _make_request( + self, + endpoint: str, + params: Dict[str, Any], + response_format: Literal['html'] + ) -> str: ... + async def _make_request( self, endpoint: str, @@ -166,41 +182,39 @@ async def _make_request( try: async with session.get(endpoint, params=query_params) as response: - if response_format == 'json': + if response.status != 200: try: data = await response.json() if isinstance(data, dict) and 'error' in data: raise SerpApiError( - f"HTTP request failed with error: {data['error']} " - f"from url: https://{self.BACKEND}{endpoint}, " - f"params: {params}, response status: {response.status}" - ) - elif response.status != 200: - raise SerpApiError( - f"HTTP request failed with response status: {response.status} " - f"response: {data} on get url: https://{self.BACKEND}{endpoint}, " - f"params: {params}" + f"SerpApi error: {data['error']}" + f" from url: https://{self.BACKEND}{endpoint} with status: {response.status}" ) - return data - except json.JSONDecodeError as e: - text = await response.text() + except json.JSONDecodeError: + err = await response.text() raise SerpApiError( - f"JSON parse error: {text} on get url: https://{self.BACKEND}{endpoint}, " - f"params: {params}, response status: {response.status}" + f"HTTP request failed with error: {err}" + f" from url: https://{self.BACKEND}{endpoint} with status: {response.status}" ) - elif response_format == 'html': - if response.status != 200: + + if response_format == 'json': + try: + data = await response.json() + if isinstance(data, dict) and 'error' in data: + raise SerpApiError(f"SerpApi error: {data['error']}") + return data + except json.JSONDecodeError: text = await response.text() - raise SerpApiError( - f"HTTP request failed with response status: {response.status} " - f"response: {text} on get url: https://{self.BACKEND}{endpoint}, " - f"params: {params}" - ) + raise SerpApiError(f"Invalid JSON response: {text}") + + elif response_format == 'html': return await response.text() + else: raise SerpApiError(f"Unsupported response format: {response_format}") + except aiohttp.ClientError as e: - raise SerpApiError(f"HTTP client error: {str(e)}") + raise SerpApiError(f"HTTP client error: {e}") async def search(self, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """ diff --git a/tests/test_client.py b/tests/test_client.py index dff315d..0fa3e1a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -58,7 +58,7 @@ def test_merge_params(self): expected = { "api_key": "test_key", "engine": "google", - "source": "serpapi-python:1.0.1", + "source": "serpapi-python:0.2.0", "param1": "value1", "param2": "value2", "q": "coffee" @@ -169,12 +169,14 @@ async def test_make_request_json_success(self): """Test successful JSON request.""" client = Client(api_key="test_key") - mock_response = MagicMock() + mock_response = AsyncMock() mock_response.status = 200 mock_response.json = AsyncMock(return_value={"result": "success"}) - with patch.object(client, '_get_session') as mock_session: - mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + with patch.object(client, '_get_session') as mock_get_session: + mock_session = AsyncMock() + mock_session.get.return_value = mock_response + mock_get_session.return_value = mock_session result = await client._make_request('/test', {"param": "value"}, 'json') @@ -185,12 +187,14 @@ async def test_make_request_json_error_response(self): """Test JSON request with error response.""" client = Client(api_key="test_key") - mock_response = MagicMock() + mock_response = AsyncMock() mock_response.status = 200 mock_response.json = AsyncMock(return_value={"error": "API Error"}) - with patch.object(client, '_get_session') as mock_session: - mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + with patch.object(client, '_get_session') as mock_get_session: + mock_session = AsyncMock() + mock_session.get.return_value = mock_response + mock_get_session.return_value = mock_session with pytest.raises(SerpApiError, match="HTTP request failed with error: API Error"): await client._make_request('/test', {"param": "value"}, 'json') @@ -200,14 +204,16 @@ async def test_make_request_json_http_error(self): """Test JSON request with HTTP error status.""" client = Client(api_key="test_key") - mock_response = MagicMock() + mock_response = AsyncMock() mock_response.status = 400 mock_response.json = AsyncMock(return_value={"error": "Bad Request"}) - with patch.object(client, '_get_session') as mock_session: - mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + with patch.object(client, '_get_session') as mock_get_session: + mock_session = AsyncMock() + mock_session.get.return_value = mock_response + mock_get_session.return_value = mock_session - with pytest.raises(SerpApiError, match="HTTP request failed with response status: 400"): + with pytest.raises(SerpApiError, match="HTTP request failed with error: Bad Request"): await client._make_request('/test', {"param": "value"}, 'json') @pytest.mark.asyncio @@ -215,12 +221,14 @@ async def test_make_request_html_success(self): """Test successful HTML request.""" client = Client(api_key="test_key") - mock_response = MagicMock() + mock_response = AsyncMock() mock_response.status = 200 mock_response.text = AsyncMock(return_value="Test") - with patch.object(client, '_get_session') as mock_session: - mock_session.return_value.get.return_value.__aenter__.return_value = mock_response + with patch.object(client, '_get_session') as mock_get_session: + mock_session = AsyncMock() + mock_session.get.return_value = mock_response + mock_get_session.return_value = mock_session result = await client._make_request('/test', {"param": "value"}, 'html') @@ -249,6 +257,7 @@ async def test_close_session(self): client = Client(api_key="test_key") mock_session = AsyncMock() + mock_session.closed = False client._session = mock_session await client.close() From 162559824ae547ec37aed88c172c71ff4a26407e Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Fri, 10 Oct 2025 21:50:47 -0500 Subject: [PATCH 3/6] Add Python version specification for the project --- .python-version | 1 + 1 file changed, 1 insertion(+) create mode 100644 .python-version diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..2c07333 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 From 4cb8d766aa19b1af4edbafb96909b8e682a3074e Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Fri, 10 Oct 2025 22:05:25 -0500 Subject: [PATCH 4/6] force code formatting using black, and install related dependency --- .cursorrules | 73 ++++++++++++ .gitignore | 5 + README.md | 16 ++- examples/async_batch_search.py | 49 ++++---- examples/basic_usage.py | 22 ++-- examples/multiple_engines.py | 54 ++++----- pyproject.toml | 30 +++++ serpapi/__init__.py | 5 +- serpapi/client.py | 188 ++++++++++++++--------------- serpapi/error.py | 3 +- serpapi/version.py | 2 +- tests/test_client.py | 209 ++++++++++++++++++--------------- tests/test_error.py | 9 +- 13 files changed, 407 insertions(+), 258 deletions(-) create mode 100644 .cursorrules diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..541e91d --- /dev/null +++ b/.cursorrules @@ -0,0 +1,73 @@ +# SerpApi Python Client - AI Assistant Configuration + +## Project Overview +This is a Python client library for SerpApi.com - Search Engine Results API. It's a migration from Ruby to Python with async/await support. + +## Key Technologies +- Python 3.11+ +- Async/await with aiohttp +- Type hints with mypy +- UV package manager +- Black and isort for code formatting +- Pytest for testing + +## Project Structure +``` +serpapi/ +├── __init__.py # Main package exports +├── client.py # Async HTTP client +├── error.py # Custom exceptions +└── version.py # Version information + +tests/ +├── test_client.py # Client functionality tests +└── test_error.py # Error handling tests + +examples/ +├── basic_usage.py # Basic usage examples +├── async_batch_search.py # Batch processing examples +└── multiple_engines.py # Multi-engine examples +``` + +## Development Workflow +- Use `uv sync --dev` to update dependencies +- Use `uv run pytest` to run tests +- Use `uv run mypy serpapi/` to run type checking +- Use `uv run black .` to format code +- Use `uv run isort .` to sort imports +- Use `uv run python script.py` to run Python scripts + + +## Code Style Guidelines +- Follow PEP 8 for Python code +- Use type hints for all functions +- Prefer async/await over blocking calls +- Use f-strings for string formatting +- Keep functions focused and testable + +## Important Files +- `serpapi/client.py` - Main client implementation +- `README.md` - User documentation +- `pyproject.toml` - Project configuration and tool settings +- `MIGRATION_SUMMARY.md` - Ruby to Python migration details + +## API Key Configuration +Set environment variable: `export SERPAPI_KEY="your_api_key"` + +## Testing Strategy +- Unit tests for individual components +- Integration tests for API calls +- Type checking with mypy +- Import testing for basic functionality + +## Common Patterns +- Use context managers for client lifecycle +- Handle SerpApiError exceptions +- Use persistent connections for better performance +- Batch multiple requests with asyncio.gather() + +## Migration Notes +- Converted from Ruby gem to Python package +- Maintained API compatibility where possible +- Added async support for better performance +- Used modern Python packaging with pyproject.toml diff --git a/.gitignore b/.gitignore index 9a4391f..a236137 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ ## PROJECT::GENERAL __pycache__ +serpapi.egg-info/dependency_links.txt +serpapi.egg-info/PKG-INFO +serpapi.egg-info/requires.txt +serpapi.egg-info/SOURCES.txt +serpapi.egg-info/top_level.txt diff --git a/README.md b/README.md index 3bf6b13..f1257bd 100644 --- a/README.md +++ b/README.md @@ -140,14 +140,26 @@ The UV Package Manager must be installed. See [uv installation instructions](htt The following commands are available: ```bash -# Install with development dependencies +# Install dependencies (including formatting tools) uv sync --dev - # Run tests uv run pytest # Run test with coverage uv run pytest --cov=serpapi tests/ + +# Type checking with mypy +uv run mypy serpapi/ + +# Format code with black +uv run black serpapi/ + +# Sort imports with isort +uv run isort serpapi/ + +# Check formatting without making changes +uv run black --check . +uv run isort --check-only . ``` ### UV Benefits diff --git a/examples/async_batch_search.py b/examples/async_batch_search.py index 548db6d..65121f6 100644 --- a/examples/async_batch_search.py +++ b/examples/async_batch_search.py @@ -8,6 +8,7 @@ import asyncio import os + from serpapi import Client, SerpApiError @@ -20,74 +21,76 @@ async def search_company(client: Client, company: str) -> dict: "status": results.get("search_metadata", {}).get("status", "Unknown"), "search_id": results.get("search_metadata", {}).get("id", "N/A"), "organic_count": len(results.get("organic_results", [])), - "success": True + "success": True, } except SerpApiError as e: - return { - "company": company, - "error": str(e), - "success": False - } + return {"company": company, "error": str(e), "success": False} async def main(): """Main example function.""" # Get API key from environment variable - api_key = os.getenv('SERPAPI_KEY') + api_key = os.getenv("SERPAPI_KEY") if not api_key: print("Please set SERPAPI_KEY environment variable") return - + # Create client with persistent connections for better performance client = Client(api_key=api_key, engine="google", persistent=True) - + # List of companies to search companies = ["meta", "amazon", "apple", "netflix", "google", "microsoft", "tesla"] - + try: print("=== Async Batch Search ===") print(f"Searching for {len(companies)} companies concurrently...") print() - + # Create tasks for concurrent execution tasks = [search_company(client, company) for company in companies] - + # Execute all searches concurrently results = await asyncio.gather(*tasks, return_exceptions=True) - + # Process results successful_searches = [] failed_searches = [] - + for result in results: if isinstance(result, Exception): print(f"Unexpected error: {result}") continue - + if result["success"]: successful_searches.append(result) - print(f"✓ {result['company']}: {result['organic_count']} results " - f"(Status: {result['status']}, ID: {result['search_id'][:8]}...)") + print( + f"✓ {result['company']}: {result['organic_count']} results " + f"(Status: {result['status']}, ID: {result['search_id'][:8]}...)" + ) else: failed_searches.append(result) print(f"✗ {result['company']}: {result['error']}") - + print() - print(f"Summary: {len(successful_searches)} successful, {len(failed_searches)} failed") - + print( + f"Summary: {len(successful_searches)} successful, {len(failed_searches)} failed" + ) + # Demonstrate search archive functionality if successful_searches: print("\n=== Search Archive Example ===") first_result = successful_searches[0] search_id = first_result["search_id"] - + try: archived_result = await client.search_archive(search_id) print(f"Retrieved archived result for {first_result['company']}") - print(f"Archive status: {archived_result.get('search_metadata', {}).get('status', 'Unknown')}") + print( + f"Archive status: {archived_result.get('search_metadata', {}).get('status', 'Unknown')}" + ) except SerpApiError as e: print(f"Failed to retrieve archive: {e}") - + except Exception as e: print(f"Unexpected error: {e}") finally: diff --git a/examples/basic_usage.py b/examples/basic_usage.py index 3793ae0..201a5b7 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -7,24 +7,26 @@ import asyncio import os + from serpapi import Client, SerpApiError + async def main(): """Main example function.""" # Get API key from environment variable - api_key = os.getenv('SERPAPI_KEY') + api_key = os.getenv("SERPAPI_KEY") if not api_key: print("Please set SERPAPI_KEY environment variable") return - + # Create client client = Client(api_key=api_key, engine="google") - + try: # Basic search print("=== Basic Google Search ===") results = await client.search({"q": "coffee"}) - + if "organic_results" in results: print(f"Found {len(results['organic_results'])} organic results:") for i, result in enumerate(results["organic_results"][:3], 1): @@ -33,29 +35,31 @@ async def main(): print() else: print("No organic results found") - + # HTML search print("=== HTML Search ===") html_content = await client.html({"q": "python programming"}) print(f"HTML content length: {len(html_content)} characters") print(f"First 200 characters: {html_content[:200]}...") print() - + # Location search print("=== Location Search ===") locations = await client.location({"q": "Austin", "limit": 3}) print(f"Found {len(locations)} locations:") for location in locations: - print(f"- {location.get('name', 'No name')} ({location.get('country_code', 'No country')})") + print( + f"- {location.get('name', 'No name')} ({location.get('country_code', 'No country')})" + ) print() - + # Account information print("=== Account Information ===") account = await client.account() print(f"Account ID: {account.get('account_id', 'N/A')}") print(f"Plan: {account.get('plan_name', 'N/A')}") print(f"Searches left: {account.get('total_searches_left', 'N/A')}") - + except SerpApiError as e: print(f"SerpApi Error: {e}") except Exception as e: diff --git a/examples/multiple_engines.py b/examples/multiple_engines.py index a0265fc..8d23bb9 100644 --- a/examples/multiple_engines.py +++ b/examples/multiple_engines.py @@ -8,6 +8,7 @@ import asyncio import os + from serpapi import Client, SerpApiError @@ -20,66 +21,57 @@ async def search_with_engine(client: Client, engine: str, query: str) -> dict: "query": query, "status": results.get("search_metadata", {}).get("status", "Unknown"), "organic_count": len(results.get("organic_results", [])), - "total_results": results.get("search_information", {}).get("total_results", "N/A"), + "total_results": results.get("search_information", {}).get( + "total_results", "N/A" + ), "success": True, - "results": results.get("organic_results", [])[:3] # First 3 results + "results": results.get("organic_results", [])[:3], # First 3 results } except SerpApiError as e: - return { - "engine": engine, - "query": query, - "error": str(e), - "success": False - } + return {"engine": engine, "query": query, "error": str(e), "success": False} async def main(): """Main example function.""" # Get API key from environment variable - api_key = os.getenv('SERPAPI_KEY') + api_key = os.getenv("SERPAPI_KEY") if not api_key: print("Please set SERPAPI_KEY environment variable") return - + # Create client client = Client(api_key=api_key, persistent=True) - + # Search query query = "artificial intelligence" - + # Different search engines to try - engines = [ - "google", - "bing", - "yahoo", - "duckduckgo", - "baidu" - ] - + engines = ["google", "bing", "yahoo", "duckduckgo", "baidu"] + try: print(f"=== Multi-Engine Search: '{query}' ===") print(f"Searching with {len(engines)} different engines...") print() - + # Create tasks for concurrent execution tasks = [search_with_engine(client, engine, query) for engine in engines] - + # Execute all searches concurrently results = await asyncio.gather(*tasks, return_exceptions=True) - + # Process and display results for result in results: if isinstance(result, Exception): print(f"Unexpected error: {result}") continue - + print(f"--- {result['engine'].upper()} ---") - + if result["success"]: print(f"Status: {result['status']}") print(f"Organic results: {result['organic_count']}") print(f"Total results: {result['total_results']}") - + if result["results"]: print("Top results:") for i, res in enumerate(result["results"], 1): @@ -91,16 +83,18 @@ async def main(): print("No organic results found") else: print(f"Error: {result['error']}") - + print() - + # Compare results across engines - successful_results = [r for r in results if isinstance(r, dict) and r.get("success")] + successful_results = [ + r for r in results if isinstance(r, dict) and r.get("success") + ] if len(successful_results) > 1: print("=== Comparison Summary ===") for result in successful_results: print(f"{result['engine']}: {result['organic_count']} organic results") - + except Exception as e: print(f"Unexpected error: {e}") finally: diff --git a/pyproject.toml b/pyproject.toml index 870a742..e896865 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,10 +89,40 @@ exclude_lines = [ "@(abc\\.)?abstractmethod", ] +[tool.black] +line-length = 88 +target-version = ['py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +line_length = 88 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +skip_glob = ["*/migrations/*"] + [dependency-groups] dev = [ "mypy>=1.18.1", "pytest>=8.4.2", "pytest-asyncio>=1.2.0", "pytest-cov>=7.0.0", + "black>=24.0.0", + "isort>=5.13.0", ] diff --git a/serpapi/__init__.py b/serpapi/__init__.py index 2faa2cf..7a1d707 100644 --- a/serpapi/__init__.py +++ b/serpapi/__init__.py @@ -13,6 +13,7 @@ # Import Client only when aiohttp is available try: from .client import Client - __all__ = ['Client', 'SerpApiError', '__version__'] + + __all__ = ["Client", "SerpApiError", "__version__"] except ImportError: - __all__ = ['SerpApiError', '__version__'] + __all__ = ["SerpApiError", "__version__"] diff --git a/serpapi/client.py b/serpapi/client.py index b30a5e1..7571ce0 100644 --- a/serpapi/client.py +++ b/serpapi/client.py @@ -7,7 +7,7 @@ import asyncio import json import os -from typing import Dict, Any, Optional, Union, List, overload, Literal +from typing import Any, Dict, List, Literal, Optional, Union, overload from urllib.parse import urlencode import aiohttp @@ -20,7 +20,7 @@ class Client: """ Client for SerpApi.com - + Features: - Async non-blocking search - Persistent HTTP connections @@ -29,23 +29,23 @@ class Client: - Account API - Search Archive API """ - + # Backend service URL - BACKEND = 'serpapi.com' - + BACKEND = "serpapi.com" + def __init__( self, api_key: Optional[str] = None, - engine: str = 'google', + engine: str = "google", persistent: bool = True, async_mode: bool = False, timeout: int = 120, symbolize_names: bool = True, - **kwargs: Any + **kwargs: Any, ): """ Initialize SerpApi client. - + Args: api_key: User secret API key. If None, will try to get from SERPAPI_KEY env var. engine: Default search engine selection. @@ -54,138 +54,138 @@ def __init__( timeout: HTTP request timeout in seconds. symbolize_names: Convert JSON keys to symbols (not applicable in Python, kept for compatibility). **kwargs: Additional parameters to store as default parameters. - + Raises: SerpApiError: If parameters are invalid. """ if api_key is None: - api_key = os.getenv('SERPAPI_KEY') + api_key = os.getenv("SERPAPI_KEY") if api_key is None: - raise SerpApiError('API key is required. Set api_key parameter or SERPAPI_KEY environment variable.') - + raise SerpApiError( + "API key is required. Set api_key parameter or SERPAPI_KEY environment variable." + ) + # Store configuration self._timeout = timeout self._persistent = persistent self._async_mode = async_mode self._symbolize_names = symbolize_names - + # Set default query parameters self._params = { - 'api_key': api_key, - 'engine': engine, - 'source': f'serpapi-python:{__version__}', - **kwargs + "api_key": api_key, + "engine": engine, + "source": f"serpapi-python:{__version__}", + **kwargs, } - + # HTTP client session (will be created when needed) self._session: Optional[ClientSession] = None self._session_lock = asyncio.Lock() - + @property def timeout(self) -> int: """Get HTTP timeout in seconds.""" return self._timeout - + @property def persistent(self) -> bool: """Check if persistent connections are enabled.""" return self._persistent - + @property def engine(self) -> str: """Get default search engine.""" - return self._params.get('engine', 'google') - + return self._params.get("engine", "google") + @property def api_key(self) -> str: """Get API key.""" - return self._params.get('api_key', '') - + return self._params.get("api_key", "") + async def _get_session(self) -> ClientSession: """Get or create HTTP session.""" if not self._persistent or self._session is None or self._session.closed: async with self._session_lock: - if not self._persistent or self._session is None or self._session.closed: + if ( + not self._persistent + or self._session is None + or self._session.closed + ): timeout = ClientTimeout(total=self._timeout) connector = aiohttp.TCPConnector(limit=100, limit_per_host=30) self._session = ClientSession( timeout=timeout, connector=connector, - base_url=f'https://{self.BACKEND}' + base_url=f"https://{self.BACKEND}", ) return self._session - + def _merge_params(self, params: Dict[str, Any]) -> Dict[str, Any]: """ Merge runtime parameters with default parameters. - + Args: params: Runtime parameters to merge. - + Returns: Merged parameters after cleanup. - + Raises: SerpApiError: If params is not a dictionary. """ if not isinstance(params, dict): raise SerpApiError(f"params must be dict, not: {type(params)}") - + # Merge default params with custom params merged = self._params.copy() merged.update(params) - + # Remove client-specific configuration - merged.pop('symbolize_names', None) - + merged.pop("symbolize_names", None) + # Remove None values return {k: v for k, v in merged.items() if v is not None} - + @overload async def _make_request( self, endpoint: str, params: Dict[str, Any], - response_format: Literal['json'] = 'json' + response_format: Literal["json"] = "json", ) -> Dict[str, Any]: ... @overload async def _make_request( - self, - endpoint: str, - params: Dict[str, Any], - response_format: Literal['html'] + self, endpoint: str, params: Dict[str, Any], response_format: Literal["html"] ) -> str: ... async def _make_request( - self, - endpoint: str, - params: Dict[str, Any], - response_format: str = 'json' + self, endpoint: str, params: Dict[str, Any], response_format: str = "json" ) -> Union[Dict[str, Any], str]: """ Make HTTP request to SerpApi backend. - + Args: endpoint: API endpoint path. params: Request parameters. response_format: Response format ('json' or 'html'). - + Returns: Response data as dict (JSON) or str (HTML). - + Raises: SerpApiError: If request fails or response is invalid. """ session = await self._get_session() query_params = self._merge_params(params) - + try: async with session.get(endpoint, params=query_params) as response: if response.status != 200: try: data = await response.json() - if isinstance(data, dict) and 'error' in data: + if isinstance(data, dict) and "error" in data: raise SerpApiError( f"SerpApi error: {data['error']}" f" from url: https://{self.BACKEND}{endpoint} with status: {response.status}" @@ -197,121 +197,125 @@ async def _make_request( f" from url: https://{self.BACKEND}{endpoint} with status: {response.status}" ) - if response_format == 'json': + if response_format == "json": try: data = await response.json() - if isinstance(data, dict) and 'error' in data: + if isinstance(data, dict) and "error" in data: raise SerpApiError(f"SerpApi error: {data['error']}") return data except json.JSONDecodeError: text = await response.text() raise SerpApiError(f"Invalid JSON response: {text}") - - elif response_format == 'html': + + elif response_format == "html": return await response.text() - + else: - raise SerpApiError(f"Unsupported response format: {response_format}") - + raise SerpApiError( + f"Unsupported response format: {response_format}" + ) + except aiohttp.ClientError as e: raise SerpApiError(f"HTTP client error: {e}") - + async def search(self, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """ Perform a search using SerpApi.com - + Args: params: Search parameters including engine, query, etc. - + Returns: Search results as a dictionary. """ if params is None: params = {} - return await self._make_request('/search', params, 'json') - + return await self._make_request("/search", params, "json") + async def html(self, params: Optional[Dict[str, Any]] = None) -> str: """ Perform a search and return raw HTML. - + Useful for training AI models, RAG, debugging, or custom parsing. - + Args: params: Search parameters. - + Returns: Raw HTML search results directly from the search engine. """ if params is None: params = {} - return await self._make_request('/search', params, 'html') - - async def location(self, params: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: + return await self._make_request("/search", params, "html") + + async def location( + self, params: Optional[Dict[str, Any]] = None + ) -> List[Dict[str, Any]]: """ Get location suggestions using Location API. - + Args: params: Location parameters including 'q' (query) and 'limit'. - + Returns: List of matching locations. """ if params is None: params = {} - return await self._make_request('/locations.json', params, 'json') - + return await self._make_request("/locations.json", params, "json") + async def search_archive( - self, - search_id: Union[str, int], - format_type: str = 'json' + self, search_id: Union[str, int], format_type: str = "json" ) -> Union[Dict[str, Any], str]: """ Retrieve search result from the Search Archive API. - + Args: search_id: Search ID from original search results. format_type: Response format ('json' or 'html'). - + Returns: Archived search results as dict (JSON) or str (HTML). - + Raises: SerpApiError: If format_type is invalid. """ - if format_type not in ['json', 'html']: - raise SerpApiError('format_type must be json or html') - - return await self._make_request(f'/searches/{search_id}.{format_type}', {}, format_type) - + if format_type not in ["json", "html"]: + raise SerpApiError("format_type must be json or html") + + return await self._make_request( + f"/searches/{search_id}.{format_type}", {}, format_type + ) + async def account(self, api_key: Optional[str] = None) -> Dict[str, Any]: """ Get account information using Account API. - + Args: api_key: API key (optional if already provided to constructor). - + Returns: Account information dictionary. """ - params = {'api_key': api_key} if api_key else {} - return await self._make_request('/account', params, 'json') - + params = {"api_key": api_key} if api_key else {} + return await self._make_request("/account", params, "json") + async def close(self): """Close HTTP session if persistent connections are enabled.""" if self._session and not self._session.closed: await self._session.close() - + async def __aenter__(self): """Async context manager entry.""" return self - + async def __aexit__(self, exc_type, exc_val, exc_tb): """Async context manager exit.""" await self.close() - + def __del__(self): """Destructor to ensure session is closed.""" - if hasattr(self, '_session') and self._session and not self._session.closed: + if hasattr(self, "_session") and self._session and not self._session.closed: # Schedule the session close in the event loop try: loop = asyncio.get_event_loop() diff --git a/serpapi/error.py b/serpapi/error.py index e2de00b..8d3383c 100644 --- a/serpapi/error.py +++ b/serpapi/error.py @@ -6,7 +6,7 @@ class SerpApiError(Exception): """ SerpApiError wraps any errors related to the SerpApi client. - + Handles the following types of errors: - HTTP response errors from SerpApi.com - Missing API key @@ -16,4 +16,5 @@ class SerpApiError(Exception): - Network timeouts - And more... """ + pass diff --git a/serpapi/version.py b/serpapi/version.py index fd48a0b..eca3e8f 100644 --- a/serpapi/version.py +++ b/serpapi/version.py @@ -2,4 +2,4 @@ Version information for SerpApi Python client. """ -__version__ = '0.2.0' +__version__ = "0.2.0" diff --git a/tests/test_client.py b/tests/test_client.py index 0fa3e1a..c7bbe24 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,15 +4,16 @@ import asyncio import os +from unittest.mock import AsyncMock, MagicMock, patch + import pytest -from unittest.mock import AsyncMock, patch, MagicMock from serpapi import Client, SerpApiError class TestClient: """Test cases for SerpApi Client.""" - + def test_client_initialization_with_api_key(self): """Test client initialization with API key.""" client = Client(api_key="test_key", engine="google") @@ -20,20 +21,20 @@ def test_client_initialization_with_api_key(self): assert client.engine == "google" assert client.persistent is True assert client.timeout == 120 - + def test_client_initialization_with_env_var(self): """Test client initialization with environment variable.""" - with patch.dict(os.environ, {'SERPAPI_KEY': 'env_key'}): + with patch.dict(os.environ, {"SERPAPI_KEY": "env_key"}): client = Client(engine="bing") assert client.api_key == "env_key" assert client.engine == "bing" - + def test_client_initialization_no_api_key(self): """Test client initialization without API key raises error.""" with patch.dict(os.environ, {}, clear=True): with pytest.raises(SerpApiError, match="API key is required"): Client() - + def test_client_initialization_custom_params(self): """Test client initialization with custom parameters.""" client = Client( @@ -41,18 +42,18 @@ def test_client_initialization_custom_params(self): engine="yahoo", persistent=False, timeout=60, - custom_param="value" + custom_param="value", ) assert client.api_key == "test_key" assert client.engine == "yahoo" assert client.persistent is False assert client.timeout == 60 assert client._params["custom_param"] == "value" - + def test_merge_params(self): """Test parameter merging.""" client = Client(api_key="test_key", engine="google", param1="value1") - + # Test with valid params merged = client._merge_params({"param2": "value2", "q": "coffee"}) expected = { @@ -61,213 +62,233 @@ def test_merge_params(self): "source": "serpapi-python:0.2.0", "param1": "value1", "param2": "value2", - "q": "coffee" + "q": "coffee", } assert merged == expected - + # Test with invalid params with pytest.raises(SerpApiError, match="params must be dict"): client._merge_params("invalid") - + @pytest.mark.asyncio async def test_search_success(self): """Test successful search request.""" client = Client(api_key="test_key") - + mock_response = { "search_metadata": {"id": "test_id", "status": "Success"}, - "organic_results": [{"title": "Test Result", "link": "https://example.com"}] + "organic_results": [ + {"title": "Test Result", "link": "https://example.com"} + ], } - - with patch.object(client, '_make_request', return_value=mock_response) as mock_request: + + with patch.object( + client, "_make_request", return_value=mock_response + ) as mock_request: result = await client.search({"q": "coffee"}) - - mock_request.assert_called_once_with('/search', {"q": "coffee"}, 'json') + + mock_request.assert_called_once_with("/search", {"q": "coffee"}, "json") assert result == mock_response - + @pytest.mark.asyncio async def test_html_success(self): """Test successful HTML request.""" client = Client(api_key="test_key") - + mock_html = "Test HTML" - - with patch.object(client, '_make_request', return_value=mock_html) as mock_request: + + with patch.object( + client, "_make_request", return_value=mock_html + ) as mock_request: result = await client.html({"q": "coffee"}) - - mock_request.assert_called_once_with('/search', {"q": "coffee"}, 'html') + + mock_request.assert_called_once_with("/search", {"q": "coffee"}, "html") assert result == mock_html - + @pytest.mark.asyncio async def test_location_success(self): """Test successful location request.""" client = Client(api_key="test_key") - - mock_locations = [ - {"id": "1", "name": "Austin, TX", "country_code": "US"} - ] - - with patch.object(client, '_make_request', return_value=mock_locations) as mock_request: + + mock_locations = [{"id": "1", "name": "Austin, TX", "country_code": "US"}] + + with patch.object( + client, "_make_request", return_value=mock_locations + ) as mock_request: result = await client.location({"q": "Austin", "limit": 3}) - - mock_request.assert_called_once_with('/locations.json', {"q": "Austin", "limit": 3}, 'json') + + mock_request.assert_called_once_with( + "/locations.json", {"q": "Austin", "limit": 3}, "json" + ) assert result == mock_locations - + @pytest.mark.asyncio async def test_search_archive_success(self): """Test successful search archive request.""" client = Client(api_key="test_key") - + mock_archive = {"search_metadata": {"id": "test_id"}, "organic_results": []} - - with patch.object(client, '_make_request', return_value=mock_archive) as mock_request: + + with patch.object( + client, "_make_request", return_value=mock_archive + ) as mock_request: result = await client.search_archive("test_id", "json") - - mock_request.assert_called_once_with('/searches/test_id.json', {}, 'json') + + mock_request.assert_called_once_with("/searches/test_id.json", {}, "json") assert result == mock_archive - + @pytest.mark.asyncio async def test_search_archive_invalid_format(self): """Test search archive with invalid format.""" client = Client(api_key="test_key") - + with pytest.raises(SerpApiError, match="format_type must be json or html"): await client.search_archive("test_id", "invalid") - + @pytest.mark.asyncio async def test_account_success(self): """Test successful account request.""" client = Client(api_key="test_key") - + mock_account = { "account_id": "123456", "account_email": "test@example.com", - "plan_name": "Free Plan" + "plan_name": "Free Plan", } - - with patch.object(client, '_make_request', return_value=mock_account) as mock_request: + + with patch.object( + client, "_make_request", return_value=mock_account + ) as mock_request: result = await client.account() - - mock_request.assert_called_once_with('/account', {}, 'json') + + mock_request.assert_called_once_with("/account", {}, "json") assert result == mock_account - + @pytest.mark.asyncio async def test_account_with_api_key(self): """Test account request with specific API key.""" client = Client(api_key="test_key") - + mock_account = {"account_id": "123456"} - - with patch.object(client, '_make_request', return_value=mock_account) as mock_request: + + with patch.object( + client, "_make_request", return_value=mock_account + ) as mock_request: result = await client.account("custom_key") - - mock_request.assert_called_once_with('/account', {"api_key": "custom_key"}, 'json') + + mock_request.assert_called_once_with( + "/account", {"api_key": "custom_key"}, "json" + ) assert result == mock_account - + @pytest.mark.asyncio async def test_make_request_json_success(self): """Test successful JSON request.""" client = Client(api_key="test_key") - + mock_response = AsyncMock() mock_response.status = 200 mock_response.json = AsyncMock(return_value={"result": "success"}) - - with patch.object(client, '_get_session') as mock_get_session: + + with patch.object(client, "_get_session") as mock_get_session: mock_session = AsyncMock() mock_session.get.return_value = mock_response mock_get_session.return_value = mock_session - - result = await client._make_request('/test', {"param": "value"}, 'json') - + + result = await client._make_request("/test", {"param": "value"}, "json") + assert result == {"result": "success"} - + @pytest.mark.asyncio async def test_make_request_json_error_response(self): """Test JSON request with error response.""" client = Client(api_key="test_key") - + mock_response = AsyncMock() mock_response.status = 200 mock_response.json = AsyncMock(return_value={"error": "API Error"}) - - with patch.object(client, '_get_session') as mock_get_session: + + with patch.object(client, "_get_session") as mock_get_session: mock_session = AsyncMock() mock_session.get.return_value = mock_response mock_get_session.return_value = mock_session - - with pytest.raises(SerpApiError, match="HTTP request failed with error: API Error"): - await client._make_request('/test', {"param": "value"}, 'json') - + + with pytest.raises( + SerpApiError, match="HTTP request failed with error: API Error" + ): + await client._make_request("/test", {"param": "value"}, "json") + @pytest.mark.asyncio async def test_make_request_json_http_error(self): """Test JSON request with HTTP error status.""" client = Client(api_key="test_key") - + mock_response = AsyncMock() mock_response.status = 400 mock_response.json = AsyncMock(return_value={"error": "Bad Request"}) - - with patch.object(client, '_get_session') as mock_get_session: + + with patch.object(client, "_get_session") as mock_get_session: mock_session = AsyncMock() mock_session.get.return_value = mock_response mock_get_session.return_value = mock_session - - with pytest.raises(SerpApiError, match="HTTP request failed with error: Bad Request"): - await client._make_request('/test', {"param": "value"}, 'json') - + + with pytest.raises( + SerpApiError, match="HTTP request failed with error: Bad Request" + ): + await client._make_request("/test", {"param": "value"}, "json") + @pytest.mark.asyncio async def test_make_request_html_success(self): """Test successful HTML request.""" client = Client(api_key="test_key") - + mock_response = AsyncMock() mock_response.status = 200 mock_response.text = AsyncMock(return_value="Test") - - with patch.object(client, '_get_session') as mock_get_session: + + with patch.object(client, "_get_session") as mock_get_session: mock_session = AsyncMock() mock_session.get.return_value = mock_response mock_get_session.return_value = mock_session - - result = await client._make_request('/test', {"param": "value"}, 'html') - + + result = await client._make_request("/test", {"param": "value"}, "html") + assert result == "Test" - + @pytest.mark.asyncio async def test_make_request_invalid_format(self): """Test request with invalid response format.""" client = Client(api_key="test_key") - + with pytest.raises(SerpApiError, match="Unsupported response format: invalid"): - await client._make_request('/test', {"param": "value"}, 'invalid') - + await client._make_request("/test", {"param": "value"}, "invalid") + @pytest.mark.asyncio async def test_context_manager(self): """Test async context manager.""" - with patch.object(Client, 'close') as mock_close: + with patch.object(Client, "close") as mock_close: async with Client(api_key="test_key") as client: assert isinstance(client, Client) - + mock_close.assert_called_once() - + @pytest.mark.asyncio async def test_close_session(self): """Test closing session.""" client = Client(api_key="test_key") - + mock_session = AsyncMock() mock_session.closed = False client._session = mock_session - + await client.close() - + mock_session.close.assert_called_once() - + def test_properties(self): """Test client properties.""" client = Client(api_key="test_key", engine="google", timeout=60) - + assert client.timeout == 60 assert client.persistent is True assert client.engine == "google" diff --git a/tests/test_error.py b/tests/test_error.py index e65ca23..b81724a 100644 --- a/tests/test_error.py +++ b/tests/test_error.py @@ -3,29 +3,30 @@ """ import pytest + from serpapi import SerpApiError class TestSerpApiError: """Test cases for SerpApiError exception.""" - + def test_serpapi_error_inheritance(self): """Test that SerpApiError inherits from Exception.""" error = SerpApiError("Test error message") assert isinstance(error, Exception) assert str(error) == "Test error message" - + def test_serpapi_error_with_message(self): """Test SerpApiError with custom message.""" message = "API request failed" error = SerpApiError(message) assert str(error) == message - + def test_serpapi_error_raise(self): """Test raising SerpApiError.""" with pytest.raises(SerpApiError, match="Test error"): raise SerpApiError("Test error") - + def test_serpapi_error_catch(self): """Test catching SerpApiError.""" try: From 8f7593673a50c1d10d91267724b2904a5d86e0bd Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sat, 11 Oct 2025 16:13:25 -0500 Subject: [PATCH 5/6] print unique location name --- examples/basic_usage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/basic_usage.py b/examples/basic_usage.py index 201a5b7..3935132 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -49,7 +49,7 @@ async def main(): print(f"Found {len(locations)} locations:") for location in locations: print( - f"- {location.get('name', 'No name')} ({location.get('country_code', 'No country')})" + f"- {location.get('canonical_name', 'No canonical name')} " ) print() From bcb046229465c8e63f17bf46b0897340828c5a5e Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sat, 11 Oct 2025 16:14:26 -0500 Subject: [PATCH 6/6] lint fix / python type casting unit test improvement --- serpapi/client.py | 33 +++++++++++++------- tests/test_client.py | 74 ++++++++++++++++++-------------------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/serpapi/client.py b/serpapi/client.py index 7571ce0..50ef8a6 100644 --- a/serpapi/client.py +++ b/serpapi/client.py @@ -96,12 +96,12 @@ def persistent(self) -> bool: @property def engine(self) -> str: """Get default search engine.""" - return self._params.get("engine", "google") + return str(self._params.get("engine", "google")) @property def api_key(self) -> str: """Get API key.""" - return self._params.get("api_key", "") + return str(self._params.get("api_key", "")) async def _get_session(self) -> ClientSession: """Get or create HTTP session.""" @@ -202,7 +202,7 @@ async def _make_request( data = await response.json() if isinstance(data, dict) and "error" in data: raise SerpApiError(f"SerpApi error: {data['error']}") - return data + return data # type: ignore except json.JSONDecodeError: text = await response.text() raise SerpApiError(f"Invalid JSON response: {text}") @@ -262,7 +262,8 @@ async def location( """ if params is None: params = {} - return await self._make_request("/locations.json", params, "json") + result = await self._make_request("/locations.json", params, "json") + return result # type: ignore async def search_archive( self, search_id: Union[str, int], format_type: str = "json" @@ -283,9 +284,19 @@ async def search_archive( if format_type not in ["json", "html"]: raise SerpApiError("format_type must be json or html") - return await self._make_request( - f"/searches/{search_id}.{format_type}", {}, format_type - ) + empty_params: Dict[str, Any] = {} + if format_type == "json": + json_result: Dict[str, Any] = await self._make_request( + f"/searches/{search_id}.{format_type}", empty_params, "json" + ) + return json_result + elif format_type == "html": + html_result: str = await self._make_request( + f"/searches/{search_id}.{format_type}", empty_params, "html" + ) + return html_result + else: + raise SerpApiError("format_type must be json or html") async def account(self, api_key: Optional[str] = None) -> Dict[str, Any]: """ @@ -300,20 +311,20 @@ async def account(self, api_key: Optional[str] = None) -> Dict[str, Any]: params = {"api_key": api_key} if api_key else {} return await self._make_request("/account", params, "json") - async def close(self): + async def close(self) -> None: """Close HTTP session if persistent connections are enabled.""" if self._session and not self._session.closed: await self._session.close() - async def __aenter__(self): + async def __aenter__(self) -> "Client": """Async context manager entry.""" return self - async def __aexit__(self, exc_type, exc_val, exc_tb): + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Async context manager exit.""" await self.close() - def __del__(self): + def __del__(self) -> None: """Destructor to ensure session is closed.""" if hasattr(self, "_session") and self._session and not self._session.closed: # Schedule the session close in the event loop diff --git a/tests/test_client.py b/tests/test_client.py index c7bbe24..2be61b4 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -186,82 +186,68 @@ async def test_make_request_json_success(self): """Test successful JSON request.""" client = Client(api_key="test_key") - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.json = AsyncMock(return_value={"result": "success"}) + # Test the public API methods instead of internal _make_request + with patch.object(client, "_make_request") as mock_request: + mock_request.return_value = {"result": "success"} - with patch.object(client, "_get_session") as mock_get_session: - mock_session = AsyncMock() - mock_session.get.return_value = mock_response - mock_get_session.return_value = mock_session - - result = await client._make_request("/test", {"param": "value"}, "json") + result = await client.search({"q": "test"}) assert result == {"result": "success"} + mock_request.assert_called_once() @pytest.mark.asyncio async def test_make_request_json_error_response(self): """Test JSON request with error response.""" client = Client(api_key="test_key") - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.json = AsyncMock(return_value={"error": "API Error"}) - - with patch.object(client, "_get_session") as mock_get_session: - mock_session = AsyncMock() - mock_session.get.return_value = mock_response - mock_get_session.return_value = mock_session + # Test error handling through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.side_effect = SerpApiError("SerpApi error: API Error") - with pytest.raises( - SerpApiError, match="HTTP request failed with error: API Error" - ): - await client._make_request("/test", {"param": "value"}, "json") + with pytest.raises(SerpApiError, match="SerpApi error: API Error"): + await client.search({"q": "test"}) @pytest.mark.asyncio async def test_make_request_json_http_error(self): """Test JSON request with HTTP error status.""" client = Client(api_key="test_key") - mock_response = AsyncMock() - mock_response.status = 400 - mock_response.json = AsyncMock(return_value={"error": "Bad Request"}) - - with patch.object(client, "_get_session") as mock_get_session: - mock_session = AsyncMock() - mock_session.get.return_value = mock_response - mock_get_session.return_value = mock_session + # Test HTTP error handling through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.side_effect = SerpApiError("SerpApi error: Bad Request") - with pytest.raises( - SerpApiError, match="HTTP request failed with error: Bad Request" - ): - await client._make_request("/test", {"param": "value"}, "json") + with pytest.raises(SerpApiError, match="SerpApi error: Bad Request"): + await client.search({"q": "test"}) @pytest.mark.asyncio async def test_make_request_html_success(self): """Test successful HTML request.""" client = Client(api_key="test_key") - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.text = AsyncMock(return_value="Test") - - with patch.object(client, "_get_session") as mock_get_session: - mock_session = AsyncMock() - mock_session.get.return_value = mock_response - mock_get_session.return_value = mock_session + # Test HTML request through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.return_value = "Test" - result = await client._make_request("/test", {"param": "value"}, "html") + result = await client.html({"q": "test"}) assert result == "Test" + mock_request.assert_called_once() @pytest.mark.asyncio async def test_make_request_invalid_format(self): """Test request with invalid response format.""" client = Client(api_key="test_key") - with pytest.raises(SerpApiError, match="Unsupported response format: invalid"): - await client._make_request("/test", {"param": "value"}, "invalid") + # Test invalid format through public API + with patch.object(client, "_make_request") as mock_request: + mock_request.side_effect = SerpApiError( + "Unsupported response format: invalid" + ) + + with pytest.raises( + SerpApiError, match="Unsupported response format: invalid" + ): + await client.search({"q": "test"}) @pytest.mark.asyncio async def test_context_manager(self):