feat: Add hooks utility for function-based hooks with Docker client integration. ref #1377
Add hooks_to_string() utility function that converts Python function objects to string representations for the Docker API, enabling developers to write hooks as regular Python functions instead of strings. Core Changes: - New hooks_to_string() utility in crawl4ai/utils.py using inspect.getsource() - Docker client now accepts both function objects and strings for hooks - Automatic detection and conversion in Crawl4aiDockerClient._prepare_request() - New hooks and hooks_timeout parameters in client.crawl() method Documentation: - Docker client examples with function-based hooks (docs/examples/docker_client_hooks_example.py) - Updated main Docker deployment guide with comprehensive hooks section - Added unit tests for hooks utility (tests/docker/test_hooks_utility.py)
This commit is contained in:
@@ -47,6 +47,7 @@ from urllib.parse import (
|
||||
urljoin, urlparse, urlunparse,
|
||||
parse_qsl, urlencode, quote, unquote
|
||||
)
|
||||
import inspect
|
||||
|
||||
|
||||
# Monkey patch to fix wildcard handling in urllib.robotparser
|
||||
@@ -3529,4 +3530,52 @@ def get_memory_stats() -> Tuple[float, float, float]:
|
||||
available_gb = get_true_available_memory_gb()
|
||||
used_percent = get_true_memory_usage_percent()
|
||||
|
||||
return used_percent, available_gb, total_gb
|
||||
return used_percent, available_gb, total_gb
|
||||
|
||||
|
||||
# Hook utilities for Docker API
|
||||
def hooks_to_string(hooks: Dict[str, Callable]) -> Dict[str, str]:
|
||||
"""
|
||||
Convert hook function objects to string representations for Docker API.
|
||||
|
||||
This utility simplifies the process of using hooks with the Docker API by converting
|
||||
Python function objects into the string format required by the API.
|
||||
|
||||
Args:
|
||||
hooks: Dictionary mapping hook point names to Python function objects.
|
||||
Functions should be async and follow hook signature requirements.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping hook point names to string representations of the functions.
|
||||
|
||||
Example:
|
||||
>>> async def my_hook(page, context, **kwargs):
|
||||
... await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
... return page
|
||||
>>>
|
||||
>>> hooks_dict = {"on_page_context_created": my_hook}
|
||||
>>> api_hooks = hooks_to_string(hooks_dict)
|
||||
>>> # api_hooks is now ready to use with Docker API
|
||||
|
||||
Raises:
|
||||
ValueError: If a hook is not callable or source cannot be extracted
|
||||
"""
|
||||
result = {}
|
||||
|
||||
for hook_name, hook_func in hooks.items():
|
||||
if not callable(hook_func):
|
||||
raise ValueError(f"Hook '{hook_name}' must be a callable function, got {type(hook_func)}")
|
||||
|
||||
try:
|
||||
# Get the source code of the function
|
||||
source = inspect.getsource(hook_func)
|
||||
# Remove any leading indentation to get clean source
|
||||
source = textwrap.dedent(source)
|
||||
result[hook_name] = source
|
||||
except (OSError, TypeError) as e:
|
||||
raise ValueError(
|
||||
f"Cannot extract source code for hook '{hook_name}'. "
|
||||
f"Make sure the function is defined in a file (not interactively). Error: {e}"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user