ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'implement-webhook-crawl-feature-011CULZY1Jy8N5MUkZqXkRVp' into develop

Browse Source
This commit is contained in:
ntohidi
2025-10-22 13:12:25 +02:00
parent 6d1a398419 d670dcde0a
commit b71d624168
14 changed files with 2579 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
deploy/docker/api.py
View File

@@ -46,6 +46,7 @@ from utils import (
get_llm_temperature,
get_llm_base_url
)
from webhook import WebhookDeliveryService
import psutil, time
@@ -120,10 +121,14 @@ async def process_llm_extraction(
schema: Optional[str] = None,
cache: str = "0",
provider: Optional[str] = None,
webhook_config: Optional[Dict] = None,
temperature: Optional[float] = None,
base_url: Optional[str] = None
) -> None:
"""Process LLM extraction in background."""
# Initialize webhook service
webhook_service = WebhookDeliveryService(config)
try:
# Validate provider
is_valid, error_msg = validate_llm_provider(config, provider)
@@ -132,6 +137,16 @@ async def process_llm_extraction(
"status": TaskStatus.FAILED,
"error": error_msg
})
# Send webhook notification on failure
await webhook_service.notify_job_completion(
task_id=task_id,
task_type="llm_extraction",
status="failed",
urls=[url],
webhook_config=webhook_config,
error=error_msg
)
return
api_key = get_llm_api_key(config, provider) # Returns None to let litellm handle it
llm_strategy = LLMExtractionStrategy(
@@ -162,17 +177,40 @@ async def process_llm_extraction(
"status": TaskStatus.FAILED,
"error": result.error_message
})
# Send webhook notification on failure
await webhook_service.notify_job_completion(
task_id=task_id,
task_type="llm_extraction",
status="failed",
urls=[url],
webhook_config=webhook_config,
error=result.error_message
)
return
try:
content = json.loads(result.extracted_content)
except json.JSONDecodeError:
content = result.extracted_content
result_data = {"extracted_content": content}
await redis.hset(f"task:{task_id}", mapping={
"status": TaskStatus.COMPLETED,
"result": json.dumps(content)
})
# Send webhook notification on successful completion
await webhook_service.notify_job_completion(
task_id=task_id,
task_type="llm_extraction",
status="completed",
urls=[url],
webhook_config=webhook_config,
result=result_data
)
except Exception as e:
logger.error(f"LLM extraction error: {str(e)}", exc_info=True)
await redis.hset(f"task:{task_id}", mapping={
@@ -180,6 +218,16 @@ async def process_llm_extraction(
"error": str(e)
})
# Send webhook notification on failure
await webhook_service.notify_job_completion(
task_id=task_id,
task_type="llm_extraction",
status="failed",
urls=[url],
webhook_config=webhook_config,
error=str(e)
)
async def handle_markdown_request(
url: str,
filter_type: FilterType,
@@ -261,6 +309,7 @@ async def handle_llm_request(
cache: str = "0",
config: Optional[dict] = None,
provider: Optional[str] = None,
webhook_config: Optional[Dict] = None,,
temperature: Optional[float] = None,
api_base_url: Optional[str] = None
) -> JSONResponse:
@@ -294,6 +343,7 @@ async def handle_llm_request(
base_url,
config,
provider,
webhook_config,
temperature,
api_base_url
)
@@ -341,6 +391,7 @@ async def create_new_task(
base_url: str,
config: dict,
provider: Optional[str] = None,
webhook_config: Optional[Dict] = None,
temperature: Optional[float] = None,
api_base_url: Optional[str] = None
) -> JSONResponse:
@@ -351,12 +402,18 @@ async def create_new_task(
from datetime import datetime
task_id = f"llm_{int(datetime.now().timestamp())}_{id(background_tasks)}"
await redis.hset(f"task:{task_id}", mapping={
task_data = {
"status": TaskStatus.PROCESSING,
"created_at": datetime.now().isoformat(),
"url": decoded_url
})
}
# Store webhook config if provided
if webhook_config:
task_data["webhook_config"] = json.dumps(webhook_config)
await redis.hset(f"task:{task_id}", mapping=task_data)
background_tasks.add_task(
process_llm_extraction,
@@ -368,6 +425,7 @@ async def create_new_task(
schema,
cache,
provider,
webhook_config,
temperature,
api_base_url
)
@@ -680,6 +738,7 @@ async def handle_crawl_job(
browser_config: Dict,
crawler_config: Dict,
config: Dict,
webhook_config: Optional[Dict] = None,
) -> Dict:
"""
Fire-and-forget version of handle_crawl_request.
@@ -687,13 +746,24 @@ async def handle_crawl_job(
lets /crawl/job/{task_id} polling fetch the result.
"""
task_id = f"crawl_{uuid4().hex[:8]}"
await redis.hset(f"task:{task_id}", mapping={
# Store task data in Redis
task_data = {
"status": TaskStatus.PROCESSING, # <-- keep enum values consistent
"created_at": datetime.now(timezone.utc).replace(tzinfo=None).isoformat(),
"url": json.dumps(urls), # store list as JSON string
"result": "",
"error": "",
})
}
# Store webhook config if provided
if webhook_config:
task_data["webhook_config"] = json.dumps(webhook_config)
await redis.hset(f"task:{task_id}", mapping=task_data)
# Initialize webhook service
webhook_service = WebhookDeliveryService(config)
async def _runner():
try:
@@ -707,6 +777,17 @@ async def handle_crawl_job(
"status": TaskStatus.COMPLETED,
"result": json.dumps(result),
})
# Send webhook notification on successful completion
await webhook_service.notify_job_completion(
task_id=task_id,
task_type="crawl",
status="completed",
urls=urls,
webhook_config=webhook_config,
result=result
)
await asyncio.sleep(5) # Give Redis time to process the update
except Exception as exc:
await redis.hset(f"task:{task_id}", mapping={
@@ -714,5 +795,15 @@ async def handle_crawl_job(
"error": str(exc),
})
# Send webhook notification on failure
await webhook_service.notify_job_completion(
task_id=task_id,
task_type="crawl",
status="failed",
urls=urls,
webhook_config=webhook_config,
error=str(exc)
)
background_tasks.add_task(_runner)
return {"task_id": task_id}