7f360577d9
Implement a privacy-first, provider-agnostic telemetry system to help improve Crawl4AI stability through anonymous crash reporting. The system is designed with user privacy as the top priority, collecting only exception information without any PII, URLs, or crawled content. Architecture & Design: - Provider-agnostic architecture with base TelemetryProvider interface - Sentry as the initial provider implementation with easy extensibility - Separate handling for sync and async code paths - Environment-aware behavior (CLI, Docker, Jupyter/Colab) Key Features: - Opt-in by default for CLI/library usage with interactive consent prompt - Opt-out by default for Docker/API server (enabled unless CRAWL4AI_TELEMETRY=0) - Jupyter/Colab support with widget-based consent (fallback to code snippets) - Persistent consent storage in ~/.crawl4ai/config.json - Optional email collection for critical issue follow-up CLI Integration: - `crwl telemetry enable [--email <email>] [--once]` - Enable telemetry - `crwl telemetry disable` - Disable telemetry - `crwl telemetry status` - Check current status Python API: - Decorators: @telemetry_decorator, @async_telemetry_decorator - Context managers: telemetry_context(), async_telemetry_context() - Manual capture: capture_exception(exc, context) - Control: telemetry.enable(), telemetry.disable(), telemetry.status() Privacy Safeguards: - No URL collection - No request/response data - No authentication tokens or cookies - No crawled content - Automatic sanitization of sensitive fields - Local consent storage only Testing: - Comprehensive test suite with 15 test cases - Coverage for all environments and consent flows - Mock providers for testing without external dependencies Documentation: - Detailed documentation in docs/md_v2/core/telemetry.md - Added to mkdocs navigation under Core section - Privacy commitment and FAQ included - Examples for all usage patterns Installation: - Optional dependency: pip install crawl4ai[telemetry] - Graceful degradation if sentry-sdk not installed - Added to pyproject.toml optional dependencies - Docker requirements updated Integration Points: - AsyncWebCrawler: Automatic exception capture in arun() and aprocess_html() - Docker server: Automatic initialization with environment control - Global exception handler for uncaught exceptions (CLI only) This implementation provides valuable error insights to improve Crawl4AI while maintaining complete transparency and user control over data collection.
104 lines
2.5 KiB
TOML
104 lines
2.5 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=64.0.0", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "Crawl4AI"
|
|
dynamic = ["version"]
|
|
description = "🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
|
|
readme = "README.md"
|
|
requires-python = ">=3.9"
|
|
license = "Apache-2.0"
|
|
authors = [
|
|
{name = "Unclecode", email = "unclecode@kidocode.com"}
|
|
]
|
|
dependencies = [
|
|
"aiofiles>=24.1.0",
|
|
"aiohttp>=3.11.11",
|
|
"aiosqlite~=0.20",
|
|
"anyio>=4.0.0",
|
|
"lxml~=5.3",
|
|
"litellm>=1.53.1",
|
|
"numpy>=1.26.0,<3",
|
|
"pillow>=10.4",
|
|
"playwright>=1.49.0",
|
|
"patchright>=1.49.0",
|
|
"python-dotenv~=1.0",
|
|
"requests~=2.26",
|
|
"beautifulsoup4~=4.12",
|
|
"tf-playwright-stealth>=1.1.0",
|
|
"xxhash~=3.4",
|
|
"rank-bm25~=0.2",
|
|
"snowballstemmer~=2.2",
|
|
"pydantic>=2.10",
|
|
"pyOpenSSL>=24.3.0",
|
|
"psutil>=6.1.1",
|
|
"PyYAML>=6.0",
|
|
"nltk>=3.9.1",
|
|
"rich>=13.9.4",
|
|
"httpx>=0.27.2",
|
|
"httpx[http2]>=0.27.2",
|
|
"fake-useragent>=2.0.3",
|
|
"click>=8.1.7",
|
|
"chardet>=5.2.0",
|
|
"brotli>=1.1.0",
|
|
"humanize>=4.10.0",
|
|
"lark>=1.2.2",
|
|
"alphashape>=1.3.1",
|
|
"shapely>=2.0.0"
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 4 - Beta",
|
|
"Intended Audience :: Developers",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.9",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
pdf = ["PyPDF2"]
|
|
torch = ["torch", "nltk", "scikit-learn"]
|
|
transformer = ["transformers", "tokenizers", "sentence-transformers"]
|
|
cosine = ["torch", "transformers", "nltk", "sentence-transformers"]
|
|
sync = ["selenium"]
|
|
telemetry = ["sentry-sdk>=2.0.0", "ipywidgets>=8.0.0"]
|
|
all = [
|
|
"PyPDF2",
|
|
"torch",
|
|
"nltk",
|
|
"scikit-learn",
|
|
"transformers",
|
|
"tokenizers",
|
|
"sentence-transformers",
|
|
"selenium",
|
|
"sentry-sdk>=2.0.0",
|
|
"ipywidgets>=8.0.0"
|
|
]
|
|
|
|
[project.scripts]
|
|
crawl4ai-download-models = "crawl4ai.model_loader:main"
|
|
crawl4ai-migrate = "crawl4ai.migrations:main"
|
|
crawl4ai-setup = "crawl4ai.install:post_install"
|
|
crawl4ai-doctor = "crawl4ai.install:doctor"
|
|
crwl = "crawl4ai.cli:main"
|
|
|
|
[tool.setuptools]
|
|
packages = {find = {where = ["."], include = ["crawl4ai*"]}}
|
|
|
|
[tool.setuptools.package-data]
|
|
crawl4ai = ["js_snippet/*.js"]
|
|
|
|
[tool.setuptools.dynamic]
|
|
version = {attr = "crawl4ai.__version__.__version__"}
|
|
|
|
[tool.uv.sources]
|
|
crawl4ai = { workspace = true }
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"crawl4ai",
|
|
]
|