diff --git a/docs/examples/v0_4_24_walkthrough.py b/docs/examples/v0_4_24_walkthrough.py index a4e1aaa3..135ac29c 100644 --- a/docs/examples/v0_4_24_walkthrough.py +++ b/docs/examples/v0_4_24_walkthrough.py @@ -169,7 +169,8 @@ async def demo_content_filtering(): ) run_config = CrawlerRunConfig( - markdown_generator=markdown_gen + markdown_generator=markdown_gen, + cache_mode=CacheMode.BYPASS ) async with AsyncWebCrawler() as crawler: @@ -436,7 +437,7 @@ async def main(): await demo_ssl_features() await demo_content_filtering() await demo_json_extraction() - await demo_input_formats() + # await demo_input_formats() if __name__ == "__main__": asyncio.run(main()) diff --git a/requirements.txt b/requirements.txt index f676100e..00ce69d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +# Note: These requirements are also specified in pyproject.toml +# This file is kept for development environment setup and compatibility aiosqlite~=0.20 lxml~=5.3 litellm>=1.53.1 diff --git a/setup.py b/setup.py index 7dd1363b..dad3199d 100644 --- a/setup.py +++ b/setup.py @@ -30,11 +30,6 @@ cache_folder.mkdir(exist_ok=True) for folder in content_folders: (crawl4ai_folder / folder).mkdir(exist_ok=True) -# Read requirements and version -__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) -with open(os.path.join(__location__, "requirements.txt")) as f: - requirements = [line for line in f.read().splitlines() if line and not line.startswith('#')] - version = "0.0.0" # This will be overridden by pyproject.toml's dynamic version try: with open("crawl4ai/__version__.py") as f: @@ -45,13 +40,6 @@ try: except Exception: pass # Let pyproject.toml handle version -# Define requirements -default_requirements = requirements -torch_requirements = ["torch", "nltk", "scikit-learn"] -transformer_requirements = ["transformers", "tokenizers"] -cosine_similarity_requirements = ["torch", "transformers", "nltk"] -sync_requirements = ["selenium"] - setup( name="Crawl4AI", version=version, @@ -64,28 +52,7 @@ setup( license="MIT", packages=find_packages(), package_data={ - 'crawl4ai': ['js_snippet/*.js'] # This matches the exact path structure - }, - install_requires=default_requirements - + ["playwright", "aiofiles"], # Added aiofiles - extras_require={ - "torch": torch_requirements, - "transformer": transformer_requirements, - "cosine": cosine_similarity_requirements, - "sync": sync_requirements, - "all": default_requirements - + torch_requirements - + transformer_requirements - + cosine_similarity_requirements - + sync_requirements, - }, - entry_points={ - "console_scripts": [ - "crawl4ai-download-models=crawl4ai.model_loader:main", - "crawl4ai-migrate=crawl4ai.migrations:main", - 'crawl4ai-setup=crawl4ai.install:post_install', - 'crawl=crawl4ai.cli:cli', - ], + 'crawl4ai': ['js_snippet/*.js'] }, classifiers=[ "Development Status :: 3 - Alpha",