refactor(build): simplify setup.py configuration

- Remove dependency management from setup.py
- Remove entry points configuration (moved to pyproject.toml)
- Keep minimal setup.py for backwards compatibility
- Clean up package metadata structure

docs/examples/v0_4_24_walkthrough.py

@@ -169,7 +169,8 @@ async def demo_content_filtering():
     )
 
     run_config = CrawlerRunConfig(
-        markdown_generator=markdown_gen
+        markdown_generator=markdown_gen,
+        cache_mode=CacheMode.BYPASS
     )
 
     async with AsyncWebCrawler() as crawler:
@@ -436,7 +437,7 @@ async def main():
     await demo_ssl_features()
     await demo_content_filtering()
     await demo_json_extraction()
-    await demo_input_formats()
+    # await demo_input_formats()
 
 if __name__ == "__main__":
     asyncio.run(main())

requirements.txt

@@ -1,3 +1,5 @@
+# Note: These requirements are also specified in pyproject.toml
+# This file is kept for development environment setup and compatibility
 aiosqlite~=0.20
 lxml~=5.3
 litellm>=1.53.1

setup.py

@@ -30,11 +30,6 @@ cache_folder.mkdir(exist_ok=True)
 for folder in content_folders:
     (crawl4ai_folder / folder).mkdir(exist_ok=True)
 
-# Read requirements and version
-__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
-with open(os.path.join(__location__, "requirements.txt")) as f:
-    requirements = [line for line in f.read().splitlines() if line and not line.startswith('#')]
-
 version = "0.0.0"  # This will be overridden by pyproject.toml's dynamic version
 try:
     with open("crawl4ai/__version__.py") as f:
@@ -45,13 +40,6 @@ try:
 except Exception:
     pass  # Let pyproject.toml handle version
 
-# Define requirements
-default_requirements = requirements
-torch_requirements = ["torch", "nltk", "scikit-learn"]
-transformer_requirements = ["transformers", "tokenizers"]
-cosine_similarity_requirements = ["torch", "transformers", "nltk"]
-sync_requirements = ["selenium"]
-
 setup(
     name="Crawl4AI",
     version=version,
@@ -64,28 +52,7 @@ setup(
     license="MIT",
     packages=find_packages(),
     package_data={
-        'crawl4ai': ['js_snippet/*.js']  # This matches the exact path structure
-    },
-    install_requires=default_requirements
-    + ["playwright", "aiofiles"],  # Added aiofiles
-    extras_require={
-        "torch": torch_requirements,
-        "transformer": transformer_requirements,
-        "cosine": cosine_similarity_requirements,
-        "sync": sync_requirements,
-        "all": default_requirements
-        + torch_requirements
-        + transformer_requirements
-        + cosine_similarity_requirements
-        + sync_requirements,
-    },
-    entry_points={
-        "console_scripts": [
-            "crawl4ai-download-models=crawl4ai.model_loader:main",
-            "crawl4ai-migrate=crawl4ai.migrations:main",  
-            'crawl4ai-setup=crawl4ai.install:post_install', 
-            'crawl=crawl4ai.cli:cli',
-        ],
+        'crawl4ai': ['js_snippet/*.js']
     },
     classifiers=[
         "Development Status :: 3 - Alpha",