feat(core): release version 0.5.0 with deep crawling and CLI
This major release adds deep crawling capabilities, memory-adaptive dispatcher, multiple crawling strategies, Docker deployment, and a new CLI. It also includes significant improvements to proxy handling, PDF processing, and LLM integration. BREAKING CHANGES: - Add memory-adaptive dispatcher as default for arun_many() - Move max_depth to CrawlerRunConfig - Replace ScrapingMode enum with strategy pattern - Update BrowserContext API - Make model fields optional with defaults - Remove content_filter parameter from CrawlerRunConfig - Remove synchronous WebCrawler and old CLI - Update Docker deployment configuration - Replace FastFilterChain with FilterChain - Change license to Apache 2.0 with attribution clause
This commit is contained in:
UncleCode
25
docs/assets/powered-by-dark.svg
Normal file
25
docs/assets/powered-by-dark.svg
Normal file
@@ -0,0 +1,25 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
|
||||
<!-- Dark Theme -->
|
||||
<g>
|
||||
<defs>
|
||||
<pattern id="halftoneDark" width="4" height="4" patternUnits="userSpaceOnUse">
|
||||
<circle cx="2" cy="2" r="1" fill="#eee" opacity="0.1"/>
|
||||
</pattern>
|
||||
<pattern id="halftoneTextDark" width="3" height="3" patternUnits="userSpaceOnUse">
|
||||
<circle cx="1.5" cy="1.5" r="2" fill="#aaa" opacity="0.2"/>
|
||||
</pattern>
|
||||
</defs>
|
||||
<!-- White border - added as outer rectangle -->
|
||||
<rect width="120" height="35" rx="5" fill="#111"/>
|
||||
<!-- Dark background slightly smaller to show thicker border -->
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="#1a1a1a"/>
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="url(#halftoneDark)"/>
|
||||
|
||||
<!-- Logo with halftone -->
|
||||
<path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#eee" stroke-width="2"/>
|
||||
<path d="M18 17.5 L27 17.5" stroke="#eee" stroke-width="2"/>
|
||||
<circle cx="22.5" cy="17.5" r="2" fill="#eee"/>
|
||||
|
||||
<text x="40" y="23" fill="#eee" font-family="Arial, sans-serif" font-weight="500" font-size="14">Crawl4AI</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
64
docs/assets/powered-by-disco.svg
Normal file
64
docs/assets/powered-by-disco.svg
Normal file
@@ -0,0 +1,64 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
|
||||
<g>
|
||||
<defs>
|
||||
<pattern id="cyberdots" width="4" height="4" patternUnits="userSpaceOnUse">
|
||||
<circle cx="2" cy="2" r="1">
|
||||
<animate attributeName="fill"
|
||||
values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
|
||||
dur="6s"
|
||||
repeatCount="indefinite"/>
|
||||
<animate attributeName="opacity"
|
||||
values="0.2;0.4;0.2"
|
||||
dur="4s"
|
||||
repeatCount="indefinite"/>
|
||||
</circle>
|
||||
</pattern>
|
||||
<filter id="neonGlow" x="-20%" y="-20%" width="140%" height="140%">
|
||||
<feGaussianBlur stdDeviation="1" result="blur"/>
|
||||
<feFlood flood-color="#FF2EC4" flood-opacity="0.2">
|
||||
<animate attributeName="flood-color"
|
||||
values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
|
||||
dur="8s"
|
||||
repeatCount="indefinite"/>
|
||||
</feFlood>
|
||||
<feComposite in2="blur" operator="in"/>
|
||||
<feMerge>
|
||||
<feMergeNode/>
|
||||
<feMergeNode in="SourceGraphic"/>
|
||||
</feMerge>
|
||||
</filter>
|
||||
</defs>
|
||||
|
||||
<rect width="120" height="35" rx="5" fill="#0A0A0F"/>
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="#16161E"/>
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="url(#cyberdots)"/>
|
||||
|
||||
<!-- Logo with animated neon -->
|
||||
<path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)">
|
||||
<animate attributeName="stroke"
|
||||
values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
|
||||
dur="8s"
|
||||
repeatCount="indefinite"/>
|
||||
</path>
|
||||
<path d="M18 17.5 L27 17.5" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)">
|
||||
<animate attributeName="stroke"
|
||||
values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
|
||||
dur="8s"
|
||||
repeatCount="indefinite"/>
|
||||
</path>
|
||||
<circle cx="22.5" cy="17.5" r="2" fill="#0BC5EA">
|
||||
<animate attributeName="fill"
|
||||
values="#0BC5EA;#FF2EC4;#8B5CF6;#0BC5EA"
|
||||
dur="8s"
|
||||
repeatCount="indefinite"/>
|
||||
</circle>
|
||||
|
||||
<text x="40" y="23" font-family="Arial, sans-serif" font-weight="500" font-size="14" filter="url(#neonGlow)">
|
||||
<animate attributeName="fill"
|
||||
values="#FF2EC4;#8B5CF6;#0BC5EA;#FF2EC4"
|
||||
dur="8s"
|
||||
repeatCount="indefinite"/>
|
||||
Crawl4AI
|
||||
</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.5 KiB |
21
docs/assets/powered-by-light.svg
Normal file
21
docs/assets/powered-by-light.svg
Normal file
@@ -0,0 +1,21 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
|
||||
<g>
|
||||
<defs>
|
||||
<pattern id="halftoneLight" width="4" height="4" patternUnits="userSpaceOnUse">
|
||||
<circle cx="2" cy="2" r="1" fill="#111" opacity="0.1"/>
|
||||
</pattern>
|
||||
</defs>
|
||||
<!-- Dark border -->
|
||||
<rect width="120" height="35" rx="5" fill="#DDD"/>
|
||||
<!-- Light background -->
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="#fff"/>
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="url(#halftoneLight)"/>
|
||||
|
||||
<!-- Logo -->
|
||||
<path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#111" stroke-width="2"/>
|
||||
<path d="M18 17.5 L27 17.5" stroke="#111" stroke-width="2"/>
|
||||
<circle cx="22.5" cy="17.5" r="2" fill="#111"/>
|
||||
|
||||
<text x="40" y="23" fill="#111" font-family="Arial, sans-serif" font-weight="500" font-size="14">Crawl4AI</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 925 B |
28
docs/assets/powered-by-night.svg
Normal file
28
docs/assets/powered-by-night.svg
Normal file
@@ -0,0 +1,28 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="120" height="35" viewBox="0 0 120 35">
|
||||
<g>
|
||||
<defs>
|
||||
<pattern id="halftoneDark" width="4" height="4" patternUnits="userSpaceOnUse">
|
||||
<circle cx="2" cy="2" r="1" fill="#8B5CF6" opacity="0.1"/>
|
||||
</pattern>
|
||||
<filter id="neonGlow" x="-20%" y="-20%" width="140%" height="140%">
|
||||
<feGaussianBlur stdDeviation="1" result="blur"/>
|
||||
<feFlood flood-color="#8B5CF6" flood-opacity="0.2"/>
|
||||
<feComposite in2="blur" operator="in"/>
|
||||
<feMerge>
|
||||
<feMergeNode/>
|
||||
<feMergeNode in="SourceGraphic"/>
|
||||
</feMerge>
|
||||
</filter>
|
||||
</defs>
|
||||
<rect width="120" height="35" rx="5" fill="#0A0A0F"/>
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="#16161E"/>
|
||||
<rect x="2" y="2" width="116" height="31" rx="4" fill="url(#halftoneDark)"/>
|
||||
|
||||
<!-- Logo with neon glow -->
|
||||
<path d="M30 17.5 a7.5 7.5 0 1 1 -15 0 a7.5 7.5 0 1 1 15 0" fill="none" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)"/>
|
||||
<path d="M18 17.5 L27 17.5" stroke="#8B5CF6" stroke-width="2" filter="url(#neonGlow)"/>
|
||||
<circle cx="22.5" cy="17.5" r="2" fill="#8B5CF6"/>
|
||||
|
||||
<text x="40" y="23" fill="#fff" font-family="Arial, sans-serif" font-weight="500" font-size="14" filter="url(#neonGlow)">Crawl4AI</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
Reference in New Issue
Block a user