ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Script Builder to Chrome Extension and reorganize LLM context files

Browse Source 
This commit introduces significant enhancements to the Crawl4AI ecosystem:

  Chrome Extension - Script Builder (Alpha):
  - Add recording functionality to capture user interactions (clicks, typing, scrolling)
  - Implement smart event grouping for cleaner script generation
  - Support export to both JavaScript and C4A script formats
  - Add timeline view for visualizing and editing recorded actions
  - Include wait commands (time-based and element-based)
  - Add saved flows functionality for reusing automation scripts
  - Update UI with consistent dark terminal theme (Dank Mono font, green/pink accents)
  - Release new extension versions: v1.1.0, v1.2.0, v1.2.1

  LLM Context Builder Improvements:
  - Reorganize context files from llmtxt/ to llm.txt/ with better structure
  - Separate diagram templates from text content (diagrams/ and txt/ subdirectories)
  - Add comprehensive context files for all major Crawl4AI components
  - Improve file naming convention for better discoverability

  Documentation Updates:
  - Update apps index page to match main documentation theme
  - Standardize color scheme: "Available" tags use primary color (#50ffff)
  - Change "Coming Soon" tags to dark gray for better visual hierarchy
  - Add interactive two-column layout for extension landing page
  - Include code examples for both Schema Builder and Script Builder features

  Technical Improvements:
  - Enhance event capture mechanism with better element selection
  - Add support for contenteditable elements and complex form interactions
  - Implement proper scroll event handling for both window and element scrolling
  - Add meta key support for keyboard shortcuts
  - Improve selector generation for more reliable element targeting

  The Script Builder is released as Alpha, acknowledging potential bugs while providing
  early access to this powerful automation recording feature.
This commit is contained in:
UncleCode
2025-06-08 22:02:12 +08:00
parent 926592649e
commit 40640badad
72 changed files with 28600 additions and 100986 deletions
Download Patch File Download Diff File Expand all files Collapse all files

496
docs/md_v2/apps/crawl4ai-assistant/assistant.css
View File

@@ -384,16 +384,20 @@ code {
.coming-features {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(350px, 1fr));
gap: 1.5rem;
margin-top: 2rem;
}
.coming-feature {
background: var(--bg-tertiary);
padding: 2rem;
padding: 1.5rem;
border-radius: 12px;
border: 1px solid var(--border-color);
transition: all 0.3s ease;
height: 100%;
display: flex;
flex-direction: column;
}
.coming-feature:hover {
@@ -429,16 +433,18 @@ code {
color: var(--text-secondary);
margin-bottom: 1rem;
line-height: 1.6;
flex-grow: 1;
}
.feature-preview {
background: var(--bg-secondary);
padding: 1rem;
padding: 0.75rem 1rem;
border-radius: 6px;
font-family: var(--font-code);
font-size: 0.875rem;
font-size: 0.8125rem;
color: var(--text-accent);
border: 1px solid var(--border-color);
margin-top: auto;
}
.stay-tuned {
@@ -533,6 +539,490 @@ code {
gap: 1rem;
}
.coming-soon-section h2 {
font-size: 1.5rem;
}
}
/* Interactive Tools Section */
.interactive-tools {
margin: 3rem 0;
}
.interactive-tools h2 {
font-size: 2rem;
margin-bottom: 2rem;
color: var(--text-primary);
}
.tools-container {
display: grid;
grid-template-columns: 300px 1fr;
gap: 2rem;
min-height: 400px;
}
/* Tool Selector Panel */
.tools-panel {
display: flex;
flex-direction: column;
gap: 1rem;
}
.tool-selector {
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
border-radius: 12px;
padding: 1.5rem;
cursor: pointer;
transition: all 0.3s ease;
display: flex;
align-items: center;
gap: 1rem;
position: relative;
}
.tool-selector:hover {
border-color: var(--primary-green);
transform: translateX(4px);
}
.tool-selector.active {
background: var(--bg-secondary);
border-color: var(--primary-green);
box-shadow: 0 0 20px rgba(15, 187, 170, 0.3);
}
.tool-icon {
font-size: 2.5rem;
flex-shrink: 0;
}
.tool-info h3 {
margin: 0;
font-size: 1.125rem;
color: var(--text-primary);
}
.tool-info p {
margin: 0.25rem 0 0;
font-size: 0.875rem;
color: var(--text-secondary);
}
.tool-status {
position: absolute;
top: 1rem;
right: 1rem;
font-size: 0.75rem;
padding: 0.25rem 0.75rem;
border-radius: 20px;
background: var(--primary-green);
color: var(--bg-dark);
font-weight: 600;
}
.tool-status.alpha {
background: var(--primary-pink);
}
/* Tool Details Panel */
.tool-details {
background: var(--bg-secondary);
border: 1px solid var(--border-color);
border-radius: 12px;
padding: 2rem;
position: relative;
overflow: hidden;
}
.tool-content {
display: none;
animation: fadeIn 0.4s ease;
}
.tool-content.active {
display: block;
}
@keyframes fadeIn {
from {
opacity: 0;
transform: translateY(10px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
.tool-header {
margin-bottom: 2rem;
}
.tool-header h3 {
font-size: 1.75rem;
margin: 0;
color: var(--text-primary);
}
.tool-tagline {
color: var(--text-secondary);
font-size: 1rem;
margin-top: 0.5rem;
display: block;
}
/* Tool Steps */
.tool-steps {
display: flex;
flex-direction: column;
gap: 1.5rem;
margin-bottom: 2rem;
}
.step-item {
display: flex;
gap: 1.5rem;
align-items: flex-start;
}
.step-number {
background: var(--primary-green);
color: var(--bg-dark);
width: 40px;
height: 40px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-weight: bold;
flex-shrink: 0;
}
.step-content h4 {
margin: 0 0 0.5rem;
color: var(--text-primary);
}
.step-content p {
margin: 0 0 0.5rem;
color: var(--text-secondary);
font-size: 0.875rem;
}
.step-visual {
display: flex;
align-items: center;
gap: 0.5rem;
font-size: 0.875rem;
color: var(--text-secondary);
}
.highlight-green {
color: var(--primary-green);
font-size: 1.25rem;
}
.highlight-pink {
color: var(--primary-pink);
font-size: 1.25rem;
}
.highlight-accent {
color: var(--primary-green);
font-size: 1.25rem;
}
.recording-dot {
color: #ff3c74;
font-size: 1.25rem;
animation: pulse 1.5s ease-in-out infinite;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.5; }
}
.action-icon {
font-size: 1.25rem;
margin: 0 0.25rem;
}
/* Tool Features */
.tool-features {
display: flex;
flex-wrap: wrap;
gap: 0.75rem;
margin-top: 2rem;
}
.feature-tag {
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
padding: 0.5rem 1rem;
border-radius: 20px;
font-size: 0.875rem;
color: var(--text-secondary);
}
.feature-tag.alpha-tag {
border-color: var(--primary-pink);
color: var(--primary-pink);
}
/* Code Showcase Section */
.code-showcase {
margin: 3rem 0;
}
.code-showcase h2 {
font-size: 2rem;
margin-bottom: 2rem;
color: var(--text-primary);
}
/* Code Tabs */
.code-tabs {
display: flex;
gap: 1rem;
margin-bottom: 2rem;
}
.code-tab {
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
padding: 0.75rem 1.5rem;
border-radius: 8px;
font-size: 1rem;
color: var(--text-secondary);
cursor: pointer;
transition: all 0.3s ease;
font-family: var(--font-primary);
}
.code-tab:hover {
border-color: var(--primary-green);
color: var(--text-primary);
}
.code-tab.active {
background: var(--primary-green);
color: var(--bg-dark);
border-color: var(--primary-green);
font-weight: 600;
}
/* Code Examples */
.code-examples {
position: relative;
min-height: 500px;
}
.code-example {
position: absolute;
width: 100%;
opacity: 0;
visibility: hidden;
transition: opacity 0.4s ease, visibility 0.4s ease;
}
.code-example.active {
opacity: 1;
visibility: visible;
position: relative;
}
/* Copy Button */
.copy-button {
position: absolute;
right: 1rem;
top: 50%;
transform: translateY(-50%);
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
color: var(--text-secondary);
padding: 0.25rem 0.75rem;
border-radius: 4px;
font-size: 0.75rem;
cursor: pointer;
transition: all 0.2s ease;
font-family: var(--font-primary);
}
.copy-button:hover {
background: var(--primary-green);
color: var(--bg-dark);
border-color: var(--primary-green);
}
.copy-button.copied {
background: var(--primary-green);
color: var(--bg-dark);
}
/* Responsive Updates */
@media (max-width: 768px) {
.tools-container {
grid-template-columns: 1fr;
}
.tools-panel {
flex-direction: row;
overflow-x: auto;
padding-bottom: 0.5rem;
}
.tool-selector {
min-width: 250px;
}
.code-tabs {
flex-wrap: wrap;
}
}
/* Script Builder Section */
.script-builder-section {
margin: 4rem 0;
}
.script-builder-section h2 {
font-size: 2rem;
margin-bottom: 2rem;
color: var(--text-primary);
}
.script-builder-section h2 span {
color: var(--primary-pink);
font-size: 0.875rem;
font-weight: normal;
margin-left: 0.5rem;
}
.script-features-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1.5rem;
margin-top: 2rem;
}
.script-feature {
background: var(--bg-tertiary);
padding: 1.5rem;
border-radius: 8px;
border: 1px solid var(--border-color);
text-align: center;
transition: all 0.2s ease;
}
.script-feature:hover {
border-color: var(--primary-green);
transform: translateY(-2px);
box-shadow: 0 4px 16px rgba(15, 187, 170, 0.2);
}
.script-feature .feature-icon {
font-size: 2.5rem;
margin-bottom: 1rem;
display: block;
}
.script-feature h4 {
font-size: 1.125rem;
margin-bottom: 0.5rem;
color: var(--text-primary);
}
.script-feature p {
font-size: 0.875rem;
color: var(--text-secondary);
}
.script-workflow {
margin-top: 2rem;
}
.workflow-step {
display: flex;
gap: 1.5rem;
margin-bottom: 2rem;
align-items: flex-start;
}
.workflow-step .step-number {
background: var(--primary-pink);
color: var(--bg-dark);
width: 40px;
height: 40px;
border-radius: 50%;
display: flex;
align-items: center;
justify-content: center;
font-weight: bold;
flex-shrink: 0;
}
.workflow-step .step-content h4 {
margin-bottom: 0.5rem;
color: var(--text-primary);
}
.workflow-step .step-content p {
color: var(--text-secondary);
}
.action-types {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
gap: 1rem;
margin-top: 1.5rem;
}
.action-type {
background: var(--bg-tertiary);
padding: 1rem;
border-radius: 6px;
font-size: 0.875rem;
color: var(--text-secondary);
border: 1px solid var(--border-color);
font-family: var(--font-code);
}
.action-type code {
color: var(--primary-green);
font-weight: 600;
margin-right: 0.5rem;
}
.alpha-note {
background: rgba(243, 128, 245, 0.1);
border: 1px solid var(--primary-pink);
border-radius: 8px;
padding: 1.5rem;
margin-top: 2rem;
color: var(--text-primary);
}
.alpha-note strong {
color: var(--primary-pink);
}
@media (max-width: 768px) {
.script-features-grid {
grid-template-columns: 1fr;
}
.workflow-step {
flex-direction: column;
gap: 1rem;
}
.action-types {
grid-template-columns: 1fr;
}
.coming-soon-section h2 {
font-size: 1.5rem;
}

2
docs/md_v2/apps/crawl4ai-assistant/background/service-worker.js
View File

@@ -2,7 +2,7 @@
// Handle messages from content script
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
if (message.action === 'downloadCode') {
if (message.action === 'downloadCode' || message.action === 'downloadScript') {
try {
// Create a data URL for the Python code
const dataUrl = 'data:text/plain;charset=utf-8,' + encodeURIComponent(message.code);

2654
docs/md_v2/apps/crawl4ai-assistant/content/content.js
View File

File diff suppressed because it is too large Load Diff

1157
docs/md_v2/apps/crawl4ai-assistant/content/overlay.css
View File

File diff suppressed because it is too large Load Diff

BIN
docs/md_v2/apps/crawl4ai-assistant/crawl4ai-assistant-v1.1.0.zip Normal file
View File

Binary file not shown.

BIN
docs/md_v2/apps/crawl4ai-assistant/crawl4ai-assistant-v1.2.0.zip Normal file
View File

Binary file not shown.

BIN
docs/md_v2/apps/crawl4ai-assistant/crawl4ai-assistant-v1.2.1.zip Normal file
View File

Binary file not shown.

354
docs/md_v2/apps/crawl4ai-assistant/index.html
View File

@@ -43,23 +43,23 @@
<span class="terminal-title">About Crawl4AI Assistant</span>
</div>
<div class="terminal-content">
<p>Transform any website into structured data with just a few clicks! The Crawl4AI Assistant Chrome Extension lets you visually select elements on any webpage and automatically generates Python code for web scraping.</p>
<p>Transform any website into structured data with just a few clicks! The Crawl4AI Assistant Chrome Extension provides two powerful tools for web scraping and automation.</p>
<div class="features-grid">
<div class="feature-card">
<span class="feature-icon">🎯</span>
<h3>Visual Selection</h3>
<p>Click on any element to select it - no CSS selectors needed</p>
<h3>Schema Builder</h3>
<p>Click to select elements and build extraction schemas visually</p>
</div>
<div class="feature-card">
<span class="feature-icon">📊</span>
<h3>Schema Builder</h3>
<p>Build extraction schemas by clicking on container and field elements</p>
<span class="feature-icon">🔴</span>
<h3>Script Builder <span style="color: #f380f5; font-size: 0.75rem;">(Alpha)</span></h3>
<p>Record browser actions to create automation scripts</p>
</div>
<div class="feature-card">
<span class="feature-icon">🐍</span>
<h3>Python Code</h3>
<p>Get production-ready Crawl4AI code with LLM extraction</p>
<p>Get production-ready Crawl4AI code instantly</p>
</div>
<div class="feature-card">
<span class="feature-icon">🎨</span>
@@ -85,9 +85,9 @@
<div class="step-content">
<h4>Download the Extension</h4>
<p>Get the latest release from GitHub or use the button below</p>
<a href="crawl4ai-assistant-v1.0.1.zip" class="download-button" download>
<a href="crawl4ai-assistant-v1.2.1.zip" class="download-button" download>
<span class="button-icon">↓</span>
Download Extension (v1.0.1)
Download Extension (v1.2.1)
</a>
</div>
</div>
@@ -110,67 +110,155 @@
</div>
</section>
<!-- Usage Guide -->
<section class="usage-section">
<h2>How to Use</h2>
<div class="terminal-window">
<div class="terminal-header">
<span class="terminal-title">Step-by-Step Guide</span>
<!-- Interactive Tools Section -->
<section class="interactive-tools">
<h2>Explore Our Tools</h2>
<div class="tools-container">
<!-- Left Panel - Tool Selector -->
<div class="tools-panel">
<div class="tool-selector active" data-tool="schema-builder">
<div class="tool-icon">📊</div>
<div class="tool-info">
<h3>Schema Builder</h3>
<p>Visual data extraction</p>
</div>
<div class="tool-status">Available</div>
</div>
<div class="tool-selector" data-tool="script-builder">
<div class="tool-icon">🔴</div>
<div class="tool-info">
<h3>Script Builder</h3>
<p>Browser automation</p>
</div>
<div class="tool-status alpha">Alpha</div>
</div>
</div>
<div class="terminal-content">
<div class="usage-flow">
<div class="usage-step">
<div class="usage-header">
<span class="usage-icon">1️⃣</span>
<h4>Start Schema Builder</h4>
</div>
<p>Click the extension icon and select "Schema Builder" to begin</p>
<!-- Right Panel - Tool Details -->
<div class="tool-details">
<!-- Schema Builder Details -->
<div class="tool-content active" id="schema-builder">
<div class="tool-header">
<h3>📊 Schema Builder</h3>
<span class="tool-tagline">Click to extract data visually</span>
</div>
<div class="usage-step">
<div class="usage-header">
<span class="usage-icon">2️⃣</span>
<h4>Select Container</h4>
<div class="tool-steps">
<div class="step-item">
<div class="step-number">1</div>
<div class="step-content">
<h4>Select Container</h4>
<p>Click on any repeating element like product cards or articles</p>
<div class="step-visual">
<span class="highlight-green">■</span> Elements highlighted in green
</div>
</div>
</div>
<p>Click on a container element (e.g., product card, article, listing)</p>
<div class="code-snippet">
<span class="comment"># Container will be highlighted in green</span>
<div class="step-item">
<div class="step-number">2</div>
<div class="step-content">
<h4>Mark Fields</h4>
<p>Click on data fields inside the container</p>
<div class="step-visual">
<span class="highlight-pink">■</span> Fields highlighted in pink
</div>
</div>
</div>
<div class="step-item">
<div class="step-number">3</div>
<div class="step-content">
<h4>Generate & Extract</h4>
<p>Get your CSS selectors and Python code instantly</p>
<div class="step-visual">
<span class="highlight-accent">⚡</span> Ready to use code
</div>
</div>
</div>
</div>
<div class="usage-step">
<div class="usage-header">
<span class="usage-icon">3️⃣</span>
<h4>Select Fields</h4>
<div class="tool-features">
<div class="feature-tag">No CSS knowledge needed</div>
<div class="feature-tag">Smart selector generation</div>
<div class="feature-tag">LLM-ready schemas</div>
</div>
</div>
<!-- Script Builder Details -->
<div class="tool-content" id="script-builder">
<div class="tool-header">
<h3>🔴 Script Builder</h3>
<span class="tool-tagline">Record actions, generate automation</span>
</div>
<div class="tool-steps">
<div class="step-item">
<div class="step-number">1</div>
<div class="step-content">
<h4>Hit Record</h4>
<p>Start capturing your browser interactions</p>
<div class="step-visual">
<span class="recording-dot">●</span> Recording indicator
</div>
</div>
</div>
<p>Click on individual fields inside the container and name them</p>
<div class="code-snippet">
<span class="comment"># Fields will be highlighted in pink</span>
<span class="comment"># Examples: title, price, description, image</span>
<div class="step-item">
<div class="step-number">2</div>
<div class="step-content">
<h4>Interact Naturally</h4>
<p>Click, type, scroll - everything is captured</p>
<div class="step-visual">
<span class="action-icon">🖱️</span> <span class="action-icon">⌨️</span> <span class="action-icon">📜</span>
</div>
</div>
</div>
<div class="step-item">
<div class="step-number">3</div>
<div class="step-content">
<h4>Export Script</h4>
<p>Get JavaScript for Crawl4AI's js_code parameter</p>
<div class="step-visual">
<span class="highlight-accent">📝</span> Automation ready
</div>
</div>
</div>
</div>
<div class="usage-step">
<div class="usage-header">
<span class="usage-icon">4️⃣</span>
<h4>Generate Code</h4>
</div>
<p>Click "Stop & Generate" to create your Python extraction code</p>
<div class="tool-features">
<div class="feature-tag">Smart action grouping</div>
<div class="feature-tag">Wait detection</div>
<div class="feature-tag">Keyboard shortcuts</div>
<div class="feature-tag alpha-tag">Alpha version</div>
</div>
</div>
</div>
</div>
</section>
<!-- Generated Code Example -->
<section class="code-section">
<h2>Generated Code Example</h2>
<div class="terminal-window">
<div class="terminal-header">
<span class="terminal-title">example_extraction.py</span>
</div>
<div class="terminal-content">
<pre><code><span class="keyword">import</span> asyncio
<!-- Interactive Code Examples -->
<section class="code-showcase">
<h2>See the Generated Code</h2>
<div class="code-tabs">
<button class="code-tab active" data-example="schema">📊 Schema Builder</button>
<button class="code-tab" data-example="script">🔴 Script Builder</button>
</div>
<div class="code-examples">
<!-- Schema Builder Code -->
<div class="code-example active" id="code-schema">
<div class="terminal-window">
<div class="terminal-header">
<span class="terminal-title">schema_extraction.py</span>
<button class="copy-button" data-code="schema">Copy</button>
</div>
<div class="terminal-content">
<pre><code><span class="keyword">import</span> asyncio
<span class="keyword">import</span> json
<span class="keyword">from</span> crawl4ai <span class="keyword">import</span> AsyncWebCrawler, CrawlerRunConfig
<span class="keyword">from</span> crawl4ai.extraction_strategy <span class="keyword">import</span> JsonCssExtractionStrategy
@@ -191,51 +279,94 @@
<span class="string">"selector"</span>: <span class="string">"span.price"</span>,
<span class="string">"type"</span>: <span class="string">"text"</span>
},
{
<span class="string">"name"</span>: <span class="string">"description"</span>,
<span class="string">"selector"</span>: <span class="string">"p.description"</span>,
<span class="string">"type"</span>: <span class="string">"text"</span>
},
{
<span class="string">"name"</span>: <span class="string">"image"</span>,
<span class="string">"selector"</span>: <span class="string">"img.product-image"</span>,
<span class="string">"selector"</span>: <span class="string">"img.product-img"</span>,
<span class="string">"type"</span>: <span class="string">"attribute"</span>,
<span class="string">"attribute"</span>: <span class="string">"src"</span>
}
]
}
<span class="comment"># Create extraction strategy</span>
extraction_strategy = JsonCssExtractionStrategy(schema, verbose=<span class="keyword">True</span>)
<span class="comment"># Configure the crawler</span>
config = CrawlerRunConfig(
extraction_strategy=extraction_strategy
extraction_strategy=JsonCssExtractionStrategy(schema)
)
<span class="keyword">async</span> <span class="keyword">with</span> AsyncWebCrawler() <span class="keyword">as</span> crawler:
result = <span class="keyword">await</span> crawler.arun(
url=<span class="string">"https://example.com/products"</span>,
config=config
)
<span class="comment"># Parse the extracted data</span>
products = json.loads(result.extracted_content)
<span class="keyword">print</span>(<span class="string">f"Extracted {len(products)} products"</span>)
<span class="comment"># Display first product</span>
<span class="keyword">if</span> products:
<span class="keyword">print</span>(json.dumps(products[0], indent=2))
<span class="keyword">return</span> products
<span class="keyword">return</span> json.loads(result.extracted_content)
<span class="comment"># Run the extraction</span>
<span class="keyword">if</span> __name__ == <span class="string">"__main__"</span>:
asyncio.run(extract_products())</code></pre>
asyncio.run(extract_products())</code></pre>
</div>
</div>
</div>
<!-- Script Builder Code -->
<div class="code-example" id="code-script">
<div class="terminal-window">
<div class="terminal-header">
<span class="terminal-title">automation_script.py</span>
<button class="copy-button" data-code="script">Copy</button>
</div>
<div class="terminal-content">
<pre><code><span class="keyword">import</span> asyncio
<span class="keyword">from</span> crawl4ai <span class="keyword">import</span> AsyncWebCrawler, CrawlerRunConfig
<span class="comment"># JavaScript generated from your recorded actions</span>
js_script = <span class="string">"""
// Search for products
document.querySelector('button.search-toggle').click();
await new Promise(r => setTimeout(r, 500));
// Type search query
const searchInput = document.querySelector('input#search');
searchInput.value = 'wireless headphones';
searchInput.dispatchEvent(new Event('input', {bubbles: true}));
// Submit search
searchInput.dispatchEvent(new KeyboardEvent('keydown', {
key: 'Enter', keyCode: 13, bubbles: true
}));
// Wait for results
await new Promise(r => setTimeout(r, 2000));
// Click first product
document.querySelector('.product-item:first-child').click();
// Wait for product page
await new Promise(r => setTimeout(r, 1000));
// Add to cart
document.querySelector('button.add-to-cart').click();
"""</span>
<span class="keyword">async</span> <span class="keyword">def</span> <span class="function">automate_shopping</span>():
config = CrawlerRunConfig(
js_code=js_script,
wait_for=<span class="string">"css:.cart-confirmation"</span>,
screenshot=<span class="keyword">True</span>
)
<span class="keyword">async</span> <span class="keyword">with</span> AsyncWebCrawler() <span class="keyword">as</span> crawler:
result = <span class="keyword">await</span> crawler.arun(
url=<span class="string">"https://shop.example.com"</span>,
config=config
)
<span class="keyword">print</span>(<span class="string">f"✓ Automation complete: {result.url}"</span>)
<span class="keyword">return</span> result
asyncio.run(automate_shopping())</code></pre>
</div>
</div>
</div>
</div>
</section>
<!-- Coming Soon Section -->
<section class="coming-soon-section">
<h2>Coming Soon: Even More Power</h2>
@@ -279,17 +410,6 @@
<code>🤖 Auto-detect fields • Smart naming • Pattern recognition</code>
</div>
</div>
<div class="coming-feature">
<div class="feature-header">
<span class="feature-badge">Script</span>
<h3>C4A Script Builder</h3>
</div>
<p>Visual automation script builder for complex interactions - fill forms, click buttons, handle pagination, all without writing code.</p>
<div class="feature-preview">
<code>🎯 Visual automation • Record & replay • Export as C4A script</code>
</div>
</div>
</div>
<div class="stay-tuned">
@@ -324,5 +444,61 @@
</footer>
</div>
</div>
<script>
// Tool Selector Interaction
document.querySelectorAll('.tool-selector').forEach(selector => {
selector.addEventListener('click', function() {
// Remove active class from all selectors
document.querySelectorAll('.tool-selector').forEach(s => s.classList.remove('active'));
document.querySelectorAll('.tool-content').forEach(c => c.classList.remove('active'));
// Add active class to clicked selector
this.classList.add('active');
// Show corresponding content
const toolId = this.getAttribute('data-tool');
document.getElementById(toolId).classList.add('active');
});
});
// Code Tab Interaction
document.querySelectorAll('.code-tab').forEach(tab => {
tab.addEventListener('click', function() {
// Remove active class from all tabs
document.querySelectorAll('.code-tab').forEach(t => t.classList.remove('active'));
document.querySelectorAll('.code-example').forEach(e => e.classList.remove('active'));
// Add active class to clicked tab
this.classList.add('active');
// Show corresponding code
const exampleId = this.getAttribute('data-example');
document.getElementById('code-' + exampleId).classList.add('active');
});
});
// Copy Button Functionality
document.querySelectorAll('.copy-button').forEach(button => {
button.addEventListener('click', async function() {
const codeType = this.getAttribute('data-code');
const codeElement = document.getElementById('code-' + codeType).querySelector('pre code');
const codeText = codeElement.textContent;
try {
await navigator.clipboard.writeText(codeText);
this.textContent = 'Copied!';
this.classList.add('copied');
setTimeout(() => {
this.textContent = 'Copy';
this.classList.remove('copied');
}, 2000);
} catch (err) {
console.error('Failed to copy code:', err);
}
});
});
</script>
</body>
</html>

4
docs/md_v2/apps/crawl4ai-assistant/manifest.json
View File

@@ -1,8 +1,8 @@
{
"manifest_version": 3,
"name": "Crawl4AI Assistant",
"version": "1.0.1",
"description": "Visual schema and script builder for Crawl4AI - Build extraction schemas by clicking on elements",
"version": "1.2.1",
"description": "Visual schema and script builder for Crawl4AI - Build extraction schemas and automation scripts by clicking and recording actions",
"permissions": [
"activeTab",
"storage",

6
docs/md_v2/apps/crawl4ai-assistant/popup/popup.html
View File

@@ -30,11 +30,11 @@
</div>
</button>
<button id="script-mode" class="mode-button script" disabled>
<button id="script-mode" class="mode-button script">
<div class="icon">🎯</div>
<div class="mode-info">
<h3>Script Builder</h3>
<p>Coming soon - Build automation scripts</p>
<h3>Script Builder <span style="color: #ff3c74; font-size: 10px;">(Alpha)</span></h3>
<p>Record actions to build automation scripts</p>
</div>
</button>
</div>

17
docs/md_v2/apps/crawl4ai-assistant/popup/popup.js
View File

@@ -18,6 +18,10 @@ document.addEventListener('DOMContentLoaded', () => {
startSchemaCapture();
});
document.getElementById('script-mode').addEventListener('click', () => {
startScriptCapture();
});
// Session actions
document.getElementById('generate-code').addEventListener('click', () => {
generateCode();
@@ -62,6 +66,19 @@ function startSchemaCapture() {
});
}
function startScriptCapture() {
chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => {
chrome.tabs.sendMessage(tabs[0].id, {
action: 'startScriptCapture'
}, (response) => {
if (response && response.success) {
// Close the popup to let user interact with the page
window.close();
}
});
});
}
function showActiveSession(stats) {
document.querySelector('.mode-selector').style.display = 'none';
document.getElementById('active-session').classList.remove('hidden');

58
docs/md_v2/apps/index.md
View File

@@ -11,8 +11,8 @@ Welcome to the Crawl4AI Apps Hub - your gateway to interactive tools and demos t
}
.app-card {
background: var(--md-code-bg-color);
border: 1px solid var(--md-default-fg-color--lightest);
background: #3f3f44;
border: 1px solid #3f3f44;
border-radius: 8px;
padding: 1.5rem;
transition: all 0.3s ease;
@@ -23,7 +23,7 @@ Welcome to the Crawl4AI Apps Hub - your gateway to interactive tools and demos t
.app-card:hover {
transform: translateY(-4px);
box-shadow: 0 8px 16px rgba(0, 0, 0, 0.3);
border-color: var(--md-primary-fg-color);
border-color: #50ffff;
}
.app-card h3 {
@@ -31,36 +31,38 @@ Welcome to the Crawl4AI Apps Hub - your gateway to interactive tools and demos t
display: flex;
align-items: center;
gap: 0.5rem;
color: #e8e9ed;
}
.app-status {
display: inline-block;
padding: 0.2rem 0.6rem;
border-radius: 4px;
padding: 0.25rem 0.75rem;
border-radius: 20px;
font-size: 0.7rem;
font-weight: bold;
font-weight: 600;
text-transform: uppercase;
margin-bottom: 1rem;
}
.status-available {
background: #22c55e;
color: #000;
background: #50ffff;
color: #070708;
}
.status-beta {
background: #f59e0b;
color: #000;
color: #070708;
}
.status-coming-soon {
background: var(--md-default-fg-color--lightest);
color: var(--md-default-bg-color);
background: #2a2a2a;
color: #888;
}
.app-description {
margin: 1rem 0;
line-height: 1.6;
color: #a3abba;
}
.app-features {
@@ -73,13 +75,15 @@ Welcome to the Crawl4AI Apps Hub - your gateway to interactive tools and demos t
padding-left: 1.5rem;
position: relative;
margin-bottom: 0.5rem;
color: #d5cec0;
font-size: 0.9rem;
}
.app-features li:before {
content: "✓";
content: "▸";
position: absolute;
left: 0;
color: var(--md-primary-fg-color);
color: #50ffff;
font-weight: bold;
}
@@ -89,35 +93,49 @@ Welcome to the Crawl4AI Apps Hub - your gateway to interactive tools and demos t
.app-btn {
display: inline-block;
padding: 0.8rem 1.5rem;
background: var(--md-primary-fg-color);
color: var(--md-primary-bg-color);
padding: 0.75rem 1.5rem;
background: #50ffff;
color: #070708;
text-decoration: none;
border-radius: 6px;
font-weight: bold;
font-weight: 600;
transition: all 0.2s ease;
font-family: dm, Monaco, monospace;
}
.app-btn:hover {
background: var(--md-primary-fg-color--dark);
background: #09b5a5;
transform: scale(1.05);
color: #070708;
}
.app-btn.disabled {
background: var(--md-default-fg-color--lightest);
background: #2a2a2a;
color: #666;
cursor: not-allowed;
transform: none;
}
.app-btn.disabled:hover {
background: #2a2a2a;
transform: none;
}
.intro-section {
background: var(--md-code-bg-color);
background: #3f3f44;
border-radius: 8px;
padding: 2rem;
margin-bottom: 3rem;
border: 1px solid #3f3f44;
}
.intro-section h2 {
margin-top: 0;
color: #50ffff;
}
.intro-section p {
color: #d5cec0;
}
</style>

37
docs/md_v2/apps/llmtxt/index.html
View File

@@ -71,33 +71,6 @@
</section>
<section class="builder">
<div class="special-contexts">
<h2>Quick Presets</h2>
<div class="preset-options">
<label class="preset-option">
<input type="radio" name="preset" value="vibe" id="preset-vibe">
<div class="preset-card">
<h3>🎯 Vibe Coding</h3>
<p>Curated context for general AI prompting - perfect for exploring capabilities</p>
</div>
</label>
<label class="preset-option">
<input type="radio" name="preset" value="all" id="preset-all">
<div class="preset-card">
<h3>📚 Complete Library</h3>
<p>Comprehensive context including all components and perspectives</p>
</div>
</label>
<label class="preset-option">
<input type="radio" name="preset" value="custom" id="preset-custom" checked>
<div class="preset-card">
<h3>🔧 Custom Selection</h3>
<p>Choose specific components and context types</p>
</div>
</label>
</div>
</div>
<div class="component-selector" id="component-selector">
<h2>Select Components & Context Types</h2>
<div class="select-all-controls">
@@ -111,9 +84,9 @@
<tr>
<th width="50"></th>
<th>Component</th>
<th class="clickable-header" data-type="memory">Memory</th>
<th class="clickable-header" data-type="reasoning">Reasoning</th>
<th class="clickable-header" data-type="examples">Examples</th>
<th class="clickable-header" data-type="memory">Memory<br><span class="header-subtitle">Full Content</span></th>
<th class="clickable-header" data-type="reasoning">Reasoning<br><span class="header-subtitle">Diagrams</span></th>
<th class="clickable-header" data-type="examples">Examples<br><span class="header-subtitle">Code</span></th>
</tr>
</thead>
<tbody id="components-tbody">
@@ -124,6 +97,10 @@
</div>
<div class="action-area">
<div class="token-summary" id="token-summary">
<span class="token-label">Estimated Tokens:</span>
<span class="token-count" id="total-tokens">0</span>
</div>
<button class="download-btn" id="download-btn">
<span class="icon">⬇</span> Generate & Download Context
</button>

56
docs/md_v2/apps/llmtxt/llmtxt.css
View File

@@ -363,6 +363,15 @@ body {
font-weight: bold;
}
.header-subtitle {
font-size: 10px;
color: var(--tertiary-color);
text-transform: none;
font-weight: normal;
display: block;
margin-top: 2px;
}
.component-selection-table th.clickable-header {
cursor: pointer;
user-select: none;
@@ -374,6 +383,16 @@ body {
color: var(--background-color);
}
.component-selection-table th.clickable-header[data-type="examples"] {
cursor: default;
opacity: 0.5;
}
.component-selection-table th.clickable-header[data-type="examples"]:hover {
background-color: var(--hover-bg);
color: var(--primary-color);
}
.component-selection-table th:nth-child(3),
.component-selection-table th:nth-child(4),
.component-selection-table th:nth-child(5) {
@@ -400,12 +419,25 @@ body {
font-weight: bold;
}
/* Token display in table cells */
.token-info {
display: block;
font-size: 11px;
color: var(--tertiary-color);
margin-top: 2px;
}
.component-selection-table input[type="checkbox"] {
cursor: pointer;
width: 16px;
height: 16px;
}
.component-selection-table input[type="checkbox"]:disabled {
cursor: not-allowed;
opacity: 0.3;
}
/* Disabled row state */
.component-selection-table tr.disabled td:not(:first-child) {
opacity: 0.5;
@@ -418,6 +450,30 @@ body {
margin: 40px 0;
}
/* Token Summary */
.token-summary {
margin-bottom: 20px;
font-size: 16px;
}
.token-label {
color: var(--tertiary-color);
margin-right: 10px;
}
.token-count {
color: var(--primary-color);
font-weight: bold;
font-size: 20px;
}
.token-count::after {
content: " est.";
font-size: 12px;
color: var(--tertiary-color);
margin-left: 4px;
}
.download-btn {
background-color: var(--primary-dimmed);
color: var(--background-color);

385
docs/md_v2/apps/llmtxt/llmtxt.js
View File

@@ -1,48 +1,61 @@
// Crawl4AI LLM Context Builder JavaScript
// Component definitions
// Component definitions - order matters
const components = [
{
id: 'all',
name: 'All Components',
description: 'All components with all context types',
special: true
id: 'installation',
name: 'Installation',
description: 'Setup and installation options'
},
{
id: 'core',
name: 'Core Functionality',
description: 'Basic crawling and scraping features'
id: 'simple_crawling',
name: 'Simple Crawling',
description: 'Basic web crawling operations'
},
{
id: 'config_objects',
name: 'Configuration Objects',
description: 'Browser and crawler configuration'
},
{
id: 'deep_crawling',
name: 'Deep Crawling',
description: 'Multi-page crawling strategies'
},
{
id: 'deployment',
name: 'Deployment',
description: 'Installation and Docker setup'
},
{
id: 'extraction',
name: 'Data Extraction',
description: 'Structured data extraction strategies'
},
{
id: 'markdown',
name: 'Markdown Generation',
description: 'Content-to-markdown conversion'
id: 'multi_urls_crawling',
name: 'Multi URLs Crawling',
description: 'Crawling multiple URLs efficiently'
},
{
id: 'vibe',
name: 'Vibe Coding',
description: 'General-purpose AI context',
special: false
id: 'deep_crawling',
name: 'Deep Crawling',
description: 'Multi-page crawling strategies'
},
{
id: 'docker',
name: 'Docker',
description: 'Docker deployment and configuration'
},
{
id: 'cli',
name: 'CLI',
description: 'Command-line interface usage'
},
{
id: 'http_based_crawler_strategy',
name: 'HTTP-based Crawler',
description: 'HTTP crawler strategy implementation'
},
{
id: 'url_seeder',
name: 'URL Seeder',
description: 'URL seeding and discovery'
},
{
id: 'deep_crawl_advanced_filters_scorers',
name: 'Advanced Filters & Scorers',
description: 'Deep crawl filtering and scoring'
}
];
@@ -51,45 +64,47 @@ const contextTypes = ['memory', 'reasoning', 'examples'];
// State management
const state = {
preset: 'custom',
selectedComponents: new Set(),
selectedContextTypes: new Map()
selectedContextTypes: new Map(),
tokenCounts: new Map() // Store token counts for each file
};
// Initialize the application
document.addEventListener('DOMContentLoaded', () => {
setupPresetHandlers();
renderComponents();
renderReferenceTable();
setupActionHandlers();
setupColumnHeaderHandlers();
// Initialize only core component as selected with all context types
state.selectedComponents.add('core');
state.selectedContextTypes.set('core', new Set(contextTypes));
// Initialize first component as selected with available context types
const firstComponent = components[0];
state.selectedComponents.add(firstComponent.id);
state.selectedContextTypes.set(firstComponent.id, new Set(['memory', 'reasoning']));
updateComponentUI();
});
// Setup preset radio button handlers
function setupPresetHandlers() {
const presetRadios = document.querySelectorAll('input[name="preset"]');
presetRadios.forEach(radio => {
radio.addEventListener('change', (e) => {
state.preset = e.target.value;
updatePresetSelection();
});
});
// Helper function to count tokens (words × 2.5)
function estimateTokens(text) {
if (!text) return 0;
const words = text.trim().split(/\s+/).length;
return Math.round(words * 2.5);
}
// Update UI based on preset selection
function updatePresetSelection() {
const componentSelector = document.getElementById('component-selector');
// Update total token count display
function updateTotalTokenCount() {
let totalTokens = 0;
if (state.preset === 'custom') {
componentSelector.style.display = 'block';
} else {
componentSelector.style.display = 'none';
}
state.selectedComponents.forEach(compId => {
const types = state.selectedContextTypes.get(compId);
if (types) {
types.forEach(type => {
const key = `${compId}-${type}`;
totalTokens += state.tokenCounts.get(key) || 0;
});
}
});
document.getElementById('total-tokens').textContent = totalTokens.toLocaleString();
}
// Render component selection table
@@ -97,10 +112,13 @@ function renderComponents() {
const tbody = document.getElementById('components-tbody');
tbody.innerHTML = '';
components.filter(c => !c.special).forEach(component => {
components.forEach(component => {
const row = createComponentRow(component);
tbody.appendChild(row);
});
// Fetch token counts for all files
fetchAllTokenCounts();
}
// Create a component table row
@@ -124,9 +142,17 @@ function createComponentRow(component) {
// Context type cells
contextTypes.forEach(type => {
const td = document.createElement('td');
const key = `${component.id}-${type}`;
const tokenCount = state.tokenCounts.get(key) || 0;
const isDisabled = type === 'examples' ? 'disabled' : '';
td.innerHTML = `
<input type="checkbox" id="check-${component.id}-${type}"
data-component="${component.id}" data-type="${type}">
data-component="${component.id}" data-type="${type}"
${isDisabled}>
<span class="token-info" id="tokens-${component.id}-${type}">
${tokenCount > 0 ? `${tokenCount.toLocaleString()} tokens` : ''}
</span>
`;
tr.appendChild(td);
});
@@ -140,9 +166,11 @@ function createComponentRow(component) {
// Add event listeners for context type checkboxes
contextTypes.forEach(type => {
const typeCheckbox = tr.querySelector(`#check-${component.id}-${type}`);
typeCheckbox.addEventListener('change', (e) => {
handleContextTypeToggle(component.id, type, e.target.checked);
});
if (!typeCheckbox.disabled) {
typeCheckbox.addEventListener('change', (e) => {
handleContextTypeToggle(component.id, type, e.target.checked);
});
}
});
return tr;
@@ -152,12 +180,12 @@ function createComponentRow(component) {
function handleComponentToggle(componentId, checked) {
if (checked) {
state.selectedComponents.add(componentId);
// Select all context types when component is selected
// Select only available context types when component is selected
if (!state.selectedContextTypes.has(componentId)) {
state.selectedContextTypes.set(componentId, new Set(contextTypes));
state.selectedContextTypes.set(componentId, new Set(['memory', 'reasoning']));
} else {
// If component was already partially selected, select all
state.selectedContextTypes.set(componentId, new Set(contextTypes));
// If component was already partially selected, select all available
state.selectedContextTypes.set(componentId, new Set(['memory', 'reasoning']));
}
} else {
state.selectedComponents.delete(componentId);
@@ -195,8 +223,10 @@ function handleContextTypeToggle(componentId, type, checked) {
// Update UI to reflect current state
function updateComponentUI() {
components.filter(c => !c.special).forEach(component => {
components.forEach(component => {
const row = document.getElementById(`component-${component.id}`);
if (!row) return;
const mainCheckbox = row.querySelector(`#check-${component.id}`);
const hasSelection = state.selectedComponents.has(component.id);
const selectedTypes = state.selectedContextTypes.get(component.id) || new Set();
@@ -213,15 +243,93 @@ function updateComponentUI() {
typeCheckbox.checked = selectedTypes.has(type);
});
});
updateTotalTokenCount();
}
// Fetch token counts for all files
async function fetchAllTokenCounts() {
const promises = [];
components.forEach(component => {
contextTypes.forEach(type => {
promises.push(fetchTokenCount(component.id, type));
});
});
await Promise.all(promises);
updateComponentUI();
renderReferenceTable(); // Update reference table with token counts
}
// Fetch token count for a specific file
async function fetchTokenCount(componentId, type) {
const key = `${componentId}-${type}`;
try {
const fileName = getFileName(componentId, type);
const baseUrl = getBaseUrl(type);
const response = await fetch(baseUrl + fileName);
if (response.ok) {
const content = await response.text();
const tokens = estimateTokens(content);
state.tokenCounts.set(key, tokens);
// Update UI
const tokenSpan = document.getElementById(`tokens-${componentId}-${type}`);
if (tokenSpan) {
tokenSpan.textContent = `${tokens.toLocaleString()} tokens`;
}
} else if (type === 'examples') {
// Examples might not exist yet
state.tokenCounts.set(key, 0);
const tokenSpan = document.getElementById(`tokens-${componentId}-${type}`);
if (tokenSpan) {
tokenSpan.textContent = '';
}
}
} catch (error) {
console.warn(`Failed to fetch token count for ${componentId}-${type}`);
if (type === 'examples') {
const tokenSpan = document.getElementById(`tokens-${componentId}-${type}`);
if (tokenSpan) {
tokenSpan.textContent = '';
}
}
}
}
// Get file name based on component and type
function getFileName(componentId, type) {
// For new structure, all files are just [componentId].txt
return `${componentId}.txt`;
}
// Get base URL based on context type
function getBaseUrl(type) {
// For MkDocs, we need to go up to the root level
const basePrefix = window.location.pathname.includes('/apps/') ? '../../' : '/';
switch(type) {
case 'memory':
return basePrefix + 'assets/llm.txt/txt/';
case 'reasoning':
return basePrefix + 'assets/llm.txt/diagrams/';
case 'examples':
return basePrefix + 'assets/llm.txt/examples/'; // Will return 404 for now
default:
return basePrefix + 'assets/llm.txt/txt/';
}
}
// Setup action button handlers
function setupActionHandlers() {
// Select/Deselect all buttons
document.getElementById('select-all').addEventListener('click', () => {
components.filter(c => !c.special).forEach(comp => {
components.forEach(comp => {
state.selectedComponents.add(comp.id);
state.selectedContextTypes.set(comp.id, new Set(contextTypes));
state.selectedContextTypes.set(comp.id, new Set(['memory', 'reasoning']));
});
updateComponentUI();
});
@@ -249,9 +357,12 @@ function setupColumnHeaderHandlers() {
// Toggle all checkboxes in a column
function toggleColumnSelection(type) {
// Don't toggle examples column
if (type === 'examples') return;
// Check if all are currently selected
let allSelected = true;
components.filter(c => !c.special).forEach(comp => {
components.forEach(comp => {
const types = state.selectedContextTypes.get(comp.id);
if (!types || !types.has(type)) {
allSelected = false;
@@ -259,7 +370,7 @@ function toggleColumnSelection(type) {
});
// Toggle all
components.filter(c => !c.special).forEach(comp => {
components.forEach(comp => {
if (!state.selectedContextTypes.has(comp.id)) {
state.selectedContextTypes.set(comp.id, new Set());
}
@@ -314,46 +425,50 @@ async function handleDownload() {
function getSelectedFiles() {
const files = [];
if (state.preset === 'vibe') {
files.push('crawl4ai_vibe.llm.full.md');
} else if (state.preset === 'all') {
// Use the dedicated aggregated files for all components
files.push('crawl4ai_all_memory_content.llm.md');
files.push('crawl4ai_all_reasoning_content.llm.md');
files.push('crawl4ai_all_examples_content.llm.md');
} else {
// Custom selection
state.selectedComponents.forEach(compId => {
const types = state.selectedContextTypes.get(compId);
if (types) {
types.forEach(type => {
files.push(`crawl4ai_${compId}_${type}_content.llm.md`);
// Build list of selected files with their context info
state.selectedComponents.forEach(compId => {
const types = state.selectedContextTypes.get(compId);
if (types) {
types.forEach(type => {
files.push({
componentId: compId,
type: type,
fileName: getFileName(compId, type),
baseUrl: getBaseUrl(type)
});
}
});
}
});
}
});
return files;
}
// Fetch multiple files
async function fetchFiles(fileNames) {
// Use /assets/llmtxt/ path with .txt extension
const baseUrl = '/assets/llmtxt/';
const promises = fileNames.map(async (fileName) => {
// Convert .md to .txt for fetching
const txtFileName = fileName.replace('.md', '.txt');
async function fetchFiles(fileInfos) {
const promises = fileInfos.map(async (fileInfo) => {
try {
const response = await fetch(baseUrl + txtFileName);
const response = await fetch(fileInfo.baseUrl + fileInfo.fileName);
if (!response.ok) {
console.warn(`Failed to fetch ${txtFileName} from ${baseUrl + txtFileName}`);
return { fileName, content: `<!-- Failed to load ${fileName} -->` };
if (fileInfo.type === 'examples') {
return {
fileInfo,
content: `<!-- Examples for ${fileInfo.componentId} coming soon -->\n\nExamples are currently being developed for this component.`
};
}
console.warn(`Failed to fetch ${fileInfo.fileName} from ${fileInfo.baseUrl + fileInfo.fileName}`);
return { fileInfo, content: `<!-- Failed to load ${fileInfo.fileName} -->` };
}
const content = await response.text();
return { fileName, content };
return { fileInfo, content };
} catch (error) {
console.warn(`Error fetching ${txtFileName} from ${baseUrl + txtFileName}:`, error);
return { fileName, content: `<!-- Error loading ${fileName} -->` };
if (fileInfo.type === 'examples') {
return {
fileInfo,
content: `<!-- Examples for ${fileInfo.componentId} coming soon -->\n\nExamples are currently being developed for this component.`
};
}
console.warn(`Error fetching ${fileInfo.fileName}:`, error);
return { fileInfo, content: `<!-- Error loading ${fileInfo.fileName} -->` };
}
});
@@ -362,20 +477,31 @@ async function fetchFiles(fileNames) {
// Combine file contents with headers
function combineContents(fileContents) {
// Calculate total tokens
let totalTokens = 0;
fileContents.forEach(({ content }) => {
totalTokens += estimateTokens(content);
});
const header = `# Crawl4AI Custom LLM Context
Generated on: ${new Date().toISOString()}
Total files: ${fileContents.length}
Estimated tokens: ${totalTokens.toLocaleString()}
---
`;
const sections = fileContents.map(({ fileName, content }) => {
const componentName = extractComponentName(fileName);
const contextType = extractContextType(fileName);
const sections = fileContents.map(({ fileInfo, content }) => {
const component = components.find(c => c.id === fileInfo.componentId);
const componentName = component ? component.name : fileInfo.componentId;
const contextType = getContextTypeName(fileInfo.type);
const tokens = estimateTokens(content);
return `## ${componentName} - ${contextType}
Source: ${fileName}
Component ID: ${fileInfo.componentId}
Context Type: ${fileInfo.type}
Estimated tokens: ${tokens.toLocaleString()}
${content}
@@ -387,25 +513,14 @@ ${content}
return header + sections.join('\n');
}
// Extract component name from filename
function extractComponentName(fileName) {
// Pattern: crawl4ai_{component}_{type}_content.llm.md
const match = fileName.match(/crawl4ai_(.+?)_(memory|reasoning|examples|llm\.full)/);
if (match) {
const compId = match[1];
const component = components.find(c => c.id === compId);
return component ? component.name : compId.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
// Get display name for context type
function getContextTypeName(type) {
switch(type) {
case 'memory': return 'Full Content';
case 'reasoning': return 'Diagrams & Workflows';
case 'examples': return 'Code Examples';
default: return type;
}
return 'Unknown Component';
}
// Extract context type from filename
function extractContextType(fileName) {
if (fileName.includes('_memory_')) return 'Memory';
if (fileName.includes('_reasoning_')) return 'Reasoning';
if (fileName.includes('_examples_')) return 'Examples';
if (fileName.includes('.llm.full')) return 'Complete Context';
return 'Context';
}
// Download file to user's computer
@@ -426,33 +541,35 @@ function renderReferenceTable() {
const tbody = document.getElementById('reference-table-body');
tbody.innerHTML = '';
// Since vibe is no longer special, just show all components the same way
// Get base path for links
const basePrefix = window.location.pathname.includes('/apps/') ? '../../' : '/';
components.forEach(component => {
const row = document.createElement('tr');
const memoryTokens = state.tokenCounts.get(`${component.id}-memory`) || 0;
const reasoningTokens = state.tokenCounts.get(`${component.id}-reasoning`) || 0;
const examplesTokens = state.tokenCounts.get(`${component.id}-examples`) || 0;
row.innerHTML = `
<td><strong>${component.name}</strong></td>
<td><a href="/assets/llmtxt/crawl4ai_${component.id}_memory_content.llm.txt" class="file-link" target="_blank">Memory</a></td>
<td><a href="/assets/llmtxt/crawl4ai_${component.id}_reasoning_content.llm.txt" class="file-link" target="_blank">Reasoning</a></td>
<td><a href="/assets/llmtxt/crawl4ai_${component.id}_examples_content.llm.txt" class="file-link" target="_blank">Examples</a></td>
<td><a href="/assets/llmtxt/crawl4ai_${component.id}.llm.full.txt" class="file-link" target="_blank">Full</a></td>
<td>
<a href="${basePrefix}assets/llm.txt/txt/${component.id}.txt" class="file-link" target="_blank">Memory</a>
${memoryTokens > 0 ? `<span class="file-size">${memoryTokens.toLocaleString()} tokens</span>` : ''}
</td>
<td>
<a href="${basePrefix}assets/llm.txt/diagrams/${component.id}.txt" class="file-link" target="_blank">Reasoning</a>
${reasoningTokens > 0 ? `<span class="file-size">${reasoningTokens.toLocaleString()} tokens</span>` : ''}
</td>
<td>
${examplesTokens > 0
? `<a href="${basePrefix}assets/llm.txt/examples/${component.id}.txt" class="file-link" target="_blank">Examples</a>
<span class="file-size">${examplesTokens.toLocaleString()} tokens</span>`
: '-'
}
</td>
<td>-</td>
`;
tbody.appendChild(row);
});
}
// Check if examples file exists (all components have examples)
function hasExamplesFile(componentId) {
// All components have examples files
return true;
}
// Check if full file exists (all components have full files)
function hasFullFile(componentId) {
// All components have full files
return true;
}
// Utility function to capitalize first letter
function capitalizeFirst(str) {
return str.charAt(0).toUpperCase() + str.slice(1);
}