`).join('');
} else {
fieldsList.style.display = 'none';
}
const hint = document.getElementById('c4ai-hint');
if (this.mode === 'container') {
hint.textContent = 'Click on a container element (e.g., product card, article, etc.)';
} else if (this.fields.length === 0) {
hint.textContent = 'Click on fields inside the container to extract (title, price, etc.)';
} else {
hint.innerHTML = `Continue selecting fields or click Stop & Generate to finish.`;
}
}
updateStats() {
chrome.runtime.sendMessage({
action: 'updateStats',
stats: {
container: !!this.container,
fields: this.fields.length
}
});
}
removeAllHighlights() {
document.querySelectorAll('.c4ai-selected-container').forEach(el => {
el.classList.remove('c4ai-selected-container');
});
document.querySelectorAll('.c4ai-selected-field').forEach(el => {
el.classList.remove('c4ai-selected-field');
});
}
generateCode() {
const fieldDescriptions = this.fields.map(f =>
`- ${f.name} (example: "${f.value.substring(0, 50)}...")`
).join('\n');
return `#!/usr/bin/env python3
"""
Generated by Crawl4AI Chrome Extension
URL: ${window.location.href}
Generated: ${new Date().toISOString()}
"""
import asyncio
import json
from pathlib import Path
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
# HTML snippet of the selected container element
HTML_SNIPPET = """
${this.container.html}
"""
# Extraction query based on your field selections
EXTRACTION_QUERY = """
Create a JSON CSS extraction schema to extract the following fields:
${fieldDescriptions}
The schema should handle multiple ${this.container.tagName} elements on the page.
Each item should be extracted as a separate object in the results array.
"""
async def generate_schema():
"""Generate extraction schema using LLM"""
print("🔧 Generating extraction schema...")
try:
# Generate the schema using Crawl4AI's built-in LLM integration
schema = JsonCssExtractionStrategy.generate_schema(
html=HTML_SNIPPET,
query=EXTRACTION_QUERY,
)
# Save the schema for reuse
schema_path = Path('generated_schema.json')
with open(schema_path, 'w') as f:
json.dump(schema, f, indent=2)
print("✅ Schema generated successfully!")
print(f"📄 Schema saved to: {schema_path}")
print("\\nGenerated schema:")
print(json.dumps(schema, indent=2))
return schema
except Exception as e:
print(f"❌ Error generating schema: {e}")
return None
async def test_extraction(url: str = "${window.location.href}"):
"""Test the generated schema on the actual webpage"""
print("\\n🧪 Testing extraction on live webpage...")
# Load the generated schema
try:
with open('generated_schema.json', 'r') as f:
schema = json.load(f)
except FileNotFoundError:
print("❌ Schema file not found. Run generate_schema() first.")
return
# Configure browser
browser_config = BrowserConfig(
headless=True,
verbose=False
)
# Configure extraction
crawler_config = CrawlerRunConfig(
extraction_strategy=JsonCssExtractionStrategy(schema=schema)
)
async with AsyncWebCrawler(config=browser_config) as crawler:
result = await crawler.arun(
url=url,
config=crawler_config
)
if result.success and result.extracted_content:
data = json.loads(result.extracted_content)
print(f"\\n✅ Successfully extracted {len(data)} items!")
# Save results
with open('extracted_data.json', 'w') as f:
json.dump(data, f, indent=2)
# Show sample results
print("\\n📊 Sample results (first 2 items):")
for i, item in enumerate(data[:2], 1):
print(f"\\nItem {i}:")
for key, value in item.items():
print(f" {key}: {value}")
else:
print("❌ Extraction failed:", result.error_message)
if __name__ == "__main__":
# Step 1: Generate the schema from HTML snippet
asyncio.run(generate_schema())
# Step 2: Test extraction on the live webpage
# Uncomment the line below to test extraction:
# asyncio.run(test_extraction())
print("\\n🎯 Next steps:")
print("1. Review the generated schema in 'generated_schema.json'")
print("2. Uncomment the test_extraction() line to test on the live site")
print("3. Use the schema in your Crawl4AI projects!")
`;
return code;
}
showCodeModal(code) {
// Create modal
this.codeModal = document.createElement('div');
this.codeModal.className = 'c4ai-code-modal';
this.codeModal.innerHTML = `
Generated Python Code
${this.escapeHtml(code)}
`;
document.body.appendChild(this.codeModal);
// Add event listeners
document.getElementById('c4ai-close-modal').addEventListener('click', () => {
this.codeModal.remove();
this.codeModal = null;
// Don't stop the capture session
});
document.getElementById('c4ai-download-code').addEventListener('click', () => {
chrome.runtime.sendMessage({
action: 'downloadCode',
code: code,
filename: `crawl4ai_schema_${Date.now()}.py`
}, (response) => {
if (response && response.success) {
const btn = document.getElementById('c4ai-download-code');
const originalHTML = btn.innerHTML;
btn.innerHTML = '✓ Downloaded!';
setTimeout(() => {
btn.innerHTML = originalHTML;
}, 2000);
} else {
console.error('Download failed:', response?.error);
alert('Download failed. Please check your browser settings.');
}
});
});
document.getElementById('c4ai-copy-code').addEventListener('click', () => {
navigator.clipboard.writeText(code).then(() => {
const btn = document.getElementById('c4ai-copy-code');
btn.innerHTML = '✓ Copied!';
setTimeout(() => {
btn.innerHTML = '📋 Copy to Clipboard';
}, 2000);
});
});
// Apply syntax highlighting if possible
this.applySyntaxHighlighting();
}
escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
applySyntaxHighlighting() {
// Simple Python syntax highlighting - using a different approach
const codeElement = this.codeModal.querySelector('.language-python');
const code = codeElement.textContent;
// Split by lines to handle line-by-line
const lines = code.split('\n');
const highlightedLines = lines.map(line => {
let highlightedLine = this.escapeHtml(line);
// Skip if line is empty
if (!highlightedLine.trim()) return highlightedLine;
// Comments (lines starting with #)
if (highlightedLine.trim().startsWith('#')) {
return `${highlightedLine}`;
}
// Triple quoted strings
if (highlightedLine.includes('"""')) {
highlightedLine = highlightedLine.replace(/(""".*?""")/g, '$1');
}
// Regular strings - single and double quotes
highlightedLine = highlightedLine.replace(/(["'])([^"']*)\1/g, '$1$2$1');
// Keywords - only highlight if not inside a string
const keywords = ['import', 'from', 'async', 'def', 'await', 'try', 'except', 'with', 'as', 'for', 'if', 'else', 'elif', 'return', 'print', 'open', 'and', 'or', 'not', 'in', 'is', 'class', 'self', 'None', 'True', 'False', '__name__', '__main__'];
keywords.forEach(keyword => {
// Use word boundaries and lookahead/lookbehind to ensure we're not in a string
const regex = new RegExp(`\\b(${keyword})\\b(?![^<]*)`, 'g');
highlightedLine = highlightedLine.replace(regex, '$1');
});
// Functions (word followed by parenthesis)
highlightedLine = highlightedLine.replace(/\b([a-zA-Z_]\w*)\s*\(/g, '$1(');
return highlightedLine;
});
codeElement.innerHTML = highlightedLines.join('\n');
}
}
// Initialize
let schemaBuilder = null;
// Listen for messages from popup
chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
switch (message.action) {
case 'startSchemaCapture':
if (!schemaBuilder) {
schemaBuilder = new SchemaBuilder();
}
schemaBuilder.start();
sendResponse({ success: true });
break;
case 'stopCapture':
if (schemaBuilder) {
schemaBuilder.stop();
schemaBuilder = null;
}
sendResponse({ success: true });
break;
case 'generateCode':
if (schemaBuilder) {
const code = schemaBuilder.generateCode();
schemaBuilder.showCodeModal(code);
}
sendResponse({ success: true });
break;
}
return true;
});
}){kind=icon}