Deploy your extraction schemas to the cloud with just one click:
โจ Instant Deployment - Your schema live in seconds
๐ API Access - RESTful endpoints for your extractions
โฐ Scheduled Runs - Automate data collection
๐ Analytics Dashboard - Monitor your extractions
๐ Auto-scaling - Handle any volume seamlessly
Be the first to know when Crawl4AI Cloud launches!
`;
document.body.appendChild(modal);
// Add event listeners
document.getElementById('c4ai-close-cloud-modal').addEventListener('click', () => modal.remove());
document.getElementById('c4ai-join-waitlist').addEventListener('click', () => {
window.open('https://crawl4ai.com/join-waiting-list', '_blank');
modal.remove();
});
// Close on escape
const escHandler = (e) => {
if (e.key === 'Escape') {
modal.remove();
document.removeEventListener('keydown', escHandler);
}
};
document.addEventListener('keydown', escHandler);
}
generatePythonCode() {
if (!this.schema) {
this.generateSchema();
}
const schemaJson = JSON.stringify(this.schema, null, 2);
return `#!/usr/bin/env python3
"""
Generated by Crawl4AI Chrome Extension
URL: ${window.location.href}
Generated: ${new Date().toISOString()}
"""
import asyncio
import json
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
# The extraction schema generated from your selections
EXTRACTION_SCHEMA = ${schemaJson}
async def extract_data(url: str = "${window.location.href}"):
"""Extract data using the generated schema"""
# Configure browser (optional)
browser_config = BrowserConfig(
headless=True, # Set to False to see the browser
verbose=False
)
# Configure extraction strategy
extraction_strategy = JsonCssExtractionStrategy(schema=EXTRACTION_SCHEMA)
# Configure crawler
crawler_config = CrawlerRunConfig(
extraction_strategy=extraction_strategy,
# Add more options as needed:
# wait_for="css:.product", # Wait for specific elements
# js_code="window.scrollTo(0, document.body.scrollHeight);", # Execute JS
)
async with AsyncWebCrawler(config=browser_config) as crawler:
result = await crawler.arun(
url=url,
config=crawler_config
)
if result.success and result.extracted_content:
data = json.loads(result.extracted_content)
print(f"\\nโ Successfully extracted {len(data)} items!")
# Save results
with open('extracted_data.json', 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
# Show sample results
print("\\n๐ Sample results (first 2 items):")
for i, item in enumerate(data[:2], 1):
print(f"\\nItem {i}:")
for key, value in item.items():
print(f" {key}: {value}")
return data
else:
print("โ Extraction failed:", result.error_message)
return None
if __name__ == "__main__":
# Run the extraction
data = asyncio.run(extract_data())
print("\\n๐ฏ Next steps:")
print("1. Install Crawl4AI: pip install crawl4ai")
print("2. Modify the URL or add multiple URLs")
print("3. Customize crawler options as needed")
print("4. Check 'extracted_data.json' for full results")
`;
}
// Legacy code generation - kept for reference but no longer used
/*
generateCode() {
const fieldDescriptions = this.fields.map(f =>
`- ${f.name} (example: "${f.value.substring(0, 50)}...")`
).join('\n');
return `#!/usr/bin/env python3
"""
Generated by Crawl4AI Chrome Extension
URL: ${window.location.href}
Generated: ${new Date().toISOString()}
"""
import asyncio
import json
from pathlib import Path
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
# HTML snippet of the selected container element
HTML_SNIPPET = """
${this.container.html}
"""
# Extraction query based on your field selections
EXTRACTION_QUERY = """
Create a JSON CSS extraction schema to extract the following fields:
${fieldDescriptions}
The schema should handle multiple ${this.container.tagName} elements on the page.
Each item should be extracted as a separate object in the results array.
"""
async def generate_schema():
"""Generate extraction schema using LLM"""
print("๐ง Generating extraction schema...")
try:
# Generate the schema using Crawl4AI's built-in LLM integration
schema = JsonCssExtractionStrategy.generate_schema(
html=HTML_SNIPPET,
query=EXTRACTION_QUERY,
)
# Save the schema for reuse
schema_path = Path('generated_schema.json')
with open(schema_path, 'w') as f:
json.dump(schema, f, indent=2)
print("โ Schema generated successfully!")
print(f"๐ Schema saved to: {schema_path}")
print("\\nGenerated schema:")
print(json.dumps(schema, indent=2))
return schema
except Exception as e:
print(f"โ Error generating schema: {e}")
return None
async def test_extraction(url: str = "${window.location.href}"):
"""Test the generated schema on the actual webpage"""
print("\\n๐งช Testing extraction on live webpage...")
# Load the generated schema
try:
with open('generated_schema.json', 'r') as f:
schema = json.load(f)
except FileNotFoundError:
print("โ Schema file not found. Run generate_schema() first.")
return
# Configure browser
browser_config = BrowserConfig(
headless=True,
verbose=False
)
# Configure extraction
crawler_config = CrawlerRunConfig(
extraction_strategy=JsonCssExtractionStrategy(schema=schema)
)
async with AsyncWebCrawler(config=browser_config) as crawler:
result = await crawler.arun(
url=url,
config=crawler_config
)
if result.success and result.extracted_content:
data = json.loads(result.extracted_content)
print(f"\\nโ Successfully extracted {len(data)} items!")
# Save results
with open('extracted_data.json', 'w') as f:
json.dump(data, f, indent=2)
# Show sample results
print("\\n๐ Sample results (first 2 items):")
for i, item in enumerate(data[:2], 1):
print(f"\\nItem {i}:")
for key, value in item.items():
print(f" {key}: {value}")
else:
print("โ Extraction failed:", result.error_message)
if __name__ == "__main__":
# Step 1: Generate the schema from HTML snippet
asyncio.run(generate_schema())
# Step 2: Test extraction on the live webpage
# Uncomment the line below to test extraction:
# asyncio.run(test_extraction())
print("\\n๐ฏ Next steps:")
print("1. Review the generated schema in 'generated_schema.json'")
print("2. Uncomment the test_extraction() line to test on the live site")
print("3. Use the schema in your Crawl4AI projects!")
`;
return code;
}
*/
/* Legacy modal - no longer used
showCodeModal(code) {
// Create modal
this.codeModal = document.createElement('div');
this.codeModal.className = 'c4ai-code-modal';
this.codeModal.innerHTML = `
}){kind=icon}