feat(extraction): add LLM-powered schema generation utility
Adds new static method generate_schema() to JsonElementExtractionStrategy classes that can automatically generate extraction schemas using LLM (OpenAI or Ollama). This provides a convenient way to bootstrap extraction schemas while maintaining the performance benefits of selector-based extraction. Key changes: - Added generate_schema() static method to base extraction strategy - Added support for both CSS and XPath schema generation - Updated documentation with examples and best practices - Added new prompt templates for schema generation
This commit is contained in:
UncleCode
@@ -1,3 +1,9 @@
|
||||
### [Added] 2025-01-20
|
||||
- New LLM-powered schema generation utility for JsonElementExtractionStrategy
|
||||
- Support for automatic CSS and XPath schema generation using OpenAI or Ollama
|
||||
- Comprehensive documentation and examples for schema generation
|
||||
- New prompt templates optimized for HTML schema analysis
|
||||
|
||||
# Changelog
|
||||
|
||||
All notable changes to Crawl4AI will be documented in this file.
|
||||
|
||||
@@ -5,7 +5,7 @@ import json
|
||||
import time
|
||||
import os
|
||||
|
||||
from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION
|
||||
from .prompts import PROMPT_EXTRACT_BLOCKS, PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION, PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION, JSON_SCHEMA_BUILDER_XPATH
|
||||
from .config import (
|
||||
DEFAULT_PROVIDER, PROVIDER_MODELS,
|
||||
CHUNK_TOKEN_THRESHOLD,
|
||||
@@ -1060,6 +1060,72 @@ class JsonElementExtractionStrategy(ExtractionStrategy):
|
||||
"""Get attribute value from element"""
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def generate_schema(
|
||||
html: str,
|
||||
schema_type: str = "CSS", # or XPATH
|
||||
query: str = None,
|
||||
provider: str = "gpt-4o",
|
||||
api_token: str = os.getenv("OPENAI_API_KEY"),
|
||||
**kwargs
|
||||
) -> dict:
|
||||
"""
|
||||
Generate extraction schema from HTML content and optional query.
|
||||
|
||||
Args:
|
||||
html (str): The HTML content to analyze
|
||||
query (str, optional): Natural language description of what data to extract
|
||||
provider (str): LLM provider to use
|
||||
api_token (str): API token for LLM provider
|
||||
prompt (str, optional): Custom prompt template to use
|
||||
**kwargs: Additional args passed to perform_completion_with_backoff
|
||||
|
||||
Returns:
|
||||
dict: Generated schema following the JsonElementExtractionStrategy format
|
||||
"""
|
||||
from .prompts import JSON_SCHEMA_BUILDER
|
||||
from .utils import perform_completion_with_backoff
|
||||
|
||||
# Use default or custom prompt
|
||||
prompt_template = JSON_SCHEMA_BUILDER if schema_type == "CSS" else JSON_SCHEMA_BUILDER_XPATH
|
||||
|
||||
# Build the prompt
|
||||
system_message = {
|
||||
"role": "system",
|
||||
"content": "You are a specialized HTML schema generator. Analyze the HTML and generate a JSON schema that follows the specified format. Only output valid JSON schema, nothing else."
|
||||
}
|
||||
|
||||
user_message = {
|
||||
"role": "user",
|
||||
"content": f"""
|
||||
Instructions:
|
||||
{prompt_template}
|
||||
|
||||
HTML to analyze:
|
||||
```html
|
||||
{html}
|
||||
```
|
||||
|
||||
{"Extract the following data: " + query if query else "Please analyze the HTML structure and create the most appropriate schema for data extraction."}
|
||||
"""
|
||||
}
|
||||
|
||||
try:
|
||||
# Call LLM with backoff handling
|
||||
response = perform_completion_with_backoff(
|
||||
provider=provider,
|
||||
prompt_with_variables="\n\n".join([system_message["content"], user_message["content"]]),
|
||||
json_response = True,
|
||||
api_token=api_token,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
# Extract and return schema
|
||||
return json.loads(response.choices[0].message.content)
|
||||
|
||||
except Exception as e:
|
||||
raise Exception(f"Failed to generate schema: {str(e)}")
|
||||
|
||||
|
||||
class JsonCssExtractionStrategy(JsonElementExtractionStrategy):
|
||||
"""
|
||||
@@ -1171,3 +1237,4 @@ class JsonXPathExtractionStrategy(JsonElementExtractionStrategy):
|
||||
|
||||
def _get_element_attribute(self, element, attribute: str):
|
||||
return element.get(attribute)
|
||||
|
||||
|
||||
@@ -256,4 +256,754 @@ Wrap your response in <content> tags. Use proper markdown throughout.
|
||||
[Your markdown content here]
|
||||
</content>
|
||||
|
||||
Begin filtering now."""
|
||||
Begin filtering now."""
|
||||
|
||||
JSON_SCHEMA_BUILDER= """
|
||||
# HTML Schema Generation Instructions
|
||||
You are a specialized model designed to analyze HTML patterns and generate extraction schemas. Your primary job is to create structured JSON schemas that can be used to extract data from HTML in a consistent and reliable way. When presented with HTML content, you must analyze its structure and generate a schema that captures all relevant data points.
|
||||
|
||||
## Your Core Responsibilities:
|
||||
1. Analyze HTML structure to identify repeating patterns and important data points
|
||||
2. Generate valid JSON schemas following the specified format
|
||||
3. Create appropriate selectors that will work reliably for data extraction
|
||||
4. Name fields meaningfully based on their content and purpose
|
||||
5. Handle both specific user requests and autonomous pattern detection
|
||||
|
||||
## Available Schema Types You Can Generate:
|
||||
|
||||
<schema_types>
|
||||
1. Basic Single-Level Schema
|
||||
- Use for simple, flat data structures
|
||||
- Example: Product cards, user profiles
|
||||
- Direct field extractions
|
||||
|
||||
2. Nested Object Schema
|
||||
- Use for hierarchical data
|
||||
- Example: Articles with author details
|
||||
- Contains objects within objects
|
||||
|
||||
3. List Schema
|
||||
- Use for repeating elements
|
||||
- Example: Comment sections, product lists
|
||||
- Handles arrays of similar items
|
||||
|
||||
4. Complex Nested Lists
|
||||
- Use for multi-level data
|
||||
- Example: Categories with subcategories
|
||||
- Multiple levels of nesting
|
||||
|
||||
5. Transformation Schema
|
||||
- Use for data requiring processing
|
||||
- Supports regex and text transformations
|
||||
- Special attribute handling
|
||||
</schema_types>
|
||||
|
||||
<schema_structure>
|
||||
Your output must always be a JSON object with this structure:
|
||||
{
|
||||
"name": "Descriptive name of the pattern",
|
||||
"baseSelector": "CSS selector for the repeating element",
|
||||
"fields": [
|
||||
{
|
||||
"name": "field_name",
|
||||
"selector": "CSS selector",
|
||||
"type": "text|attribute|nested|list|regex",
|
||||
"attribute": "attribute_name", // Optional
|
||||
"transform": "transformation_type", // Optional
|
||||
"pattern": "regex_pattern", // Optional
|
||||
"fields": [] // For nested/list types
|
||||
}
|
||||
]
|
||||
}
|
||||
</schema_structure>
|
||||
|
||||
<type_definitions>
|
||||
Available field types:
|
||||
- text: Direct text extraction
|
||||
- attribute: HTML attribute extraction
|
||||
- nested: Object containing other fields
|
||||
- list: Array of similar items
|
||||
- regex: Pattern-based extraction
|
||||
</type_definitions>
|
||||
|
||||
<behavior_rules>
|
||||
1. When given a specific query:
|
||||
- Focus on extracting requested data points
|
||||
- Use most specific selectors possible
|
||||
- Include all fields mentioned in the query
|
||||
|
||||
2. When no query is provided:
|
||||
- Identify main content areas
|
||||
- Extract all meaningful data points
|
||||
- Use semantic structure to determine importance
|
||||
- Include prices, dates, titles, and other common data types
|
||||
|
||||
3. Always:
|
||||
- Use reliable CSS selectors
|
||||
- Handle dynamic class names appropriately
|
||||
- Create descriptive field names
|
||||
- Follow consistent naming conventions
|
||||
</behavior_rules>
|
||||
|
||||
<examples>
|
||||
1. Basic Product Card Example:
|
||||
<html>
|
||||
<div class="product-card" data-cat-id="electronics" data-subcat-id="laptops">
|
||||
<h2 class="product-title">Gaming Laptop</h2>
|
||||
<span class="price">$999.99</span>
|
||||
<img src="laptop.jpg" alt="Gaming Laptop">
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Product Cards",
|
||||
"baseSelector": ".product-card",
|
||||
"baseFields": [
|
||||
{"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"},
|
||||
{"name": "data_subcat_id", "type": "attribute", "attribute": "data-subcat-id"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": ".product-title",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "price",
|
||||
"selector": ".price",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "image_url",
|
||||
"selector": "img",
|
||||
"type": "attribute",
|
||||
"attribute": "src"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
2. Article with Author Details Example:
|
||||
<html>
|
||||
<article>
|
||||
<h1>The Future of AI</h1>
|
||||
<div class="author-info">
|
||||
<span class="author-name">Dr. Smith</span>
|
||||
<img src="author.jpg" alt="Dr. Smith">
|
||||
</div>
|
||||
</article>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Article Details",
|
||||
"baseSelector": "article",
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": "h1",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "author",
|
||||
"type": "nested",
|
||||
"selector": ".author-info",
|
||||
"fields": [
|
||||
{
|
||||
"name": "name",
|
||||
"selector": ".author-name",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "avatar",
|
||||
"selector": "img",
|
||||
"type": "attribute",
|
||||
"attribute": "src"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
3. Comments Section Example:
|
||||
<html>
|
||||
<div class="comments-container">
|
||||
<div class="comment" data-user-id="123">
|
||||
<div class="user-name">John123</div>
|
||||
<p class="comment-text">Great article!</p>
|
||||
</div>
|
||||
<div class="comment" data-user-id="456">
|
||||
<div class="user-name">Alice456</div>
|
||||
<p class="comment-text">Thanks for sharing.</p>
|
||||
</div>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Comment Section",
|
||||
"baseSelector": ".comments-container",
|
||||
"baseFields": [
|
||||
{"name": "data_user_id", "type": "attribute", "attribute": "data-user-id"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "comments",
|
||||
"type": "list",
|
||||
"selector": ".comment",
|
||||
"fields": [
|
||||
{
|
||||
"name": "user",
|
||||
"selector": ".user-name",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"selector": ".comment-text",
|
||||
"type": "text"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
4. E-commerce Categories Example:
|
||||
<html>
|
||||
<div class="category-section" data-category="electronics">
|
||||
<h2>Electronics</h2>
|
||||
<div class="subcategory">
|
||||
<h3>Laptops</h3>
|
||||
<div class="product">
|
||||
<span class="product-name">MacBook Pro</span>
|
||||
<span class="price">$1299</span>
|
||||
</div>
|
||||
<div class="product">
|
||||
<span class="product-name">Dell XPS</span>
|
||||
<span class="price">$999</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "E-commerce Categories",
|
||||
"baseSelector": ".category-section",
|
||||
"baseFields": [
|
||||
{"name": "data_category", "type": "attribute", "attribute": "data-category"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "category_name",
|
||||
"selector": "h2",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "subcategories",
|
||||
"type": "nested_list",
|
||||
"selector": ".subcategory",
|
||||
"fields": [
|
||||
{
|
||||
"name": "name",
|
||||
"selector": "h3",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "products",
|
||||
"type": "list",
|
||||
"selector": ".product",
|
||||
"fields": [
|
||||
{
|
||||
"name": "name",
|
||||
"selector": ".product-name",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "price",
|
||||
"selector": ".price",
|
||||
"type": "text"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
5. Job Listings with Transformations Example:
|
||||
<html>
|
||||
<div class="job-post">
|
||||
<h3 class="job-title">Senior Developer</h3>
|
||||
<span class="salary-text">Salary: $120,000/year</span>
|
||||
<span class="location"> New York, NY </span>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Job Listings",
|
||||
"baseSelector": ".job-post",
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": ".job-title",
|
||||
"type": "text",
|
||||
"transform": "uppercase"
|
||||
},
|
||||
{
|
||||
"name": "salary",
|
||||
"selector": ".salary-text",
|
||||
"type": "regex",
|
||||
"pattern": "\\$([\\d,]+)"
|
||||
},
|
||||
{
|
||||
"name": "location",
|
||||
"selector": ".location",
|
||||
"type": "text",
|
||||
"transform": "strip"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
6. Skyscanner Place Card Example:
|
||||
<html>
|
||||
<div class="PlaceCard_descriptionContainer__M2NjN" data-testid="description-container">
|
||||
<div class="PlaceCard_nameContainer__ZjZmY" tabindex="0" role="link">
|
||||
<div class="PlaceCard_nameContent__ODUwZ">
|
||||
<span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY">Doha</span>
|
||||
</div>
|
||||
<span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY PlaceCard_subName__NTVkY">Qatar</span>
|
||||
</div>
|
||||
<span class="PlaceCard_advertLabel__YTM0N">Sunny days and the warmest welcome awaits</span>
|
||||
<a class="BpkLink_bpk-link__MmQwY PlaceCard_descriptionLink__NzYwN" href="/flights/del/doha/" data-testid="flights-link">
|
||||
<div class="PriceDescription_container__NjEzM">
|
||||
<span class="BpkText_bpk-text--heading-5__MTRjZ">₹17,559</span>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Skyscanner Place Cards",
|
||||
"baseSelector": "div[class^='PlaceCard_descriptionContainer__']",
|
||||
"baseFields": [
|
||||
{"name": "data_testid", "type": "attribute", "attribute": "data-testid"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "city_name",
|
||||
"selector": "div[class^='PlaceCard_nameContent__'] .BpkText_bpk-text--heading-4__",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "country_name",
|
||||
"selector": "span[class*='PlaceCard_subName__']",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"selector": "span[class*='PlaceCard_advertLabel__']",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "flight_price",
|
||||
"selector": "a[data-testid='flights-link'] .BpkText_bpk-text--heading-5__",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "flight_url",
|
||||
"selector": "a[data-testid='flights-link']",
|
||||
"type": "attribute",
|
||||
"attribute": "href"
|
||||
}
|
||||
]
|
||||
}
|
||||
</examples>
|
||||
|
||||
|
||||
<output_requirements>
|
||||
Your output must:
|
||||
1. Be valid JSON only
|
||||
2. Include no explanatory text
|
||||
3. Follow the exact schema structure provided
|
||||
4. Use appropriate field types
|
||||
5. Include all required fields
|
||||
6. Use valid CSS selectors
|
||||
</output_requirements>
|
||||
|
||||
"""
|
||||
|
||||
JSON_SCHEMA_BUILDER_XPATH = """
|
||||
# HTML Schema Generation Instructions
|
||||
You are a specialized model designed to analyze HTML patterns and generate extraction schemas. Your primary job is to create structured JSON schemas that can be used to extract data from HTML in a consistent and reliable way. When presented with HTML content, you must analyze its structure and generate a schema that captures all relevant data points.
|
||||
|
||||
## Your Core Responsibilities:
|
||||
1. Analyze HTML structure to identify repeating patterns and important data points
|
||||
2. Generate valid JSON schemas following the specified format
|
||||
3. Create appropriate XPath selectors that will work reliably for data extraction
|
||||
4. Name fields meaningfully based on their content and purpose
|
||||
5. Handle both specific user requests and autonomous pattern detection
|
||||
|
||||
## Available Schema Types You Can Generate:
|
||||
|
||||
<schema_types>
|
||||
1. Basic Single-Level Schema
|
||||
- Use for simple, flat data structures
|
||||
- Example: Product cards, user profiles
|
||||
- Direct field extractions
|
||||
|
||||
2. Nested Object Schema
|
||||
- Use for hierarchical data
|
||||
- Example: Articles with author details
|
||||
- Contains objects within objects
|
||||
|
||||
3. List Schema
|
||||
- Use for repeating elements
|
||||
- Example: Comment sections, product lists
|
||||
- Handles arrays of similar items
|
||||
|
||||
4. Complex Nested Lists
|
||||
- Use for multi-level data
|
||||
- Example: Categories with subcategories
|
||||
- Multiple levels of nesting
|
||||
|
||||
5. Transformation Schema
|
||||
- Use for data requiring processing
|
||||
- Supports regex and text transformations
|
||||
- Special attribute handling
|
||||
</schema_types>
|
||||
|
||||
<schema_structure>
|
||||
Your output must always be a JSON object with this structure:
|
||||
{
|
||||
"name": "Descriptive name of the pattern",
|
||||
"baseSelector": "XPath selector for the repeating element",
|
||||
"fields": [
|
||||
{
|
||||
"name": "field_name",
|
||||
"selector": "XPath selector",
|
||||
"type": "text|attribute|nested|list|regex",
|
||||
"attribute": "attribute_name", // Optional
|
||||
"transform": "transformation_type", // Optional
|
||||
"pattern": "regex_pattern", // Optional
|
||||
"fields": [] // For nested/list types
|
||||
}
|
||||
]
|
||||
}
|
||||
</schema_structure>
|
||||
|
||||
<type_definitions>
|
||||
Available field types:
|
||||
- text: Direct text extraction
|
||||
- attribute: HTML attribute extraction
|
||||
- nested: Object containing other fields
|
||||
- list: Array of similar items
|
||||
- regex: Pattern-based extraction
|
||||
</type_definitions>
|
||||
|
||||
<behavior_rules>
|
||||
1. When given a specific query:
|
||||
- Focus on extracting requested data points
|
||||
- Use most specific selectors possible
|
||||
- Include all fields mentioned in the query
|
||||
|
||||
2. When no query is provided:
|
||||
- Identify main content areas
|
||||
- Extract all meaningful data points
|
||||
- Use semantic structure to determine importance
|
||||
- Include prices, dates, titles, and other common data types
|
||||
|
||||
3. Always:
|
||||
- Use reliable XPath selectors
|
||||
- Handle dynamic element IDs appropriately
|
||||
- Create descriptive field names
|
||||
- Follow consistent naming conventions
|
||||
</behavior_rules>
|
||||
|
||||
<examples>
|
||||
1. Basic Product Card Example:
|
||||
<html>
|
||||
<div class="product-card" data-cat-id="electronics" data-subcat-id="laptops">
|
||||
<h2 class="product-title">Gaming Laptop</h2>
|
||||
<span class="price">$999.99</span>
|
||||
<img src="laptop.jpg" alt="Gaming Laptop">
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Product Cards",
|
||||
"baseSelector": "//div[@class='product-card']",
|
||||
"baseFields": [
|
||||
{"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"},
|
||||
{"name": "data_subcat_id", "type": "attribute", "attribute": "data-subcat-id"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": ".//h2[@class='product-title']",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "price",
|
||||
"selector": ".//span[@class='price']",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "image_url",
|
||||
"selector": ".//img",
|
||||
"type": "attribute",
|
||||
"attribute": "src"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
2. Article with Author Details Example:
|
||||
<html>
|
||||
<article>
|
||||
<h1>The Future of AI</h1>
|
||||
<div class="author-info">
|
||||
<span class="author-name">Dr. Smith</span>
|
||||
<img src="author.jpg" alt="Dr. Smith">
|
||||
</div>
|
||||
</article>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Article Details",
|
||||
"baseSelector": "//article",
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": ".//h1",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "author",
|
||||
"type": "nested",
|
||||
"selector": ".//div[@class='author-info']",
|
||||
"fields": [
|
||||
{
|
||||
"name": "name",
|
||||
"selector": ".//span[@class='author-name']",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "avatar",
|
||||
"selector": ".//img",
|
||||
"type": "attribute",
|
||||
"attribute": "src"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
3. Comments Section Example:
|
||||
<html>
|
||||
<div class="comments-container">
|
||||
<div class="comment" data-user-id="123">
|
||||
<div class="user-name">John123</div>
|
||||
<p class="comment-text">Great article!</p>
|
||||
</div>
|
||||
<div class="comment" data-user-id="456">
|
||||
<div class="user-name">Alice456</div>
|
||||
<p class="comment-text">Thanks for sharing.</p>
|
||||
</div>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Comment Section",
|
||||
"baseSelector": "//div[@class='comments-container']",
|
||||
"fields": [
|
||||
{
|
||||
"name": "comments",
|
||||
"type": "list",
|
||||
"selector": ".//div[@class='comment']",
|
||||
"baseFields": [
|
||||
{"name": "data_user_id", "type": "attribute", "attribute": "data-user-id"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "user",
|
||||
"selector": ".//div[@class='user-name']",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "content",
|
||||
"selector": ".//p[@class='comment-text']",
|
||||
"type": "text"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
4. E-commerce Categories Example:
|
||||
<html>
|
||||
<div class="category-section" data-category="electronics">
|
||||
<h2>Electronics</h2>
|
||||
<div class="subcategory">
|
||||
<h3>Laptops</h3>
|
||||
<div class="product">
|
||||
<span class="product-name">MacBook Pro</span>
|
||||
<span class="price">$1299</span>
|
||||
</div>
|
||||
<div class="product">
|
||||
<span class="product-name">Dell XPS</span>
|
||||
<span class="price">$999</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "E-commerce Categories",
|
||||
"baseSelector": "//div[@class='category-section']",
|
||||
"baseFields": [
|
||||
{"name": "data_category", "type": "attribute", "attribute": "data-category"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "category_name",
|
||||
"selector": ".//h2",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "subcategories",
|
||||
"type": "nested_list",
|
||||
"selector": ".//div[@class='subcategory']",
|
||||
"fields": [
|
||||
{
|
||||
"name": "name",
|
||||
"selector": ".//h3",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "products",
|
||||
"type": "list",
|
||||
"selector": ".//div[@class='product']",
|
||||
"fields": [
|
||||
{
|
||||
"name": "name",
|
||||
"selector": ".//span[@class='product-name']",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "price",
|
||||
"selector": ".//span[@class='price']",
|
||||
"type": "text"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
5. Job Listings with Transformations Example:
|
||||
<html>
|
||||
<div class="job-post">
|
||||
<h3 class="job-title">Senior Developer</h3>
|
||||
<span class="salary-text">Salary: $120,000/year</span>
|
||||
<span class="location"> New York, NY </span>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Job Listings",
|
||||
"baseSelector": "//div[@class='job-post']",
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": ".//h3[@class='job-title']",
|
||||
"type": "text",
|
||||
"transform": "uppercase"
|
||||
},
|
||||
{
|
||||
"name": "salary",
|
||||
"selector": ".//span[@class='salary-text']",
|
||||
"type": "regex",
|
||||
"pattern": "\\$([\\d,]+)"
|
||||
},
|
||||
{
|
||||
"name": "location",
|
||||
"selector": ".//span[@class='location']",
|
||||
"type": "text",
|
||||
"transform": "strip"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
6. Skyscanner Place Card Example:
|
||||
<html>
|
||||
<div class="PlaceCard_descriptionContainer__M2NjN" data-testid="description-container">
|
||||
<div class="PlaceCard_nameContainer__ZjZmY" tabindex="0" role="link">
|
||||
<div class="PlaceCard_nameContent__ODUwZ">
|
||||
<span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY">Doha</span>
|
||||
</div>
|
||||
<span class="BpkText_bpk-text__MjhhY BpkText_bpk-text--heading-4__Y2FlY PlaceCard_subName__NTVkY">Qatar</span>
|
||||
</div>
|
||||
<span class="PlaceCard_advertLabel__YTM0N">Sunny days and the warmest welcome awaits</span>
|
||||
<a class="BpkLink_bpk-link__MmQwY PlaceCard_descriptionLink__NzYwN" href="/flights/del/doha/" data-testid="flights-link">
|
||||
<div class="PriceDescription_container__NjEzM">
|
||||
<span class="BpkText_bpk-text--heading-5__MTRjZ">₹17,559</span>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</html>
|
||||
|
||||
Generated Schema:
|
||||
{
|
||||
"name": "Skyscanner Place Cards",
|
||||
"baseSelector": "//div[contains(@class, 'PlaceCard_descriptionContainer__')]",
|
||||
"baseFields": [
|
||||
{"name": "data_testid", "type": "attribute", "attribute": "data-testid"}
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "city_name",
|
||||
"selector": ".//div[contains(@class, 'PlaceCard_nameContent__')]//span[contains(@class, 'BpkText_bpk-text--heading-4__')]",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "country_name",
|
||||
"selector": ".//span[contains(@class, 'PlaceCard_subName__')]",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "description",
|
||||
"selector": ".//span[contains(@class, 'PlaceCard_advertLabel__')]",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "flight_price",
|
||||
"selector": ".//a[@data-testid='flights-link']//span[contains(@class, 'BpkText_bpk-text--heading-5__')]",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "flight_url",
|
||||
"selector": ".//a[@data-testid='flights-link']",
|
||||
"type": "attribute",
|
||||
"attribute": "href"
|
||||
}
|
||||
]
|
||||
}
|
||||
</examples>
|
||||
|
||||
<output_requirements>
|
||||
Your output must:
|
||||
1. Be valid JSON only
|
||||
2. Include no explanatory text
|
||||
3. Follow the exact schema structure provided
|
||||
4. Use appropriate field types
|
||||
5. Include all required fields
|
||||
6. Use valid XPath selectors
|
||||
</output_requirements>
|
||||
"""
|
||||
@@ -124,6 +124,36 @@ async with AsyncWebCrawler() as crawler:
|
||||
|
||||
Crawl4AI can also extract structured data (JSON) using CSS or XPath selectors. Below is a minimal CSS-based example:
|
||||
|
||||
> **New!** Crawl4AI now provides a powerful utility to automatically generate extraction schemas using LLM. This is a one-time cost that gives you a reusable schema for fast, LLM-free extractions:
|
||||
|
||||
```python
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
|
||||
# Generate a schema (one-time cost)
|
||||
html = "<div class='product'><h2>Gaming Laptop</h2><span class='price'>$999.99</span></div>"
|
||||
|
||||
# Using OpenAI (requires API token)
|
||||
schema = JsonCssExtractionStrategy.generate_schema(
|
||||
html,
|
||||
llm_provider="openai/gpt-4o", # Default provider
|
||||
api_token="your-openai-token" # Required for OpenAI
|
||||
)
|
||||
|
||||
# Or using Ollama (open source, no token needed)
|
||||
schema = JsonCssExtractionStrategy.generate_schema(
|
||||
html,
|
||||
llm_provider="ollama/llama3.3", # Open source alternative
|
||||
api_token=None # Not needed for Ollama
|
||||
)
|
||||
|
||||
# Use the schema for fast, repeated extractions
|
||||
strategy = JsonCssExtractionStrategy(schema)
|
||||
```
|
||||
|
||||
For a complete guide on schema generation and advanced usage, see [No-LLM Extraction Strategies](../extraction/no-llm-strategies.md).
|
||||
|
||||
Here's a basic extraction example:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
@@ -401,7 +401,92 @@ Then run with `JsonCssExtractionStrategy(schema)` to get an array of blog post o
|
||||
|
||||
---
|
||||
|
||||
## 8. Conclusion
|
||||
## 8. Schema Generation Utility
|
||||
|
||||
While manually crafting schemas is powerful and precise, Crawl4AI now offers a convenient utility to **automatically generate** extraction schemas using LLM. This is particularly useful when:
|
||||
|
||||
1. You're dealing with a new website structure and want a quick starting point
|
||||
2. You need to extract complex nested data structures
|
||||
3. You want to avoid the learning curve of CSS/XPath selector syntax
|
||||
|
||||
### Using the Schema Generator
|
||||
|
||||
The schema generator is available as a static method on both `JsonCssExtractionStrategy` and `JsonXPathExtractionStrategy`. You can choose between OpenAI's GPT-4 or the open-source Ollama for schema generation:
|
||||
|
||||
```python
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, JsonXPathExtractionStrategy
|
||||
|
||||
# Sample HTML with product information
|
||||
html = """
|
||||
<div class="product-card">
|
||||
<h2 class="title">Gaming Laptop</h2>
|
||||
<div class="price">$999.99</div>
|
||||
<div class="specs">
|
||||
<ul>
|
||||
<li>16GB RAM</li>
|
||||
<li>1TB SSD</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Option 1: Using OpenAI (requires API token)
|
||||
css_schema = JsonCssExtractionStrategy.generate_schema(
|
||||
html,
|
||||
schema_type="css", # This is the default
|
||||
llm_provider="openai/gpt-4o", # Default provider
|
||||
api_token="your-openai-token" # Required for OpenAI
|
||||
)
|
||||
|
||||
# Option 2: Using Ollama (open source, no token needed)
|
||||
xpath_schema = JsonXPathExtractionStrategy.generate_schema(
|
||||
html,
|
||||
schema_type="xpath",
|
||||
llm_provider="ollama/llama3.3", # Open source alternative
|
||||
api_token=None # Not needed for Ollama
|
||||
)
|
||||
|
||||
# Use the generated schema for fast, repeated extractions
|
||||
strategy = JsonCssExtractionStrategy(css_schema)
|
||||
```
|
||||
|
||||
### LLM Provider Options
|
||||
|
||||
1. **OpenAI GPT-4 (`openai/gpt4o`)**
|
||||
- Default provider
|
||||
- Requires an API token
|
||||
- Generally provides more accurate schemas
|
||||
- Set via environment variable: `OPENAI_API_KEY`
|
||||
|
||||
2. **Ollama (`ollama/llama3.3`)**
|
||||
- Open source alternative
|
||||
- No API token required
|
||||
- Self-hosted option
|
||||
- Good for development and testing
|
||||
|
||||
### Benefits of Schema Generation
|
||||
|
||||
1. **One-Time Cost**: While schema generation uses LLM, it's a one-time cost. The generated schema can be reused for unlimited extractions without further LLM calls.
|
||||
2. **Smart Pattern Recognition**: The LLM analyzes the HTML structure and identifies common patterns, often producing more robust selectors than manual attempts.
|
||||
3. **Automatic Nesting**: Complex nested structures are automatically detected and properly represented in the schema.
|
||||
4. **Learning Tool**: The generated schemas serve as excellent examples for learning how to write your own schemas.
|
||||
|
||||
### Best Practices
|
||||
|
||||
1. **Review Generated Schemas**: While the generator is smart, always review and test the generated schema before using it in production.
|
||||
2. **Provide Representative HTML**: The better your sample HTML represents the overall structure, the more accurate the generated schema will be.
|
||||
3. **Consider Both CSS and XPath**: Try both schema types and choose the one that works best for your specific case.
|
||||
4. **Cache Generated Schemas**: Since generation uses LLM, save successful schemas for reuse.
|
||||
5. **API Token Security**: Never hardcode API tokens. Use environment variables or secure configuration management.
|
||||
6. **Choose Provider Wisely**:
|
||||
- Use OpenAI for production-quality schemas
|
||||
- Use Ollama for development, testing, or when you need a self-hosted solution
|
||||
|
||||
That's it for **Extracting JSON (No LLM)**! You've seen how schema-based approaches (either CSS or XPath) can handle everything from simple lists to deeply nested product catalogs—instantly, with minimal overhead. Enjoy building robust scrapers that produce consistent, structured JSON for your data pipelines!
|
||||
|
||||
---
|
||||
|
||||
## 9. Conclusion
|
||||
|
||||
With **JsonCssExtractionStrategy** (or **JsonXPathExtractionStrategy**), you can build powerful, **LLM-free** pipelines that:
|
||||
|
||||
|
||||
111
tests/20241401/test_schema_builder.py
Normal file
111
tests/20241401/test_schema_builder.py
Normal file
@@ -0,0 +1,111 @@
|
||||
# https://claude.ai/chat/c4bbe93d-fb54-44ce-92af-76b4c8086c6b
|
||||
# https://claude.ai/chat/c24a768c-d8b2-478a-acc7-d76d42a308da
|
||||
import os, sys
|
||||
|
||||
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.append(parent_dir)
|
||||
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
|
||||
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, JsonXPathExtractionStrategy
|
||||
import json
|
||||
|
||||
# Test HTML - A complex job board with companies, departments, and positions
|
||||
test_html = """
|
||||
<div class="company-listings">
|
||||
<div class="company" data-company-id="123">
|
||||
<div class="company-header">
|
||||
<img class="company-logo" src="google.png" alt="Google">
|
||||
<h1 class="company-name">Google</h1>
|
||||
<div class="company-meta">
|
||||
<span class="company-size">10,000+ employees</span>
|
||||
<span class="company-industry">Technology</span>
|
||||
<a href="https://google.careers" class="careers-link">Careers Page</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="departments">
|
||||
<div class="department">
|
||||
<h2 class="department-name">Engineering</h2>
|
||||
<div class="positions">
|
||||
<div class="position-card" data-position-id="eng-1">
|
||||
<h3 class="position-title">Senior Software Engineer</h3>
|
||||
<span class="salary-range">$150,000 - $250,000</span>
|
||||
<div class="position-meta">
|
||||
<span class="location">Mountain View, CA</span>
|
||||
<span class="job-type">Full-time</span>
|
||||
<span class="experience">5+ years</span>
|
||||
</div>
|
||||
<div class="skills-required">
|
||||
<span class="skill">Python</span>
|
||||
<span class="skill">Kubernetes</span>
|
||||
<span class="skill">Machine Learning</span>
|
||||
</div>
|
||||
<p class="position-description">Join our core engineering team...</p>
|
||||
<div class="application-info">
|
||||
<span class="posting-date">Posted: 2024-03-15</span>
|
||||
<button class="apply-btn" data-req-id="REQ12345">Apply Now</button>
|
||||
</div>
|
||||
</div>
|
||||
<!-- More positions -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="department">
|
||||
<h2 class="department-name">Marketing</h2>
|
||||
<div class="positions">
|
||||
<div class="position-card" data-position-id="mkt-1">
|
||||
<h3 class="position-title">Growth Marketing Manager</h3>
|
||||
<span class="salary-range">$120,000 - $180,000</span>
|
||||
<div class="position-meta">
|
||||
<span class="location">New York, NY</span>
|
||||
<span class="job-type">Full-time</span>
|
||||
<span class="experience">3+ years</span>
|
||||
</div>
|
||||
<div class="skills-required">
|
||||
<span class="skill">SEO</span>
|
||||
<span class="skill">Analytics</span>
|
||||
<span class="skill">Content Strategy</span>
|
||||
</div>
|
||||
<p class="position-description">Drive our growth initiatives...</p>
|
||||
<div class="application-info">
|
||||
<span class="posting-date">Posted: 2024-03-14</span>
|
||||
<button class="apply-btn" data-req-id="REQ12346">Apply Now</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Test cases
|
||||
def test_schema_generation():
|
||||
# Test 1: No query (should extract everything)
|
||||
print("\nTest 1: No Query (Full Schema)")
|
||||
schema1 = JsonCssExtractionStrategy.generate_schema(test_html)
|
||||
print(json.dumps(schema1, indent=2))
|
||||
|
||||
# Test 2: Query for just basic job info
|
||||
print("\nTest 2: Basic Job Info Query")
|
||||
query2 = "I only need job titles, salaries, and locations"
|
||||
schema2 = JsonCssExtractionStrategy.generate_schema(test_html, query2)
|
||||
print(json.dumps(schema2, indent=2))
|
||||
|
||||
# Test 3: Query for company and department structure
|
||||
print("\nTest 3: Organizational Structure Query")
|
||||
query3 = "Extract company details and department names, without position details"
|
||||
schema3 = JsonCssExtractionStrategy.generate_schema(test_html, query3)
|
||||
print(json.dumps(schema3, indent=2))
|
||||
|
||||
# Test 4: Query for specific skills tracking
|
||||
print("\nTest 4: Skills Analysis Query")
|
||||
query4 = "I want to analyze required skills across all positions"
|
||||
schema4 = JsonCssExtractionStrategy.generate_schema(test_html, query4)
|
||||
print(json.dumps(schema4, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_schema_generation()
|
||||
Reference in New Issue
Block a user