feat(script): add new scripting capabilities and documentation
This commit introduces a comprehensive set of new scripts and examples to enhance the scripting capabilities of the crawl4ai project. The changes include the addition of several Python scripts for compiling and executing scripts, as well as a variety of example scripts demonstrating different functionalities such as login flows, data extraction, and multi-step workflows. Additionally, detailed documentation has been created to guide users on how to utilize these new features effectively. The following significant modifications were made: - Added core scripting files: , , and . - Created a new documentation file to provide an overview of the new features. - Introduced multiple example scripts in the directory to showcase various use cases. - Updated and to integrate the new functionalities. - Added font assets for improved documentation presentation. These changes significantly expand the functionality of the crawl4ai project, allowing users to create more complex and varied scripts with ease.
This commit is contained in:
UncleCode
328
crawl4ai/script/c4a_compile.py
Normal file
328
crawl4ai/script/c4a_compile.py
Normal file
@@ -0,0 +1,328 @@
|
||||
"""
|
||||
Clean C4A-Script API with Result pattern
|
||||
No exceptions - always returns results
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import pathlib
|
||||
import re
|
||||
from typing import Union, List, Optional
|
||||
|
||||
from .c4a_result import (
|
||||
CompilationResult, ValidationResult, ErrorDetail, WarningDetail,
|
||||
ErrorType, Severity, Suggestion
|
||||
)
|
||||
from .c4ai_script import Compiler
|
||||
from lark.exceptions import UnexpectedToken, UnexpectedCharacters, VisitError
|
||||
|
||||
|
||||
class C4ACompiler:
|
||||
"""Main compiler with result-based API"""
|
||||
|
||||
# Error code mapping
|
||||
ERROR_CODES = {
|
||||
"missing_then": "E001",
|
||||
"missing_paren": "E002",
|
||||
"missing_comma": "E003",
|
||||
"missing_endproc": "E004",
|
||||
"undefined_proc": "E005",
|
||||
"missing_backticks": "E006",
|
||||
"invalid_command": "E007",
|
||||
"syntax_error": "E999"
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def compile(cls, script: Union[str, List[str]], root: Optional[pathlib.Path] = None) -> CompilationResult:
|
||||
"""
|
||||
Compile C4A-Script to JavaScript
|
||||
|
||||
Args:
|
||||
script: C4A-Script as string or list of lines
|
||||
root: Root directory for includes
|
||||
|
||||
Returns:
|
||||
CompilationResult with success status and JS code or errors
|
||||
"""
|
||||
# Normalize input
|
||||
if isinstance(script, list):
|
||||
script_text = '\n'.join(script)
|
||||
script_lines = script
|
||||
else:
|
||||
script_text = script
|
||||
script_lines = script.split('\n')
|
||||
|
||||
try:
|
||||
# Try compilation
|
||||
compiler = Compiler(root)
|
||||
js_code = compiler.compile(script_text)
|
||||
|
||||
# Success!
|
||||
result = CompilationResult(
|
||||
success=True,
|
||||
js_code=js_code,
|
||||
metadata={
|
||||
"lineCount": len(script_lines),
|
||||
"statementCount": len(js_code)
|
||||
}
|
||||
)
|
||||
|
||||
# Add any warnings (future feature)
|
||||
# result.warnings = cls._check_warnings(script_text)
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
# Convert exception to ErrorDetail
|
||||
error = cls._exception_to_error(e, script_lines)
|
||||
return CompilationResult(
|
||||
success=False,
|
||||
errors=[error],
|
||||
metadata={
|
||||
"lineCount": len(script_lines)
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def validate(cls, script: Union[str, List[str]]) -> ValidationResult:
|
||||
"""
|
||||
Validate script syntax without generating code
|
||||
|
||||
Args:
|
||||
script: C4A-Script to validate
|
||||
|
||||
Returns:
|
||||
ValidationResult with validity status and any errors
|
||||
"""
|
||||
result = cls.compile(script)
|
||||
|
||||
return ValidationResult(
|
||||
valid=result.success,
|
||||
errors=result.errors,
|
||||
warnings=result.warnings
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def compile_file(cls, path: Union[str, pathlib.Path]) -> CompilationResult:
|
||||
"""
|
||||
Compile a C4A-Script file
|
||||
|
||||
Args:
|
||||
path: Path to the file
|
||||
|
||||
Returns:
|
||||
CompilationResult
|
||||
"""
|
||||
path = pathlib.Path(path)
|
||||
|
||||
if not path.exists():
|
||||
error = ErrorDetail(
|
||||
type=ErrorType.RUNTIME,
|
||||
code="E100",
|
||||
severity=Severity.ERROR,
|
||||
message=f"File not found: {path}",
|
||||
line=0,
|
||||
column=0,
|
||||
source_line=""
|
||||
)
|
||||
return CompilationResult(success=False, errors=[error])
|
||||
|
||||
try:
|
||||
script = path.read_text()
|
||||
return cls.compile(script, root=path.parent)
|
||||
except Exception as e:
|
||||
error = ErrorDetail(
|
||||
type=ErrorType.RUNTIME,
|
||||
code="E101",
|
||||
severity=Severity.ERROR,
|
||||
message=f"Error reading file: {str(e)}",
|
||||
line=0,
|
||||
column=0,
|
||||
source_line=""
|
||||
)
|
||||
return CompilationResult(success=False, errors=[error])
|
||||
|
||||
@classmethod
|
||||
def _exception_to_error(cls, exc: Exception, script_lines: List[str]) -> ErrorDetail:
|
||||
"""Convert an exception to ErrorDetail"""
|
||||
|
||||
if isinstance(exc, UnexpectedToken):
|
||||
return cls._handle_unexpected_token(exc, script_lines)
|
||||
elif isinstance(exc, UnexpectedCharacters):
|
||||
return cls._handle_unexpected_chars(exc, script_lines)
|
||||
elif isinstance(exc, ValueError):
|
||||
return cls._handle_value_error(exc, script_lines)
|
||||
else:
|
||||
# Generic error
|
||||
return ErrorDetail(
|
||||
type=ErrorType.SYNTAX,
|
||||
code=cls.ERROR_CODES["syntax_error"],
|
||||
severity=Severity.ERROR,
|
||||
message=str(exc),
|
||||
line=1,
|
||||
column=1,
|
||||
source_line=script_lines[0] if script_lines else ""
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _handle_unexpected_token(cls, exc: UnexpectedToken, script_lines: List[str]) -> ErrorDetail:
|
||||
"""Handle UnexpectedToken errors"""
|
||||
line = exc.line
|
||||
column = exc.column
|
||||
|
||||
# Get context lines
|
||||
source_line = script_lines[line - 1] if 0 < line <= len(script_lines) else ""
|
||||
line_before = script_lines[line - 2] if line > 1 and line <= len(script_lines) + 1 else None
|
||||
line_after = script_lines[line] if 0 < line < len(script_lines) else None
|
||||
|
||||
# Determine error type and suggestions
|
||||
if exc.token.type == 'CLICK' and 'THEN' in str(exc.expected):
|
||||
code = cls.ERROR_CODES["missing_then"]
|
||||
message = "Missing 'THEN' keyword after IF condition"
|
||||
suggestions = [
|
||||
Suggestion(
|
||||
"Add 'THEN' after the condition",
|
||||
source_line.replace("CLICK", "THEN CLICK") if source_line else None
|
||||
)
|
||||
]
|
||||
elif exc.token.type == '$END':
|
||||
code = cls.ERROR_CODES["missing_endproc"]
|
||||
message = "Unexpected end of script"
|
||||
suggestions = [
|
||||
Suggestion("Check for missing ENDPROC"),
|
||||
Suggestion("Ensure all procedures are properly closed")
|
||||
]
|
||||
elif 'RPAR' in str(exc.expected):
|
||||
code = cls.ERROR_CODES["missing_paren"]
|
||||
message = "Missing closing parenthesis ')'"
|
||||
suggestions = [
|
||||
Suggestion("Add closing parenthesis at the end of the condition")
|
||||
]
|
||||
elif 'COMMA' in str(exc.expected):
|
||||
code = cls.ERROR_CODES["missing_comma"]
|
||||
message = "Missing comma ',' in command"
|
||||
suggestions = [
|
||||
Suggestion("Add comma between arguments")
|
||||
]
|
||||
else:
|
||||
# Check if this might be missing backticks
|
||||
if exc.token.type == 'NAME' and 'BACKTICK_STRING' in str(exc.expected):
|
||||
code = cls.ERROR_CODES["missing_backticks"]
|
||||
message = "Selector must be wrapped in backticks"
|
||||
suggestions = [
|
||||
Suggestion(
|
||||
"Wrap the selector in backticks",
|
||||
f"`{exc.token.value}`"
|
||||
)
|
||||
]
|
||||
else:
|
||||
code = cls.ERROR_CODES["syntax_error"]
|
||||
message = f"Unexpected '{exc.token.value}'"
|
||||
if exc.expected:
|
||||
expected_list = [str(e) for e in exc.expected if not str(e).startswith('_')][:3]
|
||||
if expected_list:
|
||||
message += f". Expected: {', '.join(expected_list)}"
|
||||
suggestions = []
|
||||
|
||||
return ErrorDetail(
|
||||
type=ErrorType.SYNTAX,
|
||||
code=code,
|
||||
severity=Severity.ERROR,
|
||||
message=message,
|
||||
line=line,
|
||||
column=column,
|
||||
source_line=source_line,
|
||||
line_before=line_before,
|
||||
line_after=line_after,
|
||||
suggestions=suggestions
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _handle_unexpected_chars(cls, exc: UnexpectedCharacters, script_lines: List[str]) -> ErrorDetail:
|
||||
"""Handle UnexpectedCharacters errors"""
|
||||
line = exc.line
|
||||
column = exc.column
|
||||
|
||||
source_line = script_lines[line - 1] if 0 < line <= len(script_lines) else ""
|
||||
|
||||
# Check for missing backticks
|
||||
if "CLICK" in source_line and column > source_line.find("CLICK"):
|
||||
code = cls.ERROR_CODES["missing_backticks"]
|
||||
message = "Selector must be wrapped in backticks"
|
||||
suggestions = [
|
||||
Suggestion(
|
||||
"Wrap the selector in backticks",
|
||||
re.sub(r'CLICK\s+([^\s]+)', r'CLICK `\1`', source_line)
|
||||
)
|
||||
]
|
||||
else:
|
||||
code = cls.ERROR_CODES["syntax_error"]
|
||||
message = f"Invalid character at position {column}"
|
||||
suggestions = []
|
||||
|
||||
return ErrorDetail(
|
||||
type=ErrorType.SYNTAX,
|
||||
code=code,
|
||||
severity=Severity.ERROR,
|
||||
message=message,
|
||||
line=line,
|
||||
column=column,
|
||||
source_line=source_line,
|
||||
suggestions=suggestions
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _handle_value_error(cls, exc: ValueError, script_lines: List[str]) -> ErrorDetail:
|
||||
"""Handle ValueError (runtime errors)"""
|
||||
message = str(exc)
|
||||
|
||||
# Check for undefined procedure
|
||||
if "Unknown procedure" in message:
|
||||
proc_match = re.search(r"'([^']+)'", message)
|
||||
if proc_match:
|
||||
proc_name = proc_match.group(1)
|
||||
|
||||
# Find the line with the procedure call
|
||||
for i, line in enumerate(script_lines):
|
||||
if proc_name in line and not line.strip().startswith('PROC'):
|
||||
return ErrorDetail(
|
||||
type=ErrorType.RUNTIME,
|
||||
code=cls.ERROR_CODES["undefined_proc"],
|
||||
severity=Severity.ERROR,
|
||||
message=f"Undefined procedure '{proc_name}'",
|
||||
line=i + 1,
|
||||
column=line.find(proc_name) + 1,
|
||||
source_line=line,
|
||||
suggestions=[
|
||||
Suggestion(
|
||||
f"Define the procedure before using it",
|
||||
f"PROC {proc_name}\n # commands here\nENDPROC"
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
# Generic runtime error
|
||||
return ErrorDetail(
|
||||
type=ErrorType.RUNTIME,
|
||||
code="E999",
|
||||
severity=Severity.ERROR,
|
||||
message=message,
|
||||
line=1,
|
||||
column=1,
|
||||
source_line=script_lines[0] if script_lines else ""
|
||||
)
|
||||
|
||||
|
||||
# Convenience functions for direct use
|
||||
def compile(script: Union[str, List[str]], root: Optional[pathlib.Path] = None) -> CompilationResult:
|
||||
"""Compile C4A-Script to JavaScript"""
|
||||
return C4ACompiler.compile(script, root)
|
||||
|
||||
|
||||
def validate(script: Union[str, List[str]]) -> ValidationResult:
|
||||
"""Validate C4A-Script syntax"""
|
||||
return C4ACompiler.validate(script)
|
||||
|
||||
|
||||
def compile_file(path: Union[str, pathlib.Path]) -> CompilationResult:
|
||||
"""Compile C4A-Script file"""
|
||||
return C4ACompiler.compile_file(path)
|
||||
Reference in New Issue
Block a user