ayrisdev/crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'release/v0.7.0' - The Adaptive Intelligence Update

Browse Source
This commit is contained in:
UncleCode
2025-07-12 18:54:20 +08:00
parent 02f3127ded 0c8bb742b7
commit 7b9ba3015f
320 changed files with 115071 additions and 514 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7715
docs/apps/iseeyou/llms-full.txt Normal file
View File

File diff suppressed because it is too large Load Diff

1323
docs/apps/linkdin/Crawl4ai_Linkedin_Data_Discovery_Part_1.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

5859
docs/apps/linkdin/Crawl4ai_Linkedin_Data_Discovery_Part_2.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

4
docs/apps/linkdin/README.md
View File

@@ -1,7 +1,11 @@
# Crawl4AI Prospect‑Wizard – step‑by‑step guide
[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/10nRCwmfxPjVrRUHyJsYlX7BH5bvPoGpx?usp=sharing)
A three‑stage demo that goes from **LinkedIn scraping** ➜ **LLM reasoning** ➜ **graph visualisation**.
**Try it in Google Colab!** Click the badge above to run this demo in a cloud environment with zero setup required.
```
prospect‑wizard/
├─ c4ai_discover.py # Stage 1 – scrape companies + people

11
docs/apps/linkdin/c4ai_discover.py
View File

@@ -107,7 +107,14 @@ _COMPANY_SCHEMA_QUERY = dedent(
IMPORTANT: Do not use the base64 kind of classes to target element. It's not reliable.
The main div parent contains these li element is "div.search-results-container" you can use this.
The <ul> parent has "role" equal to "list". Using these two should be enough to target the <li> elements."
The <ul> parent has "role" equal to "list". Using these two should be enough to target the <li> elements.
IMPORTANT: Remember there might be multiple <a> tags that start with https://www.linkedin.com/company/[NAME],
so in case you refer to them for different fields, make sure to be more specific. One has the image, and one
has the person's name.
IMPORTANT: Be very smart in selecting the correct and unique way to address the element. You should ensure
your selector points to a single element and is unique to the place that contains the information.
"""
)
@@ -235,6 +242,7 @@ async def crawl_people_page(
cache_mode=CacheMode.BYPASS,
magic=True,
wait_for=".org-people-profile-card__card-spacing",
wait_for_images=5000,
delay_before_return_html=1,
session_id="people_search",
)
@@ -422,6 +430,7 @@ def main():
# decide on debug defaults
if cli_opts.debug:
opts = detect_debug_defaults(force=True)
cli_opts = opts
else:
env_defaults = detect_debug_defaults()
opts = env_defaults if env_defaults else cli_opts

58
docs/apps/linkdin/c4ai_insights.py
View File

@@ -29,9 +29,10 @@ from typing import List, Dict, Any
from rich.console import Console
from rich.logging import RichHandler
from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn
import logging
# ───────────────────────────────────────────────────────────────────────────────
BASE_DIR = pathlib.Path(__file__).resolve().parent
# ───────────────────────────────────────────────────────────────────────────────
@@ -45,6 +46,8 @@ import hashlib
from litellm import completion #Support any LLM Provider
# ───────────────────────────────────────────────────────────────────────────────
# Utils
# ───────────────────────────────────────────────────────────────────────────────
@@ -66,14 +69,16 @@ BASE_DIR = pathlib.Path(__file__).resolve().parent
# ───────────────────────────────────────────────────────────────────────────────
def dev_defaults() -> SimpleNamespace:
return SimpleNamespace(
in_dir="./debug_out",
out_dir="./insights_debug",
in_dir="./samples",
out_dir="./samples/insights",
embed_model="all-MiniLM-L6-v2",
top_k=10,
llm_provider="openai/gpt-4.1",
llm_api_key=None,
max_llm_tokens=8000,
llm_temperature=1.0,
stub=False, # Set to True to use a stub for org-chart inference
llm_base_url=None, # e.g., "https://api.openai.com/v1" for OpenAI
workers=4
)
@@ -82,8 +87,9 @@ def dev_defaults() -> SimpleNamespace:
# ───────────────────────────────────────────────────────────────────────────────
def embed_descriptions(companies, model_name:str, opts) -> np.ndarray:
from sentence_transformers import SentenceTransformer
logging.debug(f"Using embedding model: {model_name}")
console = Console()
console.print(f"Using embedding model: [bold cyan]{model_name}[/]")
cache_path = BASE_DIR / Path(opts.out_dir) / "embeds_cache.json"
cache = {}
if cache_path.exists():
@@ -122,7 +128,6 @@ def build_company_graph(companies, embeds:np.ndarray, top_k:int) -> Dict[str,Any
from sklearn.metrics.pairwise import cosine_similarity
sims = cosine_similarity(embeds)
nodes, edges = [], []
idx_of = {c["handle"]: i for i,c in enumerate(companies)}
for i,c in enumerate(companies):
node = dict(
id=c["handle"].strip("/"),
@@ -252,18 +257,18 @@ def render_html(out:Path, template_dir:Path):
# ───────────────────────────────────────────────────────────────────────────────
async def run(opts):
# ── silence SDK noise ──────────────────────────────────────────────────────
for noisy in ("openai", "httpx", "httpcore"):
lg = logging.getLogger(noisy)
lg.setLevel(logging.WARNING) # or ERROR if you want total silence
lg.propagate = False # optional: stop them reaching root
# for noisy in ("openai", "httpx", "httpcore"):
# lg = logging.getLogger(noisy)
# lg.setLevel(logging.WARNING) # or ERROR if you want total silence
# lg.propagate = False # optional: stop them reaching root
# ────────────── logging bootstrap ──────────────
console = Console()
logging.basicConfig(
level="INFO",
format="%(message)s",
handlers=[RichHandler(console=console, markup=True, rich_tracebacks=True)],
)
# logging.basicConfig(
# level="INFO",
# format="%(message)s",
# handlers=[RichHandler(console=console, markup=True, rich_tracebacks=True)],
# )
in_dir = BASE_DIR / Path(opts.in_dir)
out_dir = BASE_DIR / Path(opts.out_dir)
@@ -272,12 +277,12 @@ async def run(opts):
companies = load_jsonl(in_dir/"companies.jsonl")
people = load_jsonl(in_dir/"people.jsonl")
logging.info(f"[bold cyan]Loaded[/] {len(companies)} companies, {len(people)} people")
console.print(f"[bold cyan]Loaded[/] {len(companies)} companies, {len(people)} people")
logging.info("[bold]⇢[/] Embedding company descriptions…")
console.print("[bold]⇢[/] Embedding company descriptions…")
embeds = embed_descriptions(companies, opts.embed_model, opts)
logging.info("[bold]⇢[/] Building similarity graph")
console.print("[bold]⇢[/] Building similarity graph")
company_graph = build_company_graph(companies, embeds, opts.top_k)
dump_json(company_graph, out_dir/"company_graph.json")
@@ -286,19 +291,19 @@ async def run(opts):
for comp in companies:
handle = comp["handle"].strip("/").replace("/","_")
out_file = out_dir/f"org_chart_{handle}.json"
if out_file.exists() and False:
logging.info(f"[green]✓[/] Skipping existing {comp['name']}")
if out_file.exists():
console.print(f"[green]✓[/] Skipping existing {comp['name']}")
continue
to_process.append(comp)
if not to_process:
logging.info("[yellow]All companies already processed[/]")
console.print("[yellow]All companies already processed[/]")
else:
workers = getattr(opts, 'workers', 1)
parallel = workers > 1
logging.info(f"[bold]⇢[/] Inferring org-charts via LLM {f'(parallel={workers} workers)' if parallel else ''}")
console.print(f"[bold]⇢[/] Inferring org-charts via LLM {f'(parallel={workers} workers)' if parallel else ''}")
with Progress(
SpinnerColumn(),
@@ -341,12 +346,11 @@ async def run(opts):
# Run with concurrency control
await asyncio.gather(*(bounded_process(task) for task in tasks))
logging.info("[bold]⇢[/] Flattening decision-makers CSV")
console.print("[bold]⇢[/] Flattening decision-makers CSV")
export_decision_makers(out_dir, out_dir/"decision_makers.csv")
render_html(out_dir, template_dir=BASE_DIR/"templates")
logging.success = lambda msg, **k: console.print(f"[bold green]✓[/] {msg}", **k)
logging.success(f"Stage-2 artefacts written to {out_dir}")
console.print(f"[bold green]✓[/] Stage-2 artefacts written to {out_dir}")
# ───────────────────────────────────────────────────────────────────────────────
# CLI
@@ -369,8 +373,8 @@ def build_arg_parser():
def main():
dbg = dev_defaults()
# opts = dbg if True else build_arg_parser().parse_args()
opts = build_arg_parser().parse_args()
opts = dbg if True else build_arg_parser().parse_args()
# opts = build_arg_parser().parse_args()
asyncio.run(run(opts))
if __name__ == "__main__":

9
docs/apps/linkdin/samples/companies.jsonl Normal file
View File

@@ -0,0 +1,9 @@
{"handle": "https://www.linkedin.com/company/healthpartnersng/", "name": "Health Partners HMO", "descriptor": "Hospitals and Health Care • Ikoyi, LAGOS", "about": "Healthpartners Ltd is a leading HMO in Nigeria providing affordablehealthinsuranceandhealthmanagementservices for companies and individuals in Nigeria. We have several individual and group plans that meets yourhealthmanagementneeds. Call us now at 0807-460-9165, 0807-714-0759 or email...", "followers": null, "people_url": "https://www.linkedin.com/company/healthpartnersng/people/", "captured_at": "2025-04-29T10:46:08Z"}
{"handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "name": "Health & Insurance Management Services Organization", "descriptor": "Non-profit Organizations • Mbeya", "about": "Health&InsuranceManagementServices Organization (HIMSO) was established and registered in 2012 as a Non- Government Organization (NGO) with the aim...", "followers": 35, "people_url": "https://www.linkedin.com/company/health-insurance-management-services-organization/people/", "captured_at": "2025-04-29T13:15:04Z"}
{"handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "name": "National Health Insurance Management Authority", "descriptor": "Insurance • Lusaka, Lusaka", "about": "The NationalHealthInsuranceManagementAuthority (NHIMA) is established pursuant to section 4 of the NationalHealthInsurance(NHI) Act No. 2 of 2018. The compulsory NationalHealthInsurancescheme seeks to provide for a sound and reliable healthcare financing for Zambian households and the entirehealthsector...", "followers": null, "people_url": "https://www.linkedin.com/company/national-health-insurance-management-authority/people/", "captured_at": "2025-04-29T13:15:04Z"}
{"handle": "https://www.linkedin.com/company/health-alliance-plan/", "name": "Health Alliance Plan", "descriptor": "Hospitals and Health Care • Detroit, MI", "about": "...organizations to enhance the lives of those we touch. We offer six distincthealthinsurancelines: • Group Insured Commercial • Individual • Medicare • Medicaid • Self-Funded • Network Leasing HAP also provides: • Award-winning wellness programs • Community outreach • Digitalhealthtools • Diseasemanagement...", "followers": null, "people_url": "https://www.linkedin.com/company/health-alliance-plan/people/", "captured_at": "2025-04-29T13:15:04Z"}
{"handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "name": "Insurance Recruiting Solutions", "descriptor": "Insurance • Waukee, Iowa", "about": "InsuranceRecruiting Solutions provides staffing and recruiting services exclusively to theinsuranceindustry. We are committed to providing highly personalized recruiting services, tailored to each candidate and employer. With years ofinsuranceindustry experience, we speak your language. As a leading national...", "followers": null, "people_url": "https://www.linkedin.com/company/insurance-recruiting-solutions/people/", "captured_at": "2025-04-29T13:15:04Z"}
{"handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "name": "Health Plan of San Mateo (HPSM)", "descriptor": "Hospitals and Health Care • South San Francisco, California", "about": "TheHealthPlan of San Mateo (HPSM) is a local non-profithealthcare plan that offershealthcoverage and a provider network to San Mateo County's under-insured population. We currently serve more than 145,000 County residents.", "followers": null, "people_url": "https://www.linkedin.com/company/healthplanofsanmateo/people/", "captured_at": "2025-04-29T13:15:04Z"}
{"handle": "https://www.linkedin.com/company/insurance-management-group_2/", "name": "Insurance Management Group", "descriptor": "Insurance • Marion, Indiana", "about": "InsuranceManagementGroup is an all-riskinsuranceagency with over 140 years of experience, specializing in Home, Auto, BusinessInsurance, Individual Life &Health, and Employee Benefits. We represent highly rated and financially soundinsurancecarriers, to ensure that our clients are getting the best coverage...", "followers": null, "people_url": "https://www.linkedin.com/company/insurance-management-group_2/people/", "captured_at": "2025-04-29T13:15:04Z"}
{"handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "name": "CareCard Health Insurance Management Co", "descriptor": "Insurance • Damascus", "about": "CareCard offers Business Process Outsourcing (BPO) services toInsurance, Self Funded and Retireehealthplan market. CareCard provides operational outsourcing...", "followers": 187, "people_url": "https://www.linkedin.com/company/carecard-health-insurance-management-co/people/", "captured_at": "2025-04-29T13:15:04Z"}
{"handle": "https://www.linkedin.com/company/healthcluster/", "name": "Health Cluster", "descriptor": "Technology, Information and Internet • Dubai", "about": "..., knowledge and interaction. The company has solutions and products inHealthTech, eHealth, DigitalHealth, Revenue CycleManagement– RCM Solutions, AI & ML, Internet...", "followers": null, "people_url": "https://www.linkedin.com/company/healthcluster/people/", "captured_at": "2025-04-29T13:15:04Z"}

108
docs/apps/linkdin/samples/people.jsonl Normal file
View File

@@ -0,0 +1,108 @@
{"profile_url": null, "name": "Yahya Ipuge", "headline": "Senior Health Specialist, Independent Consultant, Certified Board Director, Board Chair in NGO and Private Entities", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQFuqPObSyLPMQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1517757008397?e=1751500800&v=beta&t=zaHc2CY7AJ-eX1MCSvazp8ny37iBAu3YsyaZjwq6gB0", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-29T13:15:33Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Field officer at Health and Insurance Management Services Organization", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5103AQEVmdDwTIhsjQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1540989154156?e=1751500800&v=beta&t=7N0baJNfZ26dbrNNbv2055sbGlacQUwQu07wUTN0whs", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-29T13:15:33Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Medical Practitioner @ Health & Insurance | Master's Degree in Infection Control", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQHjMXy7dSmmLg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1725975429410?e=1751500800&v=beta&t=lDIL2KhDw471XYvtCrRfkHAnG3Q-npDJnwDdK0sYvpA", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-29T13:15:34Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "--", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-29T13:15:38Z"}
{"profile_url": null, "name": "Fadhy Mtanga", "headline": "Executive Director at Health & Insurance Management Services Organization (HIMSO) Author | Creative Writer | Social Scientist", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQEloEreyg3qVQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1704391866585?e=1751500800&v=beta&t=86am-v3cjBPBldLTwgt8-AY-YbxFY6QZQzObwLTtMEA", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-29T13:15:38Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Business Administrator at Consultancy Business investments", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQEuKXJmknr2YA/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1714545221728?e=1751500800&v=beta&t=zJG-rDZgYJJ0eROibf-Wag-v_JecCghwU3ul4TaH2Eg", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-29T13:15:48Z"}
{"profile_url": null, "name": "Tamani Phiri", "headline": "Corporate Business Strategy | Thought Leadership | Corporate Governance", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQF4mFx8jY2n-w/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1730302954035?e=1751500800&v=beta&t=i4QIrHA6A9eLtKolwTRNhuoiaTad28sf5KHxAFuXG-w", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-29T13:15:48Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Member Service Assistant @ National Health Insurance Management Authority (NHIMA) | Clinical Officer | Health Insurance & Public Health | Claims Processing & Customer Support | Data & Policy Analyst", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQGob13KyxrB0g/profile-displayphoto-shrink_100_100/B4DZYCgreeHIAU-/0/1743798848889?e=1751500800&v=beta&t=uXxTsMLi5s7hr8FBEzVTDw7V3eJ85kpTaIC7i_5fM-Y", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-29T13:15:48Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Economist/ Development Analyst/ Planner/ Customer Care", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQFEc3EgfdpZeg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1727782989867?e=1751500800&v=beta&t=dWjKzSu5FDRgmxAVret9jQPhWF2VjcrnmEpR2LDMC1Q", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-29T13:15:48Z"}
{"profile_url": null, "name": "Samantha Ngandwe", "headline": "Quality Assurance and Accreditation Officer at National Health Insurance Management Authority", "followers": 382, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQHyOjyoz7d95g/profile-displayphoto-shrink_100_100/B4DZYvvhP5GwAY-/0/1744557712084?e=1751500800&v=beta&t=DLYRpz20zmwUWx1UY1Dn-ykvgWBnwn8XHWLaDMf199M", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-29T13:15:48Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Dental Surgery Assistant at Health Promotion Board", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-29T13:16:11Z"}
{"profile_url": null, "name": "Liz England Tucker", "headline": "Medical Performance Optimization", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQFY6yx360QunQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1713831102587?e=1751500800&v=beta&t=u-C8Ozpl_ITkTpdgt5QD-C5_Qt7MA0DagLRmiuGKngQ", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-29T13:16:11Z"}
{"profile_url": null, "name": "Merrill Hausenfluck", "headline": "Chief Financial Officer", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQGKxDKRJM_BCg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1696292650180?e=1751500800&v=beta&t=NbUVC-QP-XL3frBpQcn3GtGrZ04Fl0xdko4V-mHxPag", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-29T13:16:11Z"}
{"profile_url": null, "name": "Mike Treash", "headline": "Senior Vice President and Chief Operating Officer at Health Alliance Plan", "followers": 2000, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQH_c6tIq929gw/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1720478900599?e=1751500800&v=beta&t=l9RLnLDKBBJjJQTsFMJMa_1MpWCKcV4AUa3dcjGnSXQ", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-29T13:16:11Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Manager at Health Alliance Plan", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-29T13:16:11Z"}
{"profile_url": null, "name": "Scot Dickerson", "headline": "Insurance Industry Specialist, Insurance Recruiter, Talent Acquisition, Talent Sourcing, Hiring Consultant, Career Consultant, Staffing, Executive Recruiter at Insurance Recruiting Solutions #insurancejobs #insurance", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQGLFvtPPU3HEw/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1724950672124?e=1751500800&v=beta&t=uT4SFSMF32O1d50Z0dbnd6zRRKdABHxSGlOZdxWdXBM", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-29T13:16:24Z"}
{"profile_url": null, "name": "Steele Dickerson", "headline": "Insurance Recruiting Solutions", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQEyICWaE_PvXA/profile-displayphoto-shrink_100_100/B56ZQuDHyZH0Ac-/0/1735939358232?e=1751500800&v=beta&t=9FdnWHrjnPQ7LQ5FdwC7sY8sS6hm-R4zfWO5Vmwm46w", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-29T13:16:24Z"}
{"profile_url": null, "name": "Madeline Judas", "headline": "Recruiting Operations & Business Development Specialist", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQG6xiTaJ71UiA/profile-displayphoto-shrink_100_100/B56ZU_N_jPHoAY-/0/1740522388021?e=1751500800&v=beta&t=CxvAsYgU0zelghZsRhUJOC26ILVovP3ZPn4nMnWkEJE", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-29T13:16:24Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "All Lines Claims Adjuster / General Lines Agent (Property & Casualty : Life, Accident, Health & HMO)", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQFTjkb7SxTWWg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1725920318474?e=1751500800&v=beta&t=BGEzQg1c2l8qxuy2iKJ896nElsiYcaWnhkf-mqc-KhY", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-29T13:16:24Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Clinical Pharmacy Manager at Health Plan of San Mateo (HPSM)", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQEPO0pZOxznoA/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1551565536585?e=1751500800&v=beta&t=qwMGzWX_Zefkciq8h2m9daLMflT0WoDr5F1R5pXvyM4", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-29T13:16:40Z"}
{"profile_url": null, "name": "Tamana M.", "headline": "MPH Candidate at Brown University | Data Coordinator", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQEY3iDtFmpzlg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1714197678074?e=1751500800&v=beta&t=IsVT0uC7A-T-Tp22gZFDG9wiT7LMB5GmhccuI8f9c-I", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-29T13:16:40Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Program Manager", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-29T13:16:40Z"}
{"profile_url": null, "name": "Mackenzie Baysinger Moniz, MSW", "headline": "Program Manager at Health Plan of San Mateo", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQHAd3A4zLyuWA/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1675716742150?e=1751500800&v=beta&t=ot3fMyJFnHwwNfKJiA_YxZp6MOK_iVGtSCUgVNq867g", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-29T13:16:40Z"}
{"profile_url": null, "name": "John O.", "headline": "Healthcare Delivery Strategy Execution", "followers": null, "connection_degree": "· 3rd", "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-29T13:16:40Z"}
{"profile_url": null, "name": "Daniel McQuilkin", "headline": "Senior Vice President", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQFkScOqwhxvfQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1521406683682?e=1751500800&v=beta&t=iohhak3lrV1gpmA6dnoCxTRJidskfgmZUXKbNQbkxjs", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-29T13:17:05Z"}
{"profile_url": null, "name": "Tony Bonacuse", "headline": "Senior Vice President at Insurance Management Group", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQF_JJOFLjkZoQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1516269003018?e=1751500800&v=beta&t=0APZt5RNhvUj4IxsSdi7JO9KxezZzOH_WQCibn5Szgs", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-29T13:17:05Z"}
{"profile_url": null, "name": "Mark Bilger", "headline": "Director - Sr. Vice President at Insurance Management Group", "followers": 1000, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQEzX5qUfqhd2g/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1663842785708?e=1751500800&v=beta&t=YyKXRQol0cDntoq8vbdxyaRvEFf0vWKNHPxk0cyWiG8", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-29T13:17:05Z"}
{"profile_url": null, "name": "Adam Young, MBA", "headline": "Husband | Father | Traveler | Sports Fanatic | Food Enthusiast | Independent Insurance Professional", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQErWIq1AVyxKg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1601480475688?e=1751500800&v=beta&t=jK_mhX0PkDdG8WBZaipIIYRDm1PnWIuFR7sCKDhDi6s", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-29T13:17:05Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Senior Vice President at Insurance Management Group / Partner", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQH3dm30dXH82w/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1572228299104?e=1751500800&v=beta&t=iuBQYs4iLHJgRgjFbSA2YiNiAI8zDILqg-nVsLR9Qjk", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-29T13:17:05Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Doctor at CareCard Health Insurance Management Co", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-29T13:17:09Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Pharmacist", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQHyPi4Amu_Dkw/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1640460490377?e=1751500800&v=beta&t=q7R_b7bD9CR-1-Dvu81WoEHN_ljHK16l6ioTIA0LN7Q", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-29T13:17:09Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "IT Manager at CareCard Health Insurance Management Co", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-29T13:17:09Z"}
{"profile_url": null, "name": "Amal Shabani", "headline": "at carecard", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQFLzeP3yPkjgg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1519625412373?e=1751500800&v=beta&t=GULSoesSn83F_fYkkH_nPxWIjjs1d9Pucc3dUDNei6I", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-29T13:17:09Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "--", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-29T13:17:09Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Biologiste | Pharmaco-épidemiologie & Pharmaco-économie | Software Helath Care Management", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQHOPXrX5-oeug/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1663013895834?e=1751500800&v=beta&t=yE2RGp0rfhcJkjh_vdM0VwpaPUtoPewM80lTlr20OHU", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-29T13:17:14Z"}
{"profile_url": null, "name": "Ruqaia Ali Alkhalifa", "headline": "‎‏ RN,BSN, MSN,NE Database Officer for Scholarship Programs and Central Committee rapporteur at Al-Ahsa Health Cluster.", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQGfNujqDnuZDA/profile-displayphoto-shrink_100_100/B4EZOvsQThH0AU-/0/1733819436577?e=1751500800&v=beta&t=jleAVvhbg0H85tSi9TG96x0fqdkS1oytfaU02LHsFEI", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-29T13:17:14Z"}
{"profile_url": null, "name": "Fahad Mohyuddin", "headline": "Healthcare AI Strategist | Digital Health | SaaS | Telehealth | HIS | EHR | IoT", "followers": 7000, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQFLnPh8fu-HHg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1647320077586?e=1751500800&v=beta&t=S__knVzEVrGZuyqwszCe_5V_kawbG5tejmmEe3fkMJE", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-29T13:17:14Z"}
{"profile_url": null, "name": "Muhammad Moid Shams", "headline": "Azure DevOps | AWS Cloud Infrastructure| Freight Tech | Health Tech | HL7- NABIDH | HL7+ FHIR | KSA -NPHIES | FHIR - MOPH | HL7- Riayati | Freight Tech | Insure Tech | with Azure, Azure AI , PowerApps, D365 , M365", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQEzousRurY2Zg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1711283874675?e=1751500800&v=beta&t=ZheuoRIAkS_9M8WXafdwB1nJEuy-a5HEsrXlfOANx80", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-29T13:17:14Z"}
{"profile_url": null, "name": "Muhammad Shahzaib (PMP® - SCRUM®)", "headline": "PMP-Certified Project Manager | Health Care & Web Solutions Expert | Customer Success & Operations Management Expert | Business Transformation Expert", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D35AQFyp7WcBZinYA/profile-framedphoto-shrink_100_100/profile-framedphoto-shrink_100_100/0/1730638721808?e=1746540000&v=beta&t=QoGze1AlotUfm3K9kMWG6ZGVHS3ADu38THVPlxlYUys", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-29T13:17:14Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Music Professional at Health Options Worldwide", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5103AQGF-Dp6v6nkGw/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1585401654822?e=1751500800&v=beta&t=7yeO-dGz1p_B66cJVSlTSdAYJLMFFwxPIhwwcR8uWWo", "company_handle": "https://www.linkedin.com/company/health-options-worldwide/", "captured_at": "2025-04-29T13:17:17Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Trainer/instructor at Health Options Worldwide", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-options-worldwide/", "captured_at": "2025-04-29T13:17:17Z"}
{"profile_url": null, "name": "Michael Akpoarebe-Isaac", "headline": "Chief Operating officer, Health Partners HMO", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQE7KNFaLMyqYg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1602714385413?e=1751500800&v=beta&t=In5GaREqoXtO3sPCx9ZJJBwIPY4008ii13RPRl0w0Fw", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-29T13:17:22Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "REGISTERED NURSE/CLAIMS SUPERVISOR/HEALTH EDUCATOR/ CASE MANAGER/ Lekki.", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4E35AQEEqf5i5pD76g/profile-framedphoto-shrink_100_100/profile-framedphoto-shrink_100_100/0/1724219552412?e=1746540000&v=beta&t=1PAfKEpQFL196LZHfY0wHAZ35TH0fRjku9ihSfDdOk4", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-29T13:17:22Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Data Analyst|Dedicated Retention Officer Boosting Customer Loyalty| Business Developer/ Event planner", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D35AQHIgeS1H7w65w/profile-framedphoto-shrink_100_100/B4DZV1QfcCGcAk-/0/1741429012517?e=1746540000&v=beta&t=NCIbW7MWY7Cy4YEC2xzLoX54-Lm5CNhorbuSQe0lZSk", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-29T13:17:22Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Former Group managing director at Health Partners Ltd", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQHPQPvIQbPQPg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1583328612508?e=1751500800&v=beta&t=LpynArccJCWrdWMSBvYLH4SI5G-xae7ECoWUUAl_CeU", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-29T13:17:22Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "HEAD, FINCON, @ HEALTH PARTNERS (HMO) LTD", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQG8XOvnazEibQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1518882054975?e=1751500800&v=beta&t=5gT6GAWGTqYfpvkjOk0ArvV73I_KspkWXgoG-VhoStg", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-29T13:17:22Z"}
{"profile_url": null, "name": "Yahya Ipuge", "headline": "Senior Health Specialist, Independent Consultant, Certified Board Director, Board Chair in NGO and Private Entities", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQFuqPObSyLPMQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1517757008397?e=1751500800&v=beta&t=zaHc2CY7AJ-eX1MCSvazp8ny37iBAu3YsyaZjwq6gB0", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:36:39Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Field officer at Health and Insurance Management Services Organization", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5103AQEVmdDwTIhsjQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1540989154156?e=1751500800&v=beta&t=7N0baJNfZ26dbrNNbv2055sbGlacQUwQu07wUTN0whs", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:36:39Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Medical Practitioner @ Health & Insurance | Master's Degree in Infection Control", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQHjMXy7dSmmLg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1725975429410?e=1751500800&v=beta&t=lDIL2KhDw471XYvtCrRfkHAnG3Q-npDJnwDdK0sYvpA", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:36:39Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "--", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:36:39Z"}
{"profile_url": null, "name": "Fadhy Mtanga", "headline": "Executive Director at Health & Insurance Management Services Organization (HIMSO) Author | Creative Writer | Social Scientist", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQEloEreyg3qVQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1704391866585?e=1751500800&v=beta&t=86am-v3cjBPBldLTwgt8-AY-YbxFY6QZQzObwLTtMEA", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:36:39Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Member Service Assistant @ National Health Insurance Management Authority (NHIMA) | Clinical Officer | Health Insurance & Public Health | Claims Processing & Customer Support | Data & Policy Analyst", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQGob13KyxrB0g/profile-displayphoto-shrink_100_100/B4DZYCgreeHIAU-/0/1743798848889?e=1751500800&v=beta&t=uXxTsMLi5s7hr8FBEzVTDw7V3eJ85kpTaIC7i_5fM-Y", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:36:45Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Business Administrator at Consultancy Business investments", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQEuKXJmknr2YA/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1714545221728?e=1751500800&v=beta&t=zJG-rDZgYJJ0eROibf-Wag-v_JecCghwU3ul4TaH2Eg", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:36:45Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Economist/ Development Analyst/ Planner/ Customer Care", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQFEc3EgfdpZeg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1727782989867?e=1751500800&v=beta&t=dWjKzSu5FDRgmxAVret9jQPhWF2VjcrnmEpR2LDMC1Q", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:36:45Z"}
{"profile_url": null, "name": "Tamani Phiri", "headline": "Corporate Business Strategy | Thought Leadership | Corporate Governance", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQF4mFx8jY2n-w/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1730302954035?e=1751500800&v=beta&t=i4QIrHA6A9eLtKolwTRNhuoiaTad28sf5KHxAFuXG-w", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:36:45Z"}
{"profile_url": null, "name": "Samantha Ngandwe", "headline": "Quality Assurance and Accreditation Officer at National Health Insurance Management Authority", "followers": 382, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQHyOjyoz7d95g/profile-displayphoto-shrink_100_100/B4DZYvvhP5GwAY-/0/1744557712084?e=1751500800&v=beta&t=DLYRpz20zmwUWx1UY1Dn-ykvgWBnwn8XHWLaDMf199M", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:36:45Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Dental Surgery Assistant at Health Promotion Board", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:36:51Z"}
{"profile_url": null, "name": "Merrill Hausenfluck", "headline": "Chief Financial Officer", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQGKxDKRJM_BCg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1696292650180?e=1751500800&v=beta&t=NbUVC-QP-XL3frBpQcn3GtGrZ04Fl0xdko4V-mHxPag", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:36:51Z"}
{"profile_url": null, "name": "Mike Treash", "headline": "Senior Vice President and Chief Operating Officer at Health Alliance Plan", "followers": 2000, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQH_c6tIq929gw/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1720478900599?e=1751500800&v=beta&t=l9RLnLDKBBJjJQTsFMJMa_1MpWCKcV4AUa3dcjGnSXQ", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:36:51Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Manager at Health Alliance Plan", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:36:51Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Manager, Government Programs at Health Alliance Plan", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQF473eFGZeIpQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1654455840818?e=1751500800&v=beta&t=FllKCznSi0Ndm75QYy2i5UDtflCojNGkVzRpoChPC8c", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:36:51Z"}
{"profile_url": null, "name": "Steele Dickerson", "headline": "Insurance Recruiting Solutions", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQEyICWaE_PvXA/profile-displayphoto-shrink_100_100/B56ZQuDHyZH0Ac-/0/1735939358232?e=1751500800&v=beta&t=9FdnWHrjnPQ7LQ5FdwC7sY8sS6hm-R4zfWO5Vmwm46w", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-30T07:36:56Z"}
{"profile_url": null, "name": "Yahya Ipuge", "headline": "Senior Health Specialist, Independent Consultant, Certified Board Director, Board Chair in NGO and Private Entities", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQFuqPObSyLPMQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1517757008397?e=1751500800&v=beta&t=zaHc2CY7AJ-eX1MCSvazp8ny37iBAu3YsyaZjwq6gB0", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:44:32+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Field officer at Health and Insurance Management Services Organization", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5103AQEVmdDwTIhsjQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1540989154156?e=1751500800&v=beta&t=7N0baJNfZ26dbrNNbv2055sbGlacQUwQu07wUTN0whs", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:44:32+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Medical Practitioner @ Health & Insurance | Master's Degree in Infection Control", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQHjMXy7dSmmLg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1725975429410?e=1751500800&v=beta&t=lDIL2KhDw471XYvtCrRfkHAnG3Q-npDJnwDdK0sYvpA", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:44:32+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "--", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:44:32+00:00Z"}
{"profile_url": null, "name": "Fadhy Mtanga", "headline": "Executive Director at Health & Insurance Management Services Organization (HIMSO) Author | Creative Writer | Social Scientist", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQEloEreyg3qVQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1704391866585?e=1751500800&v=beta&t=86am-v3cjBPBldLTwgt8-AY-YbxFY6QZQzObwLTtMEA", "company_handle": "https://www.linkedin.com/company/health-insurance-management-services-organization/", "captured_at": "2025-04-30T07:44:32+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Member Service Assistant @ National Health Insurance Management Authority (NHIMA) | Clinical Officer | Health Insurance & Public Health | Claims Processing & Customer Support | Data & Policy Analyst", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQGob13KyxrB0g/profile-displayphoto-shrink_100_100/B4DZYCgreeHIAU-/0/1743798848889?e=1751500800&v=beta&t=uXxTsMLi5s7hr8FBEzVTDw7V3eJ85kpTaIC7i_5fM-Y", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:44:38+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Business Administrator at Consultancy Business investments", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQEuKXJmknr2YA/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1714545221728?e=1751500800&v=beta&t=zJG-rDZgYJJ0eROibf-Wag-v_JecCghwU3ul4TaH2Eg", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:44:38+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Economist/ Development Analyst/ Planner/ Customer Care", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQFEc3EgfdpZeg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1727782989867?e=1751500800&v=beta&t=dWjKzSu5FDRgmxAVret9jQPhWF2VjcrnmEpR2LDMC1Q", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:44:38+00:00Z"}
{"profile_url": null, "name": "Tamani Phiri", "headline": "Corporate Business Strategy | Thought Leadership | Corporate Governance", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQF4mFx8jY2n-w/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1730302954035?e=1751500800&v=beta&t=i4QIrHA6A9eLtKolwTRNhuoiaTad28sf5KHxAFuXG-w", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:44:38+00:00Z"}
{"profile_url": null, "name": "Samantha Ngandwe", "headline": "Quality Assurance and Accreditation Officer at National Health Insurance Management Authority", "followers": 382, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQHyOjyoz7d95g/profile-displayphoto-shrink_100_100/B4DZYvvhP5GwAY-/0/1744557712084?e=1751500800&v=beta&t=DLYRpz20zmwUWx1UY1Dn-ykvgWBnwn8XHWLaDMf199M", "company_handle": "https://www.linkedin.com/company/national-health-insurance-management-authority/", "captured_at": "2025-04-30T07:44:38+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Dental Surgery Assistant at Health Promotion Board", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:44:43+00:00Z"}
{"profile_url": null, "name": "Merrill Hausenfluck", "headline": "Chief Financial Officer", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQGKxDKRJM_BCg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1696292650180?e=1751500800&v=beta&t=NbUVC-QP-XL3frBpQcn3GtGrZ04Fl0xdko4V-mHxPag", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:44:43+00:00Z"}
{"profile_url": null, "name": "Mike Treash", "headline": "Senior Vice President and Chief Operating Officer at Health Alliance Plan", "followers": 2000, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQH_c6tIq929gw/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1720478900599?e=1751500800&v=beta&t=l9RLnLDKBBJjJQTsFMJMa_1MpWCKcV4AUa3dcjGnSXQ", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:44:43+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Manager at Health Alliance Plan", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:44:43+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Manager, Government Programs at Health Alliance Plan", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQF473eFGZeIpQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1654455840818?e=1751500800&v=beta&t=FllKCznSi0Ndm75QYy2i5UDtflCojNGkVzRpoChPC8c", "company_handle": "https://www.linkedin.com/company/health-alliance-plan/", "captured_at": "2025-04-30T07:44:43+00:00Z"}
{"profile_url": null, "name": "Steele Dickerson", "headline": "Insurance Recruiting Solutions", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQEyICWaE_PvXA/profile-displayphoto-shrink_100_100/B56ZQuDHyZH0Ac-/0/1735939358232?e=1751500800&v=beta&t=9FdnWHrjnPQ7LQ5FdwC7sY8sS6hm-R4zfWO5Vmwm46w", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-30T07:44:48+00:00Z"}
{"profile_url": null, "name": "Scot Dickerson", "headline": "Insurance Industry Specialist, Insurance Recruiter, Talent Acquisition, Talent Sourcing, Hiring Consultant, Career Consultant, Staffing, Executive Recruiter at Insurance Recruiting Solutions #insurancejobs #insurance", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQGLFvtPPU3HEw/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1724950672124?e=1751500800&v=beta&t=uT4SFSMF32O1d50Z0dbnd6zRRKdABHxSGlOZdxWdXBM", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-30T07:44:48+00:00Z"}
{"profile_url": null, "name": "Madeline Judas", "headline": "Recruiting Operations & Business Development Specialist", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQG6xiTaJ71UiA/profile-displayphoto-shrink_100_100/B56ZU_N_jPHoAY-/0/1740522388021?e=1751500800&v=beta&t=CxvAsYgU0zelghZsRhUJOC26ILVovP3ZPn4nMnWkEJE", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-30T07:44:48+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "All Lines Claims Adjuster / General Lines Agent (Property & Casualty : Life, Accident, Health & HMO)", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQFTjkb7SxTWWg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1725920318474?e=1751500800&v=beta&t=BGEzQg1c2l8qxuy2iKJ896nElsiYcaWnhkf-mqc-KhY", "company_handle": "https://www.linkedin.com/company/insurance-recruiting-solutions/", "captured_at": "2025-04-30T07:44:48+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Clinical Pharmacy Manager at Health Plan of San Mateo (HPSM)", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQEPO0pZOxznoA/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1551565536585?e=1751500800&v=beta&t=qwMGzWX_Zefkciq8h2m9daLMflT0WoDr5F1R5pXvyM4", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-30T07:44:54+00:00Z"}
{"profile_url": null, "name": "Tamana M.", "headline": "MPH Candidate at Brown University | Data Coordinator", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQEY3iDtFmpzlg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1714197678074?e=1751500800&v=beta&t=IsVT0uC7A-T-Tp22gZFDG9wiT7LMB5GmhccuI8f9c-I", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-30T07:44:54+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Program Manager", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-30T07:44:54+00:00Z"}
{"profile_url": null, "name": "Mackenzie Baysinger Moniz, MSW", "headline": "Program Manager at Health Plan of San Mateo", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D5603AQHAd3A4zLyuWA/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1675716742150?e=1751500800&v=beta&t=ot3fMyJFnHwwNfKJiA_YxZp6MOK_iVGtSCUgVNq867g", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-30T07:44:54+00:00Z"}
{"profile_url": null, "name": "John O.", "headline": "Healthcare Delivery Strategy Execution", "followers": null, "connection_degree": "· 3rd", "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/healthplanofsanmateo/", "captured_at": "2025-04-30T07:44:54+00:00Z"}
{"profile_url": null, "name": "Daniel McQuilkin", "headline": "Senior Vice President", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQFkScOqwhxvfQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1521406683682?e=1751500800&v=beta&t=iohhak3lrV1gpmA6dnoCxTRJidskfgmZUXKbNQbkxjs", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-30T07:44:59+00:00Z"}
{"profile_url": null, "name": "Tony Bonacuse", "headline": "Senior Vice President at Insurance Management Group", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQF_JJOFLjkZoQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1516269003018?e=1751500800&v=beta&t=0APZt5RNhvUj4IxsSdi7JO9KxezZzOH_WQCibn5Szgs", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-30T07:44:59+00:00Z"}
{"profile_url": null, "name": "Mark Bilger", "headline": "Director - Sr. Vice President at Insurance Management Group", "followers": 1000, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQEzX5qUfqhd2g/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1663842785708?e=1751500800&v=beta&t=YyKXRQol0cDntoq8vbdxyaRvEFf0vWKNHPxk0cyWiG8", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-30T07:44:59+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Senior Vice President at Insurance Management Group / Partner", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQH3dm30dXH82w/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1572228299104?e=1751500800&v=beta&t=iuBQYs4iLHJgRgjFbSA2YiNiAI8zDILqg-nVsLR9Qjk", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-30T07:44:59+00:00Z"}
{"profile_url": null, "name": "Adam Young, MBA", "headline": "Husband | Father | Traveler | Sports Fanatic | Food Enthusiast | Independent Insurance Professional", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQErWIq1AVyxKg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1601480475688?e=1751500800&v=beta&t=jK_mhX0PkDdG8WBZaipIIYRDm1PnWIuFR7sCKDhDi6s", "company_handle": "https://www.linkedin.com/company/insurance-management-group_2/", "captured_at": "2025-04-30T07:44:59+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Doctor at CareCard Health Insurance Management Co", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-30T07:45:04+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Pharmacist", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQHyPi4Amu_Dkw/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1640460490377?e=1751500800&v=beta&t=q7R_b7bD9CR-1-Dvu81WoEHN_ljHK16l6ioTIA0LN7Q", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-30T07:45:04+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "IT Manager at CareCard Health Insurance Management Co", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-30T07:45:04+00:00Z"}
{"profile_url": null, "name": "Amal Shabani", "headline": "at carecard", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C5603AQFLzeP3yPkjgg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1519625412373?e=1751500800&v=beta&t=GULSoesSn83F_fYkkH_nPxWIjjs1d9Pucc3dUDNei6I", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-30T07:45:04+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "--", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/carecard-health-insurance-management-co/", "captured_at": "2025-04-30T07:45:04+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Biologiste | Pharmaco-épidemiologie & Pharmaco-économie | Software Helath Care Management", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQHOPXrX5-oeug/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1663013895834?e=1751500800&v=beta&t=yE2RGp0rfhcJkjh_vdM0VwpaPUtoPewM80lTlr20OHU", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-30T07:45:09+00:00Z"}
{"profile_url": null, "name": "Ruqaia Ali Alkhalifa", "headline": "‎‏ RN,BSN, MSN,NE Database Officer for Scholarship Programs and Central Committee rapporteur at Al-Ahsa Health Cluster.", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4E03AQGfNujqDnuZDA/profile-displayphoto-shrink_100_100/B4EZOvsQThH0AU-/0/1733819436577?e=1751500800&v=beta&t=jleAVvhbg0H85tSi9TG96x0fqdkS1oytfaU02LHsFEI", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-30T07:45:09+00:00Z"}
{"profile_url": null, "name": "Fahad Mohyuddin", "headline": "Healthcare AI Strategist | Digital Health | SaaS | Telehealth | HIS | EHR | IoT", "followers": 7000, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQFLnPh8fu-HHg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1647320077586?e=1751500800&v=beta&t=S__knVzEVrGZuyqwszCe_5V_kawbG5tejmmEe3fkMJE", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-30T07:45:09+00:00Z"}
{"profile_url": null, "name": "Muhammad Moid Shams", "headline": "Azure DevOps | AWS Cloud Infrastructure| Freight Tech | Health Tech | HL7- NABIDH | HL7+ FHIR | KSA -NPHIES | FHIR - MOPH | HL7- Riayati | Freight Tech | Insure Tech | with Azure, Azure AI , PowerApps, D365 , M365", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D03AQEzousRurY2Zg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1711283874675?e=1751500800&v=beta&t=ZheuoRIAkS_9M8WXafdwB1nJEuy-a5HEsrXlfOANx80", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-30T07:45:09+00:00Z"}
{"profile_url": null, "name": "Muhammad Shahzaib (PMP® - SCRUM®)", "headline": "PMP-Certified Project Manager | Health Care & Web Solutions Expert | Customer Success & Operations Management Expert | Business Transformation Expert", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/D4D35AQFyp7WcBZinYA/profile-framedphoto-shrink_100_100/profile-framedphoto-shrink_100_100/0/1730638721808?e=1746604800&v=beta&t=oewST3uZcxrt48z76eiJgTxl1EPoo63Cq-JcTwrFTbs", "company_handle": "https://www.linkedin.com/company/healthcluster/", "captured_at": "2025-04-30T07:45:09+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Music Professional at Health Options Worldwide", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C5103AQGF-Dp6v6nkGw/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1585401654822?e=1751500800&v=beta&t=7yeO-dGz1p_B66cJVSlTSdAYJLMFFwxPIhwwcR8uWWo", "company_handle": "https://www.linkedin.com/company/health-options-worldwide/", "captured_at": "2025-04-30T07:45:13+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Trainer/instructor at Health Options Worldwide", "followers": null, "connection_degree": null, "avatar_url": "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "company_handle": "https://www.linkedin.com/company/health-options-worldwide/", "captured_at": "2025-04-30T07:45:13+00:00Z"}
{"profile_url": null, "name": "Michael Akpoarebe-Isaac", "headline": "Chief Operating officer, Health Partners HMO", "followers": null, "connection_degree": "· 3rd", "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQE7KNFaLMyqYg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1602714385413?e=1751500800&v=beta&t=In5GaREqoXtO3sPCx9ZJJBwIPY4008ii13RPRl0w0Fw", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-30T07:45:19+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "REGISTERED NURSE/CLAIMS SUPERVISOR/HEALTH EDUCATOR/ CASE MANAGER/ Lekki.", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4E35AQEEqf5i5pD76g/profile-framedphoto-shrink_100_100/profile-framedphoto-shrink_100_100/0/1724219552412?e=1746604800&v=beta&t=h0kqmp2KnpqQxsCCwyy7NpA8CAkSQ6qgbsZ0p0H7mXM", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-30T07:45:19+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Data Analyst|Dedicated Retention Officer Boosting Customer Loyalty| Business Developer/ Event planner", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/D4D35AQHIgeS1H7w65w/profile-framedphoto-shrink_100_100/B4DZV1QfcCGcAk-/0/1741429012517?e=1746604800&v=beta&t=zZi8WjnLpDrQD271jAId2mnfld_hO538QrN1-q2G4Zw", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-30T07:45:19+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "Former Group managing director at Health Partners Ltd", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4E03AQHPQPvIQbPQPg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1583328612508?e=1751500800&v=beta&t=LpynArccJCWrdWMSBvYLH4SI5G-xae7ECoWUUAl_CeU", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-30T07:45:19+00:00Z"}
{"profile_url": null, "name": "LinkedIn Member", "headline": "HEAD, FINCON, @ HEALTH PARTNERS (HMO) LTD", "followers": null, "connection_degree": null, "avatar_url": "https://media.licdn.com/dms/image/v2/C4D03AQG8XOvnazEibQ/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1518882054975?e=1751500800&v=beta&t=5gT6GAWGTqYfpvkjOk0ArvV73I_KspkWXgoG-VhoStg", "company_handle": "https://www.linkedin.com/company/healthpartnersng/", "captured_at": "2025-04-30T07:45:19+00:00Z"}

30
docs/apps/linkdin/schemas/company_card.json
View File

@@ -1,39 +1,51 @@
{
"name": "LinkedIn Company Card",
"baseSelector": "div.search-results-container ul[role='list'] > li",
"name": "LinkedIn Company Search Result Card",
"baseSelector": "div[data-chameleon-result-urn][data-view-name=\"search-entity-result-universal-template\"]",
"baseFields": [
{
"name": "chameleon_result_urn",
"type": "attribute",
"attribute": "data-chameleon-result-urn"
},
{
"name": "view_name",
"type": "attribute",
"attribute": "data-view-name"
}
],
"fields": [
{
"name": "handle",
"selector": "a[href*='/company/']",
"selector": "div.mb1 div.display-flex span a[data-test-app-aware-link]",
"type": "attribute",
"attribute": "href"
},
{
"name": "profile_image",
"selector": "a[href*='/company/'] img",
"selector": "div.ivm-image-view-model img",
"type": "attribute",
"attribute": "src"
},
{
"name": "name",
"selector": "span[class*='t-16'] a",
"selector": "div.mb1 div.display-flex span a[data-test-app-aware-link]",
"type": "text"
},
{
"name": "descriptor",
"selector": "div[class*='t-black t-normal']",
"selector": "div.mb1 > div[class*=\"t-14 t-black\"]",
"type": "text"
},
{
"name": "about",
"selector": "p[class*='entity-result__summary--2-lines']",
"selector": "p.entity-result__summary--2-lines",
"type": "text"
},
{
"name": "followers",
"selector": "div:contains('followers')",
"selector": "div.mb1 > div:nth-of-type(3)",
"type": "regex",
"pattern": "(\\d+)\\s*followers"
"pattern": "(\\d+[KM]?) followers"
}
]
}

31
docs/apps/linkdin/schemas/people_card.json
View File

@@ -1,38 +1,41 @@
{
"name": "LinkedIn People Card",
"name": "LinkedIn People Profile Card",
"baseSelector": "li.org-people-profile-card__profile-card-spacing",
"baseFields": [],
"fields": [
{
"name": "profile_url",
"selector": "a.eETATgYTipaVsmrBChiBJJvFsdPhNpulhPZUVLHLo",
"selector": "div.artdeco-entity-lockup__title a[data-test-app-aware-link]",
"type": "attribute",
"attribute": "href"
},
{
"name": "avatar_url",
"selector": "div.artdeco-entity-lockup__image img",
"type": "attribute",
"attribute": "src"
},
{
"name": "name",
"selector": ".artdeco-entity-lockup__title .lt-line-clamp--single-line",
"selector": "div.artdeco-entity-lockup__title a div.lt-line-clamp--single-line",
"type": "text"
},
{
"name": "headline",
"selector": ".artdeco-entity-lockup__subtitle .lt-line-clamp--multi-line",
"selector": "div.artdeco-entity-lockup__subtitle div.lt-line-clamp--multi-line",
"type": "text"
},
{
"name": "followers",
"selector": ".lt-line-clamp--multi-line.t-12",
"type": "text"
"selector": "span.text-align-center span.lt-line-clamp--multi-line",
"type": "regex",
"pattern": "(\\d+)"
},
{
"name": "connection_degree",
"selector": ".artdeco-entity-lockup__badge .artdeco-entity-lockup__degree",
"type": "text"
},
{
"name": "avatar_url",
"selector": ".artdeco-entity-lockup__image img",
"type": "attribute",
"attribute": "src"
"selector": "span.artdeco-entity-lockup__degree",
"type": "regex",
"pattern": "(\\d+\\w+)"
}
]
}

75
docs/apps/linkdin/snippets/company.html
View File

@@ -1,36 +1,34 @@
<li class="yCLWzruNprmIzaZzFFonVFBtMrbaVYnuDFA">
<li class="kZRArQqqhjjrHYceWaFbyEGWHRZbtqjTMawKA">
<!----><!---->
<div class="IxlEPbRZwQYrRltKPvHAyjBmCdIWTAoYo" data-chameleon-result-urn="urn:li:company:362492"
<div class="xAuWirHJDUTuhkfOpmJApZWziplUyPIc" data-chameleon-result-urn="urn:li:company:2095237"
data-view-name="search-entity-result-universal-template">
<div class="linked-area flex-1
cursor-pointer">
cursor-pointer">
<div class="BAEgVqVuxosMJZodcelsgPoyRcrkiqgVCGHXNQ">
<div class="afcvrbGzNuyRlhPPQWrWirJtUdHAAtUlqxwvVA">
<div class="qMGLeKnJyQnibGOueKodvnfLgWpsuA">
<div class="cBPGFfFovHsbNhBFmECDIsPgMWmtMozOUfIAbs">
<div class="display-flex align-items-center">
<!---->
<a class="eETATgYTipaVsmrBChiBJJvFsdPhNpulhPZUVLHLo scale-down " aria-hidden="true"
tabindex="-1" href="https://www.linkedin.com/company/managment-research-services-inc./"
data-test-app-aware-link="">
<a class="sDWEFrcVubKuUVGggeBOYqLlgYgPbojOc scale-down " aria-hidden="true" tabindex="-1"
href="https://www.linkedin.com/company/health-insurance/" data-test-app-aware-link="">
<div class="ivm-image-view-model ">
<div class="ivm-view-attr__img-wrapper
">
">
<!---->
<!----> <img width="48"
src="https://media.licdn.com/dms/image/v2/C560BAQFWpusEOgW-ww/company-logo_100_100/company-logo_100_100/0/1630583697877/managment_research_services_inc_logo?e=1750896000&amp;v=beta&amp;t=Ch9vyEZdfng-1D1m_XqP5kjNpVXUBKkk9cNhMZUhx0E"
loading="lazy" height="48" alt="Management Research Services, Inc. (MRS, Inc)"
id="ember28"
src="https://media.licdn.com/dms/image/v2/C560BAQEXIoLSJbShlw/company-logo_100_100/company-logo_100_100/0/1662748332921/health_insurance_logo?e=1753920000&amp;v=beta&amp;t=p2ZNMYNsC9KSlp-sIqMYuc88avBTjKF4CqDobq1Xr2M"
loading="lazy" height="48" alt="Health Insurance" id="ember28"
class="ivm-view-attr__img--centered EntityPhoto-square-3 evi-image lazy-image ember-view">
</div>
@@ -42,7 +40,7 @@
</div>
</div>
<div
class="wympnVuDByXHvafWrMGJLZuchDmCRqLmWPwg MmzCPRicJimZvjJhvqTzDcDbdHhWPzspERzA pt3 pb3 t-12 t-black--light">
class="BNxZPngZfeRnDrIUbICgBZvQjRvMAUnwCHuDrmRg yNRlrJOHDflDBnYPLbVmiAkUsCUZKUznmAc pt3 pb3 t-12 t-black--light">
<div class="mb1">
<div class="t-roman t-sans">
@@ -50,13 +48,14 @@
<div class="display-flex">
<span class="TikBXjihYvcNUoIzkslUaEjfIuLmYxfs OoHEyXgsiIqGADjcOtTmfdpoYVXrLKTvkwI ">
<span class="CgaWLOzmXNuKbRIRARSErqCJcBPYudEKo
t-16">
<a class="eETATgYTipaVsmrBChiBJJvFsdPhNpulhPZUVLHLo "
href="https://www.linkedin.com/company/managment-research-services-inc./"
<span
class="kmApjJVnFerynwITxTBSCqzqgoHwVfkiA HHGiVqODTCkszDUDWwPGPJGUPfAeRpygAKwwLePrQ ">
<span class="OjTMoZLoiuspGuWWptwqxZRcMcHZBoSDxfig
t-16">
<a class="sDWEFrcVubKuUVGggeBOYqLlgYgPbojOc "
href="https://www.linkedin.com/company/health-insurance/"
data-test-app-aware-link="">
<!---->Management Research Services, Inc. (MRS, Inc)<!---->
<!---->Health Insurance<!---->
<!----> </a>
<!----> </span>
</span>
@@ -69,14 +68,14 @@
<div class="LjmdKCEqKITHihFOiQsBAQylkdnsWhqZii
t-14 t-black t-normal">
<!---->Insurance • Milwaukee, Wisconsin<!---->
<div class="kFTZPhxHBbvnnRxiRPmTxafKGLUNSiaeInag
t-14 t-black t-normal">
<!---->Insurance • Cardiff, CA<!---->
</div>
<div class="cTPhJiHyNLmxdQYFlsEOutjznmqrVHUByZwZ
t-14 t-normal">
<!---->1K followers<!---->
<div class="FlWUwyrEUZpkVCgzGTDwUHTLntfZNseavlY
t-14 t-normal">
<!---->3K followers<!---->
</div>
@@ -86,23 +85,19 @@
</div>
<!---->
<p class="yWzlqwKNlvCWVNoKqmzoDDEnBMUuyynaLg
entity-result__summary--2-lines
t-12 t-black--light
">
<!---->MRS combines 30 years of experience supporting the Life,<span class="white-space-pre">
</span><strong><!---->Health<!----></strong><span class="white-space-pre"> </span>and
Annuities<span class="white-space-pre"> </span><strong><!---->Insurance<!----></strong><span
class="white-space-pre"> </span>Industry with customized<span class="white-space-pre">
</span><strong><!---->insurance<!----></strong><span class="white-space-pre">
</span>underwriting solutions that efficiently support clients’ workflows. Supported by the
Agenium Platform (www.agenium.ai) our innovative underwriting solutions are guaranteed to
optimize requirements...<!---->
<p class="JBUEKeXhPyClEtYwdsASPYsZsCkTvUBqsDUs
entity-result__summary--2-lines
t-12 t-black--light
">
<!---->Your<span class="white-space-pre"> </span><strong><!---->health<!----></strong><span
class="white-space-pre"> </span><!----><!----><strong><!---->insurance<!----></strong><span
class="white-space-pre"> </span>expert for all stages of your life; Medicare, Individuals,
Families, Small Groups, CoveredCA.<!---->
</p>
<!---->
</div>
<div class="qXxdnXtzRVFTnTnetmNpssucBwQBsWlUuk MmzCPRicJimZvjJhvqTzDcDbdHhWPzspERzA">
<div class="JZcKRppsWfaxfMaqtvfVwEeAtzNwryBOMdo yNRlrJOHDflDBnYPLbVmiAkUsCUZKUznmAc">
<!---->
@@ -111,7 +106,7 @@
<button aria-label="Follow Management Research Services, Inc. (MRS, Inc)" id="ember61"
<button aria-label="Follow Health Insurance" id="ember49"
class="artdeco-button artdeco-button--2 artdeco-button--secondary ember-view"
type="button"><!---->
<span class="artdeco-button__text">

51
docs/apps/linkdin/snippets/people.html
View File

@@ -2,41 +2,40 @@
<div>
<section class="artdeco-card full-width qQdPErXQkSAbwApNgNfuxukTIPPykttCcZGOHk">
<section class="artdeco-card full-width IxXiAcHfbZpayHVZUYdQwfYOkMbOirmr">
<!---->
<img width="210" src="data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"
ariarole="presentation" loading="lazy" height="210" alt="" id="ember96"
ariarole="presentation" loading="lazy" height="210" alt="" id="ember102"
class="evi-image lazy-image ghost-default ember-view org-people-profile-card__cover-photo org-people-profile-card__cover-photo--people">
<div class="org-people-profile-card__profile-info">
<div id="ember97"
<div id="ember103"
class="artdeco-entity-lockup artdeco-entity-lockup--stacked-center artdeco-entity-lockup--size-7 ember-view">
<div id="ember98"
<div id="ember104"
class="artdeco-entity-lockup__image artdeco-entity-lockup__image--type-circle ember-view"
type="circle">
<a class="eETATgYTipaVsmrBChiBJJvFsdPhNpulhPZUVLHLo "
id="org-people-profile-card__profile-image-0"
href="https://www.linkedin.com/in/speakerrayna?miniProfileUrn=urn%3Ali%3Afs_miniProfile%3AACoAABsqUBoBr5x071PuGGpNtK3NlvSARiVXPIs"
<a class="sDWEFrcVubKuUVGggeBOYqLlgYgPbojOc " id="org-people-profile-card__profile-image-0"
href="https://www.linkedin.com/in/ericweberhcbd?miniProfileUrn=urn%3Ali%3Afs_miniProfile%3AACoAAABVh2MBFoyTaAxDqYQQcW8oGxVsqsKioHw"
data-test-app-aware-link="">
<img width="104"
src="https://media.licdn.com/dms/image/v2/D5603AQGs2Vyju4xZ7A/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1681741067031?e=1750896000&amp;v=beta&amp;t=Hvj--IrrmpVIH7pec7-l_PQok8vsS__CGeUqBWOw7co"
loading="lazy" height="104" alt="Dr. Rayna S." id="ember99"
src="https://media.licdn.com/dms/image/v2/C4D03AQHNP9KoXtSrkg/profile-displayphoto-shrink_100_100/profile-displayphoto-shrink_100_100/0/1573501774845?e=1753920000&amp;v=beta&amp;t=JYsY56biGUmDzbYj2ORZMcd1dSm2IRWCA-IM3KNFLw8"
loading="lazy" height="104" alt="Eric Weber" id="ember105"
class="evi-image lazy-image ember-view">
</a>
</div>
<div id="ember100" class="artdeco-entity-lockup__content ember-view">
<div id="ember101" class="artdeco-entity-lockup__title ember-view">
<a class="eETATgYTipaVsmrBChiBJJvFsdPhNpulhPZUVLHLo link-without-visited-state"
aria-label="View Dr. Rayna S.’s profile"
href="https://www.linkedin.com/in/speakerrayna?miniProfileUrn=urn%3Ali%3Afs_miniProfile%3AACoAABsqUBoBr5x071PuGGpNtK3NlvSARiVXPIs"
<div id="ember106" class="artdeco-entity-lockup__content ember-view">
<div id="ember107" class="artdeco-entity-lockup__title ember-view">
<a class="sDWEFrcVubKuUVGggeBOYqLlgYgPbojOc link-without-visited-state"
aria-label="View Eric Weber’s profile"
href="https://www.linkedin.com/in/ericweberhcbd?miniProfileUrn=urn%3Ali%3Afs_miniProfile%3AACoAAABVh2MBFoyTaAxDqYQQcW8oGxVsqsKioHw"
data-test-app-aware-link="">
<div id="ember103" class="ember-view lt-line-clamp lt-line-clamp--single-line AGabuksChUpCmjWshSnaZryLKSthOKkwclxY
t-black" style="">
Dr. Rayna S.
<div id="ember109" class="ember-view lt-line-clamp lt-line-clamp--single-line rMKrzkehlCEvJWoQjDQJFaHmBFAYQLMGrNY
t-black" style="">
Eric Weber
<!---->
</div>
@@ -44,33 +43,33 @@
</a>
</div>
<div id="ember104" class="artdeco-entity-lockup__badge ember-view"> <span class="a11y-text">3rd+
<div id="ember110" class="artdeco-entity-lockup__badge ember-view"> <span class="a11y-text">3rd+
degree connection</span>
<span class="artdeco-entity-lockup__degree" aria-hidden="true">
·&nbsp;3rd
·&nbsp;3rd
</span>
<!----><!---->
</div>
<div id="ember105" class="artdeco-entity-lockup__subtitle ember-view">
<div id="ember111" class="artdeco-entity-lockup__subtitle ember-view">
<div class="t-14 t-black--light t-normal">
<div id="ember107" class="ember-view lt-line-clamp lt-line-clamp--multi-line"
<div id="ember113" class="ember-view lt-line-clamp lt-line-clamp--multi-line"
style="-webkit-line-clamp: 2">
Leadership and Talent Development Consultant and Professional Speaker
HIPN Executive Editor | Healthcare BizDev CEO — Health Insurance Plan News.
<!---->
</div>
</div>
</div>
<div id="ember108" class="artdeco-entity-lockup__caption ember-view"></div>
<div id="ember114" class="artdeco-entity-lockup__caption ember-view"></div>
</div>
</div>
<span class="text-align-center">
<span id="ember110"
<span id="ember116"
class="ember-view lt-line-clamp lt-line-clamp--multi-line t-12 t-black--light mt2"
style="-webkit-line-clamp: 3">
727 followers
10K followers
<!----> </span>
@@ -78,7 +77,7 @@
</div>
<footer class="ph3 pb3">
<button aria-label="Follow Dr. Rayna S." id="ember111"
<button aria-label="Follow Eric Weber" id="ember117"
class="artdeco-button artdeco-button--2 artdeco-button--secondary ember-view full-width"
type="button"><!---->
<span class="artdeco-button__text">

5
docs/apps/linkdin/templates/graph_view_template.html
View File

@@ -447,10 +447,7 @@
dragNodes: true,
dragView: true,
zoomView: true,
mouseWheel: {
speed: 0.15, // Reduced from default 1.0
smooth: true // Enable smooth zooming
}
zoomSpeed: 0.15 // Reduced from default 1.0
},
nodes: {
font: {

416
docs/blog/release-v0.7.0.md Normal file
View File

@@ -0,0 +1,416 @@
# 🚀 Crawl4AI v0.7.0: The Adaptive Intelligence Update
*January 28, 2025 • 10 min read*
---
Today I'm releasing Crawl4AI v0.7.0—the Adaptive Intelligence Update. This release introduces fundamental improvements in how Crawl4AI handles modern web complexity through adaptive learning, intelligent content discovery, and advanced extraction capabilities.
## 🎯 What's New at a Glance
- **Adaptive Crawling**: Your crawler now learns and adapts to website patterns
- **Virtual Scroll Support**: Complete content extraction from infinite scroll pages
- **Link Preview with 3-Layer Scoring**: Intelligent link analysis and prioritization
- **Async URL Seeder**: Discover thousands of URLs in seconds with intelligent filtering
- **PDF Parsing**: Extract data from PDF documents
- **Performance Optimizations**: Significant speed and memory improvements
## 🧠 Adaptive Crawling: Intelligence Through Pattern Learning
**The Problem:** Websites change. Class names shift. IDs disappear. Your carefully crafted selectors break at 3 AM, and you wake up to empty datasets and angry stakeholders.
**My Solution:** I implemented an adaptive learning system that observes patterns, builds confidence scores, and adjusts extraction strategies on the fly. It's like having a junior developer who gets better at their job with every page they scrape.
### Technical Deep-Dive
The Adaptive Crawler maintains a persistent state for each domain, tracking:
- Pattern success rates
- Selector stability over time
- Content structure variations
- Extraction confidence scores
```python
from crawl4ai import AdaptiveCrawler, AdaptiveConfig, CrawlState
# Initialize with custom learning parameters
config = AdaptiveConfig(
confidence_threshold=0.7, # Min confidence to use learned patterns
max_history=100, # Remember last 100 crawls per domain
learning_rate=0.2, # How quickly to adapt to changes
patterns_per_page=3, # Patterns to learn per page type
extraction_strategy='css' # 'css' or 'xpath'
)
adaptive_crawler = AdaptiveCrawler(config)
# First crawl - crawler learns the structure
async with AsyncWebCrawler() as crawler:
result = await crawler.arun(
"https://news.example.com/article/12345",
config=CrawlerRunConfig(
adaptive_config=config,
extraction_hints={ # Optional hints to speed up learning
"title": "article h1",
"content": "article .body-content"
}
)
)
# Crawler identifies and stores patterns
if result.success:
state = adaptive_crawler.get_state("news.example.com")
print(f"Learned {len(state.patterns)} patterns")
print(f"Confidence: {state.avg_confidence:.2%}")
# Subsequent crawls - uses learned patterns
result2 = await crawler.arun(
"https://news.example.com/article/67890",
config=CrawlerRunConfig(adaptive_config=config)
)
# Automatically extracts using learned patterns!
```
**Expected Real-World Impact:**
- **News Aggregation**: Maintain 95%+ extraction accuracy even as news sites update their templates
- **E-commerce Monitoring**: Track product changes across hundreds of stores without constant maintenance
- **Research Data Collection**: Build robust academic datasets that survive website redesigns
- **Reduced Maintenance**: Cut selector update time by 80% for frequently-changing sites
## 🌊 Virtual Scroll: Complete Content Capture
**The Problem:** Modern web apps only render what's visible. Scroll down, new content appears, old content vanishes into the void. Traditional crawlers capture that first viewport and miss 90% of the content. It's like reading only the first page of every book.
**My Solution:** I built Virtual Scroll support that mimics human browsing behavior, capturing content as it loads and preserving it before the browser's garbage collector strikes.
### Implementation Details
```python
from crawl4ai import VirtualScrollConfig
# For social media feeds (Twitter/X style)
twitter_config = VirtualScrollConfig(
container_selector="[data-testid='primaryColumn']",
scroll_count=20, # Number of scrolls
scroll_by="container_height", # Smart scrolling by container size
wait_after_scroll=1.0, # Let content load
capture_method="incremental", # Capture new content on each scroll
deduplicate=True # Remove duplicate elements
)
# For e-commerce product grids (Instagram style)
grid_config = VirtualScrollConfig(
container_selector="main .product-grid",
scroll_count=30,
scroll_by=800, # Fixed pixel scrolling
wait_after_scroll=1.5, # Images need time
stop_on_no_change=True # Smart stopping
)
# For news feeds with lazy loading
news_config = VirtualScrollConfig(
container_selector=".article-feed",
scroll_count=50,
scroll_by="page_height", # Viewport-based scrolling
wait_after_scroll=0.5,
wait_for_selector=".article-card", # Wait for specific elements
timeout=30000 # Max 30 seconds total
)
# Use it in your crawl
async with AsyncWebCrawler() as crawler:
result = await crawler.arun(
"https://twitter.com/trending",
config=CrawlerRunConfig(
virtual_scroll_config=twitter_config,
# Combine with other features
extraction_strategy=JsonCssExtractionStrategy({
"tweets": {
"selector": "[data-testid='tweet']",
"fields": {
"text": {"selector": "[data-testid='tweetText']", "type": "text"},
"likes": {"selector": "[data-testid='like']", "type": "text"}
}
}
})
)
)
print(f"Captured {len(result.extracted_content['tweets'])} tweets")
```
**Key Capabilities:**
- **DOM Recycling Awareness**: Detects and handles virtual DOM element recycling
- **Smart Scroll Physics**: Three modes - container height, page height, or fixed pixels
- **Content Preservation**: Captures content before it's destroyed
- **Intelligent Stopping**: Stops when no new content appears
- **Memory Efficient**: Streams content instead of holding everything in memory
**Expected Real-World Impact:**
- **Social Media Analysis**: Capture entire Twitter threads with hundreds of replies, not just top 10
- **E-commerce Scraping**: Extract 500+ products from infinite scroll catalogs vs. 20-50 with traditional methods
- **News Aggregation**: Get all articles from modern news sites, not just above-the-fold content
- **Research Applications**: Complete data extraction from academic databases using virtual pagination
## 🔗 Link Preview: Intelligent Link Analysis and Scoring
**The Problem:** You crawl a page and get 200 links. Which ones matter? Which lead to the content you actually want? Traditional crawlers force you to follow everything or build complex filters.
**My Solution:** I implemented a three-layer scoring system that analyzes links like a human would—considering their position, context, and relevance to your goals.
### The Three-Layer Scoring System
```python
from crawl4ai import LinkPreviewConfig
# Configure intelligent link analysis
link_config = LinkPreviewConfig(
# What to analyze
include_internal=True,
include_external=True,
max_links=100, # Analyze top 100 links
# Relevance scoring
query="machine learning tutorials", # Your interest
score_threshold=0.3, # Minimum relevance score
# Performance
concurrent_requests=10, # Parallel processing
timeout_per_link=5000, # 5s per link
# Advanced scoring weights
scoring_weights={
"intrinsic": 0.3, # Link quality indicators
"contextual": 0.5, # Relevance to query
"popularity": 0.2 # Link prominence
}
)
# Use in your crawl
result = await crawler.arun(
"https://tech-blog.example.com",
config=CrawlerRunConfig(
link_preview_config=link_config,
score_links=True
)
)
# Access scored and sorted links
for link in result.links["internal"][:10]: # Top 10 internal links
print(f"Score: {link['total_score']:.3f}")
print(f" Intrinsic: {link['intrinsic_score']:.1f}/10") # Position, attributes
print(f" Contextual: {link['contextual_score']:.1f}/1") # Relevance to query
print(f" URL: {link['href']}")
print(f" Title: {link['head_data']['title']}")
print(f" Description: {link['head_data']['meta']['description'][:100]}...")
```
**Scoring Components:**
1. **Intrinsic Score (0-10)**: Based on link quality indicators
- Position on page (navigation, content, footer)
- Link attributes (rel, title, class names)
- Anchor text quality and length
- URL structure and depth
2. **Contextual Score (0-1)**: Relevance to your query
- Semantic similarity using embeddings
- Keyword matching in link text and title
- Meta description analysis
- Content preview scoring
3. **Total Score**: Weighted combination for final ranking
**Expected Real-World Impact:**
- **Research Efficiency**: Find relevant papers 10x faster by following only high-score links
- **Competitive Analysis**: Automatically identify important pages on competitor sites
- **Content Discovery**: Build topic-focused crawlers that stay on track
- **SEO Audits**: Identify and prioritize high-value internal linking opportunities
## 🎣 Async URL Seeder: Automated URL Discovery at Scale
**The Problem:** You want to crawl an entire domain but only have the homepage. Or worse, you want specific content types across thousands of pages. Manual URL discovery? That's a job for machines, not humans.
**My Solution:** I built Async URL Seeder—a turbocharged URL discovery engine that combines multiple sources with intelligent filtering and relevance scoring.
### Technical Architecture
```python
from crawl4ai import AsyncUrlSeeder, SeedingConfig
# Basic discovery - find all product pages
seeder_config = SeedingConfig(
# Discovery sources
source="sitemap+cc", # Sitemap + Common Crawl
# Filtering
pattern="*/product/*", # URL pattern matching
ignore_patterns=["*/reviews/*", "*/questions/*"],
# Validation
live_check=True, # Verify URLs are alive
max_urls=5000, # Stop at 5000 URLs
# Performance
concurrency=100, # Parallel requests
hits_per_sec=10 # Rate limiting
)
seeder = AsyncUrlSeeder(seeder_config)
urls = await seeder.discover("https://shop.example.com")
# Advanced: Relevance-based discovery
research_config = SeedingConfig(
source="crawl+sitemap", # Deep crawl + sitemap
pattern="*/blog/*", # Blog posts only
# Content relevance
extract_head=True, # Get meta tags
query="quantum computing tutorials",
scoring_method="bm25", # Or "semantic" (coming soon)
score_threshold=0.4, # High relevance only
# Smart filtering
filter_nonsense_urls=True, # Remove .xml, .txt, etc.
min_content_length=500, # Skip thin content
force=True # Bypass cache
)
# Discover with progress tracking
discovered = []
async for batch in seeder.discover_iter("https://physics-blog.com", research_config):
discovered.extend(batch)
print(f"Found {len(discovered)} relevant URLs so far...")
# Results include scores and metadata
for url_data in discovered[:5]:
print(f"URL: {url_data['url']}")
print(f"Score: {url_data['score']:.3f}")
print(f"Title: {url_data['title']}")
```
**Discovery Methods:**
- **Sitemap Mining**: Parses robots.txt and all linked sitemaps
- **Common Crawl**: Queries the Common Crawl index for historical URLs
- **Intelligent Crawling**: Follows links with smart depth control
- **Pattern Analysis**: Learns URL structures and generates variations
**Expected Real-World Impact:**
- **Migration Projects**: Discover 10,000+ URLs from legacy sites in under 60 seconds
- **Market Research**: Map entire competitor ecosystems automatically
- **Academic Research**: Build comprehensive datasets without manual URL collection
- **SEO Audits**: Find every indexable page with content scoring
- **Content Archival**: Ensure no content is left behind during site migrations
## ⚡ Performance Optimizations
This release includes significant performance improvements through optimized resource handling, better concurrency management, and reduced memory footprint.
### What We Optimized
```python
# Before v0.7.0 (slow)
results = []
for url in urls:
result = await crawler.arun(url)
results.append(result)
# After v0.7.0 (fast)
# Automatic batching and connection pooling
results = await crawler.arun_batch(
urls,
config=CrawlerRunConfig(
# New performance options
batch_size=10, # Process 10 URLs concurrently
reuse_browser=True, # Keep browser warm
eager_loading=False, # Load only what's needed
streaming_extraction=True, # Stream large extractions
# Optimized defaults
wait_until="domcontentloaded", # Faster than networkidle
exclude_external_resources=True, # Skip third-party assets
block_ads=True # Ad blocking built-in
)
)
# Memory-efficient streaming for large crawls
async for result in crawler.arun_stream(large_url_list):
# Process results as they complete
await process_result(result)
# Memory is freed after each iteration
```
**Performance Gains:**
- **Startup Time**: 70% faster browser initialization
- **Page Loading**: 40% reduction with smart resource blocking
- **Extraction**: 3x faster with compiled CSS selectors
- **Memory Usage**: 60% reduction with streaming processing
- **Concurrent Crawls**: Handle 5x more parallel requests
## 📄 PDF Support
PDF extraction is now natively supported in Crawl4AI.
```python
# Extract data from PDF documents
result = await crawler.arun(
"https://example.com/report.pdf",
config=CrawlerRunConfig(
pdf_extraction=True,
extraction_strategy=JsonCssExtractionStrategy({
# Works on converted PDF structure
"title": {"selector": "h1", "type": "text"},
"sections": {"selector": "h2", "type": "list"}
})
)
)
```
## 🔧 Important Changes
### Breaking Changes
- `link_extractor` renamed to `link_preview` (better reflects functionality)
- Minimum Python version now 3.9
- `CrawlerConfig` split into `CrawlerRunConfig` and `BrowserConfig`
### Migration Guide
```python
# Old (v0.6.x)
from crawl4ai import CrawlerConfig
config = CrawlerConfig(timeout=30000)
# New (v0.7.0)
from crawl4ai import CrawlerRunConfig, BrowserConfig
browser_config = BrowserConfig(timeout=30000)
run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
```
## 🤖 Coming Soon: Intelligent Web Automation
I'm currently working on bringing advanced automation capabilities to Crawl4AI. This includes:
- **Crawl Agents**: Autonomous crawlers that understand your goals and adapt their strategies
- **Auto JS Generation**: Automatic JavaScript code generation for complex interactions
- **Smart Form Handling**: Intelligent form detection and filling
- **Context-Aware Actions**: Crawlers that understand page context and make decisions
These features are under active development and will revolutionize how we approach web automation. Stay tuned!
## 🚀 Get Started
```bash
pip install crawl4ai==0.7.0
```
Check out the [updated documentation](https://docs.crawl4ai.com).
Questions? Issues? I'm always listening:
- GitHub: [github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
- Discord: [discord.gg/crawl4ai](https://discord.gg/jP8KfhDhyN)
- Twitter: [@unclecode](https://x.com/unclecode)
Happy crawling! 🕷️
---
*P.S. If you're using Crawl4AI in production, I'd love to hear about it. Your use cases inspire the next features.*

85
docs/examples/adaptive_crawling/README.md Normal file
View File

@@ -0,0 +1,85 @@
# Adaptive Crawling Examples
This directory contains examples demonstrating various aspects of Crawl4AI's Adaptive Crawling feature.
## Examples Overview
### 1. `basic_usage.py`
- Simple introduction to adaptive crawling
- Uses default statistical strategy
- Shows how to get crawl statistics and relevant content
### 2. `embedding_strategy.py` ⭐ NEW
- Demonstrates the embedding-based strategy for semantic understanding
- Shows query expansion and irrelevance detection
- Includes configuration for both local and API-based embeddings
### 3. `embedding_vs_statistical.py` ⭐ NEW
- Direct comparison between statistical and embedding strategies
- Helps you choose the right strategy for your use case
- Shows performance and accuracy trade-offs
### 4. `embedding_configuration.py` ⭐ NEW
- Advanced configuration options for embedding strategy
- Parameter tuning guide for different scenarios
- Examples for research, exploration, and quality-focused crawling
### 5. `advanced_configuration.py`
- Shows various configuration options for both strategies
- Demonstrates threshold tuning and performance optimization
### 6. `custom_strategies.py`
- How to implement your own crawling strategy
- Extends the base CrawlStrategy class
- Advanced use case for specialized requirements
### 7. `export_import_kb.py`
- Export crawled knowledge base to JSONL
- Import and continue crawling from saved state
- Useful for building persistent knowledge bases
## Quick Start
For your first adaptive crawling experience, run:
```bash
python basic_usage.py
```
To try the new embedding strategy with semantic understanding:
```bash
python embedding_strategy.py
```
To compare strategies and see which works best for your use case:
```bash
python embedding_vs_statistical.py
```
## Strategy Selection Guide
### Use Statistical Strategy (Default) When:
- Working with technical documentation
- Queries contain specific terms or code
- Speed is critical
- No API access available
### Use Embedding Strategy When:
- Queries are conceptual or ambiguous
- Need semantic understanding beyond exact matches
- Want to detect irrelevant content
- Working with diverse content sources
## Requirements
- Crawl4AI installed
- For embedding strategy with local models: `sentence-transformers`
- For embedding strategy with OpenAI: Set `OPENAI_API_KEY` environment variable
## Learn More
- [Adaptive Crawling Documentation](https://docs.crawl4ai.com/core/adaptive-crawling/)
- [Mathematical Framework](https://github.com/unclecode/crawl4ai/blob/main/PROGRESSIVE_CRAWLING.md)
- [Blog: The Adaptive Crawling Revolution](https://docs.crawl4ai.com/blog/adaptive-crawling-revolution/)

207
docs/examples/adaptive_crawling/advanced_configuration.py Normal file
View File

@@ -0,0 +1,207 @@
"""
Advanced Adaptive Crawling Configuration
This example demonstrates all configuration options available for adaptive crawling,
including threshold tuning, persistence, and custom parameters.
"""
import asyncio
from pathlib import Path
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
async def main():
"""Demonstrate advanced configuration options"""
# Example 1: Custom thresholds for different use cases
print("="*60)
print("EXAMPLE 1: Custom Confidence Thresholds")
print("="*60)
# High-precision configuration (exhaustive crawling)
high_precision_config = AdaptiveConfig(
confidence_threshold=0.9, # Very high confidence required
max_pages=50, # Allow more pages
top_k_links=5, # Follow more links per page
min_gain_threshold=0.02 # Lower threshold to continue
)
# Balanced configuration (default use case)
balanced_config = AdaptiveConfig(
confidence_threshold=0.7, # Moderate confidence
max_pages=20, # Reasonable limit
top_k_links=3, # Moderate branching
min_gain_threshold=0.05 # Standard gain threshold
)
# Quick exploration configuration
quick_config = AdaptiveConfig(
confidence_threshold=0.5, # Lower confidence acceptable
max_pages=10, # Strict limit
top_k_links=2, # Minimal branching
min_gain_threshold=0.1 # High gain required
)
async with AsyncWebCrawler(verbose=False) as crawler:
# Test different configurations
for config_name, config in [
("High Precision", high_precision_config),
("Balanced", balanced_config),
("Quick Exploration", quick_config)
]:
print(f"\nTesting {config_name} configuration...")
adaptive = AdaptiveCrawler(crawler, config=config)
result = await adaptive.digest(
start_url="https://httpbin.org",
query="http headers authentication"
)
print(f" - Pages crawled: {len(result.crawled_urls)}")
print(f" - Confidence achieved: {adaptive.confidence:.2%}")
print(f" - Coverage score: {adaptive.coverage_stats['coverage']:.2f}")
# Example 2: Persistence and state management
print("\n" + "="*60)
print("EXAMPLE 2: State Persistence")
print("="*60)
state_file = "crawl_state_demo.json"
# Configuration with persistence
persistent_config = AdaptiveConfig(
confidence_threshold=0.8,
max_pages=30,
save_state=True, # Enable auto-save
state_path=state_file # Specify save location
)
async with AsyncWebCrawler(verbose=False) as crawler:
# First crawl - will be interrupted
print("\nStarting initial crawl (will interrupt after 5 pages)...")
interrupt_config = AdaptiveConfig(
confidence_threshold=0.8,
max_pages=5, # Artificially low to simulate interruption
save_state=True,
state_path=state_file
)
adaptive = AdaptiveCrawler(crawler, config=interrupt_config)
result1 = await adaptive.digest(
start_url="https://docs.python.org/3/",
query="exception handling try except finally"
)
print(f"First crawl completed: {len(result1.crawled_urls)} pages")
print(f"Confidence reached: {adaptive.confidence:.2%}")
# Resume crawl with higher page limit
print("\nResuming crawl from saved state...")
resume_config = AdaptiveConfig(
confidence_threshold=0.8,
max_pages=20, # Increase limit
save_state=True,
state_path=state_file
)
adaptive2 = AdaptiveCrawler(crawler, config=resume_config)
result2 = await adaptive2.digest(
start_url="https://docs.python.org/3/",
query="exception handling try except finally",
resume_from=state_file
)
print(f"Resumed crawl completed: {len(result2.crawled_urls)} total pages")
print(f"Final confidence: {adaptive2.confidence:.2%}")
# Clean up
Path(state_file).unlink(missing_ok=True)
# Example 3: Link selection strategies
print("\n" + "="*60)
print("EXAMPLE 3: Link Selection Strategies")
print("="*60)
# Conservative link following
conservative_config = AdaptiveConfig(
confidence_threshold=0.7,
max_pages=15,
top_k_links=1, # Only follow best link
min_gain_threshold=0.15 # High threshold
)
# Aggressive link following
aggressive_config = AdaptiveConfig(
confidence_threshold=0.7,
max_pages=15,
top_k_links=10, # Follow many links
min_gain_threshold=0.01 # Very low threshold
)
async with AsyncWebCrawler(verbose=False) as crawler:
for strategy_name, config in [
("Conservative", conservative_config),
("Aggressive", aggressive_config)
]:
print(f"\n{strategy_name} link selection:")
adaptive = AdaptiveCrawler(crawler, config=config)
result = await adaptive.digest(
start_url="https://httpbin.org",
query="api endpoints"
)
# Analyze crawl pattern
print(f" - Total pages: {len(result.crawled_urls)}")
print(f" - Unique domains: {len(set(url.split('/')[2] for url in result.crawled_urls))}")
print(f" - Max depth reached: {max(url.count('/') for url in result.crawled_urls) - 2}")
# Show saturation trend
if hasattr(result, 'new_terms_history') and result.new_terms_history:
print(f" - New terms discovered: {result.new_terms_history[:5]}...")
print(f" - Saturation trend: {'decreasing' if result.new_terms_history[-1] < result.new_terms_history[0] else 'increasing'}")
# Example 4: Monitoring crawl progress
print("\n" + "="*60)
print("EXAMPLE 4: Progress Monitoring")
print("="*60)
# Configuration with detailed monitoring
monitor_config = AdaptiveConfig(
confidence_threshold=0.75,
max_pages=10,
top_k_links=3
)
async with AsyncWebCrawler(verbose=False) as crawler:
adaptive = AdaptiveCrawler(crawler, config=monitor_config)
# Start crawl
print("\nMonitoring crawl progress...")
result = await adaptive.digest(
start_url="https://httpbin.org",
query="http methods headers"
)
# Detailed statistics
print("\nDetailed crawl analysis:")
adaptive.print_stats(detailed=True)
# Export for analysis
print("\nExporting knowledge base for external analysis...")
adaptive.export_knowledge_base("knowledge_export_demo.jsonl")
print("Knowledge base exported to: knowledge_export_demo.jsonl")
# Show sample of exported data
with open("knowledge_export_demo.jsonl", 'r') as f:
first_line = f.readline()
print(f"Sample export: {first_line[:100]}...")
# Clean up
Path("knowledge_export_demo.jsonl").unlink(missing_ok=True)
if __name__ == "__main__":
asyncio.run(main())

76
docs/examples/adaptive_crawling/basic_usage.py Normal file
View File

@@ -0,0 +1,76 @@
"""
Basic Adaptive Crawling Example
This example demonstrates the simplest use case of adaptive crawling:
finding information about a specific topic and knowing when to stop.
"""
import asyncio
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler
async def main():
"""Basic adaptive crawling example"""
# Initialize the crawler
async with AsyncWebCrawler(verbose=True) as crawler:
# Create an adaptive crawler with default settings (statistical strategy)
adaptive = AdaptiveCrawler(crawler)
# Note: You can also use embedding strategy for semantic understanding:
# from crawl4ai import AdaptiveConfig
# config = AdaptiveConfig(strategy="embedding")
# adaptive = AdaptiveCrawler(crawler, config)
# Start adaptive crawling
print("Starting adaptive crawl for Python async programming information...")
result = await adaptive.digest(
start_url="https://docs.python.org/3/library/asyncio.html",
query="async await context managers coroutines"
)
# Display crawl statistics
print("\n" + "="*50)
print("CRAWL STATISTICS")
print("="*50)
adaptive.print_stats(detailed=False)
# Get the most relevant content found
print("\n" + "="*50)
print("MOST RELEVANT PAGES")
print("="*50)
relevant_pages = adaptive.get_relevant_content(top_k=5)
for i, page in enumerate(relevant_pages, 1):
print(f"\n{i}. {page['url']}")
print(f" Relevance Score: {page['score']:.2%}")
# Show a snippet of the content
content = page['content'] or ""
if content:
snippet = content[:200].replace('\n', ' ')
if len(content) > 200:
snippet += "..."
print(f" Preview: {snippet}")
# Show final confidence
print(f"\n{'='*50}")
print(f"Final Confidence: {adaptive.confidence:.2%}")
print(f"Total Pages Crawled: {len(result.crawled_urls)}")
print(f"Knowledge Base Size: {len(adaptive.state.knowledge_base)} documents")
# Example: Check if we can answer specific questions
print(f"\n{'='*50}")
print("INFORMATION SUFFICIENCY CHECK")
print(f"{'='*50}")
if adaptive.confidence >= 0.8:
print("✓ High confidence - can answer detailed questions about async Python")
elif adaptive.confidence >= 0.6:
print("~ Moderate confidence - can answer basic questions")
else:
print("✗ Low confidence - need more information")
if __name__ == "__main__":
asyncio.run(main())

373
docs/examples/adaptive_crawling/custom_strategies.py Normal file
View File

@@ -0,0 +1,373 @@
"""
Custom Adaptive Crawling Strategies
This example demonstrates how to implement custom scoring strategies
for domain-specific crawling needs.
"""
import asyncio
import re
from typing import List, Dict, Set
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
from crawl4ai.adaptive_crawler import CrawlState, Link
import math
class APIDocumentationStrategy:
"""
Custom strategy optimized for API documentation crawling.
Prioritizes endpoint references, code examples, and parameter descriptions.
"""
def __init__(self):
# Keywords that indicate high-value API documentation
self.api_keywords = {
'endpoint', 'request', 'response', 'parameter', 'authentication',
'header', 'body', 'query', 'path', 'method', 'get', 'post', 'put',
'delete', 'patch', 'status', 'code', 'example', 'curl', 'python'
}
# URL patterns that typically contain API documentation
self.valuable_patterns = [
r'/api/',
r'/reference/',
r'/endpoints?/',
r'/methods?/',
r'/resources?/'
]
# Patterns to avoid
self.avoid_patterns = [
r'/blog/',
r'/news/',
r'/about/',
r'/contact/',
r'/legal/'
]
def score_link(self, link: Link, query: str, state: CrawlState) -> float:
"""Custom link scoring for API documentation"""
score = 1.0
url = link.href.lower()
# Boost API-related URLs
for pattern in self.valuable_patterns:
if re.search(pattern, url):
score *= 2.0
break
# Reduce score for non-API content
for pattern in self.avoid_patterns:
if re.search(pattern, url):
score *= 0.1
break
# Boost if preview contains API keywords
if link.text:
preview_lower = link.text.lower()
keyword_count = sum(1 for kw in self.api_keywords if kw in preview_lower)
score *= (1 + keyword_count * 0.2)
# Prioritize shallow URLs (likely overview pages)
depth = url.count('/') - 2 # Subtract protocol slashes
if depth <= 3:
score *= 1.5
elif depth > 6:
score *= 0.5
return score
def calculate_api_coverage(self, state: CrawlState, query: str) -> Dict[str, float]:
"""Calculate specialized coverage metrics for API documentation"""
metrics = {
'endpoint_coverage': 0.0,
'example_coverage': 0.0,
'parameter_coverage': 0.0
}
# Analyze knowledge base for API-specific content
endpoint_patterns = [r'GET\s+/', r'POST\s+/', r'PUT\s+/', r'DELETE\s+/']
example_patterns = [r'```\w+', r'curl\s+-', r'import\s+requests']
param_patterns = [r'param(?:eter)?s?\s*:', r'required\s*:', r'optional\s*:']
total_docs = len(state.knowledge_base)
if total_docs == 0:
return metrics
docs_with_endpoints = 0
docs_with_examples = 0
docs_with_params = 0
for doc in state.knowledge_base:
content = doc.markdown.raw_markdown if hasattr(doc, 'markdown') else str(doc)
# Check for endpoints
if any(re.search(pattern, content, re.IGNORECASE) for pattern in endpoint_patterns):
docs_with_endpoints += 1
# Check for examples
if any(re.search(pattern, content, re.IGNORECASE) for pattern in example_patterns):
docs_with_examples += 1
# Check for parameters
if any(re.search(pattern, content, re.IGNORECASE) for pattern in param_patterns):
docs_with_params += 1
metrics['endpoint_coverage'] = docs_with_endpoints / total_docs
metrics['example_coverage'] = docs_with_examples / total_docs
metrics['parameter_coverage'] = docs_with_params / total_docs
return metrics
class ResearchPaperStrategy:
"""
Strategy optimized for crawling research papers and academic content.
Prioritizes citations, abstracts, and methodology sections.
"""
def __init__(self):
self.academic_keywords = {
'abstract', 'introduction', 'methodology', 'results', 'conclusion',
'references', 'citation', 'paper', 'study', 'research', 'analysis',
'hypothesis', 'experiment', 'findings', 'doi'
}
self.citation_patterns = [
r'\[\d+\]', # [1] style citations
r'\(\w+\s+\d{4}\)', # (Author 2024) style
r'doi:\s*\S+', # DOI references
]
def calculate_academic_relevance(self, content: str, query: str) -> float:
"""Calculate relevance score for academic content"""
score = 0.0
content_lower = content.lower()
# Check for academic keywords
keyword_matches = sum(1 for kw in self.academic_keywords if kw in content_lower)
score += keyword_matches * 0.1
# Check for citations
citation_count = sum(
len(re.findall(pattern, content))
for pattern in self.citation_patterns
)
score += min(citation_count * 0.05, 1.0) # Cap at 1.0
# Check for query terms in academic context
query_terms = query.lower().split()
for term in query_terms:
# Boost if term appears near academic keywords
for keyword in ['abstract', 'conclusion', 'results']:
if keyword in content_lower:
section = content_lower[content_lower.find(keyword):content_lower.find(keyword) + 500]
if term in section:
score += 0.2
return min(score, 2.0) # Cap total score
async def demo_custom_strategies():
"""Demonstrate custom strategy usage"""
# Example 1: API Documentation Strategy
print("="*60)
print("EXAMPLE 1: Custom API Documentation Strategy")
print("="*60)
api_strategy = APIDocumentationStrategy()
async with AsyncWebCrawler() as crawler:
# Standard adaptive crawler
config = AdaptiveConfig(
confidence_threshold=0.8,
max_pages=15
)
adaptive = AdaptiveCrawler(crawler, config)
# Override link scoring with custom strategy
original_rank_links = adaptive._rank_links
def custom_rank_links(links, query, state):
# Apply custom scoring
scored_links = []
for link in links:
base_score = api_strategy.score_link(link, query, state)
scored_links.append((link, base_score))
# Sort by score
scored_links.sort(key=lambda x: x[1], reverse=True)
return [link for link, _ in scored_links[:config.top_k_links]]
adaptive._rank_links = custom_rank_links
# Crawl API documentation
print("\nCrawling API documentation with custom strategy...")
state = await adaptive.digest(
start_url="https://httpbin.org",
query="api endpoints authentication headers"
)
# Calculate custom metrics
api_metrics = api_strategy.calculate_api_coverage(state, "api endpoints")
print(f"\nResults:")
print(f"Pages crawled: {len(state.crawled_urls)}")
print(f"Confidence: {adaptive.confidence:.2%}")
print(f"\nAPI-Specific Metrics:")
print(f" - Endpoint coverage: {api_metrics['endpoint_coverage']:.2%}")
print(f" - Example coverage: {api_metrics['example_coverage']:.2%}")
print(f" - Parameter coverage: {api_metrics['parameter_coverage']:.2%}")
# Example 2: Combined Strategy
print("\n" + "="*60)
print("EXAMPLE 2: Hybrid Strategy Combining Multiple Approaches")
print("="*60)
class HybridStrategy:
"""Combines multiple strategies with weights"""
def __init__(self):
self.api_strategy = APIDocumentationStrategy()
self.research_strategy = ResearchPaperStrategy()
self.weights = {
'api': 0.7,
'research': 0.3
}
def score_content(self, content: str, query: str) -> float:
# Get scores from each strategy
api_score = self._calculate_api_score(content, query)
research_score = self.research_strategy.calculate_academic_relevance(content, query)
# Weighted combination
total_score = (
api_score * self.weights['api'] +
research_score * self.weights['research']
)
return total_score
def _calculate_api_score(self, content: str, query: str) -> float:
# Simplified API scoring based on keyword presence
content_lower = content.lower()
api_keywords = self.api_strategy.api_keywords
keyword_count = sum(1 for kw in api_keywords if kw in content_lower)
return min(keyword_count * 0.1, 2.0)
hybrid_strategy = HybridStrategy()
async with AsyncWebCrawler() as crawler:
adaptive = AdaptiveCrawler(crawler)
# Crawl with hybrid scoring
print("\nTesting hybrid strategy on technical documentation...")
state = await adaptive.digest(
start_url="https://docs.python.org/3/library/asyncio.html",
query="async await coroutines api"
)
# Analyze results with hybrid strategy
print(f"\nHybrid Strategy Analysis:")
total_score = 0
for doc in adaptive.get_relevant_content(top_k=5):
content = doc['content'] or ""
score = hybrid_strategy.score_content(content, "async await api")
total_score += score
print(f" - {doc['url'][:50]}... Score: {score:.2f}")
print(f"\nAverage hybrid score: {total_score/5:.2f}")
async def demo_performance_optimization():
"""Demonstrate performance optimization with custom strategies"""
print("\n" + "="*60)
print("EXAMPLE 3: Performance-Optimized Strategy")
print("="*60)
class PerformanceOptimizedStrategy:
"""Strategy that balances thoroughness with speed"""
def __init__(self):
self.url_cache: Set[str] = set()
self.domain_scores: Dict[str, float] = {}
def should_crawl_domain(self, url: str) -> bool:
"""Implement domain-level filtering"""
domain = url.split('/')[2] if url.startswith('http') else url
# Skip if we've already crawled many pages from this domain
domain_count = sum(1 for cached in self.url_cache if domain in cached)
if domain_count > 5:
return False
# Skip low-scoring domains
if domain in self.domain_scores and self.domain_scores[domain] < 0.3:
return False
return True
def update_domain_score(self, url: str, relevance: float):
"""Track domain-level performance"""
domain = url.split('/')[2] if url.startswith('http') else url
if domain not in self.domain_scores:
self.domain_scores[domain] = relevance
else:
# Moving average
self.domain_scores[domain] = (
0.7 * self.domain_scores[domain] + 0.3 * relevance
)
perf_strategy = PerformanceOptimizedStrategy()
async with AsyncWebCrawler() as crawler:
config = AdaptiveConfig(
confidence_threshold=0.7,
max_pages=10,
top_k_links=2 # Fewer links for speed
)
adaptive = AdaptiveCrawler(crawler, config)
# Track performance
import time
start_time = time.time()
state = await adaptive.digest(
start_url="https://httpbin.org",
query="http methods headers"
)
elapsed = time.time() - start_time
print(f"\nPerformance Results:")
print(f" - Time elapsed: {elapsed:.2f} seconds")
print(f" - Pages crawled: {len(state.crawled_urls)}")
print(f" - Pages per second: {len(state.crawled_urls)/elapsed:.2f}")
print(f" - Final confidence: {adaptive.confidence:.2%}")
print(f" - Efficiency: {adaptive.confidence/len(state.crawled_urls):.2%} confidence per page")
async def main():
"""Run all demonstrations"""
try:
await demo_custom_strategies()
await demo_performance_optimization()
print("\n" + "="*60)
print("All custom strategy examples completed!")
print("="*60)
except Exception as e:
print(f"Error: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
asyncio.run(main())

206
docs/examples/adaptive_crawling/embedding_configuration.py Normal file
View File

@@ -0,0 +1,206 @@
"""
Advanced Embedding Configuration Example
This example demonstrates all configuration options available for the
embedding strategy, including fine-tuning parameters for different use cases.
"""
import asyncio
import os
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
async def test_configuration(name: str, config: AdaptiveConfig, url: str, query: str):
"""Test a specific configuration"""
print(f"\n{'='*60}")
print(f"Configuration: {name}")
print(f"{'='*60}")
async with AsyncWebCrawler(verbose=False) as crawler:
adaptive = AdaptiveCrawler(crawler, config)
result = await adaptive.digest(start_url=url, query=query)
print(f"Pages crawled: {len(result.crawled_urls)}")
print(f"Final confidence: {adaptive.confidence:.1%}")
print(f"Stopped reason: {result.metrics.get('stopped_reason', 'max_pages')}")
if result.metrics.get('is_irrelevant', False):
print("⚠️ Query detected as irrelevant!")
return result
async def main():
"""Demonstrate various embedding configurations"""
print("EMBEDDING STRATEGY CONFIGURATION EXAMPLES")
print("=" * 60)
# Base URL and query for testing
test_url = "https://docs.python.org/3/library/asyncio.html"
# 1. Default Configuration
config_default = AdaptiveConfig(
strategy="embedding",
max_pages=10
)
await test_configuration(
"Default Settings",
config_default,
test_url,
"async programming patterns"
)
# 2. Strict Coverage Requirements
config_strict = AdaptiveConfig(
strategy="embedding",
max_pages=20,
# Stricter similarity requirements
embedding_k_exp=5.0, # Default is 3.0, higher = stricter
embedding_coverage_radius=0.15, # Default is 0.2, lower = stricter
# Higher validation threshold
embedding_validation_min_score=0.6, # Default is 0.3
# More query variations for better coverage
n_query_variations=15 # Default is 10
)
await test_configuration(
"Strict Coverage (Research/Academic)",
config_strict,
test_url,
"comprehensive guide async await"
)
# 3. Fast Exploration
config_fast = AdaptiveConfig(
strategy="embedding",
max_pages=10,
top_k_links=5, # Follow more links per page
# Relaxed requirements for faster convergence
embedding_k_exp=1.0, # Lower = more lenient
embedding_min_relative_improvement=0.05, # Stop earlier
# Lower quality thresholds
embedding_quality_min_confidence=0.5, # Display lower confidence
embedding_quality_max_confidence=0.85,
# Fewer query variations for speed
n_query_variations=5
)
await test_configuration(
"Fast Exploration (Quick Overview)",
config_fast,
test_url,
"async basics"
)
# 4. Irrelevance Detection Focus
config_irrelevance = AdaptiveConfig(
strategy="embedding",
max_pages=5,
# Aggressive irrelevance detection
embedding_min_confidence_threshold=0.2, # Higher threshold (default 0.1)
embedding_k_exp=5.0, # Strict similarity
# Quick stopping for irrelevant content
embedding_min_relative_improvement=0.15
)
await test_configuration(
"Irrelevance Detection",
config_irrelevance,
test_url,
"recipe for chocolate cake" # Irrelevant query
)
# 5. High-Quality Knowledge Base
config_quality = AdaptiveConfig(
strategy="embedding",
max_pages=30,
# Deduplication settings
embedding_overlap_threshold=0.75, # More aggressive deduplication
# Quality focus
embedding_validation_min_score=0.5,
embedding_quality_scale_factor=1.0, # Linear quality mapping
# Balanced parameters
embedding_k_exp=3.0,
embedding_nearest_weight=0.8, # Focus on best matches
embedding_top_k_weight=0.2
)
await test_configuration(
"High-Quality Knowledge Base",
config_quality,
test_url,
"asyncio advanced patterns best practices"
)
# 6. Custom Embedding Provider
if os.getenv('OPENAI_API_KEY'):
config_openai = AdaptiveConfig(
strategy="embedding",
max_pages=10,
# Use OpenAI embeddings
embedding_llm_config={
'provider': 'openai/text-embedding-3-small',
'api_token': os.getenv('OPENAI_API_KEY')
},
# OpenAI embeddings are high quality, can be stricter
embedding_k_exp=4.0,
n_query_variations=12
)
await test_configuration(
"OpenAI Embeddings",
config_openai,
test_url,
"event-driven architecture patterns"
)
# Parameter Guide
print("\n" + "="*60)
print("PARAMETER TUNING GUIDE")
print("="*60)
print("\n📊 Key Parameters and Their Effects:")
print("\n1. embedding_k_exp (default: 3.0)")
print(" - Lower (1-2): More lenient, faster convergence")
print(" - Higher (4-5): Stricter, better precision")
print("\n2. embedding_coverage_radius (default: 0.2)")
print(" - Lower (0.1-0.15): Requires closer matches")
print(" - Higher (0.25-0.3): Accepts broader matches")
print("\n3. n_query_variations (default: 10)")
print(" - Lower (5-7): Faster, less comprehensive")
print(" - Higher (15-20): Better coverage, slower")
print("\n4. embedding_min_confidence_threshold (default: 0.1)")
print(" - Set to 0.15-0.2 for aggressive irrelevance detection")
print(" - Set to 0.05 to crawl even barely relevant content")
print("\n5. embedding_validation_min_score (default: 0.3)")
print(" - Higher (0.5-0.6): Requires strong validation")
print(" - Lower (0.2): More permissive stopping")
print("\n💡 Tips:")
print("- For research: High k_exp, more variations, strict validation")
print("- For exploration: Low k_exp, fewer variations, relaxed thresholds")
print("- For quality: Focus on overlap_threshold and validation scores")
print("- For speed: Reduce variations, increase min_relative_improvement")
if __name__ == "__main__":
asyncio.run(main())

109
docs/examples/adaptive_crawling/embedding_strategy.py Normal file
View File

@@ -0,0 +1,109 @@
"""
Embedding Strategy Example for Adaptive Crawling
This example demonstrates how to use the embedding-based strategy
for semantic understanding and intelligent crawling.
"""
import asyncio
import os
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
async def main():
"""Demonstrate embedding strategy for adaptive crawling"""
# Configure embedding strategy
config = AdaptiveConfig(
strategy="embedding", # Use embedding strategy
embedding_model="sentence-transformers/all-MiniLM-L6-v2", # Default model
n_query_variations=10, # Generate 10 semantic variations
max_pages=15,
top_k_links=3,
min_gain_threshold=0.05,
# Embedding-specific parameters
embedding_k_exp=3.0, # Higher = stricter similarity requirements
embedding_min_confidence_threshold=0.1, # Stop if <10% relevant
embedding_validation_min_score=0.4 # Validation threshold
)
# Optional: Use OpenAI embeddings instead
if os.getenv('OPENAI_API_KEY'):
config.embedding_llm_config = {
'provider': 'openai/text-embedding-3-small',
'api_token': os.getenv('OPENAI_API_KEY')
}
print("Using OpenAI embeddings")
else:
print("Using sentence-transformers (local embeddings)")
async with AsyncWebCrawler(verbose=True) as crawler:
adaptive = AdaptiveCrawler(crawler, config)
# Test 1: Relevant query with semantic understanding
print("\n" + "="*50)
print("TEST 1: Semantic Query Understanding")
print("="*50)
result = await adaptive.digest(
start_url="https://docs.python.org/3/library/asyncio.html",
query="concurrent programming event-driven architecture"
)
print("\nQuery Expansion:")
print(f"Original query expanded to {len(result.expanded_queries)} variations")
for i, q in enumerate(result.expanded_queries[:3], 1):
print(f" {i}. {q}")
print(" ...")
print("\nResults:")
adaptive.print_stats(detailed=False)
# Test 2: Detecting irrelevant queries
print("\n" + "="*50)
print("TEST 2: Irrelevant Query Detection")
print("="*50)
# Reset crawler for new query
adaptive = AdaptiveCrawler(crawler, config)
result = await adaptive.digest(
start_url="https://docs.python.org/3/library/asyncio.html",
query="how to bake chocolate chip cookies"
)
if result.metrics.get('is_irrelevant', False):
print("\n✅ Successfully detected irrelevant query!")
print(f"Stopped after just {len(result.crawled_urls)} pages")
print(f"Reason: {result.metrics.get('stopped_reason', 'unknown')}")
else:
print("\n❌ Failed to detect irrelevance")
print(f"Final confidence: {adaptive.confidence:.1%}")
# Test 3: Semantic gap analysis
print("\n" + "="*50)
print("TEST 3: Semantic Gap Analysis")
print("="*50)
# Show how embedding strategy identifies gaps
adaptive = AdaptiveCrawler(crawler, config)
result = await adaptive.digest(
start_url="https://realpython.com",
query="python decorators advanced patterns"
)
print(f"\nSemantic gaps identified: {len(result.semantic_gaps)}")
print(f"Knowledge base embeddings shape: {result.kb_embeddings.shape if result.kb_embeddings is not None else 'None'}")
# Show coverage metrics specific to embedding strategy
print("\nEmbedding-specific metrics:")
print(f" Average best similarity: {result.metrics.get('avg_best_similarity', 0):.3f}")
print(f" Coverage score: {result.metrics.get('coverage_score', 0):.3f}")
print(f" Validation confidence: {result.metrics.get('validation_confidence', 0):.2%}")
if __name__ == "__main__":
asyncio.run(main())

167
docs/examples/adaptive_crawling/embedding_vs_statistical.py Normal file
View File

@@ -0,0 +1,167 @@
"""
Comparison: Embedding vs Statistical Strategy
This example demonstrates the differences between statistical and embedding
strategies for adaptive crawling, showing when to use each approach.
"""
import asyncio
import time
import os
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
async def crawl_with_strategy(url: str, query: str, strategy: str, **kwargs):
"""Helper function to crawl with a specific strategy"""
config = AdaptiveConfig(
strategy=strategy,
max_pages=20,
top_k_links=3,
min_gain_threshold=0.05,
**kwargs
)
async with AsyncWebCrawler(verbose=False) as crawler:
adaptive = AdaptiveCrawler(crawler, config)
start_time = time.time()
result = await adaptive.digest(start_url=url, query=query)
elapsed = time.time() - start_time
return {
'result': result,
'crawler': adaptive,
'elapsed': elapsed,
'pages': len(result.crawled_urls),
'confidence': adaptive.confidence
}
async def main():
"""Compare embedding and statistical strategies"""
# Test scenarios
test_cases = [
{
'name': 'Technical Documentation (Specific Terms)',
'url': 'https://docs.python.org/3/library/asyncio.html',
'query': 'asyncio.create_task event_loop.run_until_complete'
},
{
'name': 'Conceptual Query (Semantic Understanding)',
'url': 'https://docs.python.org/3/library/asyncio.html',
'query': 'concurrent programming patterns'
},
{
'name': 'Ambiguous Query',
'url': 'https://realpython.com',
'query': 'python performance optimization'
}
]
# Configure embedding strategy
embedding_config = {}
if os.getenv('OPENAI_API_KEY'):
embedding_config['embedding_llm_config'] = {
'provider': 'openai/text-embedding-3-small',
'api_token': os.getenv('OPENAI_API_KEY')
}
for test in test_cases:
print("\n" + "="*70)
print(f"TEST: {test['name']}")
print(f"URL: {test['url']}")
print(f"Query: '{test['query']}'")
print("="*70)
# Run statistical strategy
print("\n📊 Statistical Strategy:")
stat_result = await crawl_with_strategy(
test['url'],
test['query'],
'statistical'
)
print(f" Pages crawled: {stat_result['pages']}")
print(f" Time taken: {stat_result['elapsed']:.2f}s")
print(f" Confidence: {stat_result['confidence']:.1%}")
print(f" Sufficient: {'Yes' if stat_result['crawler'].is_sufficient else 'No'}")
# Show term coverage
if hasattr(stat_result['result'], 'term_frequencies'):
query_terms = test['query'].lower().split()
covered = sum(1 for term in query_terms
if term in stat_result['result'].term_frequencies)
print(f" Term coverage: {covered}/{len(query_terms)} query terms found")
# Run embedding strategy
print("\n🧠 Embedding Strategy:")
emb_result = await crawl_with_strategy(
test['url'],
test['query'],
'embedding',
**embedding_config
)
print(f" Pages crawled: {emb_result['pages']}")
print(f" Time taken: {emb_result['elapsed']:.2f}s")
print(f" Confidence: {emb_result['confidence']:.1%}")
print(f" Sufficient: {'Yes' if emb_result['crawler'].is_sufficient else 'No'}")
# Show semantic understanding
if emb_result['result'].expanded_queries:
print(f" Query variations: {len(emb_result['result'].expanded_queries)}")
print(f" Semantic gaps: {len(emb_result['result'].semantic_gaps)}")
# Compare results
print("\n📈 Comparison:")
efficiency_diff = ((stat_result['pages'] - emb_result['pages']) /
stat_result['pages'] * 100) if stat_result['pages'] > 0 else 0
print(f" Efficiency: ", end="")
if efficiency_diff > 0:
print(f"Embedding used {efficiency_diff:.0f}% fewer pages")
else:
print(f"Statistical used {-efficiency_diff:.0f}% fewer pages")
print(f" Speed: ", end="")
if stat_result['elapsed'] < emb_result['elapsed']:
print(f"Statistical was {emb_result['elapsed']/stat_result['elapsed']:.1f}x faster")
else:
print(f"Embedding was {stat_result['elapsed']/emb_result['elapsed']:.1f}x faster")
print(f" Confidence difference: {abs(stat_result['confidence'] - emb_result['confidence'])*100:.0f} percentage points")
# Recommendation
print("\n💡 Recommendation:")
if 'specific' in test['name'].lower() or all(len(term) > 5 for term in test['query'].split()):
print(" → Statistical strategy is likely better for this use case (specific terms)")
elif 'conceptual' in test['name'].lower() or 'semantic' in test['name'].lower():
print(" → Embedding strategy is likely better for this use case (semantic understanding)")
else:
if emb_result['confidence'] > stat_result['confidence'] + 0.1:
print(" → Embedding strategy achieved significantly better understanding")
elif stat_result['elapsed'] < emb_result['elapsed'] / 2:
print(" → Statistical strategy is much faster with similar results")
else:
print(" → Both strategies performed similarly; choose based on your priorities")
# Summary recommendations
print("\n" + "="*70)
print("STRATEGY SELECTION GUIDE")
print("="*70)
print("\n✅ Use STATISTICAL strategy when:")
print(" - Queries contain specific technical terms")
print(" - Speed is critical")
print(" - No API access available")
print(" - Working with well-structured documentation")
print("\n✅ Use EMBEDDING strategy when:")
print(" - Queries are conceptual or ambiguous")
print(" - Semantic understanding is important")
print(" - Need to detect irrelevant content")
print(" - Working with diverse content sources")
if __name__ == "__main__":
asyncio.run(main())

232
docs/examples/adaptive_crawling/export_import_kb.py Normal file
View File

@@ -0,0 +1,232 @@
"""
Knowledge Base Export and Import
This example demonstrates how to export crawled knowledge bases and
import them for reuse, sharing, or analysis.
"""
import asyncio
import json
from pathlib import Path
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
async def build_knowledge_base():
"""Build a knowledge base about web technologies"""
print("="*60)
print("PHASE 1: Building Knowledge Base")
print("="*60)
async with AsyncWebCrawler(verbose=False) as crawler:
adaptive = AdaptiveCrawler(crawler)
# Crawl information about HTTP
print("\n1. Gathering HTTP protocol information...")
await adaptive.digest(
start_url="https://httpbin.org",
query="http methods headers status codes"
)
print(f" - Pages crawled: {len(adaptive.state.crawled_urls)}")
print(f" - Confidence: {adaptive.confidence:.2%}")
# Add more information about APIs
print("\n2. Adding API documentation knowledge...")
await adaptive.digest(
start_url="https://httpbin.org/anything",
query="rest api json response request"
)
print(f" - Total pages: {len(adaptive.state.crawled_urls)}")
print(f" - Confidence: {adaptive.confidence:.2%}")
# Export the knowledge base
export_path = "web_tech_knowledge.jsonl"
print(f"\n3. Exporting knowledge base to {export_path}")
adaptive.export_knowledge_base(export_path)
# Show export statistics
export_size = Path(export_path).stat().st_size / 1024
with open(export_path, 'r') as f:
line_count = sum(1 for _ in f)
print(f" - Exported {line_count} documents")
print(f" - File size: {export_size:.1f} KB")
return export_path
async def analyze_knowledge_base(kb_path):
"""Analyze the exported knowledge base"""
print("\n" + "="*60)
print("PHASE 2: Analyzing Exported Knowledge Base")
print("="*60)
# Read and analyze JSONL
documents = []
with open(kb_path, 'r') as f:
for line in f:
documents.append(json.loads(line))
print(f"\nKnowledge base contains {len(documents)} documents:")
# Analyze document properties
total_content_length = 0
urls_by_domain = {}
for doc in documents:
# Content analysis
content_length = len(doc.get('content', ''))
total_content_length += content_length
# URL analysis
url = doc.get('url', '')
domain = url.split('/')[2] if url.startswith('http') else 'unknown'
urls_by_domain[domain] = urls_by_domain.get(domain, 0) + 1
# Show sample document
if documents.index(doc) == 0:
print(f"\nSample document structure:")
print(f" - URL: {url}")
print(f" - Content length: {content_length} chars")
print(f" - Has metadata: {'metadata' in doc}")
print(f" - Has links: {len(doc.get('links', []))} links")
print(f" - Query: {doc.get('query', 'N/A')}")
print(f"\nContent statistics:")
print(f" - Total content: {total_content_length:,} characters")
print(f" - Average per document: {total_content_length/len(documents):,.0f} chars")
print(f"\nDomain distribution:")
for domain, count in urls_by_domain.items():
print(f" - {domain}: {count} pages")
async def import_and_continue():
"""Import a knowledge base and continue crawling"""
print("\n" + "="*60)
print("PHASE 3: Importing and Extending Knowledge Base")
print("="*60)
kb_path = "web_tech_knowledge.jsonl"
async with AsyncWebCrawler(verbose=False) as crawler:
# Create new adaptive crawler
adaptive = AdaptiveCrawler(crawler)
# Import existing knowledge base
print(f"\n1. Importing knowledge base from {kb_path}")
adaptive.import_knowledge_base(kb_path)
print(f" - Imported {len(adaptive.state.knowledge_base)} documents")
print(f" - Existing URLs: {len(adaptive.state.crawled_urls)}")
# Check current state
print("\n2. Checking imported knowledge state:")
adaptive.print_stats(detailed=False)
# Continue crawling with new query
print("\n3. Extending knowledge with new query...")
await adaptive.digest(
start_url="https://httpbin.org/status/200",
query="error handling retry timeout"
)
print("\n4. Final knowledge base state:")
adaptive.print_stats(detailed=False)
# Export extended knowledge base
extended_path = "web_tech_knowledge_extended.jsonl"
adaptive.export_knowledge_base(extended_path)
print(f"\n5. Extended knowledge base exported to {extended_path}")
async def share_knowledge_bases():
"""Demonstrate sharing knowledge bases between projects"""
print("\n" + "="*60)
print("PHASE 4: Sharing Knowledge Between Projects")
print("="*60)
# Simulate two different projects
project_a_kb = "project_a_knowledge.jsonl"
project_b_kb = "project_b_knowledge.jsonl"
async with AsyncWebCrawler(verbose=False) as crawler:
# Project A: Security documentation
print("\n1. Project A: Building security knowledge...")
crawler_a = AdaptiveCrawler(crawler)
await crawler_a.digest(
start_url="https://httpbin.org/basic-auth/user/pass",
query="authentication security headers"
)
crawler_a.export_knowledge_base(project_a_kb)
print(f" - Exported {len(crawler_a.state.knowledge_base)} documents")
# Project B: API testing
print("\n2. Project B: Building testing knowledge...")
crawler_b = AdaptiveCrawler(crawler)
await crawler_b.digest(
start_url="https://httpbin.org/anything",
query="testing endpoints mocking"
)
crawler_b.export_knowledge_base(project_b_kb)
print(f" - Exported {len(crawler_b.state.knowledge_base)} documents")
# Merge knowledge bases
print("\n3. Merging knowledge bases...")
merged_crawler = AdaptiveCrawler(crawler)
# Import both knowledge bases
merged_crawler.import_knowledge_base(project_a_kb)
initial_size = len(merged_crawler.state.knowledge_base)
merged_crawler.import_knowledge_base(project_b_kb)
final_size = len(merged_crawler.state.knowledge_base)
print(f" - Project A documents: {initial_size}")
print(f" - Additional from Project B: {final_size - initial_size}")
print(f" - Total merged documents: {final_size}")
# Export merged knowledge
merged_kb = "merged_knowledge.jsonl"
merged_crawler.export_knowledge_base(merged_kb)
print(f"\n4. Merged knowledge base exported to {merged_kb}")
# Show combined coverage
print("\n5. Combined knowledge coverage:")
merged_crawler.print_stats(detailed=False)
async def main():
"""Run all examples"""
try:
# Build initial knowledge base
kb_path = await build_knowledge_base()
# Analyze the export
await analyze_knowledge_base(kb_path)
# Import and extend
await import_and_continue()
# Demonstrate sharing
await share_knowledge_bases()
print("\n" + "="*60)
print("All examples completed successfully!")
print("="*60)
finally:
# Clean up generated files
print("\nCleaning up generated files...")
for file in [
"web_tech_knowledge.jsonl",
"web_tech_knowledge_extended.jsonl",
"project_a_knowledge.jsonl",
"project_b_knowledge.jsonl",
"merged_knowledge.jsonl"
]:
Path(file).unlink(missing_ok=True)
print("Cleanup complete.")
if __name__ == "__main__":
asyncio.run(main())

2
docs/examples/amazon_product_extraction_direct_url.py
View File

@@ -5,7 +5,7 @@ prices, ratings, and other details using CSS selectors.
"""
from crawl4ai import AsyncWebCrawler
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
from crawl4ai import JsonCssExtractionStrategy
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
import json

2
docs/examples/amazon_product_extraction_using_hooks.py
View File

@@ -5,7 +5,7 @@ prices, ratings, and other details using CSS selectors.
"""
from crawl4ai import AsyncWebCrawler, CacheMode
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
from crawl4ai import JsonCssExtractionStrategy
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
import json
from playwright.async_api import Page, BrowserContext

2
docs/examples/amazon_product_extraction_using_use_javascript.py
View File

@@ -5,7 +5,7 @@ prices, ratings, and other details using CSS selectors.
"""
from crawl4ai import AsyncWebCrawler, CacheMode
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
from crawl4ai import JsonCssExtractionStrategy
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
import json

BIN
docs/examples/assets/instagram_grid_result.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.6 MiB

132
docs/examples/assets/virtual_scroll_append_only.html Normal file
View File

@@ -0,0 +1,132 @@
<!DOCTYPE html>
<html>
<head>
<title>Append-Only Scroll (Traditional Infinite Scroll)</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 0;
padding: 20px;
background-color: #f5f5f5;
}
h1 {
color: #333;
text-align: center;
}
.posts-container {
max-width: 800px;
margin: 0 auto;
background: white;
border: 1px solid #ddd;
border-radius: 5px;
padding: 20px;
}
.post {
background: #f9f9f9;
padding: 15px;
margin-bottom: 15px;
border-radius: 5px;
border: 1px solid #eee;
}
.post-title {
font-size: 18px;
font-weight: bold;
color: #2c3e50;
margin-bottom: 10px;
}
.post-content {
color: #555;
line-height: 1.6;
}
.loading {
text-align: center;
padding: 20px;
color: #888;
}
</style>
</head>
<body>
<h1>Traditional Infinite Scroll Demo</h1>
<p style="text-align: center; color: #666;">This appends new content without removing old content</p>
<div class="posts-container"></div>
<script>
// Traditional infinite scroll - APPENDS content
const container = document.querySelector('.posts-container');
const totalPosts = 200;
const postsPerPage = 20;
let loadedPosts = 0;
let isLoading = false;
// Generate fake post data
function generatePost(index) {
return {
id: index,
title: `Post Title #${index + 1}`,
content: `This is the content of post ${index + 1}. In traditional infinite scroll, new content is appended to existing content. The DOM keeps growing. Post ID: ${index}`
};
}
// Load more posts - APPENDS to existing content
function loadMorePosts() {
if (isLoading || loadedPosts >= totalPosts) return;
isLoading = true;
// Show loading indicator
const loadingDiv = document.createElement('div');
loadingDiv.className = 'loading';
loadingDiv.textContent = 'Loading more posts...';
container.appendChild(loadingDiv);
// Simulate network delay
setTimeout(() => {
// Remove loading indicator
container.removeChild(loadingDiv);
// Add new posts
const fragment = document.createDocumentFragment();
const endIndex = Math.min(loadedPosts + postsPerPage, totalPosts);
for (let i = loadedPosts; i < endIndex; i++) {
const post = generatePost(i);
const postElement = document.createElement('div');
postElement.className = 'post';
postElement.setAttribute('data-post-id', post.id);
postElement.innerHTML = `
<div class="post-title">${post.title}</div>
<div class="post-content">${post.content}</div>
`;
fragment.appendChild(postElement);
}
// APPEND new posts to existing ones
container.appendChild(fragment);
loadedPosts = endIndex;
isLoading = false;
console.log(`Loaded ${loadedPosts} of ${totalPosts} posts`);
}, 300);
}
// Initial load
loadMorePosts();
// Load more on scroll
window.addEventListener('scroll', () => {
const scrollBottom = window.innerHeight + window.scrollY;
const threshold = document.body.offsetHeight - 500;
if (scrollBottom >= threshold) {
loadMorePosts();
}
});
</script>
</body>
</html>

158
docs/examples/assets/virtual_scroll_instagram_grid.html Normal file
View File

@@ -0,0 +1,158 @@
<!DOCTYPE html>
<html>
<head>
<title>Instagram-like Grid Virtual Scroll</title>
<style>
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
margin: 0;
padding: 20px;
background-color: #fafafa;
}
h1 {
text-align: center;
color: #262626;
font-weight: 300;
}
.feed-container {
max-width: 935px;
margin: 0 auto;
height: 800px;
overflow-y: auto;
background: white;
border: 1px solid #dbdbdb;
border-radius: 3px;
}
.grid {
display: grid;
grid-template-columns: repeat(3, 1fr);
gap: 28px;
padding: 28px;
}
.post {
aspect-ratio: 1;
background: #f0f0f0;
border-radius: 3px;
position: relative;
overflow: hidden;
cursor: pointer;
}
.post:hover .overlay {
opacity: 1;
}
.post img {
width: 100%;
height: 100%;
object-fit: cover;
}
.overlay {
position: absolute;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0, 0, 0, 0.3);
display: flex;
align-items: center;
justify-content: center;
color: white;
font-size: 14px;
opacity: 0;
transition: opacity 0.2s;
}
.stats {
display: flex;
gap: 20px;
}
</style>
</head>
<body>
<h1>Instagram Grid Virtual Scroll</h1>
<p style="text-align: center; color: #8e8e8e;">Grid layout with virtual scrolling - only visible rows are rendered</p>
<div class="feed-container">
<div class="grid" id="grid"></div>
</div>
<script>
// Instagram-like grid virtual scroll
const grid = document.getElementById('grid');
const container = document.querySelector('.feed-container');
const totalPosts = 999; // Instagram style count
const postsPerRow = 3;
const rowsPerPage = 4; // 12 posts per page
const postsPerPage = postsPerRow * rowsPerPage;
let currentStartIndex = 0;
// Generate fake Instagram post data
const allPosts = [];
for (let i = 0; i < totalPosts; i++) {
allPosts.push({
id: i,
likes: Math.floor(Math.random() * 10000),
comments: Math.floor(Math.random() * 500),
imageNumber: (i % 10) + 1 // Cycle through 10 placeholder images
});
}
// Render grid - REPLACES content for performance
function renderGrid(startIndex) {
const posts = [];
const endIndex = Math.min(startIndex + postsPerPage, totalPosts);
for (let i = startIndex; i < endIndex; i++) {
const post = allPosts[i];
posts.push(`
<div class="post" data-post-id="${post.id}">
<img src="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='400' height='400'%3E%3Crect width='400' height='400' fill='%23${Math.floor(Math.random()*16777215).toString(16)}'/%3E%3Ctext x='50%25' y='50%25' text-anchor='middle' dy='.3em' font-family='Arial' font-size='48' fill='white'%3E${post.id + 1}%3C/text%3E%3C/svg%3E" alt="Post ${post.id + 1}">
<div class="overlay">
<div class="stats">
<span>❤️ ${post.likes.toLocaleString()}</span>
<span>💬 ${post.comments}</span>
</div>
</div>
</div>
`);
}
// REPLACE grid content (virtual scroll)
grid.innerHTML = posts.join('');
currentStartIndex = startIndex;
}
// Initial render
renderGrid(0);
// Handle scroll
let scrollTimeout;
container.addEventListener('scroll', () => {
clearTimeout(scrollTimeout);
scrollTimeout = setTimeout(() => {
const scrollTop = container.scrollTop;
const scrollHeight = container.scrollHeight;
const clientHeight = container.clientHeight;
// Calculate which "page" we should show
const scrollPercentage = scrollTop / (scrollHeight - clientHeight);
const targetIndex = Math.floor(scrollPercentage * (totalPosts - postsPerPage) / postsPerPage) * postsPerPage;
// When scrolled to bottom, show next page
if (scrollTop + clientHeight >= scrollHeight - 100) {
const nextIndex = currentStartIndex + postsPerPage;
if (nextIndex < totalPosts) {
renderGrid(nextIndex);
container.scrollTop = 100; // Reset scroll for continuous experience
}
}
}, 50);
});
</script>
</body>
</html>

210
docs/examples/assets/virtual_scroll_news_feed.html Normal file
View File

@@ -0,0 +1,210 @@
<!DOCTYPE html>
<html>
<head>
<title>News Feed with Mixed Scroll Behavior</title>
<style>
body {
font-family: Georgia, serif;
margin: 0;
padding: 20px;
background-color: #f8f8f8;
}
h1 {
text-align: center;
color: #1a1a1a;
font-size: 32px;
margin-bottom: 10px;
}
.description {
text-align: center;
color: #666;
margin-bottom: 20px;
}
#newsContainer {
max-width: 900px;
margin: 0 auto;
height: 700px;
overflow-y: auto;
background: white;
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
padding: 20px;
}
.article {
margin-bottom: 30px;
padding-bottom: 30px;
border-bottom: 1px solid #e0e0e0;
}
.article:last-child {
border-bottom: none;
}
.article-header {
margin-bottom: 15px;
}
.category {
display: inline-block;
background: #ff6b6b;
color: white;
padding: 4px 12px;
font-size: 12px;
text-transform: uppercase;
border-radius: 3px;
margin-bottom: 10px;
}
.headline {
font-size: 24px;
font-weight: bold;
color: #1a1a1a;
margin: 10px 0;
line-height: 1.3;
}
.meta {
color: #888;
font-size: 14px;
margin-bottom: 15px;
}
.content {
font-size: 16px;
line-height: 1.8;
color: #333;
}
.featured {
background: #fff9e6;
padding: 20px;
border-radius: 5px;
margin-bottom: 30px;
}
.featured .category {
background: #ffa500;
}
</style>
</head>
<body>
<h1>📰 Dynamic News Feed</h1>
<p class="description">Mixed behavior: Featured articles stay, regular articles use virtual scroll</p>
<div id="newsContainer"></div>
<script>
const container = document.getElementById('newsContainer');
const totalArticles = 100;
const articlesPerPage = 5;
let currentRegularIndex = 0;
// Categories for variety
const categories = ['Politics', 'Technology', 'Business', 'Science', 'Sports', 'Entertainment'];
// Generate article data
const featuredArticles = [];
const regularArticles = [];
// 3 featured articles that always stay
for (let i = 0; i < 3; i++) {
featuredArticles.push({
id: `featured-${i}`,
category: 'Featured',
headline: `Breaking: Major Story ${i + 1} That Stays Visible`,
date: new Date().toLocaleDateString(),
content: `This is featured article ${i + 1}. Featured articles remain in the DOM and are not replaced during scrolling. They provide important persistent content.`
});
}
// Regular articles that get virtualized
for (let i = 0; i < totalArticles; i++) {
regularArticles.push({
id: `article-${i}`,
category: categories[i % categories.length],
headline: `${categories[i % categories.length]} News: Article ${i + 1} of ${totalArticles}`,
date: new Date(Date.now() - i * 86400000).toLocaleDateString(),
content: `This is regular article ${i + 1}. These articles are replaced as you scroll to maintain performance. Only a subset is shown at any time. Article ID: ${i}`
});
}
// Render articles - Featured stay, regular ones are replaced
function renderArticles(regularStartIndex) {
const html = [];
// Always show featured articles
featuredArticles.forEach(article => {
html.push(`
<div class="article featured" data-article-id="${article.id}">
<div class="article-header">
<span class="category">${article.category}</span>
<h2 class="headline">${article.headline}</h2>
<div class="meta">📅 ${article.date}</div>
</div>
<div class="content">${article.content}</div>
</div>
`);
});
// Add divider
html.push('<div style="text-align: center; color: #999; margin: 20px 0;">— Latest News —</div>');
// Show current page of regular articles (virtual scroll)
const endIndex = Math.min(regularStartIndex + articlesPerPage, totalArticles);
for (let i = regularStartIndex; i < endIndex; i++) {
const article = regularArticles[i];
html.push(`
<div class="article" data-article-id="${article.id}">
<div class="article-header">
<span class="category" style="background: ${getCategoryColor(article.category)}">${article.category}</span>
<h2 class="headline">${article.headline}</h2>
<div class="meta">📅 ${article.date}</div>
</div>
<div class="content">${article.content}</div>
</div>
`);
}
container.innerHTML = html.join('');
currentRegularIndex = regularStartIndex;
}
function getCategoryColor(category) {
const colors = {
'Politics': '#e74c3c',
'Technology': '#3498db',
'Business': '#2ecc71',
'Science': '#9b59b6',
'Sports': '#f39c12',
'Entertainment': '#e91e63'
};
return colors[category] || '#95a5a6';
}
// Initial render
renderArticles(0);
// Handle scroll
container.addEventListener('scroll', () => {
const scrollTop = container.scrollTop;
const scrollHeight = container.scrollHeight;
const clientHeight = container.clientHeight;
// When near bottom, load next page of regular articles
if (scrollTop + clientHeight >= scrollHeight - 200) {
const nextIndex = currentRegularIndex + articlesPerPage;
if (nextIndex < totalArticles) {
renderArticles(nextIndex);
// Scroll to where regular articles start
const regularStart = document.querySelector('.article:not(.featured)');
if (regularStart) {
container.scrollTop = regularStart.offsetTop - 100;
}
}
}
});
</script>
</body>
</html>

122
docs/examples/assets/virtual_scroll_twitter_like.html Normal file
View File

@@ -0,0 +1,122 @@
<!DOCTYPE html>
<html>
<head>
<title>Twitter-like Virtual Scroll</title>
<style>
body {
font-family: Arial, sans-serif;
margin: 0;
padding: 20px;
background-color: #f0f2f5;
}
h1 {
color: #1da1f2;
text-align: center;
}
#timeline {
max-width: 600px;
margin: 0 auto;
height: 600px;
overflow-y: auto;
background: white;
border: 1px solid #e1e8ed;
border-radius: 10px;
}
.tweet {
padding: 15px;
border-bottom: 1px solid #e1e8ed;
min-height: 80px;
}
.tweet:hover {
background-color: #f7f9fa;
}
.author {
font-weight: bold;
color: #14171a;
margin-bottom: 5px;
}
.content {
color: #14171a;
line-height: 1.5;
}
.stats {
color: #657786;
font-size: 14px;
margin-top: 10px;
}
</style>
</head>
<body>
<h1>Virtual Scroll Demo - Twitter Style</h1>
<p style="text-align: center; color: #666;">This simulates Twitter's timeline where content is replaced as you scroll</p>
<div id="timeline"></div>
<script>
// Simulate Twitter-like virtual scrolling where DOM elements are replaced
const timeline = document.getElementById('timeline');
const totalTweets = 500;
const tweetsPerPage = 10;
let currentIndex = 0;
// Generate fake tweet data
const allTweets = [];
for (let i = 0; i < totalTweets; i++) {
allTweets.push({
id: i,
author: `User_${i + 1}`,
content: `This is tweet #${i + 1} of ${totalTweets}. Virtual scrolling replaces DOM elements to maintain performance. Unique content ID: ${i}`,
likes: Math.floor(Math.random() * 1000),
retweets: Math.floor(Math.random() * 500)
});
}
// Render tweets - REPLACES content
function renderTweets(startIndex) {
const tweets = [];
const endIndex = Math.min(startIndex + tweetsPerPage, totalTweets);
for (let i = startIndex; i < endIndex; i++) {
const tweet = allTweets[i];
tweets.push(`
<div class="tweet" data-tweet-id="${tweet.id}">
<div class="author">@${tweet.author}</div>
<div class="content">${tweet.content}</div>
<div class="stats">❤️ ${tweet.likes} | 🔁 ${tweet.retweets}</div>
</div>
`);
}
// REPLACE entire content (virtual scroll behavior)
timeline.innerHTML = tweets.join('');
currentIndex = startIndex;
}
// Initial render
renderTweets(0);
// Handle scroll
timeline.addEventListener('scroll', () => {
const scrollTop = timeline.scrollTop;
const scrollHeight = timeline.scrollHeight;
const clientHeight = timeline.clientHeight;
// When near bottom, load next page
if (scrollTop + clientHeight >= scrollHeight - 100) {
const nextIndex = currentIndex + tweetsPerPage;
if (nextIndex < totalTweets) {
renderTweets(nextIndex);
// Small scroll adjustment for continuous scrolling
timeline.scrollTop = 50;
}
}
});
</script>
</body>
</html>

171
docs/examples/c4a_script/amazon_example/README.md Normal file
View File

@@ -0,0 +1,171 @@
# Amazon R2D2 Product Search Example
A real-world demonstration of Crawl4AI's multi-step crawling with LLM-generated automation scripts.
## 🎯 What This Example Shows
This example demonstrates advanced Crawl4AI features:
- **LLM-Generated Scripts**: Automatically create C4A-Script from HTML snippets
- **Multi-Step Crawling**: Navigate through multiple pages using session persistence
- **Structured Data Extraction**: Extract product data using JSON CSS schemas
- **Visual Automation**: Watch the browser perform the search (headless=False)
## 🚀 How It Works
### 1. **Script Generation Phase**
The example uses `C4ACompiler.generate_script()` to analyze Amazon's HTML and create:
- **Search Script**: Automates filling the search box and clicking search
- **Extraction Schema**: Defines how to extract product information
### 2. **Crawling Workflow**
```
Homepage → Execute Search Script → Extract Products → Save Results
```
All steps use the same `session_id` to maintain browser state.
### 3. **Data Extraction**
Products are extracted with:
- Title, price, rating, reviews
- Delivery information
- Sponsored/Small Business badges
- Direct product URLs
## 📁 Files
- `amazon_r2d2_search.py` - Main example script
- `header.html` - Amazon search bar HTML (provided)
- `product.html` - Product card HTML (provided)
- **Generated files:**
- `generated_search_script.c4a` - Auto-generated search automation
- `generated_product_schema.json` - Auto-generated extraction rules
- `extracted_products.json` - Final scraped data
- `search_results_screenshot.png` - Visual proof of results
## 🏃 Running the Example
1. **Prerequisites**
```bash
# Ensure Crawl4AI is installed
pip install crawl4ai
# Set up LLM API key (for script generation)
export OPENAI_API_KEY="your-key-here"
```
2. **Run the scraper**
```bash
python amazon_r2d2_search.py
```
3. **Watch the magic!**
- Browser window opens (not headless)
- Navigates to Amazon.com
- Searches for "r2d2"
- Extracts all products
- Saves results to JSON
## 📊 Sample Output
```json
[
{
"title": "Death Star BB8 R2D2 Golf Balls with 20 Printed tees",
"price": "29.95",
"rating": "4.7",
"reviews_count": "184",
"delivery": "FREE delivery Thu, Jun 19",
"url": "https://www.amazon.com/Death-Star-R2D2-Balls-Printed/dp/B081XSYZMS",
"is_sponsored": true,
"small_business": true
},
...
]
```
## 🔍 Key Features Demonstrated
### Session Persistence
```python
# Same session_id across multiple arun() calls
config = CrawlerRunConfig(
session_id="amazon_r2d2_session",
# ... other settings
)
```
### LLM Script Generation
```python
# Generate automation from natural language + HTML
script = C4ACompiler.generate_script(
html=header_html,
query="Find search box, type 'r2d2', click search",
mode="c4a"
)
```
### JSON CSS Extraction
```python
# Structured data extraction with CSS selectors
schema = {
"baseSelector": "[data-component-type='s-search-result']",
"fields": [
{"name": "title", "selector": "h2 a span", "type": "text"},
{"name": "price", "selector": ".a-price-whole", "type": "text"}
]
}
```
## 🛠️ Customization
### Search Different Products
Change the search term in the script generation:
```python
search_goal = """
...
3. Type "star wars lego" into the search box
...
"""
```
### Extract More Data
Add fields to the extraction schema:
```python
"fields": [
# ... existing fields
{"name": "prime", "selector": ".s-prime", "type": "exists"},
{"name": "image_url", "selector": "img.s-image", "type": "attribute", "attribute": "src"}
]
```
### Use Different Sites
Adapt the approach for other e-commerce sites by:
1. Providing their HTML snippets
2. Adjusting the search goals
3. Updating the extraction schema
## 🎓 Learning Points
1. **No Manual Scripting**: LLM generates all automation code
2. **Session Management**: Maintain state across page navigations
3. **Robust Extraction**: Handle dynamic content and multiple products
4. **Error Handling**: Graceful fallbacks if generation fails
## 🐛 Troubleshooting
- **"No products found"**: Check if Amazon's HTML structure changed
- **"Script generation failed"**: Ensure LLM API key is configured
- **"Page timeout"**: Increase wait times in the config
- **"Session lost"**: Ensure same session_id is used consistently
## 📚 Next Steps
- Try searching for different products
- Add pagination to get more results
- Extract product details pages
- Compare prices across different sellers
- Build a price monitoring system
---
This example shows the power of combining LLM intelligence with web automation. The scripts adapt to HTML changes and natural language instructions make automation accessible to everyone!

202
docs/examples/c4a_script/amazon_example/amazon_r2d2_search.py Normal file
View File

@@ -0,0 +1,202 @@
#!/usr/bin/env python3
"""
Amazon R2D2 Product Search Example using Crawl4AI
This example demonstrates:
1. Using LLM to generate C4A-Script from HTML snippets
2. Multi-step crawling with session persistence
3. JSON CSS extraction for structured product data
4. Complete workflow: homepage → search → extract products
Requirements:
- Crawl4AI with generate_script support
- LLM API key (configured in environment)
"""
import asyncio
import json
import os
from pathlib import Path
from typing import List, Dict, Any
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
from crawl4ai import JsonCssExtractionStrategy
from crawl4ai.script.c4a_compile import C4ACompiler
class AmazonR2D2Scraper:
def __init__(self):
self.base_dir = Path(__file__).parent
self.search_script_path = self.base_dir / "generated_search_script.js"
self.schema_path = self.base_dir / "generated_product_schema.json"
self.results_path = self.base_dir / "extracted_products.json"
self.session_id = "amazon_r2d2_session"
async def generate_search_script(self) -> str:
"""Generate JavaScript for Amazon search interaction"""
print("🔧 Generating search script from header.html...")
# Check if already generated
if self.search_script_path.exists():
print("✅ Using cached search script")
return self.search_script_path.read_text()
# Read the header HTML
header_html = (self.base_dir / "header.html").read_text()
# Generate script using LLM
search_goal = """
Find the search box and search button, then:
1. Wait for the search box to be visible
2. Click on the search box to focus it
3. Clear any existing text
4. Type "r2d2" into the search box
5. Click the search submit button
6. Wait for navigation to complete and search results to appear
"""
try:
script = C4ACompiler.generate_script(
html=header_html,
query=search_goal,
mode="js"
)
# Save for future use
self.search_script_path.write_text(script)
print("✅ Search script generated and saved!")
print(f"📄 Script:\n{script}")
return script
except Exception as e:
print(f"❌ Error generating search script: {e}")
async def generate_product_schema(self) -> Dict[str, Any]:
"""Generate JSON CSS extraction schema from product HTML"""
print("\n🔧 Generating product extraction schema...")
# Check if already generated
if self.schema_path.exists():
print("✅ Using cached extraction schema")
return json.loads(self.schema_path.read_text())
# Read the product HTML
product_html = (self.base_dir / "product.html").read_text()
# Generate extraction schema using LLM
schema_goal = """
Create a JSON CSS extraction schema to extract:
- Product title (from the h2 element)
- Price (the dollar amount)
- Rating (star rating value)
- Number of reviews
- Delivery information
- Product URL (from the main product link)
- Whether it's sponsored
- Small business badge if present
The schema should handle multiple products on a search results page.
"""
try:
# Generate JavaScript that returns the schema
schema = JsonCssExtractionStrategy.generate_schema(
html=product_html,
query=schema_goal,
)
# Save for future use
self.schema_path.write_text(json.dumps(schema, indent=2))
print("✅ Extraction schema generated and saved!")
print(f"📄 Schema fields: {[f['name'] for f in schema['fields']]}")
return schema
except Exception as e:
print(f"❌ Error generating schema: {e}")
async def crawl_amazon(self):
"""Main crawling logic with 2 calls using same session"""
print("\n🚀 Starting Amazon R2D2 product search...")
# Generate scripts and schemas
search_script = await self.generate_search_script()
product_schema = await self.generate_product_schema()
# Configure browser (headless=False to see the action)
browser_config = BrowserConfig(
headless=False,
verbose=True,
viewport_width=1920,
viewport_height=1080
)
async with AsyncWebCrawler(config=browser_config) as crawler:
print("\n📍 Step 1: Navigate to Amazon and search for R2D2")
# FIRST CALL: Navigate to Amazon and execute search
search_config = CrawlerRunConfig(
session_id=self.session_id,
js_code= f"(() => {{ {search_script} }})()", # Execute generated JS
wait_for=".s-search-results", # Wait for search results
extraction_strategy=JsonCssExtractionStrategy(schema=product_schema),
delay_before_return_html=3.0 # Give time for results to load
)
results = await crawler.arun(
url="https://www.amazon.com",
config=search_config
)
if not results.success:
print("❌ Failed to search Amazon")
print(f"Error: {results.error_message}")
return
print("✅ Search completed successfully!")
print("✅ Product extraction completed!")
# Extract and save results
print("\n📍 Extracting product data")
if results[0].extracted_content:
products = json.loads(results[0].extracted_content)
print(f"🔍 Found {len(products)} products in search results")
print(f"✅ Extracted {len(products)} R2D2 products")
# Save results
self.results_path.write_text(
json.dumps(products, indent=2)
)
print(f"💾 Results saved to: {self.results_path}")
# Print sample results
print("\n📊 Sample Results:")
for i, product in enumerate(products[:3], 1):
print(f"\n{i}. {product['title'][:60]}...")
print(f" Price: ${product['price']}")
print(f" Rating: {product['rating']} ({product['number_of_reviews']} reviews)")
print(f" {'🏪 Small Business' if product['small_business_badge'] else ''}")
print(f" {'📢 Sponsored' if product['sponsored'] else ''}")
else:
print("❌ No products extracted")
async def main():
"""Run the Amazon scraper"""
scraper = AmazonR2D2Scraper()
await scraper.crawl_amazon()
print("\n🎉 Amazon R2D2 search example completed!")
print("Check the generated files:")
print(" - generated_search_script.js")
print(" - generated_product_schema.json")
print(" - extracted_products.json")
print(" - search_results_screenshot.png")
if __name__ == "__main__":
asyncio.run(main())

114
docs/examples/c4a_script/amazon_example/extracted_products.json Normal file
View File

@@ -0,0 +1,114 @@
[
{
"title": "Death Star BB8 R2D2 Golf Balls with 20 Printed tees \u2022 Great Gift IDEA from Moms, DADS and Kids -",
"price": "$29.95",
"rating": "4.7 out of 5 stars",
"number_of_reviews": "184",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfYXRmOjIwMDA2NzY0ODgwMjc5ODo6MDo6&url=%2FDeath-Star-R2D2-Balls-Printed%2Fdp%2FB081XSYZMS%2Fref%3Dsr_1_1_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-1-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9hdGY%26psc%3D1",
"sponsored": "Sponsored",
"small_business_badge": "Small Business"
},
{
"title": "TEENKON French Press Insulated 304 Stainless Steel Coffee Maker, 32 Oz Robot R2D2 Hand Home Coffee Presser, with Filter Screen for Brew Coffee and Tea (White)",
"price": "$49.99",
"rating": "4.3 out of 5 stars",
"number_of_reviews": "82",
"delivery_info": "Delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjMwMDAzNzc4Njg4MDAwMjo6MDo6&url=%2FTEENKON-French-Insulated-Stainless-Presser%2Fdp%2FB0CD3HH5PN%2Fref%3Dsr_1_17_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-17-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored"
},
{
"title": "3D Illusion LED Night Light,7 Colors Gradual Changing Touch Switch USB Table Lamp for Holiday Gifts or Home Decorations (R2-D2)",
"price": "$9.97",
"rating": "4.3 out of 5 stars",
"number_of_reviews": "235",
"delivery_info": "Delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjIwMDA0NjMwMTQwODA4MTo6MDo6&url=%2FIllusion-Gradual-Changing-Holiday-Decorations%2Fdp%2FB089NMBKF2%2Fref%3Dsr_1_18_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-18-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored"
},
{
"title": "Paladone Star Wars R2-D2 Headlamp with Droid Sounds, Officially Licensed Disney Star Wars Head Lamp and Reading Light",
"price": "$21.99",
"rating": "4.1 out of 5 stars",
"number_of_reviews": "66",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjMwMDI1NjA0MDQwMTUwMjo6MDo6&url=%2FSounds-Officially-Licensed-Headlamp-Flashlight%2Fdp%2FB09RTDZF8J%2Fref%3Dsr_1_19_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-19-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored"
},
{
"title": "4 Pcs Set Star Wars Kylo Ren BB8 Stormtrooper R2D2 Silicone Travel Luggage Baggage Identification Labels ID Tag for Bag Suitcase Plane Cruise Ships with Belt Strap",
"price": "$16.99",
"rating": "4.7 out of 5 stars",
"number_of_reviews": "3,414",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjIwMDAyMzk3ODkwMzIxMTo6MDo6&url=%2FFinex-Set-Suitcase-Adjustable-Stormtrooper%2Fdp%2FB01D1CBFJS%2Fref%3Dsr_1_24_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-24-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored",
"small_business_badge": "Small Business"
},
{
"title": "Papyrus Star Wars Birthday Card Assortment, Darth Vader, Storm Trooper, and R2-D2 (3-Count)",
"price": "$23.16",
"rating": "4.8 out of 5 stars",
"number_of_reviews": "328",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjMwMDcwNzI4MjA1MzcwMjo6MDo6&url=%2FPapyrus-Birthday-Assortment-Characters-3-Count%2Fdp%2FB07YT2ZPKX%2Fref%3Dsr_1_25_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-25-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored"
},
{
"title": "STAR WARS R2-D2 Artoo 3D Top Motion Lamp, Mood Light | 18 Inches",
"price": "$69.99",
"rating": "4.5 out of 5 stars",
"number_of_reviews": "520",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjIwMDA5NDc3MzczMTQ0MTo6MDo6&url=%2FR2-D2-Artoo-Motion-Light-Inches%2Fdp%2FB08MCWPHQR%2Fref%3Dsr_1_26_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-26-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored"
},
{
"title": "Saturday Park Star Wars Droids Full Sheet Set - 4 Piece 100% Organic Cotton Sheets Features R2-D2 & BB-8 - GOTS & Oeko-TEX Certified (Star Wars Official)",
"price": "$70.00",
"rating": "4.5 out of 5 stars",
"number_of_reviews": "388",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjMwMDAyMzI0NDI5MDQwMjo6MDo6&url=%2FSaturday-Park-Star-Droids-Sheet%2Fdp%2FB0BBSFX4J2%2Fref%3Dsr_1_27_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-27-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored",
"small_business_badge": "1 sustainability feature"
},
{
"title": "AQUARIUS Star Wars R2D2 Action Figure Funky Chunky Novelty Magnet for Refrigerator, Locker, Whiteboard & Game Room Officially Licensed Merchandise & Collectibles",
"price": "$11.94",
"rating": "4.3 out of 5 stars",
"number_of_reviews": "10",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjMwMDA5MDMwMzY5NjEwMjo6MDo6&url=%2FAQUARIUS-Refrigerator-Whiteboard-Merchandise-Collectibles%2Fdp%2FB09W8VKXGC%2Fref%3Dsr_1_32_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-32-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored"
},
{
"title": "STAR WARS C-3PO and R2-D2 Men's Crew Socks 2 Pair Pack",
"price": "$11.95",
"rating": "4.7 out of 5 stars",
"number_of_reviews": "1,272",
"delivery_info": "Delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjIwMDAxMDk5NDkyMTg2MTo6MDo6&url=%2FStar-Wars-R2-D2-C-3PO-Socks%2Fdp%2FB0178IU1GY%2Fref%3Dsr_1_33_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-33-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored"
},
{
"title": "Buckle-Down Belt Women's Cinch Star Wars R2D2 Bounding Parts3 White Black Blue Gray Available In Adjustable Sizes",
"price": "$24.95",
"rating": "4.3 out of 5 stars",
"number_of_reviews": "32",
"delivery_info": "FREE delivery",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjMwMDY1OTQ5NTQ4MzkwMjo6MDo6&url=%2FWomens-Cinch-Bounding-Parts3-Inches%2Fdp%2FB07WK7RG4D%2Fref%3Dsr_1_34_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-34-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored",
"small_business_badge": "Small Business"
},
{
"title": "Star Wars R2D2 Metal Head Vintage Disney+ T-Shirt",
"price": "$22.99",
"rating": "4.8 out of 5 stars",
"number_of_reviews": "869",
"product_url": "/sspa/click?ie=UTF8&spc=MToxNDMzMjA0MzA4MzEzMjAxOjE3NDkzMDI3NDY6c3BfbXRmOjIwMDA1OTUyMzgzNDMyMTo6MDo6&url=%2FStar-Wars-Vintage-Graphic-T-Shirt%2Fdp%2FB07H9PSNXS%2Fref%3Dsr_1_35_sspa%3Fdib%3DeyJ2IjoiMSJ9.iiJYY01upNMdD4BNNt8CYLZEIMXulNkcBlKEMJlr_U_h9eSGqChxwcIiCKUbJeEO_plLkXZvB7Yx-v4UDOCdiUFI-sHFgcTznXrP7tdD8xHpRaMKmaBDWMCAFwzPmVcgK_6Q9qIRoN4sp8tunKX26j5EC_8LiK-D5QximGkE8i8f-R5GhSUo__DaSkAP1cnzxUtSESfA8fYfewsZ1iSol9_zohE6r1ZZeawnWHPmDTkLqzCW3uK44EnvJbPFvzMlpiKcs9p9Eh9w5Rc5rrumMihdaWkC63B0cz5jU-S2Ieg._D8d5nv3hOExHPbZ04L-vaC7YwJjEZM-vu5AED5sz0U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749302746%26sr%3D8-35-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9tdGY%26psc%3D1",
"sponsored": "Sponsored",
"small_business_badge": "1 sustainability feature"
}
]

47
docs/examples/c4a_script/amazon_example/generated_product_schema.json Normal file
View File

@@ -0,0 +1,47 @@
{
"name": "Amazon Product Search Results",
"baseSelector": "div[data-component-type='s-impression-counter']",
"fields": [
{
"name": "title",
"selector": "h2.a-size-base-plus.a-spacing-none.a-color-base.a-text-normal span",
"type": "text"
},
{
"name": "price",
"selector": "span.a-price > span.a-offscreen",
"type": "text"
},
{
"name": "rating",
"selector": "i.a-icon-star-small span.a-icon-alt",
"type": "text"
},
{
"name": "number_of_reviews",
"selector": "a.a-link-normal.s-underline-text span.a-size-base",
"type": "text"
},
{
"name": "delivery_info",
"selector": "div[data-cy='delivery-recipe'] span.a-color-base",
"type": "text"
},
{
"name": "product_url",
"selector": "a.a-link-normal.s-no-outline",
"type": "attribute",
"attribute": "href"
},
{
"name": "sponsored",
"selector": "span.puis-label-popover-default span.a-color-secondary",
"type": "text"
},
{
"name": "small_business_badge",
"selector": "span.a-size-base.a-color-base",
"type": "text"
}
]
}

9
docs/examples/c4a_script/amazon_example/generated_search_script.js Normal file
View File

@@ -0,0 +1,9 @@
const searchBox = document.querySelector('#twotabsearchtextbox');
const searchButton = document.querySelector('#nav-search-submit-button');
if (searchBox && searchButton) {
searchBox.focus();
searchBox.value = '';
searchBox.value = 'r2d2';
searchButton.click();
}

214
docs/examples/c4a_script/amazon_example/header.html Normal file
View File

@@ -0,0 +1,214 @@
<div id="nav-belt" style="width: 100%;">
<div class="nav-left">
<script type="text/javascript">window.navmet.tmp = +new Date();</script>
<div id="nav-logo">
<a href="/ref=nav_logo" id="nav-logo-sprites" class="nav-logo-link nav-progressive-attribute"
aria-label="Amazon" lang="en">
<span class="nav-sprite nav-logo-base"></span>
<span id="logo-ext" class="nav-sprite nav-logo-ext nav-progressive-content"></span>
<span class="nav-logo-locale">.us</span>
</a>
</div>
<script
type="text/javascript">window.navmet.push({ key: 'Logo', end: +new Date(), begin: window.navmet.tmp });</script>
<div id="nav-global-location-slot">
<span id="nav-global-location-data-modal-action" class="a-declarative nav-progressive-attribute"
data-a-modal="{&quot;width&quot;:375, &quot;closeButton&quot;:&quot;true&quot;,&quot;popoverLabel&quot;:&quot;Choose your location&quot;, &quot;ajaxHeaders&quot;:{&quot;anti-csrftoken-a2z&quot;:&quot;hHBwllskaYQrylaW9ifYQIdmqBZOtGdKro0TWb5kDoPKAAAAAGhEMhsAAAAB&quot;}, &quot;name&quot;:&quot;glow-modal&quot;, &quot;url&quot;:&quot;/portal-migration/hz/glow/get-rendered-address-selections?deviceType=desktop&amp;pageType=Gateway&amp;storeContext=NoStoreName&amp;actionSource=desktop-modal&quot;, &quot;footer&quot;:&quot;<span class=\&quot;a-declarative\&quot; data-action=\&quot;a-popover-close\&quot; data-a-popover-close=\&quot;{}\&quot;><span class=\&quot;a-button a-button-primary\&quot;><span class=\&quot;a-button-inner\&quot;><button name=\&quot;glowDoneButton\&quot; class=\&quot;a-button-text\&quot; type=\&quot;button\&quot;>Done</button></span></span></span>&quot;,&quot;header&quot;:&quot;Choose your location&quot;}"
data-action="a-modal">
<a id="nav-global-location-popover-link" role="button" tabindex="0"
class="nav-a nav-a-2 a-popover-trigger a-declarative nav-progressive-attribute" href="">
<div class="nav-sprite nav-progressive-attribute" id="nav-packard-glow-loc-icon"></div>
<div id="glow-ingress-block">
<span class="nav-line-1 nav-progressive-content" id="glow-ingress-line1">
Deliver to
</span>
<span class="nav-line-2 nav-progressive-content" id="glow-ingress-line2">
Malaysia
</span>
</div>
</a>
</span>
<input data-addnewaddress="add-new" id="unifiedLocation1ClickAddress" name="dropdown-selection"
type="hidden" value="add-new" class="nav-progressive-attribute">
<input data-addnewaddress="add-new" id="ubbShipTo" name="dropdown-selection-ubb" type="hidden"
value="add-new" class="nav-progressive-attribute">
<input id="glowValidationToken" name="glow-validation-token" type="hidden"
value="hHBwllskaYQrylaW9ifYQIdmqBZOtGdKro0TWb5kDoPKAAAAAGhEMhsAAAAB" class="nav-progressive-attribute">
<input id="glowDestinationType" name="glow-destination-type" type="hidden" value="COUNTRY"
class="nav-progressive-attribute">
</div>
<div id="nav-global-location-toaster-script-container" class="nav-progressive-content">
<!-- NAVYAAN-GLOW-NAV-TOASTER -->
<script>
P.when('glow-toaster-strings').execute(function (S) {
S.load({ "glow-toaster-address-change-error": "An error has occurred and the address has not been updated. Please try again.", "glow-toaster-unknown-error": "An error has occurred. Please try again." });
});
</script>
<script>
P.when('glow-toaster-manager').execute(function (M) {
M.create({ "pageType": "Gateway", "aisTransitionState": null, "rancorLocationSource": "REALM_DEFAULT" })
});
</script>
</div>
</div>
<div class="nav-fill" id="nav-fill-search">
<script type="text/javascript">window.navmet.tmp = +new Date();</script>
<div id="nav-search">
<div id="nav-bar-left"></div>
<form id="nav-search-bar-form" accept-charset="utf-8" action="/s/ref=nb_sb_noss_1"
class="nav-searchbar nav-progressive-attribute" method="GET" name="site-search" role="search">
<div class="nav-left">
<div id="nav-search-dropdown-card">
<div class="nav-search-scope nav-sprite">
<div class="nav-search-facade" data-value="search-alias=aps">
<span id="nav-search-label-id" class="nav-search-label nav-progressive-content"
style="width: auto;">All</span>
<i class="nav-icon"></i>
</div>
<label id="searchDropdownDescription" for="searchDropdownBox"
class="nav-progressive-attribute" style="display:none">Select the department you want to
search in</label>
<select aria-describedby="searchDropdownDescription"
class="nav-search-dropdown searchSelect nav-progressive-attrubute nav-progressive-search-dropdown"
data-nav-digest="k+fyIAyB82R9jVEmroQ0OWwSW3A=" data-nav-selected="0"
id="searchDropdownBox" name="url" style="display: block; top: 2.5px;" tabindex="0"
title="Search in">
<option selected="selected" value="search-alias=aps">All Departments</option>
<option value="search-alias=arts-crafts-intl-ship">Arts &amp; Crafts</option>
<option value="search-alias=automotive-intl-ship">Automotive</option>
<option value="search-alias=baby-products-intl-ship">Baby</option>
<option value="search-alias=beauty-intl-ship">Beauty &amp; Personal Care</option>
<option value="search-alias=stripbooks-intl-ship">Books</option>
<option value="search-alias=fashion-boys-intl-ship">Boys' Fashion</option>
<option value="search-alias=computers-intl-ship">Computers</option>
<option value="search-alias=deals-intl-ship">Deals</option>
<option value="search-alias=digital-music">Digital Music</option>
<option value="search-alias=electronics-intl-ship">Electronics</option>
<option value="search-alias=fashion-girls-intl-ship">Girls' Fashion</option>
<option value="search-alias=hpc-intl-ship">Health &amp; Household</option>
<option value="search-alias=kitchen-intl-ship">Home &amp; Kitchen</option>
<option value="search-alias=industrial-intl-ship">Industrial &amp; Scientific</option>
<option value="search-alias=digital-text">Kindle Store</option>
<option value="search-alias=luggage-intl-ship">Luggage</option>
<option value="search-alias=fashion-mens-intl-ship">Men's Fashion</option>
<option value="search-alias=movies-tv-intl-ship">Movies &amp; TV</option>
<option value="search-alias=music-intl-ship">Music, CDs &amp; Vinyl</option>
<option value="search-alias=pets-intl-ship">Pet Supplies</option>
<option value="search-alias=instant-video">Prime Video</option>
<option value="search-alias=software-intl-ship">Software</option>
<option value="search-alias=sporting-intl-ship">Sports &amp; Outdoors</option>
<option value="search-alias=tools-intl-ship">Tools &amp; Home Improvement</option>
<option value="search-alias=toys-and-games-intl-ship">Toys &amp; Games</option>
<option value="search-alias=videogames-intl-ship">Video Games</option>
<option value="search-alias=fashion-womens-intl-ship">Women's Fashion</option>
</select>
</div>
</div>
</div>
<div class="nav-fill">
<div class="nav-search-field ">
<label for="twotabsearchtextbox" style="display: none;">Search Amazon</label>
<input type="text" id="twotabsearchtextbox" value="" name="field-keywords" autocomplete="off"
placeholder="Search Amazon" class="nav-input nav-progressive-attribute" dir="auto"
tabindex="0" aria-label="Search Amazon" role="searchbox" aria-autocomplete="list"
aria-controls="sac-autocomplete-results-container" aria-expanded="false"
aria-haspopup="grid" spellcheck="false">
</div>
<div id="nav-iss-attach"></div>
</div>
<div class="nav-right">
<div class="nav-search-submit nav-sprite">
<span id="nav-search-submit-text"
class="nav-search-submit-text nav-sprite nav-progressive-attribute" aria-label="Go">
<input id="nav-search-submit-button" type="submit"
class="nav-input nav-progressive-attribute" value="Go" tabindex="0">
</span>
</div>
</div>
<input type="hidden" id="isscrid" name="crid" value="15O5T5OCG5OZE"><input type="hidden" id="issprefix"
name="sprefix" value="r2d2,aps,588">
</form>
</div>
<script
type="text/javascript">window.navmet.push({ key: 'Search', end: +new Date(), begin: window.navmet.tmp });</script>
</div>
<div class="nav-right">
<script type="text/javascript">window.navmet.tmp = +new Date();</script>
<div id="nav-tools" class="layoutToolbarPadding">
<div class="nav-div" id="icp-nav-flyout">
<a href="/customer-preferences/edit?ie=UTF8&amp;preferencesReturnUrl=%2F&amp;ref_=topnav_lang_ais"
class="nav-a nav-a-2 icp-link-style-2" aria-label="Choose a language for shopping in Amazon United States. The current selection is English (EN).
">
<span class="icp-nav-link-inner">
<span class="nav-line-1">
</span>
<span class="nav-line-2">
<span class="icp-nav-flag icp-nav-flag-us icp-nav-flag-lop" role="img"
aria-label="United States"></span>
<div>EN</div>
</span>
</span>
</a>
<button class="nav-flyout-button nav-icon nav-arrow" aria-label="Expand to Change Language or Country"
tabindex="0" style="visibility: visible;"></button>
</div>
<div class="nav-div" id="nav-link-accountList">
<a href="https://www.amazon.com/ap/signin?openid.pape.max_auth_age=0&amp;openid.return_to=https%3A%2F%2Fwww.amazon.com%2F%3Fref_%3Dnav_ya_signin&amp;openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&amp;openid.assoc_handle=usflex&amp;openid.mode=checkid_setup&amp;openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&amp;openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0"
class="nav-a nav-a-2 nav-progressive-attribute" data-nav-ref="nav_ya_signin"
data-nav-role="signin" data-ux-jq-mouseenter="true" tabindex="0" data-csa-c-type="link"
data-csa-c-slot-id="nav-link-accountList" data-csa-c-content-id="nav_ya_signin"
aria-controls="nav-flyout-accountList" data-csa-c-id="37vs0l-z575id-52hnw3-x34ncp">
<div class="nav-line-1-container"><span id="nav-link-accountList-nav-line-1"
class="nav-line-1 nav-progressive-content">Hello, sign in</span></div>
<span class="nav-line-2 ">Account &amp; Lists
</span>
</a>
<button class="nav-flyout-button nav-icon nav-arrow" aria-label="Expand Account and Lists" tabindex="0"
style="visibility: visible;"></button>
</div>
<a href="/gp/css/order-history?ref_=nav_orders_first" class="nav-a nav-a-2 nav-progressive-attribute"
id="nav-orders" tabindex="0">
<span class="nav-line-1">Returns</span>
<span class="nav-line-2">&amp; Orders<span class="nav-icon nav-arrow"></span></span>
</a>
<a href="/gp/cart/view.html?ref_=nav_cart" aria-label="0 items in cart"
class="nav-a nav-a-2 nav-progressive-attribute" id="nav-cart">
<div id="nav-cart-count-container">
<span id="nav-cart-count" aria-hidden="true"
class="nav-cart-count nav-cart-0 nav-progressive-attribute nav-progressive-content">0</span>
<span class="nav-cart-icon nav-sprite"></span>
</div>
<div id="nav-cart-text-container" class=" nav-progressive-attribute">
<span aria-hidden="true" class="nav-line-1">
</span>
<span aria-hidden="true" class="nav-line-2">
Cart
<span class="nav-icon nav-arrow"></span>
</span>
</div>
</a>
</div>
<script
type="text/javascript">window.navmet.push({ key: 'Tools', end: +new Date(), begin: window.navmet.tmp });</script>
</div>
</div>

206
docs/examples/c4a_script/amazon_example/product.html Normal file
View File

@@ -0,0 +1,206 @@
<div class="sg-col-inner">
<div cel_widget_id="MAIN-SEARCH_RESULTS-2"
class="s-widget-container s-spacing-small s-widget-container-height-small celwidget slot=MAIN template=SEARCH_RESULTS widgetId=search-results_1"
data-csa-c-pos="1" data-csa-c-item-id="amzn1.asin.1.B081XSYZMS" data-csa-op-log-render="" data-csa-c-type="item"
data-csa-c-id="dp9zuy-vyww1v-brlmmq-fmgitb" data-cel-widget="MAIN-SEARCH_RESULTS-2">
<div data-component-type="s-impression-logger"
data-component-props="{&quot;percentageShownToFire&quot;:&quot;50&quot;,&quot;batchable&quot;:true,&quot;requiredElementSelector&quot;:&quot;.s-image:visible&quot;,&quot;url&quot;:&quot;https://unagi-na.amazon.com/1/events/com.amazon.eel.SponsoredProductsEventTracking.prod?qualifier=1749299833&amp;id=1740514893473797&amp;widgetName=sp_atf&amp;adId=200067648802798&amp;eventType=1&amp;adIndex=0&quot;}"
class="rush-component s-expand-height" data-component-id="6">
<div data-component-type="s-impression-counter"
data-component-props="{&quot;presenceCounterName&quot;:&quot;sp_delivered&quot;,&quot;testElementSelector&quot;:&quot;.s-image&quot;,&quot;hiddenCounterName&quot;:&quot;sp_hidden&quot;}"
class="rush-component s-featured-result-item s-expand-height" data-component-id="7">
<span class="a-declarative" data-version-id="v2dwi5hq8xzthf26x0gg1mcl2oj"
data-render-id="r3o8bgr5zt3kmy2jv4su6fn4kyw" data-action="puis-card-container-declarative"
data-csa-c-func-deps="aui-da-puis-card-container-declarative"
data-csa-c-item-id="amzn1.asin.B081XSYZMS" data-csa-c-posx="1" data-csa-c-type="item"
data-csa-c-owner="puis" data-csa-c-id="88w0j1-kcbf5g-80v4i9-96cv88">
<div class="puis-card-container s-card-container s-overflow-hidden aok-relative puis-expand-height puis-include-content-margin puis puis-v2dwi5hq8xzthf26x0gg1mcl2oj s-latency-cf-section puis-card-border"
data-cy="asin-faceout-container">
<div class="a-section a-spacing-base">
<div class="s-product-image-container aok-relative s-text-center s-image-overlay-grey puis-image-overlay-grey s-padding-left-small s-padding-right-small puis-spacing-small s-height-equalized puis puis-v2dwi5hq8xzthf26x0gg1mcl2oj"
data-cy="image-container" style="padding-top: 0px !important;"><span
data-component-type="s-product-image" class="rush-component"
data-version-id="v2dwi5hq8xzthf26x0gg1mcl2oj"
data-render-id="r3o8bgr5zt3kmy2jv4su6fn4kyw"><a aria-hidden="true"
class="a-link-normal s-no-outline" tabindex="-1"
href="/sspa/click?ie=UTF8&amp;spc=MToxNzQwNTE0ODkzNDczNzk3OjE3NDkyOTk4MzM6c3BfYXRmOjIwMDA2NzY0ODgwMjc5ODo6MDo6&amp;url=%2FDeath-Star-R2D2-Balls-Printed%2Fdp%2FB081XSYZMS%2Fref%3Dsr_1_1_sspa%3Fcrid%3D3C1EXMXN59Q9G%26dib%3DeyJ2IjoiMSJ9.7tBl5bhZh59L9qIPZUe9SLa2fy_HvzboxuQxvrRcAc0VUXayi9fxQFsMLyFplDE9vMkIJbP76AVpa-5-fxhNza3DqhX4tss4NlB49WPi_dA00Hw6O8qK5pDzdetYlhGgOyXOLBe7mTG9oJ5W0wcvQhEVoX9mpJk_SGeqRLWGA0dBSjYCZtiyrY8_B-DP53S7fbYwiSYtq-g7sQDXKVadRpGvUyKq7yxA0SLsU42uvoqSGb0qcd6udL1wbnTEkKmwNjNSb7xIUb-8PyE7DTPMt1ScJksn70sFQMJNkM2aK5M.x9_jYvKPnSibV1d0umUStZBxlSTSXrzVIFKqFzS8c-U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749299833%26sprefix%3Dr2d2%252Caps%252C548%26sr%3D8-1-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9hdGY%26psc%3D1">
<div class="a-section aok-relative s-image-square-aspect"><img class="s-image"
src="https://m.media-amazon.com/images/I/61kAC69zQUL._AC_UL320_.jpg"
srcset="https://m.media-amazon.com/images/I/61kAC69zQUL._AC_UL320_.jpg 1x, https://m.media-amazon.com/images/I/61kAC69zQUL._AC_UL480_FMwebp_QL65_.jpg 1.5x, https://m.media-amazon.com/images/I/61kAC69zQUL._AC_UL640_FMwebp_QL65_.jpg 2x, https://m.media-amazon.com/images/I/61kAC69zQUL._AC_UL800_FMwebp_QL65_.jpg 2.5x, https://m.media-amazon.com/images/I/61kAC69zQUL._AC_UL960_FMwebp_QL65_.jpg 3x"
alt="Sponsored Ad - Death Star BB8 R2D2 Golf Balls with 20 Printed tees • Great Gift IDEA from Moms, DADS and Kids -"
aria-hidden="true" data-image-index="1" data-image-load=""
data-image-latency="s-product-image" data-image-source-density="1">
</div>
</a></span></div>
<div class="a-section a-spacing-small puis-padding-left-small puis-padding-right-small">
<div data-cy="title-recipe"
class="a-section a-spacing-none a-spacing-top-small s-title-instructions-style">
<div class="a-row a-spacing-micro"><span class="a-declarative"
data-version-id="v2dwi5hq8xzthf26x0gg1mcl2oj"
data-render-id="r3o8bgr5zt3kmy2jv4su6fn4kyw" data-action="a-popover"
data-csa-c-func-deps="aui-da-a-popover"
data-a-popover="{&quot;name&quot;:&quot;sp-info-popover-B081XSYZMS&quot;,&quot;position&quot;:&quot;triggerVertical&quot;,&quot;popoverLabel&quot;:&quot;View Sponsored information or leave ad feedback&quot;,&quot;closeButtonLabel&quot;:&quot;Close popup&quot;,&quot;closeButton&quot;:&quot;true&quot;,&quot;dataStrategy&quot;:&quot;preload&quot;}"
data-csa-c-type="widget" data-csa-c-id="wqddan-z1l67e-lissct-rciw65"><a
href="javascript:void(0)" role="button" style="text-decoration: none;"
class="puis-label-popover puis-sponsored-label-text"><span
class="puis-label-popover-default"><span
aria-label="View Sponsored information or leave ad feedback"
class="a-color-secondary">Sponsored</span></span><span
class="puis-label-popover-hover"><span aria-hidden="true"
class="a-color-base">Sponsored</span></span> <span
class="aok-inline-block puis-sponsored-label-info-icon"></span></a></span>
<div class="a-popover-preload" id="a-popover-sp-info-popover-B081XSYZMS">
<div class="puis puis-v2dwi5hq8xzthf26x0gg1mcl2oj"><span>You’re seeing this
ad based on the product’s relevance to your search query.</span>
<div class="a-row"><span class="a-declarative"
data-version-id="v2dwi5hq8xzthf26x0gg1mcl2oj"
data-render-id="r3o8bgr5zt3kmy2jv4su6fn4kyw"
data-action="s-safe-ajax-modal-trigger"
data-csa-c-func-deps="aui-da-s-safe-ajax-modal-trigger"
data-s-safe-ajax-modal-trigger="{&quot;header&quot;:&quot;Leave feedback&quot;,&quot;dataStrategy&quot;:&quot;ajax&quot;,&quot;ajaxUrl&quot;:&quot;/af/sp-loom/feedback-form?pl=%7B%22adPlacementMetaData%22%3A%7B%22searchTerms%22%3A%22cjJkMg%3D%3D%22%2C%22pageType%22%3A%22Search%22%2C%22feedbackType%22%3A%22sponsoredProductsLoom%22%2C%22slotName%22%3A%22TOP%22%7D%2C%22adCreativeMetaData%22%3A%7B%22adProgramId%22%3A1024%2C%22adCreativeDetails%22%3A%5B%7B%22asin%22%3A%22B081XSYZMS%22%2C%22title%22%3A%22Death+Star+BB8+R2D2+Golf+Balls+with+20+Printed+tees+%E2%80%A2+Great+Gift+IDEA+from+Moms%2C+DADS+and+Kids+-%22%2C%22priceInfo%22%3A%7B%22amount%22%3A29.95%2C%22currencyCode%22%3A%22USD%22%7D%2C%22sku%22%3A%22starwars3pk20tees%22%2C%22adId%22%3A%22A03790291PREH7M3Q3SVS%22%2C%22campaignId%22%3A%22A01050612Q0SQZ2PTMGO9%22%2C%22advertiserIdNS%22%3Anull%2C%22selectionSignals%22%3Anull%7D%5D%7D%7D&quot;}"
data-csa-c-type="widget"
data-csa-c-id="ygslsp-ir23ei-7k9x6z-73l1tp"><a
class="a-link-normal s-underline-text s-underline-link-text s-link-style"
href="#"><span>Leave ad feedback</span> </a> </span></div>
</div>
</div>
</div><a class="a-link-normal s-line-clamp-4 s-link-style a-text-normal"
href="/sspa/click?ie=UTF8&amp;spc=MToxNzQwNTE0ODkzNDczNzk3OjE3NDkyOTk4MzM6c3BfYXRmOjIwMDA2NzY0ODgwMjc5ODo6MDo6&amp;url=%2FDeath-Star-R2D2-Balls-Printed%2Fdp%2FB081XSYZMS%2Fref%3Dsr_1_1_sspa%3Fcrid%3D3C1EXMXN59Q9G%26dib%3DeyJ2IjoiMSJ9.7tBl5bhZh59L9qIPZUe9SLa2fy_HvzboxuQxvrRcAc0VUXayi9fxQFsMLyFplDE9vMkIJbP76AVpa-5-fxhNza3DqhX4tss4NlB49WPi_dA00Hw6O8qK5pDzdetYlhGgOyXOLBe7mTG9oJ5W0wcvQhEVoX9mpJk_SGeqRLWGA0dBSjYCZtiyrY8_B-DP53S7fbYwiSYtq-g7sQDXKVadRpGvUyKq7yxA0SLsU42uvoqSGb0qcd6udL1wbnTEkKmwNjNSb7xIUb-8PyE7DTPMt1ScJksn70sFQMJNkM2aK5M.x9_jYvKPnSibV1d0umUStZBxlSTSXrzVIFKqFzS8c-U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749299833%26sprefix%3Dr2d2%252Caps%252C548%26sr%3D8-1-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9hdGY%26psc%3D1">
<h2 aria-label="Sponsored Ad - Death Star BB8 R2D2 Golf Balls with 20 Printed tees • Great Gift IDEA from Moms, DADS and Kids -"
class="a-size-base-plus a-spacing-none a-color-base a-text-normal">
<span>Death Star BB8 R2D2 Golf Balls with 20 Printed tees • Great Gift IDEA
from Moms, DADS and Kids -</span></h2>
</a>
</div>
<div data-cy="reviews-block" class="a-section a-spacing-none a-spacing-top-micro">
<div class="a-row a-size-small"><span class="a-declarative"
data-version-id="v2dwi5hq8xzthf26x0gg1mcl2oj"
data-render-id="r3o8bgr5zt3kmy2jv4su6fn4kyw" data-action="a-popover"
data-csa-c-func-deps="aui-da-a-popover"
data-a-popover="{&quot;position&quot;:&quot;triggerBottom&quot;,&quot;popoverLabel&quot;:&quot;4.7 out of 5 stars, rating details&quot;,&quot;url&quot;:&quot;/review/widgets/average-customer-review/popover/ref=acr_search__popover?ie=UTF8&amp;asin=B081XSYZMS&amp;ref_=acr_search__popover&amp;contextId=search&quot;,&quot;closeButton&quot;:true,&quot;closeButtonLabel&quot;:&quot;&quot;}"
data-csa-c-type="widget" data-csa-c-id="oykdvt-8s1ebj-2kegf2-7ii7tp"><a
aria-label="4.7 out of 5 stars, rating details"
href="javascript:void(0)" role="button"
class="a-popover-trigger a-declarative"><i
data-cy="reviews-ratings-slot" aria-hidden="true"
class="a-icon a-icon-star-small a-star-small-4-5"><span
class="a-icon-alt">4.7 out of 5 stars</span></i><i
class="a-icon a-icon-popover"></i></a></span> <span
data-component-type="s-client-side-analytics" class="rush-component"
data-version-id="v2dwi5hq8xzthf26x0gg1mcl2oj"
data-render-id="r3o8bgr5zt3kmy2jv4su6fn4kyw" data-component-id="8">
<div style="display: inline-block"
class="s-csa-instrumentation-wrapper alf-search-csa-instrumentation-wrapper"
data-csa-c-type="alf-af-component"
data-csa-c-content-id="alf-customer-ratings-count-component"
data-csa-c-slot-id="alf-reviews" data-csa-op-log-render=""
data-csa-c-layout="GRID" data-csa-c-asin="B081XSYZMS"
data-csa-c-id="6l5wc4-ngelan-hd9x4t-d4a2k7"><a aria-label="184 ratings"
class="a-link-normal s-underline-text s-underline-link-text s-link-style"
href="/sspa/click?ie=UTF8&amp;spc=MToxNzQwNTE0ODkzNDczNzk3OjE3NDkyOTk4MzM6c3BfYXRmOjIwMDA2NzY0ODgwMjc5ODo6MDo6&amp;url=%2FDeath-Star-R2D2-Balls-Printed%2Fdp%2FB081XSYZMS%2Fref%3Dsr_1_1_sspa%3Fcrid%3D3C1EXMXN59Q9G%26dib%3DeyJ2IjoiMSJ9.7tBl5bhZh59L9qIPZUe9SLa2fy_HvzboxuQxvrRcAc0VUXayi9fxQFsMLyFplDE9vMkIJbP76AVpa-5-fxhNza3DqhX4tss4NlB49WPi_dA00Hw6O8qK5pDzdetYlhGgOyXOLBe7mTG9oJ5W0wcvQhEVoX9mpJk_SGeqRLWGA0dBSjYCZtiyrY8_B-DP53S7fbYwiSYtq-g7sQDXKVadRpGvUyKq7yxA0SLsU42uvoqSGb0qcd6udL1wbnTEkKmwNjNSb7xIUb-8PyE7DTPMt1ScJksn70sFQMJNkM2aK5M.x9_jYvKPnSibV1d0umUStZBxlSTSXrzVIFKqFzS8c-U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749299833%26sprefix%3Dr2d2%252Caps%252C548%26sr%3D8-1-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9hdGY%26psc%3D1#customerReviews"><span
aria-hidden="true"
class="a-size-base s-underline-text">184</span> </a> </div>
</span></div>
<div class="a-row a-size-base"><span class="a-size-base a-color-secondary">50+
bought in past month</span></div>
</div>
<div data-cy="price-recipe"
class="a-section a-spacing-none a-spacing-top-small s-price-instructions-style">
<div class="a-row a-size-base a-color-base">
<div class="a-row"><span id="price-link" class="aok-offscreen">Price, product
page</span><a aria-describedby="price-link"
class="a-link-normal s-no-hover s-underline-text s-underline-link-text s-link-style a-text-normal"
href="/sspa/click?ie=UTF8&amp;spc=MToxNzQwNTE0ODkzNDczNzk3OjE3NDkyOTk4MzM6c3BfYXRmOjIwMDA2NzY0ODgwMjc5ODo6MDo6&amp;url=%2FDeath-Star-R2D2-Balls-Printed%2Fdp%2FB081XSYZMS%2Fref%3Dsr_1_1_sspa%3Fcrid%3D3C1EXMXN59Q9G%26dib%3DeyJ2IjoiMSJ9.7tBl5bhZh59L9qIPZUe9SLa2fy_HvzboxuQxvrRcAc0VUXayi9fxQFsMLyFplDE9vMkIJbP76AVpa-5-fxhNza3DqhX4tss4NlB49WPi_dA00Hw6O8qK5pDzdetYlhGgOyXOLBe7mTG9oJ5W0wcvQhEVoX9mpJk_SGeqRLWGA0dBSjYCZtiyrY8_B-DP53S7fbYwiSYtq-g7sQDXKVadRpGvUyKq7yxA0SLsU42uvoqSGb0qcd6udL1wbnTEkKmwNjNSb7xIUb-8PyE7DTPMt1ScJksn70sFQMJNkM2aK5M.x9_jYvKPnSibV1d0umUStZBxlSTSXrzVIFKqFzS8c-U%26dib_tag%3Dse%26keywords%3Dr2d2%26qid%3D1749299833%26sprefix%3Dr2d2%252Caps%252C548%26sr%3D8-1-spons%26sp_csd%3Dd2lkZ2V0TmFtZT1zcF9hdGY%26psc%3D1"><span
class="a-price" data-a-size="xl" data-a-color="base"><span
class="a-offscreen">$29.95</span><span aria-hidden="true"><span
class="a-price-symbol">$</span><span
class="a-price-whole">29<span
class="a-price-decimal">.</span></span><span
class="a-price-fraction">95</span></span></span></a></div>
<div class="a-row"></div>
</div>
</div>
<div data-cy="delivery-recipe" class="a-section a-spacing-none a-spacing-top-micro">
<div class="a-row a-size-base a-color-secondary s-align-children-center"><span
aria-label="FREE delivery Thu, Jun 19 to Malaysia on $49 of eligible items"><span
class="a-color-base">FREE delivery </span><span
class="a-color-base a-text-bold">Thu, Jun 19 </span><span
class="a-color-base">to Malaysia on $49 of eligible items</span></span>
</div>
</div>
<div data-cy="certification-recipe"
class="a-section a-spacing-none a-spacing-top-micro">
<div class="a-row">
<div class="a-section a-spacing-none s-align-children-center">
<div class="a-section a-spacing-none s-pc-faceout-container">
<div>
<div class="s-align-children-center"><span class="a-declarative"
data-version-id="v2dwi5hq8xzthf26x0gg1mcl2oj"
data-render-id="r3o8bgr5zt3kmy2jv4su6fn4kyw"
data-action="s-pc-sidesheet-open"
data-csa-c-func-deps="aui-da-s-pc-sidesheet-open"
data-s-pc-sidesheet-open="{&quot;preloadDomId&quot;:&quot;pc-side-sheet-B081XSYZMS&quot;,&quot;popoverLabel&quot;:&quot;Product certifications&quot;,&quot;interactLoggingMetricsList&quot;:[&quot;provenanceCertifications_desktop_sbe_badge&quot;],&quot;closeButtonLabel&quot;:&quot;Close popup&quot;,&quot;dwellMetric&quot;:&quot;provenanceCertifications_desktop_sbe_badge_t&quot;}"
data-csa-c-type="widget"
data-csa-c-id="hdfxi6-bjlgup-5dql15-88t9ao"><a
data-cy="s-pc-faceout-badge"
class="a-link-normal s-no-underline s-pc-badge s-align-children-center aok-block"
href="javascript:void(0)" role="button">
<div
class="a-section s-pc-attribute-pill-text s-margin-bottom-none s-margin-bottom-none aok-block s-pc-certification-faceout">
<span class="faceout-image-view"></span><img alt=""
src="https://m.media-amazon.com/images/I/111mHoVK0kL._SS200_.png"
class="s-image" height="18px" width="18px">
<span class="a-size-base a-color-base">Small
Business</span>
<div
class="s-margin-bottom-none s-pc-sidesheet-chevron aok-nowrap">
<i class="a-icon a-icon-popover aok-align-center"
role="presentation"></i></div>
</div>
</a></span></div>
</div>
</div>
</div>
<div id="pc-side-sheet-B081XSYZMS"
class="a-section puis puis-v2dwi5hq8xzthf26x0gg1mcl2oj aok-hidden">
<div class="a-section s-pc-container-side-sheet">
<div class="s-align-children-center a-spacing-small">
<div class="s-align-children-center s-pc-certification"
role="heading" aria-level="2"><span
class="faceout-image-view"></span>
<div alt="" style="height: 24px; width: 24px;"
class="a-image-wrapper a-lazy-loaded a-manually-loaded s-image"
data-a-image-source="https://m.media-amazon.com/images/I/111mHoVK0kL._SS200_.png">
<noscript><img alt=""
src="https://m.media-amazon.com/images/I/111mHoVK0kL._SS200_.png"
height="24px" width="24px" /></noscript></div> <span
class="a-size-medium-plus a-color-base a-text-bold">Small
Business</span>
</div>
</div>
<div class="a-spacing-medium s-pc-link-container"><span
class="a-size-base a-color-secondary">Shop products from small
business brands sold in Amazon’s store. Discover more about the
small businesses partnering with Amazon and Amazon’s commitment
to empowering them.</span> <a
class="a-size-base a-link-normal s-link-style"
href="https://www.amazon.com/b/ref=s9_acss_bw_cg_sbp22c_1e1_w/ref=SBE_navbar_5?pf_rd_r=6W5X52VNZRB7GK1E1VX2&amp;pf_rd_p=56621c3d-cff4-45e1-9bf4-79bbeb8006fc&amp;pf_rd_m=ATVPDKIKX0DER&amp;pf_rd_s=merchandised-search-top-3&amp;pf_rd_t=30901&amp;pf_rd_i=17879387011&amp;node=18018208011">Learn
more</a> </div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</span>
</div>
</div>
</div>
</div>

217
docs/examples/c4a_script/api_usage_examples.py Normal file
View File

@@ -0,0 +1,217 @@
"""
C4A-Script API Usage Examples
Shows how to use the new Result-based API in various scenarios
"""
from c4a_compile import compile, validate, compile_file
from c4a_result import CompilationResult, ValidationResult
import json
print("C4A-Script API Usage Examples")
print("=" * 80)
# Example 1: Basic compilation
print("\n1. Basic Compilation")
print("-" * 40)
script = """
GO https://example.com
WAIT 2
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
REPEAT (SCROLL DOWN 300, 3)
"""
result = compile(script)
print(f"Success: {result.success}")
print(f"Statements generated: {len(result.js_code) if result.js_code else 0}")
# Example 2: Error handling
print("\n\n2. Error Handling")
print("-" * 40)
error_script = """
GO https://example.com
IF (EXISTS `.modal`) CLICK `.close`
undefined_procedure
"""
result = compile(error_script)
if not result.success:
# Access error details
error = result.first_error
print(f"Error on line {error.line}: {error.message}")
print(f"Error code: {error.code}")
# Show suggestions if available
if error.suggestions:
print("Suggestions:")
for suggestion in error.suggestions:
print(f" - {suggestion.message}")
# Example 3: Validation only
print("\n\n3. Validation (no code generation)")
print("-" * 40)
validation_script = """
PROC validate_form
IF (EXISTS `#email`) THEN TYPE "test@example.com"
PRESS Tab
ENDPROC
validate_form
"""
validation = validate(validation_script)
print(f"Valid: {validation.valid}")
if validation.errors:
print(f"Errors found: {len(validation.errors)}")
# Example 4: JSON output for UI
print("\n\n4. JSON Output for UI Integration")
print("-" * 40)
ui_script = """
CLICK button.submit
"""
result = compile(ui_script)
if not result.success:
# Get JSON for UI
error_json = result.to_dict()
print("Error data for UI:")
print(json.dumps(error_json["errors"][0], indent=2))
# Example 5: File compilation
print("\n\n5. File Compilation")
print("-" * 40)
# Create a test file
test_file = "test_script.c4a"
with open(test_file, "w") as f:
f.write("""
GO https://example.com
WAIT `.content` 5
CLICK `.main-button`
""")
result = compile_file(test_file)
print(f"File compilation: {'Success' if result.success else 'Failed'}")
if result.success:
print(f"Generated {len(result.js_code)} JavaScript statements")
# Clean up
import os
os.remove(test_file)
# Example 6: Batch processing
print("\n\n6. Batch Processing Multiple Scripts")
print("-" * 40)
scripts = [
"GO https://example1.com\nCLICK `.button`",
"GO https://example2.com\nWAIT 2",
"GO https://example3.com\nINVALID_CMD"
]
results = []
for i, script in enumerate(scripts, 1):
result = compile(script)
results.append(result)
status = "✓" if result.success else "✗"
print(f"Script {i}: {status}")
# Summary
successful = sum(1 for r in results if r.success)
print(f"\nBatch result: {successful}/{len(scripts)} successful")
# Example 7: Custom error formatting
print("\n\n7. Custom Error Formatting")
print("-" * 40)
def format_error_for_ide(error):
"""Format error for IDE integration"""
return f"{error.source_line}:{error.line}:{error.column}: {error.type.value}: {error.message} [{error.code}]"
error_script = "IF EXISTS `.button` THEN CLICK `.button`"
result = compile(error_script)
if not result.success:
error = result.first_error
print("IDE format:", format_error_for_ide(error))
print("Simple format:", error.simple_message)
print("Full format:", error.formatted_message)
# Example 8: Working with warnings (future feature)
print("\n\n8. Handling Warnings")
print("-" * 40)
# In the future, we might have warnings
result = compile("GO https://example.com\nWAIT 100") # Very long wait
print(f"Success: {result.success}")
print(f"Warnings: {len(result.warnings)}")
# Example 9: Metadata usage
print("\n\n9. Using Metadata")
print("-" * 40)
complex_script = """
PROC helper1
CLICK `.btn1`
ENDPROC
PROC helper2
CLICK `.btn2`
ENDPROC
GO https://example.com
helper1
helper2
"""
result = compile(complex_script)
if result.success:
print(f"Script metadata:")
for key, value in result.metadata.items():
print(f" {key}: {value}")
# Example 10: Integration patterns
print("\n\n10. Integration Patterns")
print("-" * 40)
# Web API endpoint simulation
def api_compile(request_body):
"""Simulate API endpoint"""
script = request_body.get("script", "")
result = compile(script)
response = {
"status": "success" if result.success else "error",
"data": result.to_dict()
}
return response
# CLI tool simulation
def cli_compile(script, output_format="text"):
"""Simulate CLI tool"""
result = compile(script)
if output_format == "json":
return result.to_json()
elif output_format == "simple":
if result.success:
return f"OK: {len(result.js_code)} statements"
else:
return f"ERROR: {result.first_error.simple_message}"
else:
return str(result)
# Test the patterns
api_response = api_compile({"script": "GO https://example.com"})
print(f"API response status: {api_response['status']}")
cli_output = cli_compile("WAIT 2", "simple")
print(f"CLI output: {cli_output}")
print("\n" + "=" * 80)
print("All examples completed successfully!")

53
docs/examples/c4a_script/c4a_script_hello_world.py Normal file
View File

@@ -0,0 +1,53 @@
"""
C4A-Script Hello World
A concise example showing how to use the C4A-Script compiler
"""
from c4a_compile import compile
# Define your C4A-Script
script = """
GO https://example.com
WAIT `#content` 5
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
CLICK `button.submit`
"""
# Compile the script
result = compile(script)
# Check if compilation was successful
if result.success:
# Success! Use the generated JavaScript
print("✅ Compilation successful!")
print(f"Generated {len(result.js_code)} JavaScript statements:\n")
for i, js in enumerate(result.js_code, 1):
print(f"{i}. {js}\n")
# In real usage, you'd pass result.js_code to Crawl4AI:
# config = CrawlerRunConfig(js_code=result.js_code)
else:
# Error! Handle the compilation error
print("❌ Compilation failed!")
# Get the first error (there might be multiple)
error = result.first_error
# Show error details
print(f"Error at line {error.line}, column {error.column}")
print(f"Message: {error.message}")
# Show the problematic code
print(f"\nCode: {error.source_line}")
print(" " * (6 + error.column) + "^")
# Show suggestions if available
if error.suggestions:
print("\n💡 How to fix:")
for suggestion in error.suggestions:
print(f" {suggestion.message}")
# For debugging or logging, you can also get JSON
# error_json = result.to_json()

53
docs/examples/c4a_script/c4a_script_hello_world_error.py Normal file
View File

@@ -0,0 +1,53 @@
"""
C4A-Script Hello World - Error Example
Shows how error handling works
"""
from c4a_compile import compile
# Define a script with an error (missing THEN)
script = """
GO https://example.com
WAIT `#content` 5
IF (EXISTS `.cookie-banner`) CLICK `.accept`
CLICK `button.submit`
"""
# Compile the script
result = compile(script)
# Check if compilation was successful
if result.success:
# Success! Use the generated JavaScript
print("✅ Compilation successful!")
print(f"Generated {len(result.js_code)} JavaScript statements:\n")
for i, js in enumerate(result.js_code, 1):
print(f"{i}. {js}\n")
# In real usage, you'd pass result.js_code to Crawl4AI:
# config = CrawlerRunConfig(js_code=result.js_code)
else:
# Error! Handle the compilation error
print("❌ Compilation failed!")
# Get the first error (there might be multiple)
error = result.first_error
# Show error details
print(f"Error at line {error.line}, column {error.column}")
print(f"Message: {error.message}")
# Show the problematic code
print(f"\nCode: {error.source_line}")
print(" " * (6 + error.column) + "^")
# Show suggestions if available
if error.suggestions:
print("\n💡 How to fix:")
for suggestion in error.suggestions:
print(f" {suggestion.message}")
# For debugging or logging, you can also get JSON
# error_json = result.to_json()

285
docs/examples/c4a_script/demo_c4a_crawl4ai.py Normal file
View File

@@ -0,0 +1,285 @@
"""
Demonstration of C4A-Script integration with Crawl4AI
Shows various use cases and features
"""
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai import c4a_compile, CompilationResult
async def example_basic_usage():
"""Basic C4A-Script usage with Crawl4AI"""
print("\n" + "="*60)
print("Example 1: Basic C4A-Script Usage")
print("="*60)
# Define your automation script
c4a_script = """
# Wait for page to load
WAIT `body` 2
# Handle cookie banner if present
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-btn`
# Scroll down to load more content
SCROLL DOWN 500
WAIT 1
# Click load more button if exists
IF (EXISTS `.load-more`) THEN CLICK `.load-more`
"""
# Create crawler config with C4A script
config = CrawlerRunConfig(
url="https://example.com",
c4a_script=c4a_script,
wait_for="css:.content",
verbose=False
)
print("✅ C4A Script compiled successfully!")
print(f"Generated {len(config.js_code)} JavaScript commands")
# In production, you would run:
# async with AsyncWebCrawler() as crawler:
# result = await crawler.arun(config=config)
async def example_form_filling():
"""Form filling with C4A-Script"""
print("\n" + "="*60)
print("Example 2: Form Filling with C4A-Script")
print("="*60)
# Form automation script
form_script = """
# Set form values
SET email = "test@example.com"
SET message = "This is a test message"
# Fill the form
CLICK `#email-input`
TYPE $email
CLICK `#message-textarea`
TYPE $message
# Submit the form
CLICK `button[type="submit"]`
# Wait for success message
WAIT `.success-message` 10
"""
config = CrawlerRunConfig(
url="https://example.com/contact",
c4a_script=form_script
)
print("✅ Form filling script ready")
print("Script will:")
print(" - Fill email field")
print(" - Fill message textarea")
print(" - Submit form")
print(" - Wait for confirmation")
async def example_dynamic_loading():
"""Handle dynamic content loading"""
print("\n" + "="*60)
print("Example 3: Dynamic Content Loading")
print("="*60)
# Script for infinite scroll or pagination
pagination_script = """
# Initial wait
WAIT `.product-list` 5
# Load all products by clicking "Load More" repeatedly
REPEAT (CLICK `.load-more`, `document.querySelector('.load-more') !== null`)
# Alternative: Scroll to load (infinite scroll)
# REPEAT (SCROLL DOWN 1000, `document.querySelectorAll('.product').length < 100`)
# Extract count
EVAL `console.log('Products loaded: ' + document.querySelectorAll('.product').length)`
"""
config = CrawlerRunConfig(
url="https://example.com/products",
c4a_script=pagination_script,
screenshot=True # Capture final state
)
print("✅ Dynamic loading script ready")
print("Script will load all products by repeatedly clicking 'Load More'")
async def example_multi_step_workflow():
"""Complex multi-step workflow with procedures"""
print("\n" + "="*60)
print("Example 4: Multi-Step Workflow with Procedures")
print("="*60)
# Complex workflow with reusable procedures
workflow_script = """
# Define login procedure
PROC login
CLICK `#username`
TYPE "demo_user"
CLICK `#password`
TYPE "demo_pass"
CLICK `#login-btn`
WAIT `.dashboard` 10
ENDPROC
# Define search procedure
PROC search_product
CLICK `.search-box`
TYPE "laptop"
PRESS Enter
WAIT `.search-results` 5
ENDPROC
# Main workflow
GO https://example.com
login
search_product
# Process results
IF (EXISTS `.no-results`) THEN EVAL `console.log('No products found')`
ELSE REPEAT (CLICK `.add-to-cart`, 3)
"""
# Compile to check for errors
result = c4a_compile(workflow_script)
if result.success:
print("✅ Complex workflow compiled successfully!")
print("Workflow includes:")
print(" - Login procedure")
print(" - Product search")
print(" - Conditional cart additions")
config = CrawlerRunConfig(
url="https://example.com",
c4a_script=workflow_script
)
else:
print("❌ Compilation error:")
error = result.first_error
print(f" Line {error.line}: {error.message}")
async def example_error_handling():
"""Demonstrate error handling"""
print("\n" + "="*60)
print("Example 5: Error Handling")
print("="*60)
# Script with intentional error
bad_script = """
WAIT body 2
CLICK button
IF (EXISTS .modal) CLICK .close
"""
try:
config = CrawlerRunConfig(
url="https://example.com",
c4a_script=bad_script
)
except ValueError as e:
print("✅ Error caught as expected:")
print(f" {e}")
# Fixed version
good_script = """
WAIT `body` 2
CLICK `button`
IF (EXISTS `.modal`) THEN CLICK `.close`
"""
config = CrawlerRunConfig(
url="https://example.com",
c4a_script=good_script
)
print("\n✅ Fixed script compiled successfully!")
async def example_combining_with_extraction():
"""Combine C4A-Script with extraction strategies"""
print("\n" + "="*60)
print("Example 6: C4A-Script + Extraction Strategies")
print("="*60)
from crawl4ai import JsonCssExtractionStrategy
# Script to prepare page for extraction
prep_script = """
# Expand all collapsed sections
REPEAT (CLICK `.expand-btn`, `document.querySelectorAll('.expand-btn:not(.expanded)').length > 0`)
# Load all comments
IF (EXISTS `.load-comments`) THEN CLICK `.load-comments`
WAIT `.comments-section` 5
# Close any popups
IF (EXISTS `.popup-close`) THEN CLICK `.popup-close`
"""
# Define extraction schema
schema = {
"name": "article",
"selector": "article.main",
"fields": {
"title": {"selector": "h1", "type": "text"},
"content": {"selector": ".content", "type": "text"},
"comments": {
"selector": ".comment",
"type": "list",
"fields": {
"author": {"selector": ".author", "type": "text"},
"text": {"selector": ".text", "type": "text"}
}
}
}
}
config = CrawlerRunConfig(
url="https://example.com/article",
c4a_script=prep_script,
extraction_strategy=JsonCssExtractionStrategy(schema),
wait_for="css:.comments-section"
)
print("✅ Combined C4A + Extraction ready")
print("Workflow:")
print(" 1. Expand collapsed sections")
print(" 2. Load comments")
print(" 3. Extract structured data")
async def main():
"""Run all examples"""
print("\n🚀 C4A-Script + Crawl4AI Integration Demo\n")
# Run all examples
await example_basic_usage()
await example_form_filling()
await example_dynamic_loading()
await example_multi_step_workflow()
await example_error_handling()
await example_combining_with_extraction()
print("\n" + "="*60)
print("✅ All examples completed successfully!")
print("="*60)
print("\nTo run actual crawls, uncomment the AsyncWebCrawler sections")
print("or create your own scripts using these examples as templates.")
if __name__ == "__main__":
asyncio.run(main())

89
docs/examples/c4a_script/generate_script_hello_world.py Normal file
View File

@@ -0,0 +1,89 @@
#!/usr/bin/env python3
"""
Hello World Example: LLM-Generated C4A-Script
This example shows how to use the new generate_script() function to automatically
create C4A-Script automation from natural language descriptions and HTML.
"""
from crawl4ai.script.c4a_compile import C4ACompiler
def main():
print("🤖 C4A-Script Generation Hello World")
print("=" * 50)
# Example 1: Simple login form
html = """
<html>
<body>
<form id="login">
<input id="email" type="email" placeholder="Email">
<input id="password" type="password" placeholder="Password">
<button id="submit">Login</button>
</form>
</body>
</html>
"""
goal = "Fill in email 'user@example.com', password 'secret123', and submit the form"
print("📝 Goal:", goal)
print("🌐 HTML: Simple login form")
print()
# Generate C4A-Script
print("🔧 Generated C4A-Script:")
print("-" * 30)
c4a_script = C4ACompiler.generate_script(
html=html,
query=goal,
mode="c4a"
)
print(c4a_script)
print()
# Generate JavaScript
print("🔧 Generated JavaScript:")
print("-" * 30)
js_script = C4ACompiler.generate_script(
html=html,
query=goal,
mode="js"
)
print(js_script)
print()
# Example 2: Simple button click
html2 = """
<html>
<body>
<div class="content">
<h1>Welcome!</h1>
<button id="start-btn" class="primary">Get Started</button>
</div>
</body>
</html>
"""
goal2 = "Click the 'Get Started' button"
print("=" * 50)
print("📝 Goal:", goal2)
print("🌐 HTML: Simple button")
print()
print("🔧 Generated C4A-Script:")
print("-" * 30)
c4a_script2 = C4ACompiler.generate_script(
html=html2,
query=goal2,
mode="c4a"
)
print(c4a_script2)
print()
print("✅ Done! The LLM automatically converted natural language goals")
print(" into executable automation scripts.")
if __name__ == "__main__":
main()

111
docs/examples/c4a_script/github_search/extracted_repositories.json Normal file
View File

@@ -0,0 +1,111 @@
[
{
"repository_name": "unclecode/crawl4ai",
"repository_owner": "unclecode/crawl4ai",
"repository_url": "/unclecode/crawl4ai",
"description": "\ud83d\ude80\ud83e\udd16Crawl4AI: Open-source LLM Friendly Web Crawler & Scraper. Don't be shy, join here:https://discord.gg/jP8KfhDhyN",
"primary_language": "Python",
"star_count": "45.1k",
"topics": [],
"last_updated": "23 hours ago"
},
{
"repository_name": "coleam00/mcp-crawl4ai-rag",
"repository_owner": "coleam00/mcp-crawl4ai-rag",
"repository_url": "/coleam00/mcp-crawl4ai-rag",
"description": "Web Crawling and RAG Capabilities for AI Agents and AI Coding Assistants",
"primary_language": "Python",
"star_count": "748",
"topics": [],
"last_updated": "yesterday"
},
{
"repository_name": "pdichone/crawl4ai-rag-system",
"repository_owner": "pdichone/crawl4ai-rag-system",
"repository_url": "/pdichone/crawl4ai-rag-system",
"primary_language": "Python",
"star_count": "44",
"topics": [],
"last_updated": "on 21 Jan"
},
{
"repository_name": "weidwonder/crawl4ai-mcp-server",
"repository_owner": "weidwonder/crawl4ai-mcp-server",
"repository_url": "/weidwonder/crawl4ai-mcp-server",
"description": "\u7528\u4e8e\u63d0\u4f9b\u7ed9\u672c\u5730\u5f00\u53d1\u8005\u7684 LLM\u7684\u9ad8\u6548\u4e92\u8054\u7f51\u641c\u7d22&\u5185\u5bb9\u83b7\u53d6\u7684MCP Server\uff0c \u8282\u7701\u4f60\u7684token",
"primary_language": "Python",
"star_count": "87",
"topics": [],
"last_updated": "24 days ago"
},
{
"repository_name": "leonardogrig/crawl4ai-deepseek-example",
"repository_owner": "leonardogrig/crawl4ai-deepseek-example",
"repository_url": "/leonardogrig/crawl4ai-deepseek-example",
"primary_language": "Python",
"star_count": "29",
"topics": [],
"last_updated": "on 18 Jan"
},
{
"repository_name": "laurentvv/crawl4ai-mcp",
"repository_owner": "laurentvv/crawl4ai-mcp",
"repository_url": "/laurentvv/crawl4ai-mcp",
"description": "Web crawling tool that integrates with AI assistants via the MCP",
"primary_language": "Python",
"star_count": "10",
"topics": [
{},
{},
{},
{},
{}
],
"last_updated": "on 16 Mar"
},
{
"repository_name": "kaymen99/ai-web-scraper",
"repository_owner": "kaymen99/ai-web-scraper",
"repository_url": "/kaymen99/ai-web-scraper",
"description": "AI web scraper built withCrawl4AIfor extracting structured leads data from websites.",
"primary_language": "Python",
"star_count": "30",
"topics": [
{},
{},
{},
{},
{}
],
"last_updated": "on 13 Feb"
},
{
"repository_name": "atakkant/ai_web_crawler",
"repository_owner": "atakkant/ai_web_crawler",
"repository_url": "/atakkant/ai_web_crawler",
"description": "crawl4ai, DeepSeek, Groq",
"primary_language": "Python",
"star_count": "9",
"topics": [],
"last_updated": "on 19 Feb"
},
{
"repository_name": "Croups/auto-scraper-with-llms",
"repository_owner": "Croups/auto-scraper-with-llms",
"repository_url": "/Croups/auto-scraper-with-llms",
"description": "Web scraping AI that leverages thecrawl4ailibrary to extract structured data from web pages using various large language models (LLMs).",
"primary_language": "Python",
"star_count": "49",
"topics": [],
"last_updated": "on 8 Apr"
},
{
"repository_name": "leonardogrig/crawl4ai_llm_examples",
"repository_owner": "leonardogrig/crawl4ai_llm_examples",
"repository_url": "/leonardogrig/crawl4ai_llm_examples",
"primary_language": "Python",
"star_count": "8",
"topics": [],
"last_updated": "on 29 Jan"
}
]

66
docs/examples/c4a_script/github_search/generated_result_schema.json Normal file
View File

@@ -0,0 +1,66 @@
{
"name": "GitHub Repository Cards",
"baseSelector": "div.Box-sc-g0xbh4-0.iwUbcA",
"fields": [
{
"name": "repository_name",
"selector": "div.search-title a span",
"type": "text",
"transform": "strip"
},
{
"name": "repository_owner",
"selector": "div.search-title a span",
"type": "text",
"transform": "split",
"pattern": "/"
},
{
"name": "repository_url",
"selector": "div.search-title a",
"type": "attribute",
"attribute": "href",
"transform": "prepend",
"pattern": "https://github.com"
},
{
"name": "description",
"selector": "div.dcdlju span",
"type": "text"
},
{
"name": "primary_language",
"selector": "ul.bZkODq li span[aria-label]",
"type": "text"
},
{
"name": "star_count",
"selector": "ul.bZkODq li a[href*='stargazers'] span",
"type": "text",
"transform": "strip"
},
{
"name": "topics",
"type": "list",
"selector": "div.jgRnBg div a",
"fields": [
{
"name": "topic_name",
"selector": "a",
"type": "text"
}
]
},
{
"name": "last_updated",
"selector": "ul.bZkODq li span[title]",
"type": "text"
},
{
"name": "has_sponsor_button",
"selector": "button[aria-label*='Sponsor']",
"type": "text",
"transform": "exists"
}
]
}

39
docs/examples/c4a_script/github_search/generated_search_script.js Normal file
View File

@@ -0,0 +1,39 @@
(async () => {
const waitForElement = (selector, timeout = 10000) => new Promise((resolve, reject) => {
const el = document.querySelector(selector);
if (el) return resolve(el);
const observer = new MutationObserver(() => {
const el = document.querySelector(selector);
if (el) {
observer.disconnect();
resolve(el);
}
});
observer.observe(document.body, { childList: true, subtree: true });
setTimeout(() => {
observer.disconnect();
reject(new Error(`Timeout waiting for ${selector}`));
}, timeout);
});
try {
const searchInput = await waitForElement('#adv_code_search input[type="text"]');
searchInput.value = 'crawl4AI';
searchInput.dispatchEvent(new Event('input', { bubbles: true }));
const languageSelect = await waitForElement('#search_language');
languageSelect.value = 'Python';
languageSelect.dispatchEvent(new Event('change', { bubbles: true }));
const starsInput = await waitForElement('#search_stars');
starsInput.value = '>10000';
starsInput.dispatchEvent(new Event('input', { bubbles: true }));
const searchButton = await waitForElement('#adv_code_search button[type="submit"]');
searchButton.click();
await waitForElement('.codesearch-results, #search-results');
} catch (e) {
console.error('Search script failed:', e.message);
}
})();

211
docs/examples/c4a_script/github_search/github_search_crawler.py Normal file
View File

@@ -0,0 +1,211 @@
#!/usr/bin/env python3
"""
GitHub Advanced Search Example using Crawl4AI
This example demonstrates:
1. Using LLM to generate C4A-Script from HTML snippets
2. Single arun() call with navigation, search form filling, and extraction
3. JSON CSS extraction for structured repository data
4. Complete workflow: navigate → fill form → submit → extract results
Requirements:
- Crawl4AI with generate_script support
- LLM API key (configured in environment)
"""
import asyncio
import json
import os
from pathlib import Path
from typing import List, Dict, Any
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
from crawl4ai import JsonCssExtractionStrategy
from crawl4ai.script.c4a_compile import C4ACompiler
class GitHubSearchScraper:
def __init__(self):
self.base_dir = Path(__file__).parent
self.search_script_path = self.base_dir / "generated_search_script.js"
self.schema_path = self.base_dir / "generated_result_schema.json"
self.results_path = self.base_dir / "extracted_repositories.json"
self.session_id = "github_search_session"
async def generate_search_script(self) -> str:
"""Generate JavaScript for GitHub advanced search interaction"""
print("🔧 Generating search script from search_form.html...")
# Check if already generated
if self.search_script_path.exists():
print("✅ Using cached search script")
return self.search_script_path.read_text()
# Read the search form HTML
search_form_html = (self.base_dir / "search_form.html").read_text()
# Generate script using LLM
search_goal = """
Search for crawl4AI repositories written in Python with more than 10000 stars:
1. Wait for the main search input to be visible
2. Type "crawl4AI" into the main search box
3. Select "Python" from the language dropdown (#search_language)
4. Type ">10000" into the stars input field (#search_stars)
5. Click the search button to submit the form
6. Wait for the search results to appear
"""
try:
script = C4ACompiler.generate_script(
html=search_form_html,
query=search_goal,
mode="js"
)
# Save for future use
self.search_script_path.write_text(script)
print("✅ Search script generated and saved!")
print(f"📄 Script preview:\n{script[:500]}...")
return script
except Exception as e:
print(f"❌ Error generating search script: {e}")
raise
async def generate_result_schema(self) -> Dict[str, Any]:
"""Generate JSON CSS extraction schema from result HTML"""
print("\n🔧 Generating result extraction schema...")
# Check if already generated
if self.schema_path.exists():
print("✅ Using cached extraction schema")
return json.loads(self.schema_path.read_text())
# Read the result HTML
result_html = (self.base_dir / "result.html").read_text()
# Generate extraction schema using LLM
schema_goal = """
Create a JSON CSS extraction schema to extract from each repository card:
- Repository name (the repository name only, not including owner)
- Repository owner (organization or username)
- Repository URL (full GitHub URL)
- Description
- Primary programming language
- Star count (numeric value)
- Topics/tags (array of topic names)
- Last updated (time ago string)
- Whether it has a sponsor button
The schema should handle multiple repository results on the search results page.
"""
try:
# Generate schema
schema = JsonCssExtractionStrategy.generate_schema(
html=result_html,
query=schema_goal,
)
# Save for future use
self.schema_path.write_text(json.dumps(schema, indent=2))
print("✅ Extraction schema generated and saved!")
print(f"📄 Schema fields: {[f['name'] for f in schema['fields']]}")
return schema
except Exception as e:
print(f"❌ Error generating schema: {e}")
raise
async def crawl_github(self):
"""Main crawling logic with single arun() call"""
print("\n🚀 Starting GitHub repository search...")
# Generate scripts and schemas
search_script = await self.generate_search_script()
result_schema = await self.generate_result_schema()
# Configure browser (headless=False to see the action)
browser_config = BrowserConfig(
headless=False,
verbose=True,
viewport_width=1920,
viewport_height=1080
)
async with AsyncWebCrawler(config=browser_config) as crawler:
print("\n📍 Navigating to GitHub advanced search and executing search...")
# Single call: Navigate, execute search, and extract results
search_config = CrawlerRunConfig(
session_id=self.session_id,
js_code=search_script, # Execute generated JS
# wait_for="[data-testid='results-list']", # Wait for search results
wait_for=".Box-sc-g0xbh4-0.iwUbcA", # Wait for search results
extraction_strategy=JsonCssExtractionStrategy(schema=result_schema),
delay_before_return_html=3.0, # Give time for results to fully load
cache_mode=CacheMode.BYPASS # Don't cache for fresh results
)
result = await crawler.arun(
url="https://github.com/search/advanced",
config=search_config
)
if not result.success:
print("❌ Failed to search GitHub")
print(f"Error: {result.error_message}")
return
print("✅ Search and extraction completed successfully!")
# Extract and save results
if result.extracted_content:
repositories = json.loads(result.extracted_content)
print(f"\n🔍 Found {len(repositories)} repositories matching criteria")
# Save results
self.results_path.write_text(
json.dumps(repositories, indent=2)
)
print(f"💾 Results saved to: {self.results_path}")
# Print sample results
print("\n📊 Sample Results:")
for i, repo in enumerate(repositories[:5], 1):
print(f"\n{i}. {repo.get('owner', 'Unknown')}/{repo.get('name', 'Unknown')}")
print(f" Description: {repo.get('description', 'No description')[:80]}...")
print(f" Language: {repo.get('language', 'Unknown')}")
print(f" Stars: {repo.get('stars', 'Unknown')}")
print(f" Updated: {repo.get('last_updated', 'Unknown')}")
if repo.get('topics'):
print(f" Topics: {', '.join(repo['topics'][:5])}")
print(f" URL: {repo.get('url', 'Unknown')}")
else:
print("❌ No repositories extracted")
# Save screenshot for reference
if result.screenshot:
screenshot_path = self.base_dir / "search_results_screenshot.png"
with open(screenshot_path, "wb") as f:
f.write(result.screenshot)
print(f"\n📸 Screenshot saved to: {screenshot_path}")
async def main():
"""Run the GitHub search scraper"""
scraper = GitHubSearchScraper()
await scraper.crawl_github()
print("\n🎉 GitHub search example completed!")
print("Check the generated files:")
print(" - generated_search_script.js")
print(" - generated_result_schema.json")
print(" - extracted_repositories.json")
print(" - search_results_screenshot.png")
if __name__ == "__main__":
asyncio.run(main())

54
docs/examples/c4a_script/github_search/result.html Normal file
View File

@@ -0,0 +1,54 @@
<div class="Box-sc-g0xbh4-0 iwUbcA"><div class="Box-sc-g0xbh4-0 cSURfY"><div class="Box-sc-g0xbh4-0 gPrlij"><h3 class="Box-sc-g0xbh4-0 cvnppv"><div class="Box-sc-g0xbh4-0 kYLlPM"><div class="Box-sc-g0xbh4-0 eurdCD"><img data-component="Avatar" class="prc-Avatar-Avatar-ZRS-m" alt="" data-square="" width="20" height="20" src="https://github.com/TheAlgorithms.png?size=40" data-testid="github-avatar" style="--avatarSize-regular: 20px;"></div><div class="Box-sc-g0xbh4-0 MHoGG search-title"><a class="prc-Link-Link-85e08" href="/TheAlgorithms/Python"><span class="Box-sc-g0xbh4-0 kzfhBO search-match prc-Text-Text-0ima0">TheAlgorithms/<em>Python</em></span></a></div></div></h3><div class="Box-sc-g0xbh4-0 dcdlju"><span class="Box-sc-g0xbh4-0 gKFdvh search-match prc-Text-Text-0ima0">All Algorithms implemented in <em>Python</em></span></div><div class="Box-sc-g0xbh4-0 jgRnBg"><div><a class="Box-sc-g0xbh4-0 hIVEGR prc-Link-Link-85e08" href="/topics/python">python</a></div><div><a class="Box-sc-g0xbh4-0 hIVEGR prc-Link-Link-85e08" href="/topics/education">education</a></div><div><a class="Box-sc-g0xbh4-0 hIVEGR prc-Link-Link-85e08" href="/topics/algorithm">algorithm</a></div><div><a class="Box-sc-g0xbh4-0 hIVEGR prc-Link-Link-85e08" href="/topics/practice">practice</a></div><div><a class="Box-sc-g0xbh4-0 hIVEGR prc-Link-Link-85e08" href="/topics/interview">interview</a></div></div><ul class="Box-sc-g0xbh4-0 bZkODq"><li class="Box-sc-g0xbh4-0 eCfCAC"><div class="Box-sc-g0xbh4-0 hjDqIa"><div class="Box-sc-g0xbh4-0 fwSYsx"></div></div><span aria-label="Python language">Python</span></li><span class="Box-sc-g0xbh4-0 eXQoFa prc-Text-Text-0ima0" aria-hidden="true">·</span><li class="Box-sc-g0xbh4-0 eCfCAC"><a class="Box-sc-g0xbh4-0 iPuHRc prc-Link-Link-85e08" href="/TheAlgorithms/Python/stargazers" aria-label="201161 stars"><svg aria-hidden="true" focusable="false" class="octicon octicon-star Octicon-sc-9kayk9-0 kHVtWu" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align: text-bottom;"><path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path></svg><span class="prc-Text-Text-0ima0">201k</span></a></li><span class="Box-sc-g0xbh4-0 eXQoFa prc-Text-Text-0ima0" aria-hidden="true">·</span><li class="Box-sc-g0xbh4-0 eCfCAC"><span>Updated <div title="3 Jun 2025, 01:57 GMT+8" class="Truncate__StyledTruncate-sc-23o1d2-0 liVpTx"><span class="prc-Text-Text-0ima0" title="3 Jun 2025, 01:57 GMT+8">4 days ago</span></div></span></li></ul></div><div class="Box-sc-g0xbh4-0 gtlRHe"><div class="Box-sc-g0xbh4-0 fvaNTI"><button type="button" class="prc-Button-ButtonBase-c50BI" data-loading="false" data-size="small" data-variant="default" aria-describedby=":r1c:-loading-announcement"><span data-component="buttonContent" data-align="center" class="prc-Button-ButtonContent-HKbr-"><span data-component="leadingVisual" class="prc-Button-Visual-2epfX prc-Button-VisualWrap-Db-eB"><svg aria-hidden="true" focusable="false" class="octicon octicon-star" viewBox="0 0 16 16" width="16" height="16" fill="currentColor" display="inline-block" overflow="visible" style="vertical-align: text-bottom;"><path d="M8 .25a.75.75 0 0 1 .673.418l1.882 3.815 4.21.612a.75.75 0 0 1 .416 1.279l-3.046 2.97.719 4.192a.751.751 0 0 1-1.088.791L8 12.347l-3.766 1.98a.75.75 0 0 1-1.088-.79l.72-4.194L.818 6.374a.75.75 0 0 1 .416-1.28l4.21-.611L7.327.668A.75.75 0 0 1 8 .25Zm0 2.445L6.615 5.5a.75.75 0 0 1-.564.41l-3.097.45 2.24 2.184a.75.75 0 0 1 .216.664l-.528 3.084 2.769-1.456a.75.75 0 0 1 .698 0l2.77 1.456-.53-3.084a.75.75 0 0 1 .216-.664l2.24-2.183-3.096-.45a.75.75 0 0 1-.564-.41L8 2.694Z"></path></svg></span><span data-component="text" class="prc-Button-Label-pTQ3x">Star</span></span></button></div><div class="Box-sc-g0xbh4-0 llZEgI"><div class="Box-sc-g0xbh4-0"> <button id="dialog-show-funding-links-modal-TheAlgorithms-Python" aria-label="Sponsor TheAlgorithms/Python" data-show-dialog-id="funding-links-modal-TheAlgorithms-Python" type="button" data-view-component="true" class="Button--secondary Button--small Button"> <span class="Button-content">
<span class="Button-label"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-heart icon-sponsor mr-1 color-fg-sponsors">
<path d="m8 14.25.345.666a.75.75 0 0 1-.69 0l-.008-.004-.018-.01a7.152 7.152 0 0 1-.31-.17 22.055 22.055 0 0 1-3.434-2.414C2.045 10.731 0 8.35 0 5.5 0 2.836 2.086 1 4.25 1 5.797 1 7.153 1.802 8 3.02 8.847 1.802 10.203 1 11.75 1 13.914 1 16 2.836 16 5.5c0 2.85-2.045 5.231-3.885 6.818a22.066 22.066 0 0 1-3.744 2.584l-.018.01-.006.003h-.002ZM4.25 2.5c-1.336 0-2.75 1.164-2.75 3 0 2.15 1.58 4.144 3.365 5.682A20.58 20.58 0 0 0 8 13.393a20.58 20.58 0 0 0 3.135-2.211C12.92 9.644 14.5 7.65 14.5 5.5c0-1.836-1.414-3-2.75-3-1.373 0-2.609.986-3.029 2.456a.749.749 0 0 1-1.442 0C6.859 3.486 5.623 2.5 4.25 2.5Z"></path>
</svg> <span data-view-component="true">Sponsor</span></span>
</span>
</button>
<dialog-helper>
<dialog id="funding-links-modal-TheAlgorithms-Python" aria-modal="true" aria-labelledby="funding-links-modal-TheAlgorithms-Python-title" aria-describedby="funding-links-modal-TheAlgorithms-Python-description" data-view-component="true" class="Overlay Overlay-whenNarrow Overlay--size-medium Overlay--motion-scaleFade Overlay--disableScroll">
<div data-view-component="true" class="Overlay-header">
<div class="Overlay-headerContentWrap">
<div class="Overlay-titleWrap">
<h1 class="Overlay-title " id="funding-links-modal-TheAlgorithms-Python-title">
Sponsor TheAlgorithms/Python
</h1>
</div>
<div class="Overlay-actionWrap">
<button data-close-dialog-id="funding-links-modal-TheAlgorithms-Python" aria-label="Close" type="button" data-view-component="true" class="close-button Overlay-closeButton"><svg aria-hidden="true" height="16" viewBox="0 0 16 16" version="1.1" width="16" data-view-component="true" class="octicon octicon-x">
<path d="M3.72 3.72a.75.75 0 0 1 1.06 0L8 6.94l3.22-3.22a.749.749 0 0 1 1.275.326.749.749 0 0 1-.215.734L9.06 8l3.22 3.22a.749.749 0 0 1-.326 1.275.749.749 0 0 1-.734-.215L8 9.06l-3.22 3.22a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042L6.94 8 3.72 4.78a.75.75 0 0 1 0-1.06Z"></path>
</svg></button>
</div>
</div>
</div>
<scrollable-region data-labelled-by="funding-links-modal-TheAlgorithms-Python-title" data-catalyst="" style="overflow: auto;">
<div data-view-component="true" class="Overlay-body"> <div class="text-left f5">
<div class="pt-3 color-bg-overlay">
<h5 class="flex-auto mb-3 mt-0">External links</h5>
<div class="d-flex mb-3">
<div class="circle mr-2 border d-flex flex-justify-center flex-items-center flex-shrink-0" style="width:24px;height:24px;">
<img width="16" height="16" class="octicon rounded-2 d-block" alt="liberapay" src="https://github.githubassets.com/assets/liberapay-48108ded7267.svg">
</div>
<div class="flex-auto min-width-0">
<a target="_blank" data-ga-click="Dashboard, click, Nav menu - item:org-profile context:organization" data-hydro-click="{&quot;event_type&quot;:&quot;sponsors.repo_funding_links_link_click&quot;,&quot;payload&quot;:{&quot;platform&quot;:{&quot;platform_type&quot;:&quot;LIBERAPAY&quot;,&quot;platform_url&quot;:&quot;https://liberapay.com/TheAlgorithms&quot;},&quot;platforms&quot;:[{&quot;platform_type&quot;:&quot;LIBERAPAY&quot;,&quot;platform_url&quot;:&quot;https://liberapay.com/TheAlgorithms&quot;}],&quot;repo_id&quot;:63476337,&quot;owner_id&quot;:20487725,&quot;user_id&quot;:12494079,&quot;originating_url&quot;:&quot;https://github.com/TheAlgorithms/Python/funding_links?fragment=1&quot;}}" data-hydro-click-hmac="123b5aa7d5ffff5ef0530f8e7fbaebcb564e8de1af26f1b858a19b0e1d4f9e5f" href="https://liberapay.com/TheAlgorithms"><span>liberapay.com/<strong>TheAlgorithms</strong></span></a>
</div>
</div>
</div>
<div class="text-small p-3 border-top">
<p class="my-0">
<a class="Link--inTextBlock" href="https://docs.github.com/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/displaying-a-sponsor-button-in-your-repository">Learn more about funding links in repositories</a>.
</p>
<p class="my-0">
<a class="Link--secondary" href="/contact/report-abuse?report=TheAlgorithms%2FPython+%28Repository+Funding+Links%29">Report abuse</a>
</p>
</div>
</div>
</div>
</scrollable-region>
</dialog></dialog-helper>
</div></div></div></div></div>

336
docs/examples/c4a_script/github_search/search_form.html Normal file
View File

@@ -0,0 +1,336 @@
<form id="search_form" class="search_repos" data-turbo="false" action="/search" accept-charset="UTF-8" method="get">
<div class="pagehead codesearch-head color-border-muted">
<div class="container-lg p-responsive d-flex flex-column flex-md-row">
<h1 class="flex-shrink-0" id="search-title">Advanced search</h1>
<div class="search-form-fluid flex-auto d-flex flex-column flex-md-row pt-2 pt-md-0" id="adv_code_search">
<div class="flex-auto pr-md-2">
<label class="form-control search-page-label js-advanced-search-label">
<input aria-labelledby="search-title" class="form-control input-block search-page-input js-advanced-search-input js-advanced-search-prefix" data-search-prefix="" type="text" value="">
<p class="completed-query js-advanced-query top-0 right-0 left-0"><span></span> </p>
</label>
<input class="js-search-query" type="hidden" name="q" value="">
<input class="js-type-value" type="hidden" name="type" value="Repositories">
<input type="hidden" name="ref" value="advsearch">
</div>
<div class="d-flex d-md-block flex-shrink-0 pt-2 pt-md-0">
<button type="submit" data-view-component="true" class="btn flex-auto"> Search
</button>
</div>
</div>
</div>
</div>
<div class="container-lg p-responsive advanced-search-form">
<fieldset class="pb-3 mb-4 border-bottom color-border-muted min-width-0">
<h3>Advanced options</h3>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_from">From these owners</label></dt>
<dd><input id="search_from" type="text" class="form-control js-advanced-search-prefix" placeholder="github, atom, electron, octokit" data-search-prefix="user:"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_repos">In these repositories</label></dt>
<dd><input id="search_repos" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="twbs/bootstrap, rails/rails" data-search-prefix="repo:"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_date">Created on the dates</label></dt>
<dd><input id="search_date" type="text" class="form-control js-advanced-search-prefix" value="" placeholder=">YYYY-MM-DD, YYYY-MM-DD" data-search-prefix="created:"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_language">Written in this language</label></dt>
<dd>
<select id="search_language" name="l" class="form-select js-advanced-search-prefix" data-search-prefix="language:">
<option value="">Any language</option>
<optgroup label="Popular">
<option value="C">C</option>
<option value="C#">C#</option>
<option value="C++">C++</option>
<option value="CoffeeScript">CoffeeScript</option>
<option value="CSS">CSS</option>
<option value="Dart">Dart</option>
<option value="DM">DM</option>
<option value="Elixir">Elixir</option>
<option value="Go">Go</option>
<option value="Groovy">Groovy</option>
<option value="HTML">HTML</option>
<option value="Java">Java</option>
<option value="JavaScript">JavaScript</option>
<option value="Kotlin">Kotlin</option>
<option value="Objective-C">Objective-C</option>
<option value="Perl">Perl</option>
<option value="PHP">PHP</option>
<option value="PowerShell">PowerShell</option>
<option value="Python">Python</option>
<option value="Ruby">Ruby</option>
<option value="Rust">Rust</option>
<option value="Scala">Scala</option>
<option value="Shell">Shell</option>
<option value="Swift">Swift</option>
<option value="TypeScript">TypeScript</option>
</optgroup>
<optgroup label="Everything else">
<option value="1C Enterprise">1C Enterprise</option>
<option value="2-Dimensional Array">2-Dimensional Array</option>
<option value="4D">4D</option>
<option value="ABAP">ABAP</option>
<option value="ABAP CDS">ABAP CDS</option>
<option value="ABNF">ABNF</option>
<option value="ActionScript">ActionScript</option>
<option value="Ada">Ada</option>
<option value="Adblock Filter List">Adblock Filter List</option>
<option value="Adobe Font Metrics">Adobe Font Metrics</option>
<option value="Agda">Agda</option>
<option value="AGS Script">AGS Script</option>
<option value="AIDL">AIDL</option>
<option value="Aiken">Aiken</option>
</optgroup>
</select>
</dd>
</dl>
</fieldset>
<fieldset class="pb-3 mb-4 border-bottom color-border-muted min-width-0">
<h3>Repositories options</h3>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_stars">With this many stars</label></dt>
<dd><input id="search_stars" type="text" class="form-control js-advanced-search-prefix" placeholder="0..100, 200, >1000" data-search-prefix="stars:" data-search-type="Repositories"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_forks">With this many forks</label></dt>
<dd><input id="search_forks" type="text" class="form-control js-advanced-search-prefix" placeholder="50..100, 200, <5" data-search-prefix="forks:" data-search-type="Repositories"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_size">Of this size</label></dt>
<dd><input id="search_size" type="text" class="form-control js-advanced-search-prefix" placeholder="Repository size in KB" data-search-prefix="size:" data-search-type="Repositories"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_push">Pushed to</label></dt>
<dd><input id="search_push" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="<YYYY-MM-DD" data-search-prefix="pushed:" data-search-type="Repositories"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_license">With this license</label></dt>
<dd>
<select id="search_license" class="form-select js-advanced-search-prefix" data-search-prefix="license:" data-search-type="Repositories">
<option value="">Any license</option>
<optgroup label="Licenses">
<option value="0bsd">BSD Zero Clause License</option>
<option value="afl-3.0">Academic Free License v3.0</option>
<option value="agpl-3.0">GNU Affero General Public License v3.0</option>
<option value="apache-2.0">Apache License 2.0</option>
<option value="artistic-2.0">Artistic License 2.0</option>
<option value="blueoak-1.0.0">Blue Oak Model License 1.0.0</option>
<option value="bsd-2-clause">BSD 2-Clause "Simplified" License</option>
<option value="bsd-2-clause-patent">BSD-2-Clause Plus Patent License</option>
<option value="bsd-3-clause">BSD 3-Clause "New" or "Revised" License</option>
<option value="bsd-3-clause-clear">BSD 3-Clause Clear License</option>
<option value="bsd-4-clause">BSD 4-Clause "Original" or "Old" License</option>
<option value="bsl-1.0">Boost Software License 1.0</option>
<option value="cc-by-4.0">Creative Commons Attribution 4.0 International</option>
<option value="cc-by-sa-4.0">Creative Commons Attribution Share Alike 4.0 International</option>
<option value="cc0-1.0">Creative Commons Zero v1.0 Universal</option>
<option value="cecill-2.1">CeCILL Free Software License Agreement v2.1</option>
<option value="cern-ohl-p-2.0">CERN Open Hardware Licence Version 2 - Permissive</option>
<option value="cern-ohl-s-2.0">CERN Open Hardware Licence Version 2 - Strongly Reciprocal</option>
<option value="cern-ohl-w-2.0">CERN Open Hardware Licence Version 2 - Weakly Reciprocal</option>
<option value="ecl-2.0">Educational Community License v2.0</option>
<option value="epl-1.0">Eclipse Public License 1.0</option>
<option value="epl-2.0">Eclipse Public License 2.0</option>
<option value="eupl-1.1">European Union Public License 1.1</option>
<option value="eupl-1.2">European Union Public License 1.2</option>
<option value="gfdl-1.3">GNU Free Documentation License v1.3</option>
<option value="gpl-2.0">GNU General Public License v2.0</option>
<option value="gpl-3.0">GNU General Public License v3.0</option>
<option value="isc">ISC License</option>
<option value="lgpl-2.1">GNU Lesser General Public License v2.1</option>
<option value="lgpl-3.0">GNU Lesser General Public License v3.0</option>
<option value="lppl-1.3c">LaTeX Project Public License v1.3c</option>
<option value="mit">MIT License</option>
<option value="mit-0">MIT No Attribution</option>
<option value="mpl-2.0">Mozilla Public License 2.0</option>
<option value="ms-pl">Microsoft Public License</option>
<option value="ms-rl">Microsoft Reciprocal License</option>
<option value="mulanpsl-2.0">Mulan Permissive Software License, Version 2</option>
<option value="ncsa">University of Illinois/NCSA Open Source License</option>
<option value="odbl-1.0">Open Data Commons Open Database License v1.0</option>
<option value="ofl-1.1">SIL Open Font License 1.1</option>
<option value="osl-3.0">Open Software License 3.0</option>
<option value="postgresql">PostgreSQL License</option>
<option value="unlicense">The Unlicense</option>
<option value="upl-1.0">Universal Permissive License v1.0</option>
<option value="vim">Vim License</option>
<option value="wtfpl">Do What The F*ck You Want To Public License</option>
<option value="zlib">zlib License</option>
</optgroup>
<optgroup label="License families">
<option value="cc">Creative Commons</option>
<option value="gpl">GNU General Public License</option>
<option value="lgpl">GNU Lesser General Public License</option>
</optgroup>
</select>
</dd>
</dl>
<label>
Return repositories <select class="form-select js-advanced-search-prefix" data-search-prefix="fork:" data-search-type="Repositories">
<option value="">not</option>
<option value="true">and</option>
<option value="only">only</option>
</select> including forks.
</label>
</fieldset>
<fieldset class="pb-3 mb-4 border-bottom color-border-muted min-width-0">
<h3>Code options</h3>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_extension">With this extension</label></dt>
<dd>
<input id="search_extension" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="rb, py, jpg" data-search-type="Code" data-search-prefix="path:" data-glob-pattern="*.$0" data-regex-pattern="/.$0$/" data-use-or="true">
</dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_path">In this path</label></dt>
<dd><input id="search_path" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="/foo/bar/baz/qux" data-search-prefix="path:" data-search-type="Code" data-use-or=""></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_filename">With this file name</label></dt>
<dd>
<input id="search_filename" type="text" class="form-control js-advanced-search-prefix" placeholder="app.rb, footer.erb" data-search-type="code:" data-search-prefix="path:" data-glob-pattern="**/$0" data-regex-pattern="/(^|/)$0$/" data-use-or="true">
</dd>
</dl>
<label>
Return code <select class="form-select js-advanced-search-prefix" data-search-prefix="fork:" data-search-type="Code">
<option value="">not</option>
<option value="true">and</option>
<option value="only">only</option>
</select> including forks.
</label>
</fieldset>
<fieldset class="pb-3 mb-4 border-bottom color-border-muted min-width-0">
<h3>Issues options</h3>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_state">In the state</label></dt>
<dd><select id="search_state" class="form-select js-advanced-search-prefix" data-search-prefix="state:" data-search-type="Issues">
<option value="">open/closed</option>
<option value="open">open</option>
<option value="closed">closed</option>
</select></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_state_reason">With the reason</label></dt>
<dd><select id="search_state_reason" class="form-select js-advanced-search-prefix" data-search-prefix="reason:" data-search-type="Issues">
<option value="">any reason</option>
<option value="completed">completed</option>
<option value="not planned">not planned</option>
<option value="reopened">reopened</option>
</select></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_comments">With this many comments</label></dt>
<dd><input id="search_comments" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="0..100, >442" data-search-prefix="comments:" data-search-type="Issues"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_labels">With the labels</label></dt>
<dd><input id="search_labels" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="bug, ie6" data-search-prefix="label:" data-search-type="Issues"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_author">Opened by the author</label></dt>
<dd><input id="search_author" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="hubot, octocat" data-search-prefix="author:" data-search-type="Issues"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_mention">Mentioning the users</label></dt>
<dd><input id="search_mention" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="tpope, mattt" data-search-prefix="mentions:" data-search-type="Issues"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_assignment">Assigned to the users</label></dt>
<dd><input id="search_assignment" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="twp, jim" data-search-prefix="assignee:" data-search-type="Issues"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_updated_date">Updated before the date</label></dt>
<dd><input id="search_updated_date" type="text" class="form-control js-advanced-search-prefix" value="" placeholder="<YYYY-MM-DD" data-search-prefix="updated:" data-search-type="Issues"></dd>
</dl>
</fieldset>
<fieldset class="pb-3 mb-4 border-bottom color-border-muted min-width-0">
<h3>Users options</h3>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_full_name">With this full name</label></dt>
<dd><input id="search_full_name" type="text" class="form-control js-advanced-search-prefix" placeholder="Grace Hopper" data-search-prefix="fullname:" data-search-type="Users"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_location">From this location</label></dt>
<dd><input id="search_location" type="text" class="form-control js-advanced-search-prefix" placeholder="San Francisco, CA" data-search-prefix="location:" data-search-type="Users"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_followers">With this many followers</label></dt>
<dd><input id="search_followers" type="text" class="form-control js-advanced-search-prefix" placeholder="20..50, >200, <2" data-search-prefix="followers:" data-search-type="Users"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_public_repos">With this many public repositories</label></dt>
<dd><input id="search_public_repos" type="text" class="form-control js-advanced-search-prefix" placeholder="0, <42, >5" data-search-prefix="repos:" data-search-type="Users"></dd>
</dl>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_user_language">Working in this language</label></dt>
<dd>
<select id="search_user_language" name="l" class="form-select js-advanced-search-prefix" data-search-prefix="language:">
<option value="">Any language</option>
<optgroup label="Popular">
<option value="C">C</option>
<option value="C#">C#</option>
<option value="C++">C++</option>
<option value="CoffeeScript">CoffeeScript</option>
<option value="CSS">CSS</option>
<option value="Dart">Dart</option>
<option value="DM">DM</option>
<option value="Elixir">Elixir</option>
<option value="Go">Go</option>
<option value="Groovy">Groovy</option>
<option value="HTML">HTML</option>
<option value="Java">Java</option>
<option value="JavaScript">JavaScript</option>
<option value="Kotlin">Kotlin</option>
<option value="Objective-C">Objective-C</option>
<option value="Perl">Perl</option>
<option value="PHP">PHP</option>
<option value="PowerShell">PowerShell</option>
<option value="Python">Python</option>
<option value="Ruby">Ruby</option>
<option value="Rust">Rust</option>
<option value="Scala">Scala</option>
<option value="Shell">Shell</option>
<option value="Swift">Swift</option>
<option value="TypeScript">TypeScript</option>
</optgroup>
<optgroup label="Everything else">
<option value="1C Enterprise">1C Enterprise</option>
<option value="2-Dimensional Array">2-Dimensional Array</option>
<option value="4D">4D</option>
<option value="ABAP">ABAP</option>
<option value="ABAP CDS">ABAP CDS</option>
<option value="ABNF">ABNF</option>
<option value="ActionScript">ActionScript</option>
<option value="Ada">Ada</option>
<option value="Yul">Yul</option>
<option value="ZAP">ZAP</option>
<option value="Zeek">Zeek</option>
<option value="ZenScript">ZenScript</option>
<option value="Zephir">Zephir</option>
<option value="Zig">Zig</option>
<option value="ZIL">ZIL</option>
<option value="Zimpl">Zimpl</option>
<option value="Zmodel">Zmodel</option>
</optgroup>
</select>
</dd>
</dl>
</fieldset>
<fieldset class="pb-3 mb-4 border-bottom color-border-muted min-width-0">
<h3>Wiki options</h3>
<dl class="form-group flattened d-flex d-md-block flex-column">
<dt><label for="search_wiki_updated_date">Updated before the date</label></dt>
<dd><input id="search_wiki_updated_date" type="text" class="form-control js-advanced-search-prefix" placeholder="<YYYY-MM-DD" data-search-prefix="updated:" data-search-type="Wiki"></dd>
</dl>
</fieldset>
<div class="form-group flattened">
<div class="d-flex d-md-block"> <button type="submit" data-view-component="true" class="btn flex-auto"> Search
</button></div>
</div>
</div>
</form>

7
docs/examples/c4a_script/script_samples/add_to_cart.c4a Normal file
View File

@@ -0,0 +1,7 @@
GO https://store.example.com/product/laptop
WAIT `.product-details` 8
CLICK `button.add-to-cart`
WAIT `.cart-notification` 3
CLICK `.cart-icon`
WAIT `.checkout-btn` 5
CLICK `.checkout-btn`

43
docs/examples/c4a_script/script_samples/advanced_control_flow.c4a Normal file
View File

@@ -0,0 +1,43 @@
# Advanced control flow with IF, EXISTS, and REPEAT
# Define reusable procedures
PROC handle_cookie_banner
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-cookies`
IF (EXISTS `.privacy-notice`) THEN CLICK `.dismiss-privacy`
ENDPROC
PROC scroll_to_load
SCROLL DOWN 500
WAIT 0.5
ENDPROC
PROC try_login
CLICK `#email`
TYPE "user@example.com"
CLICK `#password`
TYPE "secure123"
CLICK `button[type="submit"]`
WAIT 2
ENDPROC
# Main script
GO https://example.com
WAIT 2
# Handle popups
handle_cookie_banner
# Conditional navigation based on login state
IF (EXISTS `.user-menu`) THEN CLICK `.dashboard-link` ELSE try_login
# Repeat scrolling based on content count
REPEAT (scroll_to_load, 5)
# Load more content while button exists
REPEAT (CLICK `.load-more`, `document.querySelector('.load-more') && !document.querySelector('.no-more-content')`)
# Process items conditionally
IF (`document.querySelectorAll('.item').length > 10`) THEN EVAL `console.log('Found ' + document.querySelectorAll('.item').length + ' items')`
# Complex condition with viewport check
IF (`window.innerWidth < 768 && document.querySelector('.mobile-menu')`) THEN CLICK `.mobile-menu-toggle`

8
docs/examples/c4a_script/script_samples/conditional_login.c4a Normal file
View File

@@ -0,0 +1,8 @@
GO https://myapp.com
WAIT 2
IF (EXISTS `.user-avatar`) THEN CLICK `.logout` ELSE CLICK `.login`
WAIT `#auth-form` 5
IF (EXISTS `#auth-form`) THEN TYPE "user@example.com"
IF (EXISTS `#auth-form`) THEN PRESS Tab
IF (EXISTS `#auth-form`) THEN TYPE "password123"
IF (EXISTS `#auth-form`) THEN CLICK `button[type="submit"]`

56
docs/examples/c4a_script/script_samples/data_extraction.c4a Normal file
View File

@@ -0,0 +1,56 @@
# Data extraction example
# Scrapes product information from an e-commerce site
# Navigate to products page
GO https://shop.example.com/products
WAIT `.product-list` 10
# Scroll to load lazy-loaded content
SCROLL DOWN 500
WAIT 1
SCROLL DOWN 500
WAIT 1
SCROLL DOWN 500
WAIT 2
# Extract product data
EVAL `
// Extract all product information
const products = Array.from(document.querySelectorAll('.product-card')).map((card, index) => {
return {
id: index + 1,
name: card.querySelector('.product-title')?.textContent?.trim() || 'N/A',
price: card.querySelector('.price')?.textContent?.trim() || 'N/A',
rating: card.querySelector('.rating')?.textContent?.trim() || 'N/A',
availability: card.querySelector('.in-stock') ? 'In Stock' : 'Out of Stock',
image: card.querySelector('img')?.src || 'N/A'
};
});
// Log results
console.log('=== Product Extraction Results ===');
console.log('Total products found:', products.length);
console.log(JSON.stringify(products, null, 2));
// Save to localStorage for retrieval
localStorage.setItem('scraped_products', JSON.stringify(products));
`
# Optional: Click on first product for details
CLICK `.product-card:first-child`
WAIT `.product-details` 5
# Extract detailed information
EVAL `
const details = {
description: document.querySelector('.product-description')?.textContent?.trim(),
specifications: Array.from(document.querySelectorAll('.spec-item')).map(spec => ({
label: spec.querySelector('.spec-label')?.textContent,
value: spec.querySelector('.spec-value')?.textContent
})),
reviews: document.querySelector('.review-count')?.textContent
};
console.log('=== Product Details ===');
console.log(JSON.stringify(details, null, 2));
`

8
docs/examples/c4a_script/script_samples/fill_contact.c4a Normal file
View File

@@ -0,0 +1,8 @@
GO https://company.com/contact
WAIT `form#contact` 10
TYPE "John Smith"
PRESS Tab
TYPE "john@email.com"
PRESS Tab
TYPE "Need help with my order"
CLICK `button[type="submit"]`

7
docs/examples/c4a_script/script_samples/load_more_content.c4a Normal file
View File

@@ -0,0 +1,7 @@
GO https://news.example.com
WAIT `.article-list` 5
REPEAT (SCROLL DOWN 500, 3)
WAIT 1
REPEAT (CLICK `.load-more`, `document.querySelector('.load-more') !== null`)
WAIT 2
IF (`document.querySelectorAll('.article').length > 20`) THEN EVAL `console.log('Loaded enough articles')`

36
docs/examples/c4a_script/script_samples/login_flow.c4a Normal file
View File

@@ -0,0 +1,36 @@
# Login flow with error handling
# Demonstrates procedures, variables, and conditional checks
# Define login procedure
PROC perform_login
CLICK `input#email`
TYPE $email
CLICK `input#password`
TYPE $password
CLICK `button.login-submit`
ENDPROC
# Set credentials
SET email = "user@example.com"
SET password = "securePassword123"
# Navigate to login page
GO https://app.example.com/login
WAIT `.login-container` 15
# Attempt login
perform_login
# Wait for page to load
WAIT 3
# Check if login was successful
EVAL `
if (document.querySelector('.dashboard')) {
console.log('Login successful - on dashboard');
} else if (document.querySelector('.error-message')) {
console.log('Login failed:', document.querySelector('.error-message').textContent);
} else {
console.log('Unknown state after login');
}
`

106
docs/examples/c4a_script/script_samples/multi_step_workflow.c4a Normal file
View File

@@ -0,0 +1,106 @@
# Multi-step e-commerce workflow
# Complete purchase flow with procedures and error handling
# Reusable procedures
PROC search_product
CLICK `input.search-bar`
TYPE $search_term
PRESS Enter
WAIT `.search-results` 10
ENDPROC
PROC add_first_item_to_cart
CLICK `.product-item:first-child .add-to-cart`
WAIT ".added-to-cart-notification" 3
ENDPROC
PROC go_to_checkout
CLICK `.cart-icon`
WAIT `.cart-drawer` 5
CLICK `button.proceed-to-checkout`
WAIT `.checkout-page` 10
ENDPROC
PROC fill_customer_info
# Billing information
CLICK `#billing-firstname`
TYPE $first_name
CLICK `#billing-lastname`
TYPE $last_name
CLICK `#billing-email`
TYPE $email
CLICK `#billing-phone`
TYPE $phone
# Address
CLICK `#billing-address`
TYPE $address
CLICK `#billing-city`
TYPE $city
CLICK `#billing-state`
TYPE $state
CLICK `#billing-zip`
TYPE $zip
ENDPROC
PROC select_shipping
CLICK `input[value="standard"]`
WAIT 1
ENDPROC
# Set all required variables
SET search_term = "wireless headphones"
SET first_name = "John"
SET last_name = "Doe"
SET email = "john.doe@example.com"
SET phone = "555-0123"
SET address = "123 Main Street"
SET city = "San Francisco"
SET state = "CA"
SET zip = "94105"
# Main workflow starts here
GO https://shop.example.com
WAIT `.homepage-loaded` 10
# Step 1: Search and add to cart
search_product
EVAL `console.log('Found', document.querySelectorAll('.product-item').length, 'products')`
add_first_item_to_cart
# Add a second item
CLICK `.product-item:nth-child(2) .add-to-cart`
WAIT 2
# Step 2: Go to checkout
go_to_checkout
# Step 3: Fill customer information
fill_customer_info
# Step 4: Select shipping method
select_shipping
# Step 5: Continue to payment
CLICK `button.continue-to-payment`
WAIT `.payment-section` 10
# Log order summary
EVAL `
const orderTotal = document.querySelector('.order-total')?.textContent;
const itemCount = document.querySelectorAll('.order-item').length;
console.log('=== Order Summary ===');
console.log('Items:', itemCount);
console.log('Total:', orderTotal);
// Get all items
const items = Array.from(document.querySelectorAll('.order-item')).map(item => ({
name: item.querySelector('.item-name')?.textContent,
quantity: item.querySelector('.item-quantity')?.textContent,
price: item.querySelector('.item-price')?.textContent
}));
console.log('Items:', JSON.stringify(items, null, 2));
`
# Note: Stopping here before actual payment submission
EVAL `console.log('Workflow completed - stopped before payment submission')`

8
docs/examples/c4a_script/script_samples/navigate_tabs.c4a Normal file
View File

@@ -0,0 +1,8 @@
GO https://app.example.com
WAIT `.nav-menu` 8
CLICK `a[href="/products"]`
WAIT 2
CLICK `a[href="/about"]`
WAIT 2
BACK
WAIT 1

8
docs/examples/c4a_script/script_samples/quick_login.c4a Normal file
View File

@@ -0,0 +1,8 @@
GO https://myapp.com/login
WAIT `input#email` 5
CLICK `input#email`
TYPE "user@example.com"
PRESS Tab
TYPE "password123"
CLICK `button.login-btn`
WAIT `.dashboard` 10

7
docs/examples/c4a_script/script_samples/responsive_actions.c4a Normal file
View File

@@ -0,0 +1,7 @@
GO https://responsive.site.com
WAIT 2
IF (`window.innerWidth < 768`) THEN CLICK `.mobile-menu`
IF (`window.innerWidth < 768`) THEN WAIT `.mobile-nav` 3
IF (`window.innerWidth >= 768`) THEN CLICK `.desktop-menu li:nth-child(2)`
REPEAT (CLICK `.next-slide`, 5)
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-cookies`

8
docs/examples/c4a_script/script_samples/scroll_and_click.c4a Normal file
View File

@@ -0,0 +1,8 @@
GO https://news.site.com
WAIT `.article-list` 10
SCROLL DOWN 500
WAIT 1
SCROLL DOWN 500
WAIT 1
CLICK `.article:nth-child(5)`
WAIT `.article-content` 5

7
docs/examples/c4a_script/script_samples/search_product.c4a Normal file
View File

@@ -0,0 +1,7 @@
GO https://shop.example.com
WAIT `.search-bar` 10
CLICK `.search-bar`
TYPE "wireless headphones"
PRESS Enter
WAIT `.results` 5
CLICK `.product-card:first-child`

19
docs/examples/c4a_script/script_samples/simple_form.c4a Normal file
View File

@@ -0,0 +1,19 @@
# Simple form submission example
# This script fills out a contact form and submits it
GO https://example.com/contact
WAIT `form#contact-form` 10
# Fill out the form fields
CLICK `input[name="name"]`
TYPE "Alice Smith"
PRESS Tab
TYPE "alice@example.com"
PRESS Tab
TYPE "I'd like to learn more about your services"
# Submit the form
CLICK `button[type="submit"]`
# Wait for success message
WAIT "Thank you for your message" 5

11
docs/examples/c4a_script/script_samples/smart_form_fill.c4a Normal file
View File

@@ -0,0 +1,11 @@
PROC fill_field
TYPE "test@example.com"
PRESS Tab
ENDPROC
GO https://forms.example.com
WAIT `form` 5
IF (EXISTS `input[type="email"]`) THEN CLICK `input[type="email"]`
IF (EXISTS `input[type="email"]`) THEN fill_field
REPEAT (PRESS Tab, `document.activeElement.type !== 'submit'`)
CLICK `button[type="submit"]`

396
docs/examples/c4a_script/tutorial/README.md Normal file
View File

@@ -0,0 +1,396 @@
# C4A-Script Interactive Tutorial
A comprehensive web-based tutorial for learning and experimenting with C4A-Script - Crawl4AI's visual web automation language.
## 🚀 Quick Start
### Prerequisites
- Python 3.7+
- Modern web browser (Chrome, Firefox, Safari, Edge)
### Running the Tutorial
1. **Clone and Navigate**
```bash
git clone https://github.com/unclecode/crawl4ai.git
cd crawl4ai/docs/examples/c4a_script/tutorial/
```
2. **Install Dependencies**
```bash
pip install flask
```
3. **Launch the Server**
```bash
python server.py
```
4. **Open in Browser**
```
http://localhost:8080
```
**🌐 Try Online**: [Live Demo](https://docs.crawl4ai.com/c4a-script/demo)
### 2. Try Your First Script
```c4a
# Basic interaction
GO playground/
WAIT `body` 2
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
CLICK `#start-tutorial`
```
## 🎯 What You'll Learn
### Core Features
- **📝 Text Editor**: Write C4A-Script with syntax highlighting
- **🧩 Visual Editor**: Build scripts using drag-and-drop Blockly interface
- **🎬 Recording Mode**: Capture browser actions and auto-generate scripts
- **⚡ Live Execution**: Run scripts in real-time with instant feedback
- **📊 Timeline View**: Visualize and edit automation steps
## 📚 Tutorial Content
### Basic Commands
- **Navigation**: `GO url`
- **Waiting**: `WAIT selector timeout` or `WAIT seconds`
- **Clicking**: `CLICK selector`
- **Typing**: `TYPE "text"`
- **Scrolling**: `SCROLL DOWN/UP amount`
### Control Flow
- **Conditionals**: `IF (condition) THEN action`
- **Loops**: `REPEAT (action, condition)`
- **Procedures**: Define reusable command sequences
### Advanced Features
- **JavaScript evaluation**: `EVAL code`
- **Variables**: `SET name = "value"`
- **Complex selectors**: CSS selectors in backticks
## 🎮 Interactive Playground Features
The tutorial includes a fully interactive web app with:
### 1. **Authentication System**
- Login form with validation
- Session management
- Protected content
### 2. **Dynamic Content**
- Infinite scroll products
- Pagination controls
- Load more buttons
### 3. **Complex Forms**
- Multi-step wizards
- Dynamic field visibility
- Form validation
### 4. **Interactive Elements**
- Tabs and accordions
- Modals and popups
- Expandable content
### 5. **Data Tables**
- Sortable columns
- Search functionality
- Export options
## 🛠️ Tutorial Features
### Live Code Editor
- Syntax highlighting
- Real-time compilation
- Error messages with suggestions
### JavaScript Output Viewer
- See generated JavaScript code
- Edit and test JS directly
- Understand the compilation
### Visual Execution
- Step-by-step progress
- Element highlighting
- Console output
### Example Scripts
Load pre-written examples demonstrating:
- Cookie banner handling
- Login workflows
- Infinite scroll automation
- Multi-step form completion
- Complex interaction sequences
## 📖 Tutorial Sections
### 1. Getting Started
Learn basic commands and syntax:
```c4a
GO https://example.com
WAIT `.content` 5
CLICK `.button`
```
### 2. Handling Dynamic Content
Master waiting strategies and conditionals:
```c4a
IF (EXISTS `.popup`) THEN CLICK `.close`
WAIT `.results` 10
```
### 3. Form Automation
Fill and submit forms:
```c4a
CLICK `#email`
TYPE "user@example.com"
CLICK `button[type="submit"]`
```
### 4. Advanced Workflows
Build complex automation flows:
```c4a
PROC login
CLICK `#username`
TYPE $username
CLICK `#password`
TYPE $password
CLICK `#login-btn`
ENDPROC
SET username = "demo"
SET password = "pass123"
login
```
## 🎯 Practice Challenges
### Challenge 1: Cookie & Popups
Handle the cookie banner and newsletter popup that appear on page load.
### Challenge 2: Complete Login
Successfully log into the application using the demo credentials.
### Challenge 3: Load All Products
Use infinite scroll to load all 100 products in the catalog.
### Challenge 4: Multi-step Survey
Complete the entire multi-step survey form.
### Challenge 5: Full Workflow
Create a script that logs in, browses products, and exports data.
## 💡 Tips & Tricks
### 1. Use Specific Selectors
```c4a
# Good - specific
CLICK `button.submit-order`
# Bad - too generic
CLICK `button`
```
### 2. Always Handle Popups
```c4a
# Check for common popups
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
IF (EXISTS `.newsletter-modal`) THEN CLICK `.close`
```
### 3. Add Appropriate Waits
```c4a
# Wait for elements before interacting
WAIT `.form` 5
CLICK `#submit`
```
### 4. Use Procedures for Reusability
```c4a
PROC handle_popups
IF (EXISTS `.popup`) THEN CLICK `.close`
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
ENDPROC
# Use anywhere
handle_popups
```
## 🔧 Troubleshooting
### Common Issues
1. **"Element not found"**
- Add a WAIT before clicking
- Check selector specificity
- Verify element exists with IF
2. **"Timeout waiting for selector"**
- Increase timeout value
- Check if element is dynamically loaded
- Verify selector is correct
3. **"Missing THEN keyword"**
- All IF statements need THEN
- Format: `IF (condition) THEN action`
## 🚀 Using with Crawl4AI
Once you've mastered C4A-Script in the tutorial, use it with Crawl4AI:
```python
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
config = CrawlerRunConfig(
url="https://example.com",
c4a_script="""
WAIT `.content` 5
IF (EXISTS `.load-more`) THEN CLICK `.load-more`
WAIT `.new-content` 3
"""
)
async with AsyncWebCrawler() as crawler:
result = await crawler.arun(config=config)
```
## 📝 Example Scripts
Check the `scripts/` folder for complete examples:
- `01-basic-interaction.c4a` - Getting started
- `02-login-flow.c4a` - Authentication
- `03-infinite-scroll.c4a` - Dynamic content
- `04-multi-step-form.c4a` - Complex forms
- `05-complex-workflow.c4a` - Full automation
## 🏗️ Developer Guide
### Project Architecture
```
tutorial/
├── server.py # Flask application server
├── assets/ # Tutorial-specific assets
│ ├── app.js # Main application logic
│ ├── c4a-blocks.js # Custom Blockly blocks
│ ├── c4a-generator.js # Code generation
│ ├── blockly-manager.js # Blockly integration
│ └── styles.css # Main styling
├── playground/ # Interactive demo environment
│ ├── index.html # Demo web application
│ ├── app.js # Demo app logic
│ └── styles.css # Demo styling
├── scripts/ # Example C4A scripts
└── index.html # Main tutorial interface
```
### Key Components
#### 1. TutorialApp (`assets/app.js`)
Main application controller managing:
- Code editor integration (CodeMirror)
- Script execution and browser preview
- Tutorial navigation and lessons
- State management and persistence
#### 2. BlocklyManager (`assets/blockly-manager.js`)
Visual programming interface:
- Custom C4A-Script block definitions
- Bidirectional sync between visual blocks and text
- Real-time code generation
- Dark theme integration
#### 3. Recording System
Powers the recording functionality:
- Browser event capture
- Smart event grouping and filtering
- Automatic C4A-Script generation
- Timeline visualization
### Customization
#### Adding New Commands
1. **Define Block** (`assets/c4a-blocks.js`)
2. **Add Generator** (`assets/c4a-generator.js`)
3. **Update Parser** (`assets/blockly-manager.js`)
#### Themes and Styling
- Main styles: `assets/styles.css`
- Theme variables: CSS custom properties
- Dark mode: Auto-applied based on system preference
### Configuration
```python
# server.py configuration
PORT = 8080
DEBUG = True
THREADED = True
```
### API Endpoints
- `GET /` - Main tutorial interface
- `GET /playground/` - Interactive demo environment
- `POST /execute` - Script execution endpoint
- `GET /examples/<script>` - Load example scripts
## 🔧 Troubleshooting
### Common Issues
**Port Already in Use**
```bash
# Kill existing process
lsof -ti:8080 | xargs kill -9
# Or use different port
python server.py --port 8081
```
**Blockly Not Loading**
- Check browser console for JavaScript errors
- Verify all static files are served correctly
- Ensure proper script loading order
**Recording Issues**
- Verify iframe permissions
- Check cross-origin communication
- Ensure event listeners are attached
### Debug Mode
Enable detailed logging by setting `DEBUG = True` in `assets/app.js`
## 📚 Additional Resources
- **[C4A-Script Documentation](../../md_v2/core/c4a-script.md)** - Complete language guide
- **[API Reference](../../md_v2/api/c4a-script-reference.md)** - Detailed command documentation
- **[Live Demo](https://docs.crawl4ai.com/c4a-script/demo)** - Try without installation
- **[Example Scripts](../)** - More automation examples
## 🤝 Contributing
### Bug Reports
1. Check existing issues on GitHub
2. Provide minimal reproduction steps
3. Include browser and system information
4. Add relevant console logs
### Feature Requests
1. Fork the repository
2. Create feature branch: `git checkout -b feature/my-feature`
3. Test thoroughly with different browsers
4. Update documentation
5. Submit pull request
### Code Style
- Use consistent indentation (2 spaces for JS, 4 for Python)
- Add comments for complex logic
- Follow existing naming conventions
- Test with multiple browsers
---
**Happy Automating!** 🎉
Need help? Check our [documentation](https://docs.crawl4ai.com) or open an issue on [GitHub](https://github.com/unclecode/crawl4ai).

BIN
docs/examples/c4a_script/tutorial/assets/DankMono-Bold.woff2 Normal file
View File

Binary file not shown.

BIN
docs/examples/c4a_script/tutorial/assets/DankMono-Italic.woff2 Normal file
View File

Binary file not shown.

BIN
docs/examples/c4a_script/tutorial/assets/DankMono-Regular.woff2 Normal file
View File

Binary file not shown.

906
docs/examples/c4a_script/tutorial/assets/app.css Normal file
View File

@@ -0,0 +1,906 @@
/* ================================================================
C4A-Script Tutorial - App Layout CSS
Terminal theme with Dank Mono font
================================================================ */
/* CSS Variables */
:root {
--bg-primary: #070708;
--bg-secondary: #0e0e10;
--bg-tertiary: #1a1a1b;
--border-color: #2a2a2c;
--border-hover: #3a3a3c;
--text-primary: #e0e0e0;
--text-secondary: #8b8b8d;
--text-muted: #606065;
--primary-color: #0fbbaa;
--primary-hover: #0da89a;
--primary-dim: #0a8577;
--error-color: #ff5555;
--warning-color: #ffb86c;
--success-color: #50fa7b;
--info-color: #8be9fd;
--code-bg: #1e1e20;
--modal-overlay: rgba(0, 0, 0, 0.8);
}
/* Base Reset */
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
/* Fonts */
@font-face {
font-family: 'Dank Mono';
src: url('DankMono-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
}
@font-face {
font-family: 'Dank Mono';
src: url('DankMono-Bold.woff2') format('woff2');
font-weight: 700;
font-style: normal;
}
@font-face {
font-family: 'Dank Mono';
src: url('DankMono-Italic.woff2') format('woff2');
font-weight: 400;
font-style: italic;
}
/* Body & App Container */
body {
font-family: 'Dank Mono', 'Monaco', 'Consolas', monospace;
background: var(--bg-primary);
color: var(--text-primary);
font-size: 14px;
line-height: 1.6;
overflow: hidden;
}
.app-container {
display: flex;
height: 100vh;
width: 100vw;
overflow: hidden;
}
/* Panels */
.editor-panel,
.playground-panel {
display: flex;
flex-direction: column;
height: 100%;
overflow: hidden;
}
.editor-panel {
flex: 1;
background: var(--bg-secondary);
border-right: 1px solid var(--border-color);
min-width: 400px;
}
.playground-panel {
flex: 1;
background: var(--bg-primary);
min-width: 400px;
}
/* Panel Headers */
.panel-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 12px 16px;
background: var(--bg-tertiary);
border-bottom: 1px solid var(--border-color);
flex-shrink: 0;
}
.panel-header h2 {
font-size: 16px;
font-weight: 600;
color: var(--primary-color);
margin: 0;
}
.header-actions {
display: flex;
gap: 8px;
}
/* Action Buttons */
.action-btn {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 12px;
background: var(--bg-secondary);
color: var(--text-secondary);
border: 1px solid var(--border-color);
border-radius: 4px;
font-family: inherit;
font-size: 13px;
cursor: pointer;
transition: all 0.2s;
}
.action-btn:hover {
background: var(--bg-tertiary);
color: var(--text-primary);
border-color: var(--border-hover);
}
.action-btn.primary {
background: var(--primary-color);
color: var(--bg-primary);
border-color: var(--primary-color);
}
.action-btn.primary:hover {
background: var(--primary-hover);
border-color: var(--primary-hover);
}
.action-btn .icon {
font-size: 16px;
}
/* Editor Wrapper */
.editor-wrapper {
flex: 1;
display: flex;
overflow: hidden;
position: relative;
z-index: 1; /* Ensure it's above any potential overlays */
}
.editor-wrapper .CodeMirror {
flex: 1;
height: 100%;
font-family: 'Dank Mono', monospace;
font-size: 14px;
line-height: 1.5;
}
/* Ensure CodeMirror is interactive */
.CodeMirror {
background: var(--bg-primary) !important;
}
.CodeMirror-scroll {
overflow: auto !important;
}
/* Make cursor more visible */
.CodeMirror-cursor {
border-left: 2px solid var(--primary-color) !important;
border-left-width: 2px !important;
opacity: 1 !important;
visibility: visible !important;
}
/* Ensure cursor is visible when focused */
.CodeMirror-focused .CodeMirror-cursor {
visibility: visible !important;
}
/* Fix for CodeMirror in flex container */
.CodeMirror-sizer {
min-height: auto !important;
}
/* Remove aggressive pointer-events override */
.CodeMirror-code {
cursor: text;
}
.editor-wrapper textarea {
display: none;
}
/* Output Section (Bottom of Editor) */
.output-section {
height: 250px;
border-top: 1px solid var(--border-color);
display: flex;
flex-direction: column;
flex-shrink: 0;
}
/* Tabs */
.tabs {
display: flex;
background: var(--bg-tertiary);
border-bottom: 1px solid var(--border-color);
flex-shrink: 0;
}
.tab {
padding: 8px 20px;
background: transparent;
color: var(--text-secondary);
border: none;
border-bottom: 2px solid transparent;
font-family: inherit;
font-size: 13px;
cursor: pointer;
transition: all 0.2s;
}
.tab:hover {
color: var(--text-primary);
background: var(--bg-secondary);
}
.tab.active {
color: var(--primary-color);
border-bottom-color: var(--primary-color);
}
/* Tab Content */
.tab-content {
flex: 1;
overflow: hidden;
}
.tab-pane {
display: none;
height: 100%;
overflow-y: auto;
}
.tab-pane.active {
display: block;
}
/* Console */
.console {
padding: 12px;
background: var(--bg-primary);
font-size: 13px;
min-height: 100%;
}
.console-line {
margin-bottom: 8px;
display: flex;
align-items: flex-start;
gap: 8px;
}
.console-prompt {
color: var(--primary-color);
flex-shrink: 0;
}
.console-text {
color: var(--text-primary);
}
.console-error {
color: var(--error-color);
}
.console-warning {
color: var(--warning-color);
}
.console-success {
color: var(--success-color);
}
/* JavaScript Output */
.js-output-header {
display: flex;
justify-content: flex-end;
padding: 8px 12px;
background: var(--bg-tertiary);
border-bottom: 1px solid var(--border-color);
}
.js-actions {
display: flex;
gap: 8px;
}
.mini-btn {
padding: 4px 8px;
background: var(--bg-secondary);
color: var(--text-secondary);
border: 1px solid var(--border-color);
border-radius: 3px;
font-size: 12px;
cursor: pointer;
transition: all 0.2s;
}
.mini-btn:hover {
background: var(--bg-primary);
color: var(--text-primary);
}
.js-output {
padding: 12px;
background: var(--code-bg);
color: var(--text-primary);
font-family: 'Dank Mono', monospace;
font-size: 13px;
line-height: 1.5;
white-space: pre-wrap;
margin: 0;
min-height: calc(100% - 44px);
}
/* Execution Progress */
.execution-progress {
padding: 12px;
background: var(--bg-primary);
}
.progress-item {
display: flex;
align-items: center;
gap: 8px;
margin-bottom: 8px;
font-size: 13px;
}
.progress-icon {
color: var(--text-muted);
}
.progress-item.active .progress-icon {
color: var(--info-color);
animation: pulse 1s infinite;
}
.progress-item.completed .progress-icon {
color: var(--success-color);
}
.progress-item.error .progress-icon {
color: var(--error-color);
}
/* Playground */
.playground-wrapper {
flex: 1;
overflow: hidden;
}
#playground-frame {
width: 100%;
height: 100%;
border: none;
background: var(--bg-secondary);
}
/* Tutorial Intro Modal */
.tutorial-intro-modal {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: var(--modal-overlay);
display: flex;
align-items: center;
justify-content: center;
z-index: 2000;
transition: opacity 0.3s;
}
.tutorial-intro-modal.hidden {
display: none;
}
.intro-content {
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
border-radius: 8px;
padding: 32px;
max-width: 500px;
box-shadow: 0 16px 48px rgba(0, 0, 0, 0.6);
}
.intro-content h2 {
color: var(--primary-color);
margin-bottom: 16px;
font-size: 24px;
}
.intro-content p {
color: var(--text-primary);
margin-bottom: 16px;
line-height: 1.6;
}
.intro-content ul {
list-style: none;
margin-bottom: 24px;
}
.intro-content li {
color: var(--text-secondary);
margin-bottom: 8px;
padding-left: 20px;
position: relative;
}
.intro-content li:before {
content: "▸";
position: absolute;
left: 0;
color: var(--primary-color);
}
.intro-actions {
display: flex;
gap: 12px;
justify-content: flex-end;
}
.intro-btn {
padding: 10px 24px;
background: var(--bg-secondary);
color: var(--text-primary);
border: 1px solid var(--border-color);
border-radius: 4px;
font-family: inherit;
font-size: 14px;
cursor: pointer;
transition: all 0.2s;
}
.intro-btn:hover {
background: var(--bg-primary);
border-color: var(--border-hover);
}
.intro-btn.primary {
background: var(--primary-color);
color: var(--bg-primary);
border-color: var(--primary-color);
}
.intro-btn.primary:hover {
background: var(--primary-hover);
border-color: var(--primary-hover);
}
/* Tutorial Navigation Bar */
.tutorial-nav {
position: fixed;
top: 0;
left: 0;
right: 0;
background: var(--bg-tertiary);
border-bottom: 1px solid var(--primary-color);
z-index: 1000;
transition: transform 0.3s;
}
.tutorial-nav.hidden {
transform: translateY(-100%);
}
.tutorial-nav-content {
display: flex;
align-items: center;
justify-content: space-between;
padding: 16px 24px;
}
.tutorial-left {
flex: 1;
}
.tutorial-step-title {
display: flex;
align-items: center;
gap: 16px;
margin-bottom: 8px;
}
.tutorial-step-title span:first-child {
color: var(--text-secondary);
font-size: 12px;
text-transform: uppercase;
}
.tutorial-step-title span:last-child {
color: var(--primary-color);
font-weight: 600;
font-size: 16px;
}
.tutorial-description {
color: var(--text-primary);
margin: 0;
font-size: 14px;
max-width: 600px;
}
.tutorial-right {
display: flex;
align-items: center;
}
.tutorial-progress-bar {
height: 3px;
background: var(--bg-secondary);
position: absolute;
bottom: 0;
left: 0;
right: 0;
}
.tutorial-progress-bar .progress-fill {
height: 100%;
background: var(--primary-color);
transition: width 0.3s;
}
/* Adjust app container when tutorial is active */
.app-container.tutorial-active {
padding-top: 80px;
}
.tutorial-controls {
display: flex;
gap: 12px;
}
.nav-btn {
padding: 8px 16px;
background: var(--bg-secondary);
color: var(--text-primary);
border: 1px solid var(--border-color);
border-radius: 4px;
font-family: inherit;
font-size: 13px;
cursor: pointer;
transition: all 0.2s;
}
.nav-btn:hover:not(:disabled) {
background: var(--bg-primary);
border-color: var(--border-hover);
}
.nav-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.nav-btn.primary {
background: var(--primary-color);
color: var(--bg-primary);
border-color: var(--primary-color);
}
.nav-btn.primary:hover {
background: var(--primary-hover);
border-color: var(--primary-hover);
}
.exit-btn {
width: 32px;
height: 32px;
background: transparent;
color: var(--text-secondary);
border: none;
font-size: 20px;
cursor: pointer;
border-radius: 4px;
transition: all 0.2s;
margin-left: 16px;
}
.exit-btn:hover {
background: var(--bg-secondary);
color: var(--text-primary);
}
/* Fullscreen Mode */
.playground-panel.fullscreen {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
z-index: 1500;
}
/* Animations */
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.5; }
}
/* Scrollbar Styling */
::-webkit-scrollbar {
width: 10px;
height: 10px;
}
::-webkit-scrollbar-track {
background: var(--bg-secondary);
}
::-webkit-scrollbar-thumb {
background: var(--border-color);
border-radius: 5px;
}
::-webkit-scrollbar-thumb:hover {
background: var(--border-hover);
}
/* Responsive */
@media (max-width: 768px) {
.app-container {
flex-direction: column;
}
.editor-panel,
.playground-panel {
min-width: auto;
width: 100%;
}
.editor-panel {
border-right: none;
border-bottom: 1px solid var(--border-color);
}
.output-section {
height: 200px;
}
}
/* ================================================================
Recording Timeline Styles
================================================================ */
.action-btn.record {
background: var(--bg-tertiary);
border-color: var(--error-color);
}
.action-btn.record:hover {
background: var(--error-color);
border-color: var(--error-color);
}
.action-btn.record.recording {
background: var(--error-color);
animation: pulse 1.5s infinite;
}
.action-btn.record.recording .icon {
animation: blink 1s infinite;
}
@keyframes pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.8; }
}
@keyframes blink {
0%, 100% { opacity: 1; }
50% { opacity: 0.3; }
}
.editor-container {
flex: 1;
display: flex;
flex-direction: column;
overflow: hidden;
}
#editor-view,
#timeline-view {
flex: 1;
display: flex;
flex-direction: column;
overflow: hidden;
}
.recording-timeline {
background: var(--bg-secondary);
display: flex;
flex-direction: column;
height: 100%;
}
.timeline-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 10px 15px;
border-bottom: 1px solid var(--border-color);
background: var(--bg-tertiary);
}
.timeline-header h3 {
font-size: 14px;
font-weight: 600;
color: var(--text-primary);
margin: 0;
}
.timeline-actions {
display: flex;
gap: 8px;
}
.timeline-events {
flex: 1;
overflow-y: auto;
padding: 10px;
}
.timeline-event {
display: flex;
align-items: center;
padding: 8px 10px;
margin-bottom: 6px;
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
border-radius: 4px;
transition: all 0.2s;
cursor: pointer;
}
.timeline-event:hover {
border-color: var(--border-hover);
background: var(--code-bg);
}
.timeline-event.selected {
border-color: var(--primary-color);
background: rgba(15, 187, 170, 0.1);
}
.event-checkbox {
margin-right: 10px;
width: 16px;
height: 16px;
cursor: pointer;
}
.event-time {
font-size: 11px;
color: var(--text-muted);
margin-right: 10px;
font-family: 'Dank Mono', monospace;
min-width: 45px;
}
.event-command {
flex: 1;
font-family: 'Dank Mono', monospace;
font-size: 13px;
color: var(--text-primary);
}
.event-command .cmd-name {
color: var(--primary-color);
font-weight: 600;
}
.event-command .cmd-selector {
color: var(--info-color);
}
.event-command .cmd-value {
color: var(--warning-color);
}
.event-command .cmd-detail {
color: var(--text-secondary);
font-size: 11px;
margin-left: 5px;
}
.event-edit {
margin-left: 10px;
padding: 2px 8px;
font-size: 11px;
background: var(--bg-secondary);
border: 1px solid var(--border-color);
color: var(--text-secondary);
cursor: pointer;
border-radius: 3px;
transition: all 0.2s;
}
.event-edit:hover {
border-color: var(--primary-color);
color: var(--primary-color);
}
/* Event Editor Modal */
.modal-overlay {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: var(--modal-overlay);
z-index: 999;
}
.event-editor-modal {
position: fixed;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
background: var(--bg-secondary);
border: 1px solid var(--border-color);
border-radius: 8px;
padding: 20px;
z-index: 1000;
min-width: 400px;
}
.event-editor-modal h4 {
margin: 0 0 15px 0;
color: var(--text-primary);
font-family: 'Dank Mono', monospace;
}
.editor-field {
margin-bottom: 15px;
}
.editor-field label {
display: block;
margin-bottom: 5px;
font-size: 12px;
color: var(--text-secondary);
font-family: 'Dank Mono', monospace;
}
.editor-field input,
.editor-field select {
width: 100%;
padding: 8px;
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
color: var(--text-primary);
border-radius: 4px;
font-family: 'Dank Mono', monospace;
font-size: 13px;
}
.editor-field input:focus,
.editor-field select:focus {
outline: none;
border-color: var(--primary-color);
}
.editor-actions {
display: flex;
justify-content: flex-end;
gap: 10px;
margin-top: 20px;
}
/* Blockly Button */
#blockly-btn .icon {
font-size: 16px;
}
/* Hidden State */
.hidden {
display: none !important;
}

1485
docs/examples/c4a_script/tutorial/assets/app.js Normal file
View File

File diff suppressed because it is too large Load Diff

591
docs/examples/c4a_script/tutorial/assets/blockly-manager.js Normal file
View File

@@ -0,0 +1,591 @@
// Blockly Manager for C4A-Script
// Handles Blockly workspace, code generation, and synchronization with text editor
class BlocklyManager {
constructor(tutorialApp) {
this.app = tutorialApp;
this.workspace = null;
this.isUpdating = false; // Prevent circular updates
this.blocklyVisible = false;
this.toolboxXml = this.generateToolbox();
this.init();
}
init() {
this.setupBlocklyContainer();
this.initializeWorkspace();
this.setupEventHandlers();
this.setupSynchronization();
}
setupBlocklyContainer() {
// Create blockly container div
const editorContainer = document.querySelector('.editor-container');
const blocklyDiv = document.createElement('div');
blocklyDiv.id = 'blockly-view';
blocklyDiv.className = 'blockly-workspace hidden';
blocklyDiv.style.height = '100%';
blocklyDiv.style.width = '100%';
editorContainer.appendChild(blocklyDiv);
}
generateToolbox() {
return `
<xml id="toolbox" style="display: none">
<category name="Navigation" colour="${BlockColors.NAVIGATION}">
<block type="c4a_go"></block>
<block type="c4a_reload"></block>
<block type="c4a_back"></block>
<block type="c4a_forward"></block>
</category>
<category name="Wait" colour="${BlockColors.WAIT}">
<block type="c4a_wait_time">
<field name="SECONDS">3</field>
</block>
<block type="c4a_wait_selector">
<field name="SELECTOR">#content</field>
<field name="TIMEOUT">10</field>
</block>
<block type="c4a_wait_text">
<field name="TEXT">Loading complete</field>
<field name="TIMEOUT">5</field>
</block>
</category>
<category name="Mouse Actions" colour="${BlockColors.ACTIONS}">
<block type="c4a_click">
<field name="SELECTOR">button.submit</field>
</block>
<block type="c4a_click_xy"></block>
<block type="c4a_double_click"></block>
<block type="c4a_right_click"></block>
<block type="c4a_move"></block>
<block type="c4a_drag"></block>
<block type="c4a_scroll">
<field name="DIRECTION">DOWN</field>
<field name="AMOUNT">500</field>
</block>
</category>
<category name="Keyboard" colour="${BlockColors.KEYBOARD}">
<block type="c4a_type">
<field name="TEXT">hello@example.com</field>
</block>
<block type="c4a_type_var">
<field name="VAR">email</field>
</block>
<block type="c4a_clear"></block>
<block type="c4a_set">
<field name="SELECTOR">#email</field>
<field name="VALUE">user@example.com</field>
</block>
<block type="c4a_press">
<field name="KEY">Tab</field>
</block>
<block type="c4a_key_down">
<field name="KEY">Shift</field>
</block>
<block type="c4a_key_up">
<field name="KEY">Shift</field>
</block>
</category>
<category name="Control Flow" colour="${BlockColors.CONTROL}">
<block type="c4a_if_exists">
<field name="SELECTOR">.cookie-banner</field>
</block>
<block type="c4a_if_exists_else">
<field name="SELECTOR">#user</field>
</block>
<block type="c4a_if_not_exists">
<field name="SELECTOR">.modal</field>
</block>
<block type="c4a_if_js">
<field name="CONDITION">window.innerWidth < 768</field>
</block>
<block type="c4a_repeat_times">
<field name="TIMES">5</field>
</block>
<block type="c4a_repeat_while">
<field name="CONDITION">document.querySelector('.load-more')</field>
</block>
</category>
<category name="Variables" colour="${BlockColors.VARIABLES}">
<block type="c4a_setvar">
<field name="NAME">username</field>
<field name="VALUE">john@example.com</field>
</block>
<block type="c4a_eval">
<field name="CODE">console.log('Hello')</field>
</block>
</category>
<category name="Procedures" colour="${BlockColors.PROCEDURES}">
<block type="c4a_proc_def">
<field name="NAME">login</field>
</block>
<block type="c4a_proc_call">
<field name="NAME">login</field>
</block>
</category>
<category name="Comments" colour="#9E9E9E">
<block type="c4a_comment">
<field name="TEXT">Add comment here</field>
</block>
</category>
</xml>`;
}
initializeWorkspace() {
const blocklyDiv = document.getElementById('blockly-view');
// Dark theme configuration
const theme = Blockly.Theme.defineTheme('c4a-dark', {
'base': Blockly.Themes.Classic,
'componentStyles': {
'workspaceBackgroundColour': '#0e0e10',
'toolboxBackgroundColour': '#1a1a1b',
'toolboxForegroundColour': '#e0e0e0',
'flyoutBackgroundColour': '#1a1a1b',
'flyoutForegroundColour': '#e0e0e0',
'flyoutOpacity': 0.9,
'scrollbarColour': '#2a2a2c',
'scrollbarOpacity': 0.5,
'insertionMarkerColour': '#0fbbaa',
'insertionMarkerOpacity': 0.3,
'markerColour': '#0fbbaa',
'cursorColour': '#0fbbaa',
'selectedGlowColour': '#0fbbaa',
'selectedGlowOpacity': 0.4,
'replacementGlowColour': '#0fbbaa',
'replacementGlowOpacity': 0.5
},
'fontStyle': {
'family': 'Dank Mono, Monaco, Consolas, monospace',
'weight': 'normal',
'size': 13
}
});
this.workspace = Blockly.inject(blocklyDiv, {
toolbox: this.toolboxXml,
theme: theme,
grid: {
spacing: 20,
length: 3,
colour: '#2a2a2c',
snap: true
},
zoom: {
controls: true,
wheel: true,
startScale: 1.0,
maxScale: 3,
minScale: 0.3,
scaleSpeed: 1.2
},
trashcan: true,
sounds: false,
media: 'https://unpkg.com/blockly/media/'
});
// Add workspace change listener
this.workspace.addChangeListener((event) => {
if (!this.isUpdating && event.type !== Blockly.Events.UI) {
this.syncBlocksToCode();
}
});
}
setupEventHandlers() {
// Add blockly toggle button
const headerActions = document.querySelector('.editor-panel .header-actions');
const blocklyBtn = document.createElement('button');
blocklyBtn.id = 'blockly-btn';
blocklyBtn.className = 'action-btn';
blocklyBtn.title = 'Toggle Blockly Mode';
blocklyBtn.innerHTML = '<span class="icon">🧩</span>';
// Insert before the Run button
const runBtn = document.getElementById('run-btn');
headerActions.insertBefore(blocklyBtn, runBtn);
blocklyBtn.addEventListener('click', () => this.toggleBlocklyView());
}
setupSynchronization() {
// Listen to CodeMirror changes
this.app.editor.on('change', (instance, changeObj) => {
if (!this.isUpdating && this.blocklyVisible && changeObj.origin !== 'setValue') {
this.syncCodeToBlocks();
}
});
}
toggleBlocklyView() {
const editorView = document.getElementById('editor-view');
const blocklyView = document.getElementById('blockly-view');
const timelineView = document.getElementById('timeline-view');
const blocklyBtn = document.getElementById('blockly-btn');
this.blocklyVisible = !this.blocklyVisible;
if (this.blocklyVisible) {
// Show Blockly
editorView.classList.add('hidden');
timelineView.classList.add('hidden');
blocklyView.classList.remove('hidden');
blocklyBtn.classList.add('active');
// Resize workspace
Blockly.svgResize(this.workspace);
// Sync current code to blocks
this.syncCodeToBlocks();
} else {
// Show editor
blocklyView.classList.add('hidden');
editorView.classList.remove('hidden');
blocklyBtn.classList.remove('active');
// Refresh CodeMirror
setTimeout(() => this.app.editor.refresh(), 100);
}
}
syncBlocksToCode() {
if (this.isUpdating) return;
try {
this.isUpdating = true;
// Generate C4A-Script from blocks using our custom generator
if (typeof c4aGenerator !== 'undefined') {
const code = c4aGenerator.workspaceToCode(this.workspace);
// Process the code to maintain proper formatting
const lines = code.split('\n');
const formattedLines = [];
let lastWasComment = false;
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
if (!line) continue;
const isComment = line.startsWith('#');
// Add blank line when transitioning between comments and commands
if (formattedLines.length > 0 && lastWasComment !== isComment) {
formattedLines.push('');
}
formattedLines.push(line);
lastWasComment = isComment;
}
const cleanCode = formattedLines.join('\n');
// Update CodeMirror
this.app.editor.setValue(cleanCode);
}
} catch (error) {
console.error('Error syncing blocks to code:', error);
} finally {
this.isUpdating = false;
}
}
syncCodeToBlocks() {
if (this.isUpdating) return;
try {
this.isUpdating = true;
// Clear workspace
this.workspace.clear();
// Parse C4A-Script and generate blocks
const code = this.app.editor.getValue();
const blocks = this.parseC4AToBlocks(code);
if (blocks) {
Blockly.Xml.domToWorkspace(blocks, this.workspace);
}
} catch (error) {
console.error('Error syncing code to blocks:', error);
// Show error in console
this.app.addConsoleMessage(`Blockly sync error: ${error.message}`, 'warning');
} finally {
this.isUpdating = false;
}
}
parseC4AToBlocks(code) {
const lines = code.split('\n');
const xml = document.createElement('xml');
let yPos = 20;
let previousBlock = null;
let rootBlock = null;
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();
// Skip empty lines
if (!line) continue;
// Handle comments
if (line.startsWith('#')) {
const commentBlock = this.parseLineToBlock(line, i, lines);
if (commentBlock) {
if (previousBlock) {
// Connect to previous block
const next = document.createElement('next');
next.appendChild(commentBlock);
previousBlock.appendChild(next);
} else {
// First block - set position
commentBlock.setAttribute('x', 20);
commentBlock.setAttribute('y', yPos);
xml.appendChild(commentBlock);
rootBlock = commentBlock;
yPos += 60;
}
previousBlock = commentBlock;
}
continue;
}
const block = this.parseLineToBlock(line, i, lines);
if (block) {
if (previousBlock) {
// Connect to previous block using <next>
const next = document.createElement('next');
next.appendChild(block);
previousBlock.appendChild(next);
} else {
// First block - set position
block.setAttribute('x', 20);
block.setAttribute('y', yPos);
xml.appendChild(block);
rootBlock = block;
yPos += 60;
}
previousBlock = block;
}
}
return xml;
}
parseLineToBlock(line, index, allLines) {
// Navigation commands
if (line.startsWith('GO ')) {
const url = line.substring(3).trim();
return this.createBlock('c4a_go', { 'URL': url });
}
if (line === 'RELOAD') {
return this.createBlock('c4a_reload');
}
if (line === 'BACK') {
return this.createBlock('c4a_back');
}
if (line === 'FORWARD') {
return this.createBlock('c4a_forward');
}
// Wait commands
if (line.startsWith('WAIT ')) {
const parts = line.substring(5).trim();
// Check if it's just a number (wait time)
if (/^\d+(\.\d+)?$/.test(parts)) {
return this.createBlock('c4a_wait_time', { 'SECONDS': parts });
}
// Check for selector wait
const selectorMatch = parts.match(/^`([^`]+)`\s+(\d+)$/);
if (selectorMatch) {
return this.createBlock('c4a_wait_selector', {
'SELECTOR': selectorMatch[1],
'TIMEOUT': selectorMatch[2]
});
}
// Check for text wait
const textMatch = parts.match(/^"([^"]+)"\s+(\d+)$/);
if (textMatch) {
return this.createBlock('c4a_wait_text', {
'TEXT': textMatch[1],
'TIMEOUT': textMatch[2]
});
}
}
// Click commands
if (line.startsWith('CLICK ')) {
const target = line.substring(6).trim();
// Check for coordinates
const coordMatch = target.match(/^(\d+)\s+(\d+)$/);
if (coordMatch) {
return this.createBlock('c4a_click_xy', {
'X': coordMatch[1],
'Y': coordMatch[2]
});
}
// Selector click
const selectorMatch = target.match(/^`([^`]+)`$/);
if (selectorMatch) {
return this.createBlock('c4a_click', {
'SELECTOR': selectorMatch[1]
});
}
}
// Other mouse actions
if (line.startsWith('DOUBLE_CLICK ')) {
const selector = line.substring(13).trim().match(/^`([^`]+)`$/);
if (selector) {
return this.createBlock('c4a_double_click', {
'SELECTOR': selector[1]
});
}
}
if (line.startsWith('RIGHT_CLICK ')) {
const selector = line.substring(12).trim().match(/^`([^`]+)`$/);
if (selector) {
return this.createBlock('c4a_right_click', {
'SELECTOR': selector[1]
});
}
}
// Scroll
if (line.startsWith('SCROLL ')) {
const match = line.match(/^SCROLL\s+(UP|DOWN|LEFT|RIGHT)(?:\s+(\d+))?$/);
if (match) {
return this.createBlock('c4a_scroll', {
'DIRECTION': match[1],
'AMOUNT': match[2] || '500'
});
}
}
// Type commands
if (line.startsWith('TYPE ')) {
const content = line.substring(5).trim();
// Variable type
if (content.startsWith('$')) {
return this.createBlock('c4a_type_var', {
'VAR': content.substring(1)
});
}
// Text type
const textMatch = content.match(/^"([^"]*)"$/);
if (textMatch) {
return this.createBlock('c4a_type', {
'TEXT': textMatch[1]
});
}
}
// SET command
if (line.startsWith('SET ')) {
const match = line.match(/^SET\s+`([^`]+)`\s+"([^"]*)"$/);
if (match) {
return this.createBlock('c4a_set', {
'SELECTOR': match[1],
'VALUE': match[2]
});
}
}
// CLEAR command
if (line.startsWith('CLEAR ')) {
const match = line.match(/^CLEAR\s+`([^`]+)`$/);
if (match) {
return this.createBlock('c4a_clear', {
'SELECTOR': match[1]
});
}
}
// SETVAR command
if (line.startsWith('SETVAR ')) {
const match = line.match(/^SETVAR\s+(\w+)\s*=\s*"([^"]*)"$/);
if (match) {
return this.createBlock('c4a_setvar', {
'NAME': match[1],
'VALUE': match[2]
});
}
}
// IF commands (simplified - only single line)
if (line.startsWith('IF ')) {
// IF EXISTS
const existsMatch = line.match(/^IF\s+\(EXISTS\s+`([^`]+)`\)\s+THEN\s+(.+?)(?:\s+ELSE\s+(.+))?$/);
if (existsMatch) {
if (existsMatch[3]) {
// Has ELSE
const block = this.createBlock('c4a_if_exists_else', {
'SELECTOR': existsMatch[1]
});
// Parse then and else commands - simplified for now
return block;
} else {
// No ELSE
const block = this.createBlock('c4a_if_exists', {
'SELECTOR': existsMatch[1]
});
return block;
}
}
// IF NOT EXISTS
const notExistsMatch = line.match(/^IF\s+\(NOT\s+EXISTS\s+`([^`]+)`\)\s+THEN\s+(.+)$/);
if (notExistsMatch) {
const block = this.createBlock('c4a_if_not_exists', {
'SELECTOR': notExistsMatch[1]
});
return block;
}
}
// Comments
if (line.startsWith('#')) {
return this.createBlock('c4a_comment', {
'TEXT': line.substring(1).trim()
});
}
// If we can't parse it, return null
return null;
}
createBlock(type, fields = {}) {
const block = document.createElement('block');
block.setAttribute('type', type);
// Add fields
for (const [name, value] of Object.entries(fields)) {
const field = document.createElement('field');
field.setAttribute('name', name);
field.textContent = value;
block.appendChild(field);
}
return block;
}
}

238
docs/examples/c4a_script/tutorial/assets/blockly-theme.css Normal file
View File

@@ -0,0 +1,238 @@
/* Blockly Theme CSS for C4A-Script */
/* Blockly workspace container */
.blockly-workspace {
position: relative;
width: 100%;
height: 100%;
background: var(--bg-primary);
}
/* Blockly button active state */
#blockly-btn.active {
background: var(--primary-color);
color: var(--bg-primary);
border-color: var(--primary-color);
}
#blockly-btn.active:hover {
background: var(--primary-hover);
border-color: var(--primary-hover);
}
/* Override Blockly's default styles for dark theme */
.blocklyToolboxDiv {
background-color: var(--bg-tertiary) !important;
border-right: 1px solid var(--border-color) !important;
}
.blocklyFlyout {
background-color: var(--bg-secondary) !important;
}
.blocklyFlyoutBackground {
fill: var(--bg-secondary) !important;
}
.blocklyMainBackground {
stroke: none !important;
}
.blocklyTreeRow {
color: var(--text-primary) !important;
font-family: 'Dank Mono', monospace !important;
padding: 4px 16px !important;
margin: 2px 0 !important;
}
.blocklyTreeRow:hover {
background-color: var(--bg-secondary) !important;
}
.blocklyTreeSelected {
background-color: var(--primary-dim) !important;
}
.blocklyTreeLabel {
cursor: pointer;
}
/* Blockly scrollbars */
.blocklyScrollbarHorizontal,
.blocklyScrollbarVertical {
background-color: transparent !important;
}
.blocklyScrollbarHandle {
fill: var(--border-color) !important;
opacity: 0.5 !important;
}
.blocklyScrollbarHandle:hover {
fill: var(--border-hover) !important;
opacity: 0.8 !important;
}
/* Blockly zoom controls */
.blocklyZoom > image {
opacity: 0.6;
}
.blocklyZoom > image:hover {
opacity: 1;
}
/* Blockly trash can */
.blocklyTrash {
opacity: 0.6;
}
.blocklyTrash:hover {
opacity: 1;
}
/* Blockly context menus */
.blocklyContextMenu {
background-color: var(--bg-tertiary) !important;
border: 1px solid var(--border-color) !important;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3) !important;
}
.blocklyMenuItem {
color: var(--text-primary) !important;
font-family: 'Dank Mono', monospace !important;
}
.blocklyMenuItemDisabled {
color: var(--text-muted) !important;
}
.blocklyMenuItem:hover {
background-color: var(--bg-secondary) !important;
}
/* Blockly text inputs */
.blocklyHtmlInput {
background-color: var(--bg-tertiary) !important;
color: var(--text-primary) !important;
border: 1px solid var(--border-color) !important;
font-family: 'Dank Mono', monospace !important;
font-size: 13px !important;
padding: 4px 8px !important;
}
.blocklyHtmlInput:focus {
border-color: var(--primary-color) !important;
outline: none !important;
}
/* Blockly dropdowns */
.blocklyDropDownDiv {
background-color: var(--bg-tertiary) !important;
border: 1px solid var(--border-color) !important;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3) !important;
}
.blocklyDropDownContent {
color: var(--text-primary) !important;
}
.blocklyDropDownDiv .goog-menuitem {
color: var(--text-primary) !important;
font-family: 'Dank Mono', monospace !important;
padding: 4px 16px !important;
}
.blocklyDropDownDiv .goog-menuitem-highlight,
.blocklyDropDownDiv .goog-menuitem-hover {
background-color: var(--bg-secondary) !important;
}
/* Custom block colors are defined in the block definitions */
/* Block text styling */
.blocklyText {
fill: #ffffff !important;
font-family: 'Dank Mono', monospace !important;
font-size: 13px !important;
}
.blocklyEditableText > .blocklyText {
fill: #ffffff !important;
}
.blocklyEditableText:hover > rect {
stroke: var(--primary-color) !important;
stroke-width: 2px !important;
}
/* Improve visibility of connection highlights */
.blocklyHighlightedConnectionPath {
stroke: var(--primary-color) !important;
stroke-width: 4px !important;
}
.blocklyInsertionMarker > .blocklyPath {
fill-opacity: 0.3 !important;
stroke-opacity: 0.6 !important;
}
/* Workspace grid pattern */
.blocklyWorkspace > .blocklyBlockCanvas > .blocklyGridCanvas {
opacity: 0.1;
}
/* Smooth transitions */
.blocklyDraggable {
transition: transform 0.1s ease;
}
/* Field labels */
.blocklyFieldLabel {
font-weight: normal !important;
}
/* Comment blocks styling */
.blocklyCommentText {
font-style: italic !important;
}
/* Make comment blocks slightly transparent */
g[data-category="Comments"] .blocklyPath {
fill-opacity: 0.8 !important;
}
/* Better visibility for disabled blocks */
.blocklyDisabled > .blocklyPath {
fill-opacity: 0.3 !important;
}
.blocklyDisabled > .blocklyText {
fill-opacity: 0.5 !important;
}
/* Warning and error text */
.blocklyWarningText,
.blocklyErrorText {
font-family: 'Dank Mono', monospace !important;
font-size: 12px !important;
}
/* Workspace scrollbar improvement for dark theme */
::-webkit-scrollbar {
width: 10px;
height: 10px;
}
::-webkit-scrollbar-track {
background: var(--bg-secondary);
}
::-webkit-scrollbar-thumb {
background: var(--border-color);
border-radius: 5px;
}
::-webkit-scrollbar-thumb:hover {
background: var(--border-hover);
}

549
docs/examples/c4a_script/tutorial/assets/c4a-blocks.js Normal file
View File

@@ -0,0 +1,549 @@
// C4A-Script Blockly Block Definitions
// This file defines all custom blocks for C4A-Script commands
// Color scheme for different block categories
const BlockColors = {
NAVIGATION: '#1E88E5', // Blue
ACTIONS: '#43A047', // Green
CONTROL: '#FB8C00', // Orange
VARIABLES: '#8E24AA', // Purple
WAIT: '#E53935', // Red
KEYBOARD: '#00ACC1', // Cyan
PROCEDURES: '#6A1B9A' // Deep Purple
};
// Helper to create selector input with backticks
Blockly.Blocks['c4a_selector_input'] = {
init: function() {
this.appendDummyInput()
.appendField("`")
.appendField(new Blockly.FieldTextInput("selector"), "SELECTOR")
.appendField("`");
this.setOutput(true, "Selector");
this.setColour(BlockColors.ACTIONS);
this.setTooltip("CSS selector for element");
}
};
// ============================================
// NAVIGATION BLOCKS
// ============================================
Blockly.Blocks['c4a_go'] = {
init: function() {
this.appendDummyInput()
.appendField("GO")
.appendField(new Blockly.FieldTextInput("https://example.com"), "URL");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.NAVIGATION);
this.setTooltip("Navigate to URL");
}
};
Blockly.Blocks['c4a_reload'] = {
init: function() {
this.appendDummyInput()
.appendField("RELOAD");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.NAVIGATION);
this.setTooltip("Reload current page");
}
};
Blockly.Blocks['c4a_back'] = {
init: function() {
this.appendDummyInput()
.appendField("BACK");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.NAVIGATION);
this.setTooltip("Go back in browser history");
}
};
Blockly.Blocks['c4a_forward'] = {
init: function() {
this.appendDummyInput()
.appendField("FORWARD");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.NAVIGATION);
this.setTooltip("Go forward in browser history");
}
};
// ============================================
// WAIT BLOCKS
// ============================================
Blockly.Blocks['c4a_wait_time'] = {
init: function() {
this.appendDummyInput()
.appendField("WAIT")
.appendField(new Blockly.FieldNumber(1, 0), "SECONDS")
.appendField("seconds");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.WAIT);
this.setTooltip("Wait for specified seconds");
}
};
Blockly.Blocks['c4a_wait_selector'] = {
init: function() {
this.appendDummyInput()
.appendField("WAIT for")
.appendField("`")
.appendField(new Blockly.FieldTextInput("selector"), "SELECTOR")
.appendField("`")
.appendField("max")
.appendField(new Blockly.FieldNumber(10, 1), "TIMEOUT")
.appendField("sec");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.WAIT);
this.setTooltip("Wait for element to appear");
}
};
Blockly.Blocks['c4a_wait_text'] = {
init: function() {
this.appendDummyInput()
.appendField("WAIT for text")
.appendField(new Blockly.FieldTextInput("Loading complete"), "TEXT")
.appendField("max")
.appendField(new Blockly.FieldNumber(5, 1), "TIMEOUT")
.appendField("sec");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.WAIT);
this.setTooltip("Wait for text to appear on page");
}
};
// ============================================
// MOUSE ACTION BLOCKS
// ============================================
Blockly.Blocks['c4a_click'] = {
init: function() {
this.appendDummyInput()
.appendField("CLICK")
.appendField("`")
.appendField(new Blockly.FieldTextInput("button"), "SELECTOR")
.appendField("`");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.ACTIONS);
this.setTooltip("Click on element");
}
};
Blockly.Blocks['c4a_click_xy'] = {
init: function() {
this.appendDummyInput()
.appendField("CLICK at")
.appendField("X:")
.appendField(new Blockly.FieldNumber(100, 0), "X")
.appendField("Y:")
.appendField(new Blockly.FieldNumber(100, 0), "Y");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.ACTIONS);
this.setTooltip("Click at coordinates");
}
};
Blockly.Blocks['c4a_double_click'] = {
init: function() {
this.appendDummyInput()
.appendField("DOUBLE_CLICK")
.appendField("`")
.appendField(new Blockly.FieldTextInput(".item"), "SELECTOR")
.appendField("`");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.ACTIONS);
this.setTooltip("Double click on element");
}
};
Blockly.Blocks['c4a_right_click'] = {
init: function() {
this.appendDummyInput()
.appendField("RIGHT_CLICK")
.appendField("`")
.appendField(new Blockly.FieldTextInput("#menu"), "SELECTOR")
.appendField("`");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.ACTIONS);
this.setTooltip("Right click on element");
}
};
Blockly.Blocks['c4a_move'] = {
init: function() {
this.appendDummyInput()
.appendField("MOVE to")
.appendField("X:")
.appendField(new Blockly.FieldNumber(500, 0), "X")
.appendField("Y:")
.appendField(new Blockly.FieldNumber(300, 0), "Y");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.ACTIONS);
this.setTooltip("Move mouse to position");
}
};
Blockly.Blocks['c4a_drag'] = {
init: function() {
this.appendDummyInput()
.appendField("DRAG from")
.appendField("X:")
.appendField(new Blockly.FieldNumber(100, 0), "X1")
.appendField("Y:")
.appendField(new Blockly.FieldNumber(100, 0), "Y1");
this.appendDummyInput()
.appendField("to")
.appendField("X:")
.appendField(new Blockly.FieldNumber(500, 0), "X2")
.appendField("Y:")
.appendField(new Blockly.FieldNumber(300, 0), "Y2");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.ACTIONS);
this.setTooltip("Drag from one point to another");
}
};
Blockly.Blocks['c4a_scroll'] = {
init: function() {
this.appendDummyInput()
.appendField("SCROLL")
.appendField(new Blockly.FieldDropdown([
["DOWN", "DOWN"],
["UP", "UP"],
["LEFT", "LEFT"],
["RIGHT", "RIGHT"]
]), "DIRECTION")
.appendField(new Blockly.FieldNumber(500, 0), "AMOUNT")
.appendField("pixels");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.ACTIONS);
this.setTooltip("Scroll in direction");
}
};
// ============================================
// KEYBOARD BLOCKS
// ============================================
Blockly.Blocks['c4a_type'] = {
init: function() {
this.appendDummyInput()
.appendField("TYPE")
.appendField(new Blockly.FieldTextInput("text to type"), "TEXT");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.KEYBOARD);
this.setTooltip("Type text");
}
};
Blockly.Blocks['c4a_type_var'] = {
init: function() {
this.appendDummyInput()
.appendField("TYPE")
.appendField("$")
.appendField(new Blockly.FieldTextInput("variable"), "VAR");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.KEYBOARD);
this.setTooltip("Type variable value");
}
};
Blockly.Blocks['c4a_clear'] = {
init: function() {
this.appendDummyInput()
.appendField("CLEAR")
.appendField("`")
.appendField(new Blockly.FieldTextInput("input"), "SELECTOR")
.appendField("`");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.KEYBOARD);
this.setTooltip("Clear input field");
}
};
Blockly.Blocks['c4a_set'] = {
init: function() {
this.appendDummyInput()
.appendField("SET")
.appendField("`")
.appendField(new Blockly.FieldTextInput("#input"), "SELECTOR")
.appendField("`")
.appendField("to")
.appendField(new Blockly.FieldTextInput("value"), "VALUE");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.KEYBOARD);
this.setTooltip("Set input field value");
}
};
Blockly.Blocks['c4a_press'] = {
init: function() {
this.appendDummyInput()
.appendField("PRESS")
.appendField(new Blockly.FieldDropdown([
["Tab", "Tab"],
["Enter", "Enter"],
["Escape", "Escape"],
["Space", "Space"],
["ArrowUp", "ArrowUp"],
["ArrowDown", "ArrowDown"],
["ArrowLeft", "ArrowLeft"],
["ArrowRight", "ArrowRight"],
["Delete", "Delete"],
["Backspace", "Backspace"]
]), "KEY");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.KEYBOARD);
this.setTooltip("Press and release key");
}
};
Blockly.Blocks['c4a_key_down'] = {
init: function() {
this.appendDummyInput()
.appendField("KEY_DOWN")
.appendField(new Blockly.FieldDropdown([
["Shift", "Shift"],
["Control", "Control"],
["Alt", "Alt"],
["Meta", "Meta"]
]), "KEY");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.KEYBOARD);
this.setTooltip("Hold key down");
}
};
Blockly.Blocks['c4a_key_up'] = {
init: function() {
this.appendDummyInput()
.appendField("KEY_UP")
.appendField(new Blockly.FieldDropdown([
["Shift", "Shift"],
["Control", "Control"],
["Alt", "Alt"],
["Meta", "Meta"]
]), "KEY");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.KEYBOARD);
this.setTooltip("Release key");
}
};
// ============================================
// CONTROL FLOW BLOCKS
// ============================================
Blockly.Blocks['c4a_if_exists'] = {
init: function() {
this.appendDummyInput()
.appendField("IF EXISTS")
.appendField("`")
.appendField(new Blockly.FieldTextInput(".element"), "SELECTOR")
.appendField("`")
.appendField("THEN");
this.appendStatementInput("THEN")
.setCheck(null);
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.CONTROL);
this.setTooltip("If element exists, then do something");
}
};
Blockly.Blocks['c4a_if_exists_else'] = {
init: function() {
this.appendDummyInput()
.appendField("IF EXISTS")
.appendField("`")
.appendField(new Blockly.FieldTextInput(".element"), "SELECTOR")
.appendField("`")
.appendField("THEN");
this.appendStatementInput("THEN")
.setCheck(null);
this.appendDummyInput()
.appendField("ELSE");
this.appendStatementInput("ELSE")
.setCheck(null);
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.CONTROL);
this.setTooltip("If element exists, then do something, else do something else");
}
};
Blockly.Blocks['c4a_if_not_exists'] = {
init: function() {
this.appendDummyInput()
.appendField("IF NOT EXISTS")
.appendField("`")
.appendField(new Blockly.FieldTextInput(".element"), "SELECTOR")
.appendField("`")
.appendField("THEN");
this.appendStatementInput("THEN")
.setCheck(null);
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.CONTROL);
this.setTooltip("If element does not exist, then do something");
}
};
Blockly.Blocks['c4a_if_js'] = {
init: function() {
this.appendDummyInput()
.appendField("IF")
.appendField("`")
.appendField(new Blockly.FieldTextInput("window.innerWidth < 768"), "CONDITION")
.appendField("`")
.appendField("THEN");
this.appendStatementInput("THEN")
.setCheck(null);
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.CONTROL);
this.setTooltip("If JavaScript condition is true");
}
};
Blockly.Blocks['c4a_repeat_times'] = {
init: function() {
this.appendDummyInput()
.appendField("REPEAT")
.appendField(new Blockly.FieldNumber(5, 1), "TIMES")
.appendField("times");
this.appendStatementInput("DO")
.setCheck(null);
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.CONTROL);
this.setTooltip("Repeat commands N times");
}
};
Blockly.Blocks['c4a_repeat_while'] = {
init: function() {
this.appendDummyInput()
.appendField("REPEAT WHILE")
.appendField("`")
.appendField(new Blockly.FieldTextInput("document.querySelector('.load-more')"), "CONDITION")
.appendField("`");
this.appendStatementInput("DO")
.setCheck(null);
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.CONTROL);
this.setTooltip("Repeat while condition is true");
}
};
// ============================================
// VARIABLE BLOCKS
// ============================================
Blockly.Blocks['c4a_setvar'] = {
init: function() {
this.appendDummyInput()
.appendField("SETVAR")
.appendField(new Blockly.FieldTextInput("username"), "NAME")
.appendField("=")
.appendField(new Blockly.FieldTextInput("value"), "VALUE");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.VARIABLES);
this.setTooltip("Set variable value");
}
};
// ============================================
// ADVANCED BLOCKS
// ============================================
Blockly.Blocks['c4a_eval'] = {
init: function() {
this.appendDummyInput()
.appendField("EVAL")
.appendField("`")
.appendField(new Blockly.FieldTextInput("console.log('Hello')"), "CODE")
.appendField("`");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.VARIABLES);
this.setTooltip("Execute JavaScript code");
}
};
Blockly.Blocks['c4a_comment'] = {
init: function() {
this.appendDummyInput()
.appendField("#")
.appendField(new Blockly.FieldTextInput("Comment", null, {
spellcheck: false,
class: 'blocklyCommentText'
}), "TEXT");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour("#616161");
this.setTooltip("Add a comment");
this.setStyle('comment_blocks');
}
};
// ============================================
// PROCEDURE BLOCKS
// ============================================
Blockly.Blocks['c4a_proc_def'] = {
init: function() {
this.appendDummyInput()
.appendField("PROC")
.appendField(new Blockly.FieldTextInput("procedure_name"), "NAME");
this.appendStatementInput("BODY")
.setCheck(null);
this.appendDummyInput()
.appendField("ENDPROC");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.PROCEDURES);
this.setTooltip("Define a procedure");
}
};
Blockly.Blocks['c4a_proc_call'] = {
init: function() {
this.appendDummyInput()
.appendField("Call")
.appendField(new Blockly.FieldTextInput("procedure_name"), "NAME");
this.setPreviousStatement(true, null);
this.setNextStatement(true, null);
this.setColour(BlockColors.PROCEDURES);
this.setTooltip("Call a procedure");
}
};
// Code generators have been moved to c4a-generator.js

261
docs/examples/c4a_script/tutorial/assets/c4a-generator.js Normal file
View File

@@ -0,0 +1,261 @@
// C4A-Script Code Generator for Blockly
// Compatible with latest Blockly API
// Create a custom code generator for C4A-Script
const c4aGenerator = new Blockly.Generator('C4A');
// Helper to get field value with proper escaping
c4aGenerator.getFieldValue = function(block, fieldName) {
return block.getFieldValue(fieldName);
};
// Navigation generators
c4aGenerator.forBlock['c4a_go'] = function(block, generator) {
const url = generator.getFieldValue(block, 'URL');
return `GO ${url}\n`;
};
c4aGenerator.forBlock['c4a_reload'] = function(block, generator) {
return 'RELOAD\n';
};
c4aGenerator.forBlock['c4a_back'] = function(block, generator) {
return 'BACK\n';
};
c4aGenerator.forBlock['c4a_forward'] = function(block, generator) {
return 'FORWARD\n';
};
// Wait generators
c4aGenerator.forBlock['c4a_wait_time'] = function(block, generator) {
const seconds = generator.getFieldValue(block, 'SECONDS');
return `WAIT ${seconds}\n`;
};
c4aGenerator.forBlock['c4a_wait_selector'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
const timeout = generator.getFieldValue(block, 'TIMEOUT');
return `WAIT \`${selector}\` ${timeout}\n`;
};
c4aGenerator.forBlock['c4a_wait_text'] = function(block, generator) {
const text = generator.getFieldValue(block, 'TEXT');
const timeout = generator.getFieldValue(block, 'TIMEOUT');
return `WAIT "${text}" ${timeout}\n`;
};
// Mouse action generators
c4aGenerator.forBlock['c4a_click'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
return `CLICK \`${selector}\`\n`;
};
c4aGenerator.forBlock['c4a_click_xy'] = function(block, generator) {
const x = generator.getFieldValue(block, 'X');
const y = generator.getFieldValue(block, 'Y');
return `CLICK ${x} ${y}\n`;
};
c4aGenerator.forBlock['c4a_double_click'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
return `DOUBLE_CLICK \`${selector}\`\n`;
};
c4aGenerator.forBlock['c4a_right_click'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
return `RIGHT_CLICK \`${selector}\`\n`;
};
c4aGenerator.forBlock['c4a_move'] = function(block, generator) {
const x = generator.getFieldValue(block, 'X');
const y = generator.getFieldValue(block, 'Y');
return `MOVE ${x} ${y}\n`;
};
c4aGenerator.forBlock['c4a_drag'] = function(block, generator) {
const x1 = generator.getFieldValue(block, 'X1');
const y1 = generator.getFieldValue(block, 'Y1');
const x2 = generator.getFieldValue(block, 'X2');
const y2 = generator.getFieldValue(block, 'Y2');
return `DRAG ${x1} ${y1} ${x2} ${y2}\n`;
};
c4aGenerator.forBlock['c4a_scroll'] = function(block, generator) {
const direction = generator.getFieldValue(block, 'DIRECTION');
const amount = generator.getFieldValue(block, 'AMOUNT');
return `SCROLL ${direction} ${amount}\n`;
};
// Keyboard generators
c4aGenerator.forBlock['c4a_type'] = function(block, generator) {
const text = generator.getFieldValue(block, 'TEXT');
return `TYPE "${text}"\n`;
};
c4aGenerator.forBlock['c4a_type_var'] = function(block, generator) {
const varName = generator.getFieldValue(block, 'VAR');
return `TYPE $${varName}\n`;
};
c4aGenerator.forBlock['c4a_clear'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
return `CLEAR \`${selector}\`\n`;
};
c4aGenerator.forBlock['c4a_set'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
const value = generator.getFieldValue(block, 'VALUE');
return `SET \`${selector}\` "${value}"\n`;
};
c4aGenerator.forBlock['c4a_press'] = function(block, generator) {
const key = generator.getFieldValue(block, 'KEY');
return `PRESS ${key}\n`;
};
c4aGenerator.forBlock['c4a_key_down'] = function(block, generator) {
const key = generator.getFieldValue(block, 'KEY');
return `KEY_DOWN ${key}\n`;
};
c4aGenerator.forBlock['c4a_key_up'] = function(block, generator) {
const key = generator.getFieldValue(block, 'KEY');
return `KEY_UP ${key}\n`;
};
// Control flow generators
c4aGenerator.forBlock['c4a_if_exists'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
const thenCode = generator.statementToCode(block, 'THEN').trim();
if (thenCode.includes('\n')) {
// Multi-line then block
const lines = thenCode.split('\n').filter(line => line.trim());
return lines.map(line => `IF (EXISTS \`${selector}\`) THEN ${line}`).join('\n') + '\n';
} else if (thenCode) {
// Single line
return `IF (EXISTS \`${selector}\`) THEN ${thenCode}\n`;
}
return '';
};
c4aGenerator.forBlock['c4a_if_exists_else'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
const thenCode = generator.statementToCode(block, 'THEN').trim();
const elseCode = generator.statementToCode(block, 'ELSE').trim();
// For simplicity, only handle single-line then/else
const thenLine = thenCode.split('\n')[0];
const elseLine = elseCode.split('\n')[0];
if (thenLine && elseLine) {
return `IF (EXISTS \`${selector}\`) THEN ${thenLine} ELSE ${elseLine}\n`;
} else if (thenLine) {
return `IF (EXISTS \`${selector}\`) THEN ${thenLine}\n`;
}
return '';
};
c4aGenerator.forBlock['c4a_if_not_exists'] = function(block, generator) {
const selector = generator.getFieldValue(block, 'SELECTOR');
const thenCode = generator.statementToCode(block, 'THEN').trim();
if (thenCode.includes('\n')) {
const lines = thenCode.split('\n').filter(line => line.trim());
return lines.map(line => `IF (NOT EXISTS \`${selector}\`) THEN ${line}`).join('\n') + '\n';
} else if (thenCode) {
return `IF (NOT EXISTS \`${selector}\`) THEN ${thenCode}\n`;
}
return '';
};
c4aGenerator.forBlock['c4a_if_js'] = function(block, generator) {
const condition = generator.getFieldValue(block, 'CONDITION');
const thenCode = generator.statementToCode(block, 'THEN').trim();
if (thenCode.includes('\n')) {
const lines = thenCode.split('\n').filter(line => line.trim());
return lines.map(line => `IF (\`${condition}\`) THEN ${line}`).join('\n') + '\n';
} else if (thenCode) {
return `IF (\`${condition}\`) THEN ${thenCode}\n`;
}
return '';
};
c4aGenerator.forBlock['c4a_repeat_times'] = function(block, generator) {
const times = generator.getFieldValue(block, 'TIMES');
const doCode = generator.statementToCode(block, 'DO').trim();
if (doCode) {
// Get first command for repeat
const firstLine = doCode.split('\n')[0];
return `REPEAT (${firstLine}, ${times})\n`;
}
return '';
};
c4aGenerator.forBlock['c4a_repeat_while'] = function(block, generator) {
const condition = generator.getFieldValue(block, 'CONDITION');
const doCode = generator.statementToCode(block, 'DO').trim();
if (doCode) {
// Get first command for repeat
const firstLine = doCode.split('\n')[0];
return `REPEAT (${firstLine}, \`${condition}\`)\n`;
}
return '';
};
// Variable generators
c4aGenerator.forBlock['c4a_setvar'] = function(block, generator) {
const name = generator.getFieldValue(block, 'NAME');
const value = generator.getFieldValue(block, 'VALUE');
return `SETVAR ${name} = "${value}"\n`;
};
// Advanced generators
c4aGenerator.forBlock['c4a_eval'] = function(block, generator) {
const code = generator.getFieldValue(block, 'CODE');
return `EVAL \`${code}\`\n`;
};
c4aGenerator.forBlock['c4a_comment'] = function(block, generator) {
const text = generator.getFieldValue(block, 'TEXT');
return `# ${text}\n`;
};
// Procedure generators
c4aGenerator.forBlock['c4a_proc_def'] = function(block, generator) {
const name = generator.getFieldValue(block, 'NAME');
const body = generator.statementToCode(block, 'BODY');
return `PROC ${name}\n${body}ENDPROC\n`;
};
c4aGenerator.forBlock['c4a_proc_call'] = function(block, generator) {
const name = generator.getFieldValue(block, 'NAME');
return `${name}\n`;
};
// Override scrub_ to handle our custom format
c4aGenerator.scrub_ = function(block, code, opt_thisOnly) {
const nextBlock = block.nextConnection && block.nextConnection.targetBlock();
let nextCode = '';
if (nextBlock) {
if (!opt_thisOnly) {
nextCode = c4aGenerator.blockToCode(nextBlock);
// Add blank line between comment and non-comment blocks
const currentIsComment = block.type === 'c4a_comment';
const nextIsComment = nextBlock.type === 'c4a_comment';
// Add blank line when transitioning from command to comment or vice versa
if (currentIsComment !== nextIsComment && code.trim() && nextCode.trim()) {
nextCode = '\n' + nextCode;
}
}
}
return code + nextCode;
};

531
docs/examples/c4a_script/tutorial/assets/styles.css Normal file
View File

@@ -0,0 +1,531 @@
/* DankMono Font Faces */
@font-face {
font-family: 'DankMono';
src: url('DankMono-Regular.woff2') format('woff2');
font-weight: 400;
font-style: normal;
}
@font-face {
font-family: 'DankMono';
src: url('DankMono-Bold.woff2') format('woff2');
font-weight: 700;
font-style: normal;
}
@font-face {
font-family: 'DankMono';
src: url('DankMono-Italic.woff2') format('woff2');
font-weight: 400;
font-style: italic;
}
/* Root Variables - Matching docs theme */
:root {
--global-font-size: 14px;
--global-code-font-size: 13px;
--global-line-height: 1.5em;
--global-space: 10px;
--font-stack: DankMono, Monaco, Courier New, monospace;
--mono-font-stack: DankMono, Monaco, Courier New, monospace;
--background-color: #070708;
--font-color: #e8e9ed;
--invert-font-color: #222225;
--secondary-color: #d5cec0;
--tertiary-color: #a3abba;
--primary-color: #0fbbaa;
--error-color: #ff3c74;
--progress-bar-background: #3f3f44;
--progress-bar-fill: #09b5a5;
--code-bg-color: #3f3f44;
--block-background-color: #202020;
--header-height: 55px;
}
/* Base Styles */
* {
box-sizing: border-box;
}
body {
margin: 0;
padding: 0;
font-family: var(--font-stack);
font-size: var(--global-font-size);
line-height: var(--global-line-height);
color: var(--font-color);
background-color: var(--background-color);
}
/* Terminal Framework */
.terminal {
min-height: 100vh;
}
.container {
width: 100%;
margin: 0 auto;
}
/* Header */
.header-container {
position: fixed;
top: 0;
left: 0;
right: 0;
height: var(--header-height);
background-color: var(--background-color);
border-bottom: 1px solid var(--progress-bar-background);
z-index: 1000;
padding: 0 calc(var(--global-space) * 2);
}
.terminal-nav {
display: flex;
align-items: center;
justify-content: space-between;
height: 100%;
}
.terminal-logo h1 {
margin: 0;
font-size: 1.2em;
color: var(--primary-color);
font-weight: 400;
}
.terminal-menu ul {
list-style: none;
margin: 0;
padding: 0;
display: flex;
gap: 2em;
}
.terminal-menu a {
color: var(--secondary-color);
text-decoration: none;
transition: color 0.2s;
}
.terminal-menu a:hover,
.terminal-menu a.active {
color: var(--primary-color);
}
/* Main Container */
.main-container {
padding-top: calc(var(--header-height) + 2em);
padding-left: 2em;
padding-right: 2em;
max-width: 1400px;
margin: 0 auto;
}
/* Tutorial Grid */
.tutorial-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 2em;
align-items: start;
}
/* Terminal Cards */
.terminal-card {
background-color: var(--block-background-color);
border: 1px solid var(--progress-bar-background);
margin-bottom: 1.5em;
}
.terminal-card header {
background-color: var(--progress-bar-background);
padding: 0.8em 1em;
font-weight: 700;
color: var(--font-color);
display: flex;
justify-content: space-between;
align-items: center;
}
.terminal-card > div {
padding: 1.5em;
}
/* Editor Section */
.editor-controls {
display: flex;
gap: 0.5em;
}
.editor-container {
height: 300px;
overflow: hidden;
}
#c4a-editor {
width: 100%;
height: 100%;
font-family: var(--mono-font-stack);
font-size: var(--global-code-font-size);
background-color: var(--code-bg-color);
color: var(--font-color);
border: none;
padding: 1em;
resize: none;
}
/* JS Output */
.js-output-container {
max-height: 300px;
overflow-y: auto;
}
.js-output-container pre {
margin: 0;
padding: 1em;
background-color: var(--code-bg-color);
}
.js-output-container code {
font-family: var(--mono-font-stack);
font-size: var(--global-code-font-size);
color: var(--font-color);
white-space: pre-wrap;
}
/* Console Output */
.console-output {
font-family: var(--mono-font-stack);
font-size: var(--global-code-font-size);
max-height: 200px;
overflow-y: auto;
padding: 1em;
}
.console-line {
margin-bottom: 0.5em;
}
.console-prompt {
color: var(--primary-color);
margin-right: 0.5em;
}
.console-text {
color: var(--font-color);
}
.console-error {
color: var(--error-color);
}
.console-success {
color: var(--primary-color);
}
/* Playground */
.playground-container {
height: 600px;
background-color: #fff;
border: 1px solid var(--progress-bar-background);
}
#playground-frame {
width: 100%;
height: 100%;
border: none;
}
/* Execution Progress */
.execution-progress {
padding: 1em;
}
.progress-item {
display: flex;
align-items: center;
gap: 0.8em;
margin-bottom: 0.8em;
color: var(--secondary-color);
}
.progress-item.active {
color: var(--primary-color);
}
.progress-item.completed {
color: var(--tertiary-color);
}
.progress-item.error {
color: var(--error-color);
}
.progress-icon {
font-size: 1.2em;
}
/* Buttons */
.btn {
background-color: var(--primary-color);
color: var(--background-color);
border: none;
padding: 0.5em 1em;
font-family: var(--font-stack);
font-size: 0.9em;
cursor: pointer;
transition: all 0.2s;
}
.btn:hover {
background-color: var(--progress-bar-fill);
}
.btn-sm {
padding: 0.3em 0.8em;
font-size: 0.85em;
}
.btn-ghost {
background-color: transparent;
color: var(--secondary-color);
border: 1px solid var(--progress-bar-background);
}
.btn-ghost:hover {
background-color: var(--progress-bar-background);
color: var(--font-color);
}
/* Scrollbars */
::-webkit-scrollbar {
width: 8px;
height: 8px;
}
::-webkit-scrollbar-track {
background: var(--block-background-color);
}
::-webkit-scrollbar-thumb {
background: var(--progress-bar-background);
}
::-webkit-scrollbar-thumb:hover {
background: var(--secondary-color);
}
/* CodeMirror Theme Override */
.CodeMirror {
font-family: var(--mono-font-stack) !important;
font-size: var(--global-code-font-size) !important;
background-color: var(--code-bg-color) !important;
color: var(--font-color) !important;
height: 100% !important;
}
.CodeMirror-gutters {
background-color: var(--progress-bar-background) !important;
border-right: 1px solid var(--progress-bar-background) !important;
}
/* Responsive */
@media (max-width: 1200px) {
.tutorial-grid {
grid-template-columns: 1fr;
}
.playground-section {
order: -1;
}
}
/* Links */
a {
color: var(--primary-color);
text-decoration: none;
}
a:hover {
text-decoration: underline;
}
/* Lists */
ul, ol {
padding-left: 2em;
}
li {
margin-bottom: 0.5em;
}
/* Code */
code {
background-color: var(--code-bg-color);
padding: 0.2em 0.4em;
font-family: var(--mono-font-stack);
font-size: 0.9em;
}
/* Headings */
h1, h2, h3, h4, h5, h6 {
font-weight: 700;
margin-top: 1.5em;
margin-bottom: 0.8em;
}
h3 {
color: var(--primary-color);
font-size: 1.1em;
}
/* Tutorial Panel */
.tutorial-panel {
position: absolute;
top: 60px;
right: 20px;
width: 380px;
background: #1a1a1b;
border: 1px solid #2a2a2c;
border-radius: 8px;
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.5);
z-index: 1000;
transition: all 0.3s ease;
}
.tutorial-panel.hidden {
display: none;
}
.tutorial-header {
display: flex;
justify-content: space-between;
align-items: center;
padding: 16px 20px;
border-bottom: 1px solid #2a2a2c;
}
.tutorial-header h3 {
margin: 0;
color: #0fbbaa;
font-size: 18px;
}
.close-btn {
background: none;
border: none;
color: #8b8b8d;
font-size: 24px;
cursor: pointer;
padding: 0;
width: 30px;
height: 30px;
display: flex;
align-items: center;
justify-content: center;
border-radius: 4px;
transition: all 0.2s;
}
.close-btn:hover {
background: #2a2a2c;
color: #e0e0e0;
}
.tutorial-content {
padding: 20px;
}
.tutorial-content p {
margin: 0 0 16px 0;
color: #e0e0e0;
line-height: 1.6;
}
.tutorial-progress {
margin-top: 16px;
}
.tutorial-progress span {
display: block;
margin-bottom: 8px;
color: #8b8b8d;
font-size: 12px;
text-transform: uppercase;
}
.progress-bar {
height: 4px;
background: #2a2a2c;
border-radius: 2px;
overflow: hidden;
}
.progress-fill {
height: 100%;
background: #0fbbaa;
transition: width 0.3s ease;
}
.tutorial-actions {
display: flex;
gap: 12px;
padding: 0 20px 20px;
}
.tutorial-btn {
flex: 1;
padding: 10px 16px;
background: #2a2a2c;
color: #e0e0e0;
border: 1px solid #3a3a3c;
border-radius: 6px;
font-size: 14px;
cursor: pointer;
transition: all 0.2s;
}
.tutorial-btn:hover:not(:disabled) {
background: #3a3a3c;
transform: translateY(-1px);
}
.tutorial-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.tutorial-btn.primary {
background: #0fbbaa;
color: #070708;
border-color: #0fbbaa;
}
.tutorial-btn.primary:hover {
background: #0da89a;
border-color: #0da89a;
}
/* Tutorial Highlights */
.tutorial-highlight {
position: relative;
animation: pulse 2s infinite;
}
@keyframes pulse {
0% {
box-shadow: 0 0 0 0 rgba(15, 187, 170, 0.4);
}
50% {
box-shadow: 0 0 0 10px rgba(15, 187, 170, 0);
}
100% {
box-shadow: 0 0 0 0 rgba(15, 187, 170, 0);
}
}
.editor-card {
position: relative;
}

21
docs/examples/c4a_script/tutorial/blockly-demo.c4a Normal file
View File

@@ -0,0 +1,21 @@
# Demo: Login Flow with Blockly
# This script can be created visually using Blockly blocks
GO https://example.com/login
WAIT `#login-form` 5
# Check if already logged in
IF (EXISTS `.user-avatar`) THEN GO https://example.com/dashboard
# Fill login form
CLICK `#email`
TYPE "demo@example.com"
CLICK `#password`
TYPE "password123"
# Submit form
CLICK `button[type="submit"]`
WAIT `.dashboard` 10
# Success message
EVAL `console.log('Login successful!')`

205
docs/examples/c4a_script/tutorial/index.html Normal file
View File

@@ -0,0 +1,205 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>C4A-Script Interactive Tutorial | Crawl4AI</title>
<link rel="stylesheet" href="assets/app.css">
<link rel="stylesheet" href="assets/blockly-theme.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/codemirror.min.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/theme/material-darker.min.css">
</head>
<body>
<!-- Tutorial Intro Modal -->
<div id="tutorial-intro" class="tutorial-intro-modal">
<div class="intro-content">
<h2>Welcome to C4A-Script Tutorial!</h2>
<p>C4A-Script is a simple language for web automation. This interactive tutorial will teach you:</p>
<ul>
<li>How to handle popups and banners</li>
<li>Form filling and navigation</li>
<li>Advanced automation techniques</li>
</ul>
<div class="intro-actions">
<button id="start-tutorial-btn" class="intro-btn primary">Start Tutorial</button>
<button id="skip-tutorial-btn" class="intro-btn">Skip</button>
</div>
</div>
</div>
<!-- Event Editor Modal -->
<div id="event-editor-overlay" class="modal-overlay hidden"></div>
<div id="event-editor-modal" class="event-editor-modal hidden">
<h4>Edit Event</h4>
<div class="editor-field">
<label>Command Type</label>
<select id="edit-command-type" disabled>
<option value="CLICK">CLICK</option>
<option value="DOUBLE_CLICK">DOUBLE_CLICK</option>
<option value="RIGHT_CLICK">RIGHT_CLICK</option>
<option value="TYPE">TYPE</option>
<option value="SET">SET</option>
<option value="SCROLL">SCROLL</option>
<option value="WAIT">WAIT</option>
</select>
</div>
<div id="edit-selector-field" class="editor-field">
<label>Selector</label>
<input type="text" id="edit-selector" placeholder=".class or #id">
</div>
<div id="edit-value-field" class="editor-field">
<label>Value</label>
<input type="text" id="edit-value" placeholder="Text or number">
</div>
<div id="edit-direction-field" class="editor-field hidden">
<label>Direction</label>
<select id="edit-direction">
<option value="UP">UP</option>
<option value="DOWN">DOWN</option>
<option value="LEFT">LEFT</option>
<option value="RIGHT">RIGHT</option>
</select>
</div>
<div class="editor-actions">
<button id="edit-cancel" class="mini-btn">Cancel</button>
<button id="edit-save" class="mini-btn primary">Save</button>
</div>
</div>
<!-- Main App Layout -->
<div class="app-container">
<!-- Left Panel: Editor -->
<div class="editor-panel">
<div class="panel-header">
<h2>C4A-Script Editor</h2>
<div class="header-actions">
<button id="tutorial-btn" class="action-btn" title="Tutorial">
<span class="icon">📚</span>
</button>
<button id="examples-btn" class="action-btn" title="Examples">
<span class="icon">📋</span>
</button>
<button id="clear-btn" class="action-btn" title="Clear">
<span class="icon">🗑</span>
</button>
<button id="run-btn" class="action-btn primary">
<span class="icon">▶</span>Run
</button>
<button id="record-btn" class="action-btn record">
<span class="icon">⏺</span>Record
</button>
<button id="timeline-btn" class="action-btn timeline hidden" title="View Timeline">
<span class="icon">📊</span>
</button>
</div>
</div>
<div class="editor-container">
<div id="editor-view" class="editor-wrapper">
<textarea id="c4a-editor" placeholder="# Write your C4A script here..."></textarea>
</div>
<!-- Recording Timeline -->
<div id="timeline-view" class="recording-timeline hidden">
<div class="timeline-header">
<h3>Recording Timeline</h3>
<div class="timeline-actions">
<button id="back-to-editor" class="mini-btn">← Back</button>
<button id="select-all-events" class="mini-btn">Select All</button>
<button id="clear-events" class="mini-btn">Clear</button>
<button id="generate-script" class="mini-btn primary">Generate Script</button>
</div>
</div>
<div id="timeline-events" class="timeline-events">
<!-- Events will be added here dynamically -->
</div>
</div>
</div>
<!-- Bottom: Output Tabs -->
<div class="output-section">
<div class="tabs">
<button class="tab active" data-tab="console">Console</button>
<button class="tab" data-tab="javascript">Generated JS</button>
</div>
<div class="tab-content">
<div id="console-tab" class="tab-pane active">
<div id="console-output" class="console">
<div class="console-line">
<span class="console-prompt">$</span>
<span class="console-text">Ready to run C4A scripts...</span>
</div>
</div>
</div>
<div id="javascript-tab" class="tab-pane">
<div class="js-output-header">
<div class="js-actions">
<button id="copy-js-btn" class="mini-btn" title="Copy">
<span>📋</span>
</button>
<button id="edit-js-btn" class="mini-btn" title="Edit">
<span>✏️</span>
</button>
</div>
</div>
<pre id="js-output" class="js-output">// JavaScript will appear here...</pre>
</div>
</div>
</div>
</div>
<!-- Right Panel: Playground -->
<div class="playground-panel">
<div class="panel-header">
<h2>Playground</h2>
<div class="header-actions">
<button id="reset-playground" class="action-btn" title="Reset">
<span class="icon">🔄</span>
</button>
<button id="fullscreen-btn" class="action-btn" title="Fullscreen">
<span class="icon">⛶</span>
</button>
</div>
</div>
<div class="playground-wrapper">
<iframe id="playground-frame" src="playground/" title="Playground"></iframe>
</div>
</div>
</div>
<!-- Tutorial Navigation Bar -->
<div id="tutorial-nav" class="tutorial-nav hidden">
<div class="tutorial-nav-content">
<div class="tutorial-left">
<div class="tutorial-step-title">
<span id="tutorial-step-info">Step 1 of 9</span>
<span id="tutorial-title">Welcome</span>
</div>
<p id="tutorial-description" class="tutorial-description">Let's start by waiting for the page to load.</p>
</div>
<div class="tutorial-right">
<div class="tutorial-controls">
<button id="tutorial-prev" class="nav-btn" disabled>← Previous</button>
<button id="tutorial-next" class="nav-btn primary">Next →</button>
</div>
<button id="tutorial-exit" class="exit-btn" title="Exit Tutorial">×</button>
</div>
</div>
<div class="tutorial-progress-bar">
<div id="tutorial-progress-fill" class="progress-fill"></div>
</div>
</div>
<!-- Scripts -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/codemirror.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/mode/javascript/javascript.min.js"></script>
<!-- Blockly -->
<script src="https://unpkg.com/blockly/blockly.min.js"></script>
<script src="assets/c4a-blocks.js"></script>
<script src="assets/c4a-generator.js"></script>
<script src="assets/blockly-manager.js"></script>
<script src="assets/app.js"></script>
</body>
</html>

604
docs/examples/c4a_script/tutorial/playground/app.js Normal file
View File

@@ -0,0 +1,604 @@
// Playground App JavaScript
class PlaygroundApp {
constructor() {
this.isLoggedIn = false;
this.currentSection = 'home';
this.productsLoaded = 0;
this.maxProducts = 100;
this.tableRowsLoaded = 10;
this.inspectorMode = false;
this.tooltip = null;
this.init();
}
init() {
this.setupCookieBanner();
this.setupNewsletterPopup();
this.setupNavigation();
this.setupAuth();
this.setupProductCatalog();
this.setupForms();
this.setupTabs();
this.setupDataTable();
this.setupInspector();
this.loadInitialData();
}
// Cookie Banner
setupCookieBanner() {
const banner = document.getElementById('cookie-banner');
const acceptBtn = banner.querySelector('.accept');
const declineBtn = banner.querySelector('.decline');
acceptBtn.addEventListener('click', () => {
banner.style.display = 'none';
console.log('✅ Cookies accepted');
});
declineBtn.addEventListener('click', () => {
banner.style.display = 'none';
console.log('❌ Cookies declined');
});
}
// Newsletter Popup
setupNewsletterPopup() {
const popup = document.getElementById('newsletter-popup');
const closeBtn = popup.querySelector('.close');
const subscribeBtn = popup.querySelector('.subscribe');
// Show popup after 3 seconds
setTimeout(() => {
popup.style.display = 'flex';
}, 3000);
closeBtn.addEventListener('click', () => {
popup.style.display = 'none';
});
subscribeBtn.addEventListener('click', () => {
const email = popup.querySelector('input').value;
if (email) {
console.log(`📧 Subscribed: ${email}`);
popup.style.display = 'none';
}
});
// Close on outside click
popup.addEventListener('click', (e) => {
if (e.target === popup) {
popup.style.display = 'none';
}
});
}
// Navigation
setupNavigation() {
const navLinks = document.querySelectorAll('.nav-link');
const sections = document.querySelectorAll('.section');
navLinks.forEach(link => {
link.addEventListener('click', (e) => {
e.preventDefault();
const targetId = link.getAttribute('href').substring(1);
// Update active states
navLinks.forEach(l => l.classList.remove('active'));
link.classList.add('active');
// Show target section
sections.forEach(s => s.classList.remove('active'));
const targetSection = document.getElementById(targetId);
if (targetSection) {
targetSection.classList.add('active');
this.currentSection = targetId;
// Load content for specific sections
this.loadSectionContent(targetId);
}
});
});
// Start tutorial button
const startBtn = document.getElementById('start-tutorial');
if (startBtn) {
startBtn.addEventListener('click', () => {
console.log('🚀 Tutorial started!');
alert('Tutorial started! Check the console for progress.');
});
}
}
// Authentication
setupAuth() {
const loginBtn = document.getElementById('login-btn');
const logoutBtn = document.getElementById('logout-btn');
const loginModal = document.getElementById('login-modal');
const loginForm = document.getElementById('login-form');
const closeBtn = loginModal.querySelector('.close');
loginBtn.addEventListener('click', () => {
loginModal.style.display = 'flex';
});
closeBtn.addEventListener('click', () => {
loginModal.style.display = 'none';
});
loginForm.addEventListener('submit', (e) => {
e.preventDefault();
const email = document.getElementById('email').value;
const password = document.getElementById('password').value;
const rememberMe = document.getElementById('remember-me').checked;
const messageEl = document.getElementById('login-message');
// Simple validation
if (email === 'demo@example.com' && password === 'demo123') {
this.isLoggedIn = true;
messageEl.textContent = '✅ Login successful!';
messageEl.className = 'form-message success';
setTimeout(() => {
loginModal.style.display = 'none';
document.getElementById('login-btn').style.display = 'none';
document.getElementById('user-info').style.display = 'flex';
document.getElementById('username-display').textContent = 'Demo User';
console.log(`✅ Logged in${rememberMe ? ' (remembered)' : ''}`);
}, 1000);
} else {
messageEl.textContent = '❌ Invalid credentials. Try demo@example.com / demo123';
messageEl.className = 'form-message error';
}
});
logoutBtn.addEventListener('click', () => {
this.isLoggedIn = false;
document.getElementById('login-btn').style.display = 'block';
document.getElementById('user-info').style.display = 'none';
console.log('👋 Logged out');
});
// Close modal on outside click
loginModal.addEventListener('click', (e) => {
if (e.target === loginModal) {
loginModal.style.display = 'none';
}
});
}
// Product Catalog
setupProductCatalog() {
// View toggle
const infiniteBtn = document.getElementById('infinite-scroll-btn');
const paginationBtn = document.getElementById('pagination-btn');
const infiniteView = document.getElementById('infinite-scroll-view');
const paginationView = document.getElementById('pagination-view');
infiniteBtn.addEventListener('click', () => {
infiniteBtn.classList.add('active');
paginationBtn.classList.remove('active');
infiniteView.style.display = 'block';
paginationView.style.display = 'none';
this.setupInfiniteScroll();
});
paginationBtn.addEventListener('click', () => {
paginationBtn.classList.add('active');
infiniteBtn.classList.remove('active');
paginationView.style.display = 'block';
infiniteView.style.display = 'none';
});
// Load more button
const loadMoreBtn = paginationView.querySelector('.load-more');
loadMoreBtn.addEventListener('click', () => {
this.loadMoreProducts();
});
// Collapsible filters
const collapsibles = document.querySelectorAll('.collapsible');
collapsibles.forEach(header => {
header.addEventListener('click', () => {
const content = header.nextElementSibling;
const toggle = header.querySelector('.toggle');
content.style.display = content.style.display === 'none' ? 'block' : 'none';
toggle.textContent = content.style.display === 'none' ? '▶' : '▼';
});
});
}
setupInfiniteScroll() {
const container = document.querySelector('.products-container');
const loadingIndicator = document.getElementById('loading-indicator');
container.addEventListener('scroll', () => {
if (container.scrollTop + container.clientHeight >= container.scrollHeight - 100) {
if (this.productsLoaded < this.maxProducts) {
loadingIndicator.style.display = 'block';
setTimeout(() => {
this.loadMoreProducts();
loadingIndicator.style.display = 'none';
}, 1000);
}
}
});
}
loadMoreProducts() {
const grid = document.getElementById('product-grid');
const batch = 10;
for (let i = 0; i < batch && this.productsLoaded < this.maxProducts; i++) {
const product = this.createProductCard(this.productsLoaded + 1);
grid.appendChild(product);
this.productsLoaded++;
}
console.log(`📦 Loaded ${batch} more products. Total: ${this.productsLoaded}`);
}
createProductCard(id) {
const card = document.createElement('div');
card.className = 'product-card';
card.innerHTML = `
<div class="product-image">📦</div>
<div class="product-name">Product ${id}</div>
<div class="product-price">$${(Math.random() * 100 + 10).toFixed(2)}</div>
<button class="btn btn-sm">Quick View</button>
`;
// Quick view functionality
const quickViewBtn = card.querySelector('button');
quickViewBtn.addEventListener('click', () => {
alert(`Quick view for Product ${id}`);
});
return card;
}
// Forms
setupForms() {
// Contact Form
const contactForm = document.getElementById('contact-form');
const subjectSelect = document.getElementById('contact-subject');
const departmentGroup = document.getElementById('department-group');
const departmentSelect = document.getElementById('department');
subjectSelect.addEventListener('change', () => {
if (subjectSelect.value === 'support') {
departmentGroup.style.display = 'block';
departmentSelect.innerHTML = `
<option value="">Select department</option>
<option value="technical">Technical Support</option>
<option value="billing">Billing Support</option>
<option value="general">General Support</option>
`;
} else {
departmentGroup.style.display = 'none';
}
});
contactForm.addEventListener('submit', (e) => {
e.preventDefault();
const messageDisplay = document.getElementById('contact-message-display');
messageDisplay.textContent = '✅ Message sent successfully!';
messageDisplay.className = 'form-message success';
console.log('📧 Contact form submitted');
});
// Multi-step Form
const surveyForm = document.getElementById('survey-form');
const steps = surveyForm.querySelectorAll('.form-step');
const progressFill = document.getElementById('progress-fill');
let currentStep = 1;
surveyForm.addEventListener('click', (e) => {
if (e.target.classList.contains('next-step')) {
if (currentStep < 3) {
steps[currentStep - 1].style.display = 'none';
currentStep++;
steps[currentStep - 1].style.display = 'block';
progressFill.style.width = `${(currentStep / 3) * 100}%`;
}
} else if (e.target.classList.contains('prev-step')) {
if (currentStep > 1) {
steps[currentStep - 1].style.display = 'none';
currentStep--;
steps[currentStep - 1].style.display = 'block';
progressFill.style.width = `${(currentStep / 3) * 100}%`;
}
}
});
surveyForm.addEventListener('submit', (e) => {
e.preventDefault();
document.getElementById('survey-success').style.display = 'block';
console.log('📋 Survey submitted successfully!');
});
}
// Tabs
setupTabs() {
const tabBtns = document.querySelectorAll('.tab-btn');
const tabPanes = document.querySelectorAll('.tab-pane');
tabBtns.forEach(btn => {
btn.addEventListener('click', () => {
const targetTab = btn.getAttribute('data-tab');
// Update active states
tabBtns.forEach(b => b.classList.remove('active'));
btn.classList.add('active');
// Show target pane
tabPanes.forEach(pane => {
pane.style.display = pane.id === targetTab ? 'block' : 'none';
});
});
});
// Show more functionality
const showMoreBtn = document.querySelector('.show-more');
const hiddenText = document.querySelector('.hidden-text');
if (showMoreBtn) {
showMoreBtn.addEventListener('click', () => {
if (hiddenText.style.display === 'none') {
hiddenText.style.display = 'block';
showMoreBtn.textContent = 'Show Less';
} else {
hiddenText.style.display = 'none';
showMoreBtn.textContent = 'Show More';
}
});
}
// Load comments
const loadCommentsBtn = document.querySelector('.load-comments');
const commentsSection = document.querySelector('.comments-section');
if (loadCommentsBtn) {
loadCommentsBtn.addEventListener('click', () => {
commentsSection.style.display = 'block';
commentsSection.innerHTML = `
<div class="comment">
<div class="comment-author">John Doe</div>
<div class="comment-text">Great product! Highly recommended.</div>
</div>
<div class="comment">
<div class="comment-author">Jane Smith</div>
<div class="comment-text">Excellent quality and fast shipping.</div>
</div>
`;
loadCommentsBtn.style.display = 'none';
console.log('💬 Comments loaded');
});
}
}
// Data Table
setupDataTable() {
const loadMoreBtn = document.querySelector('.load-more-rows');
const searchInput = document.querySelector('.search-input');
const exportBtn = document.getElementById('export-btn');
const sortableHeaders = document.querySelectorAll('.sortable');
// Load more rows
loadMoreBtn.addEventListener('click', () => {
this.loadMoreTableRows();
});
// Search functionality
searchInput.addEventListener('input', (e) => {
const searchTerm = e.target.value.toLowerCase();
const rows = document.querySelectorAll('#table-body tr');
rows.forEach(row => {
const text = row.textContent.toLowerCase();
row.style.display = text.includes(searchTerm) ? '' : 'none';
});
});
// Export functionality
exportBtn.addEventListener('click', () => {
console.log('📊 Exporting table data...');
alert('Table data exported! (Check console)');
});
// Sorting
sortableHeaders.forEach(header => {
header.addEventListener('click', () => {
console.log(`🔄 Sorting by ${header.getAttribute('data-sort')}`);
});
});
}
loadMoreTableRows() {
const tbody = document.getElementById('table-body');
const batch = 10;
for (let i = 0; i < batch; i++) {
const row = document.createElement('tr');
const id = this.tableRowsLoaded + i + 1;
row.innerHTML = `
<td>User ${id}</td>
<td>user${id}@example.com</td>
<td>${new Date().toLocaleDateString()}</td>
<td><button class="btn btn-sm">Edit</button></td>
`;
tbody.appendChild(row);
}
this.tableRowsLoaded += batch;
console.log(`📄 Loaded ${batch} more rows. Total: ${this.tableRowsLoaded}`);
}
// Load initial data
loadInitialData() {
// Load initial products
this.loadMoreProducts();
// Load initial table rows
this.loadMoreTableRows();
}
// Load content when navigating to sections
loadSectionContent(sectionId) {
switch(sectionId) {
case 'catalog':
// Ensure products are loaded in catalog
if (this.productsLoaded === 0) {
this.loadMoreProducts();
}
break;
case 'data-tables':
// Ensure table rows are loaded
if (this.tableRowsLoaded === 0) {
this.loadMoreTableRows();
}
break;
case 'forms':
// Forms are already set up
break;
case 'tabs':
// Tabs content is static
break;
}
}
// Inspector Mode
setupInspector() {
const inspectorBtn = document.getElementById('inspector-btn');
// Create tooltip element
this.tooltip = document.createElement('div');
this.tooltip.className = 'inspector-tooltip';
this.tooltip.style.cssText = `
position: fixed;
background: rgba(0, 0, 0, 0.9);
color: white;
padding: 8px 12px;
border-radius: 4px;
font-size: 12px;
font-family: monospace;
pointer-events: none;
z-index: 10000;
display: none;
max-width: 300px;
`;
document.body.appendChild(this.tooltip);
inspectorBtn.addEventListener('click', () => {
this.toggleInspector();
});
// Add mouse event listeners
document.addEventListener('mousemove', this.handleMouseMove.bind(this));
document.addEventListener('mouseout', this.handleMouseOut.bind(this));
}
toggleInspector() {
this.inspectorMode = !this.inspectorMode;
const inspectorBtn = document.getElementById('inspector-btn');
if (this.inspectorMode) {
inspectorBtn.classList.add('active');
inspectorBtn.style.background = '#0fbbaa';
document.body.style.cursor = 'crosshair';
} else {
inspectorBtn.classList.remove('active');
inspectorBtn.style.background = '';
document.body.style.cursor = '';
this.tooltip.style.display = 'none';
this.removeHighlight();
}
}
handleMouseMove(e) {
if (!this.inspectorMode) return;
const element = e.target;
if (element === this.tooltip) return;
// Highlight element
this.highlightElement(element);
// Show tooltip with element info
const info = this.getElementInfo(element);
this.tooltip.innerHTML = info;
this.tooltip.style.display = 'block';
// Position tooltip
const x = e.clientX + 15;
const y = e.clientY + 15;
// Adjust position if tooltip would go off screen
const rect = this.tooltip.getBoundingClientRect();
const adjustedX = x + rect.width > window.innerWidth ? x - rect.width - 30 : x;
const adjustedY = y + rect.height > window.innerHeight ? y - rect.height - 30 : y;
this.tooltip.style.left = adjustedX + 'px';
this.tooltip.style.top = adjustedY + 'px';
}
handleMouseOut(e) {
if (!this.inspectorMode) return;
if (e.target === document.body) {
this.removeHighlight();
this.tooltip.style.display = 'none';
}
}
highlightElement(element) {
this.removeHighlight();
element.style.outline = '2px solid #0fbbaa';
element.style.outlineOffset = '1px';
element.setAttribute('data-inspector-highlighted', 'true');
}
removeHighlight() {
const highlighted = document.querySelector('[data-inspector-highlighted]');
if (highlighted) {
highlighted.style.outline = '';
highlighted.style.outlineOffset = '';
highlighted.removeAttribute('data-inspector-highlighted');
}
}
getElementInfo(element) {
const tagName = element.tagName.toLowerCase();
const id = element.id ? `#${element.id}` : '';
const classes = element.className ?
`.${element.className.split(' ').filter(c => c).join('.')}` : '';
let selector = tagName;
if (id) {
selector = id;
} else if (classes) {
selector = `${tagName}${classes}`;
}
// Build info HTML
let info = `<strong>${selector}</strong>`;
// Add additional attributes
const attrs = [];
if (element.name) attrs.push(`name="${element.name}"`);
if (element.type) attrs.push(`type="${element.type}"`);
if (element.href) attrs.push(`href="${element.href}"`);
if (element.value && element.tagName === 'INPUT') attrs.push(`value="${element.value}"`);
if (attrs.length > 0) {
info += `<br><span style="color: #888;">${attrs.join(' ')}</span>`;
}
return info;
}
}
// Initialize app when DOM is ready
document.addEventListener('DOMContentLoaded', () => {
window.playgroundApp = new PlaygroundApp();
console.log('🎮 Playground app initialized!');
});

328
docs/examples/c4a_script/tutorial/playground/index.html Normal file
View File

@@ -0,0 +1,328 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>C4A-Script Playground</title>
<link rel="stylesheet" href="styles.css">
</head>
<body>
<!-- Cookie Banner -->
<div class="cookie-banner" id="cookie-banner">
<div class="cookie-content">
<p>🍪 We use cookies to enhance your experience. By continuing, you agree to our cookie policy.</p>
<div class="cookie-actions">
<button class="btn accept">Accept All</button>
<button class="btn btn-secondary decline">Decline</button>
</div>
</div>
</div>
<!-- Newsletter Popup (appears after 3 seconds) -->
<div class="modal" id="newsletter-popup" style="display: none;">
<div class="modal-content">
<span class="close">&times;</span>
<h2>📬 Subscribe to Our Newsletter</h2>
<p>Get the latest updates on web automation!</p>
<input type="email" placeholder="Enter your email" class="input">
<button class="btn subscribe">Subscribe</button>
</div>
</div>
<!-- Header -->
<header class="site-header">
<nav class="nav-menu">
<a href="#home" class="nav-link active">Home</a>
<a href="#catalog" class="nav-link" id="catalog-link">Products</a>
<a href="#forms" class="nav-link">Forms</a>
<a href="#data-tables" class="nav-link">Data Tables</a>
<div class="dropdown">
<a href="#" class="nav-link dropdown-toggle">More ▼</a>
<div class="dropdown-content">
<a href="#tabs">Tabs Demo</a>
<a href="#accordion">FAQ</a>
<a href="#gallery">Gallery</a>
</div>
</div>
</nav>
<div class="auth-section">
<button class="btn btn-sm" id="inspector-btn" title="Toggle Inspector">🔍</button>
<button class="btn btn-sm" id="login-btn">Login</button>
<div class="user-info" id="user-info" style="display: none;">
<span class="user-avatar">👤</span>
<span class="welcome-message">Welcome, <span id="username-display">User</span>!</span>
<button class="btn btn-sm btn-secondary" id="logout-btn">Logout</button>
</div>
</div>
</header>
<!-- Main Content -->
<main class="main-content">
<!-- Home Section -->
<section id="home" class="section active">
<h1>Welcome to C4A-Script Playground</h1>
<p>This is an interactive demo for testing C4A-Script commands. Each section contains different challenges for web automation.</p>
<button class="btn btn-primary" id="start-tutorial">Start Tutorial</button>
<div class="feature-grid">
<div class="feature-card">
<h3>🔐 Authentication</h3>
<p>Test login forms and user sessions</p>
</div>
<div class="feature-card">
<h3>📜 Dynamic Content</h3>
<p>Infinite scroll and pagination</p>
</div>
<div class="feature-card">
<h3>📝 Forms</h3>
<p>Complex form interactions</p>
</div>
<div class="feature-card">
<h3>📊 Data Tables</h3>
<p>Sortable and filterable data</p>
</div>
</div>
</section>
<!-- Login Modal -->
<div class="modal" id="login-modal" style="display: none;">
<div class="modal-content login-form">
<span class="close">&times;</span>
<h2>Login</h2>
<form id="login-form">
<div class="form-group">
<label>Email</label>
<input type="email" id="email" class="input" placeholder="demo@example.com">
</div>
<div class="form-group">
<label>Password</label>
<input type="password" id="password" class="input" placeholder="demo123">
</div>
<div class="form-group">
<label class="checkbox-label">
<input type="checkbox" id="remember-me">
Remember me
</label>
</div>
<button type="submit" class="btn btn-primary">Login</button>
<div class="form-message" id="login-message"></div>
</form>
</div>
</div>
<!-- Product Catalog Section -->
<section id="catalog" class="section">
<h1>Product Catalog</h1>
<div class="view-toggle">
<button class="btn btn-sm active" id="infinite-scroll-btn">Infinite Scroll</button>
<button class="btn btn-sm" id="pagination-btn">Pagination</button>
</div>
<!-- Filters Sidebar -->
<div class="catalog-layout">
<aside class="filters-sidebar">
<h3>Filters</h3>
<div class="filter-group">
<h4 class="collapsible">Category <span class="toggle">▼</span></h4>
<div class="filter-content">
<label><input type="checkbox"> Electronics</label>
<label><input type="checkbox"> Clothing</label>
<label><input type="checkbox"> Books</label>
</div>
</div>
<div class="filter-group">
<h4 class="collapsible">Price Range <span class="toggle">▼</span></h4>
<div class="filter-content">
<input type="range" min="0" max="1000" value="500">
<span>$0 - $500</span>
</div>
</div>
</aside>
<!-- Products Grid -->
<div class="products-container">
<div class="product-grid" id="product-grid">
<!-- Products will be loaded here -->
</div>
<!-- Infinite Scroll View -->
<div id="infinite-scroll-view" class="view-mode">
<div class="loading-indicator" id="loading-indicator" style="display: none;">
<div class="spinner"></div>
<p>Loading more products...</p>
</div>
</div>
<!-- Pagination View -->
<div id="pagination-view" class="view-mode" style="display: none;">
<button class="btn load-more">Load More</button>
<div class="pagination">
<button class="page-btn">1</button>
<button class="page-btn">2</button>
<button class="page-btn">3</button>
</div>
</div>
</div>
</div>
</section>
<!-- Forms Section -->
<section id="forms" class="section">
<h1>Form Examples</h1>
<!-- Contact Form -->
<div class="form-card">
<h2>Contact Form</h2>
<form id="contact-form">
<div class="form-group">
<label>Name</label>
<input type="text" class="input" id="contact-name">
</div>
<div class="form-group">
<label>Email</label>
<input type="email" class="input" id="contact-email">
</div>
<div class="form-group">
<label>Subject</label>
<select class="input" id="contact-subject">
<option value="">Select a subject</option>
<option value="support">Support</option>
<option value="sales">Sales</option>
<option value="feedback">Feedback</option>
</select>
</div>
<div class="form-group" id="department-group" style="display: none;">
<label>Department</label>
<select class="input" id="department">
<option value="">Select department</option>
</select>
</div>
<div class="form-group">
<label>Message</label>
<textarea class="input" id="contact-message" rows="4"></textarea>
</div>
<button type="submit" class="btn btn-primary">Send Message</button>
<div class="form-message" id="contact-message-display"></div>
</form>
</div>
<!-- Multi-step Form -->
<div class="form-card">
<h2>Multi-step Survey</h2>
<div class="progress-bar">
<div class="progress-fill" id="progress-fill" style="width: 33%"></div>
</div>
<form id="survey-form">
<!-- Step 1 -->
<div class="form-step active" data-step="1">
<h3>Step 1: Basic Information</h3>
<div class="form-group">
<label>Full Name</label>
<input type="text" class="input" id="full-name">
</div>
<div class="form-group">
<label>Email</label>
<input type="email" class="input" id="survey-email">
</div>
<button type="button" class="btn next-step">Next</button>
</div>
<!-- Step 2 -->
<div class="form-step" data-step="2" style="display: none;">
<h3>Step 2: Preferences</h3>
<div class="form-group">
<label>Interests (select multiple)</label>
<select multiple class="input" id="interests">
<option value="tech">Technology</option>
<option value="sports">Sports</option>
<option value="music">Music</option>
<option value="travel">Travel</option>
</select>
</div>
<button type="button" class="btn prev-step">Previous</button>
<button type="button" class="btn next-step">Next</button>
</div>
<!-- Step 3 -->
<div class="form-step" data-step="3" style="display: none;">
<h3>Step 3: Confirmation</h3>
<p>Please review your information and submit.</p>
<button type="button" class="btn prev-step">Previous</button>
<button type="submit" class="btn btn-primary" id="submit-survey">Submit Survey</button>
</div>
</form>
<div class="form-message success-message" id="survey-success" style="display: none;">
✅ Survey submitted successfully!
</div>
</div>
</section>
<!-- Tabs Section -->
<section id="tabs" class="section">
<h1>Tabs Demo</h1>
<div class="tabs-container">
<div class="tabs-header">
<button class="tab-btn active" data-tab="description">Description</button>
<button class="tab-btn" data-tab="reviews">Reviews</button>
<button class="tab-btn" data-tab="specs">Specifications</button>
</div>
<div class="tabs-content">
<div class="tab-pane active" id="description">
<h3>Product Description</h3>
<p>This is a detailed description of the product...</p>
<div class="expandable-text">
<p class="text-preview">Lorem ipsum dolor sit amet, consectetur adipiscing elit...</p>
<button class="btn btn-sm show-more">Show More</button>
<div class="hidden-text" style="display: none;">
<p>This is the hidden text that appears when you click "Show More". It contains additional details about the product that weren't visible initially.</p>
</div>
</div>
</div>
<div class="tab-pane" id="reviews" style="display: none;">
<h3>Customer Reviews</h3>
<button class="btn btn-sm load-comments">Load Comments</button>
<div class="comments-section" style="display: none;">
<!-- Comments will be loaded here -->
</div>
</div>
<div class="tab-pane" id="specs" style="display: none;">
<h3>Technical Specifications</h3>
<table class="specs-table">
<tr><td>Model</td><td>XYZ-2000</td></tr>
<tr><td>Weight</td><td>2.5 kg</td></tr>
<tr><td>Dimensions</td><td>30 x 20 x 10 cm</td></tr>
</table>
</div>
</div>
</div>
</section>
<!-- Data Tables Section -->
<section id="data-tables" class="section">
<h1>Data Tables</h1>
<div class="table-controls">
<input type="text" class="input search-input" placeholder="Search...">
<button class="btn btn-sm" id="export-btn">Export</button>
</div>
<table class="data-table" id="data-table">
<thead>
<tr>
<th class="sortable" data-sort="name">Name ↕</th>
<th class="sortable" data-sort="email">Email ↕</th>
<th class="sortable" data-sort="date">Date ↕</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="table-body">
<!-- Table rows will be loaded here -->
</tbody>
</table>
<button class="btn load-more-rows">Load More Rows</button>
</section>
</main>
<script src="app.js"></script>
</body>
</html>

627
docs/examples/c4a_script/tutorial/playground/styles.css Normal file
View File

@@ -0,0 +1,627 @@
/* Playground Styles - Modern Web App Theme */
:root {
--primary-color: #0fbbaa;
--secondary-color: #3f3f44;
--background-color: #ffffff;
--text-color: #333333;
--border-color: #e0e0e0;
--error-color: #ff3c74;
--success-color: #0fbbaa;
--warning-color: #ffa500;
}
* {
box-sizing: border-box;
}
body {
margin: 0;
padding: 0;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 16px;
line-height: 1.6;
color: var(--text-color);
background-color: var(--background-color);
}
/* Cookie Banner */
.cookie-banner {
position: fixed;
bottom: 0;
left: 0;
right: 0;
background-color: #2c3e50;
color: white;
padding: 1rem;
z-index: 1000;
box-shadow: 0 -2px 10px rgba(0,0,0,0.1);
}
.cookie-content {
max-width: 1200px;
margin: 0 auto;
display: flex;
align-items: center;
justify-content: space-between;
flex-wrap: wrap;
gap: 1rem;
}
.cookie-actions {
display: flex;
gap: 0.5rem;
}
/* Header */
.site-header {
background-color: #fff;
border-bottom: 1px solid var(--border-color);
padding: 1rem 2rem;
position: sticky;
top: 0;
z-index: 100;
display: flex;
justify-content: space-between;
align-items: center;
}
.nav-menu {
display: flex;
gap: 2rem;
align-items: center;
}
.nav-link {
text-decoration: none;
color: var(--text-color);
font-weight: 500;
transition: color 0.2s;
}
.nav-link:hover,
.nav-link.active {
color: var(--primary-color);
}
/* Dropdown */
.dropdown {
position: relative;
}
.dropdown-content {
display: none;
position: absolute;
background-color: white;
min-width: 160px;
box-shadow: 0 8px 16px rgba(0,0,0,0.1);
z-index: 1;
border-radius: 4px;
top: 100%;
margin-top: 0.5rem;
}
.dropdown:hover .dropdown-content {
display: block;
}
.dropdown-content a {
color: var(--text-color);
padding: 0.75rem 1rem;
text-decoration: none;
display: block;
}
.dropdown-content a:hover {
background-color: #f5f5f5;
}
/* Auth Section */
.auth-section {
display: flex;
align-items: center;
gap: 1rem;
}
.user-info {
display: flex;
align-items: center;
gap: 0.5rem;
}
.user-avatar {
font-size: 1.5rem;
}
/* Main Content */
.main-content {
padding: 2rem;
max-width: 1200px;
margin: 0 auto;
}
.section {
display: none;
}
.section.active {
display: block;
}
/* Buttons */
.btn {
background-color: var(--primary-color);
color: white;
border: none;
padding: 0.5rem 1rem;
border-radius: 4px;
cursor: pointer;
font-size: 1rem;
font-weight: 500;
transition: all 0.2s;
}
.btn:hover {
background-color: #0aa599;
transform: translateY(-1px);
}
.btn-sm {
padding: 0.25rem 0.75rem;
font-size: 0.875rem;
}
.btn-secondary {
background-color: var(--secondary-color);
}
.btn-secondary:hover {
background-color: #333;
}
.btn-primary {
background-color: var(--primary-color);
}
/* Feature Grid */
.feature-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1.5rem;
margin-top: 2rem;
}
.feature-card {
background-color: #f8f9fa;
padding: 1.5rem;
border-radius: 8px;
text-align: center;
transition: transform 0.2s;
}
.feature-card:hover {
transform: translateY(-4px);
box-shadow: 0 4px 12px rgba(0,0,0,0.1);
}
.feature-card h3 {
margin-top: 0;
}
/* Modal */
.modal {
position: fixed;
z-index: 1000;
left: 0;
top: 0;
width: 100%;
height: 100%;
background-color: rgba(0,0,0,0.5);
display: flex;
align-items: center;
justify-content: center;
}
.modal-content {
background-color: white;
padding: 2rem;
border-radius: 8px;
max-width: 500px;
width: 90%;
position: relative;
animation: modalFadeIn 0.3s;
}
@keyframes modalFadeIn {
from { opacity: 0; transform: translateY(-20px); }
to { opacity: 1; transform: translateY(0); }
}
.close {
position: absolute;
right: 1rem;
top: 1rem;
font-size: 1.5rem;
cursor: pointer;
color: #999;
}
.close:hover {
color: #333;
}
/* Forms */
.form-group {
margin-bottom: 1rem;
}
.form-group label {
display: block;
margin-bottom: 0.5rem;
font-weight: 500;
}
.input {
width: 100%;
padding: 0.5rem;
border: 1px solid var(--border-color);
border-radius: 4px;
font-size: 1rem;
}
.input:focus {
outline: none;
border-color: var(--primary-color);
}
.checkbox-label {
display: flex;
align-items: center;
gap: 0.5rem;
}
.form-message {
margin-top: 1rem;
padding: 0.75rem;
border-radius: 4px;
display: none;
}
.form-message.error {
background-color: #ffe6e6;
color: var(--error-color);
display: block;
}
.form-message.success {
background-color: #e6fff6;
color: var(--success-color);
display: block;
}
/* Product Catalog */
.view-toggle {
margin-bottom: 1rem;
}
.catalog-layout {
display: grid;
grid-template-columns: 250px 1fr;
gap: 2rem;
}
.filters-sidebar {
background-color: #f8f9fa;
padding: 1rem;
border-radius: 8px;
}
.filter-group {
margin-bottom: 1.5rem;
}
.collapsible {
cursor: pointer;
display: flex;
justify-content: space-between;
align-items: center;
}
.filter-content {
margin-top: 0.5rem;
}
.filter-content label {
display: block;
margin-bottom: 0.5rem;
}
/* Product Grid */
.product-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
gap: 1.5rem;
}
.product-card {
background-color: white;
border: 1px solid var(--border-color);
border-radius: 8px;
padding: 1rem;
text-align: center;
transition: transform 0.2s;
}
.product-card:hover {
transform: translateY(-4px);
box-shadow: 0 4px 12px rgba(0,0,0,0.1);
}
.product-image {
width: 100%;
height: 150px;
background-color: #f0f0f0;
margin-bottom: 1rem;
display: flex;
align-items: center;
justify-content: center;
font-size: 3rem;
}
.product-name {
font-weight: 600;
margin-bottom: 0.5rem;
}
.product-price {
color: var(--primary-color);
font-size: 1.2rem;
font-weight: 700;
}
/* Loading Indicator */
.loading-indicator {
text-align: center;
padding: 2rem;
}
.spinner {
border: 3px solid #f3f3f3;
border-top: 3px solid var(--primary-color);
border-radius: 50%;
width: 40px;
height: 40px;
animation: spin 1s linear infinite;
margin: 0 auto;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
/* Pagination */
.pagination {
display: flex;
gap: 0.5rem;
justify-content: center;
margin-top: 2rem;
}
.page-btn {
padding: 0.5rem 1rem;
border: 1px solid var(--border-color);
background-color: white;
cursor: pointer;
border-radius: 4px;
}
.page-btn:hover,
.page-btn.active {
background-color: var(--primary-color);
color: white;
}
/* Multi-step Form */
.progress-bar {
width: 100%;
height: 8px;
background-color: #e0e0e0;
border-radius: 4px;
margin-bottom: 2rem;
}
.progress-fill {
height: 100%;
background-color: var(--primary-color);
border-radius: 4px;
transition: width 0.3s;
}
.form-step {
display: none;
}
.form-step.active {
display: block;
}
/* Tabs */
.tabs-container {
margin-top: 2rem;
}
.tabs-header {
display: flex;
border-bottom: 2px solid var(--border-color);
}
.tab-btn {
background: none;
border: none;
padding: 1rem 2rem;
cursor: pointer;
font-size: 1rem;
font-weight: 500;
color: var(--text-color);
position: relative;
}
.tab-btn:hover {
color: var(--primary-color);
}
.tab-btn.active {
color: var(--primary-color);
}
.tab-btn.active::after {
content: '';
position: absolute;
bottom: -2px;
left: 0;
right: 0;
height: 2px;
background-color: var(--primary-color);
}
.tabs-content {
padding: 2rem 0;
}
.tab-pane {
display: none;
}
.tab-pane.active {
display: block;
}
/* Expandable Text */
.expandable-text {
margin-top: 1rem;
}
.text-preview {
margin-bottom: 0.5rem;
}
.show-more {
margin-top: 0.5rem;
}
/* Comments Section */
.comments-section {
margin-top: 1rem;
}
.comment {
background-color: #f8f9fa;
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.comment-author {
font-weight: 600;
margin-bottom: 0.5rem;
}
/* Data Table */
.table-controls {
display: flex;
gap: 1rem;
margin-bottom: 1rem;
}
.search-input {
flex: 1;
max-width: 300px;
}
.data-table {
width: 100%;
border-collapse: collapse;
background-color: white;
}
.data-table th,
.data-table td {
padding: 0.75rem;
text-align: left;
border-bottom: 1px solid var(--border-color);
}
.data-table th {
background-color: #f8f9fa;
font-weight: 600;
}
.sortable {
cursor: pointer;
}
.sortable:hover {
color: var(--primary-color);
}
/* Form Cards */
.form-card {
background-color: white;
border: 1px solid var(--border-color);
border-radius: 8px;
padding: 2rem;
margin-bottom: 2rem;
}
.form-card h2 {
margin-top: 0;
}
/* Success Message */
.success-message {
background-color: #e6fff6;
color: var(--success-color);
padding: 1rem;
border-radius: 4px;
text-align: center;
font-weight: 500;
}
/* Load More Button */
.load-more,
.load-more-rows {
display: block;
margin: 2rem auto;
}
/* Responsive */
@media (max-width: 768px) {
.catalog-layout {
grid-template-columns: 1fr;
}
.feature-grid {
grid-template-columns: 1fr;
}
.nav-menu {
flex-wrap: wrap;
gap: 1rem;
}
.cookie-content {
flex-direction: column;
text-align: center;
}
}
/* Inspector Mode */
#inspector-btn.active {
background: var(--primary-color) !important;
color: var(--bg-primary) !important;
}
.inspector-tooltip {
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3);
border: 1px solid rgba(255, 255, 255, 0.1);
}

2
docs/examples/c4a_script/tutorial/requirements.txt Normal file
View File

@@ -0,0 +1,2 @@
flask>=2.3.0
flask-cors>=4.0.0

18
docs/examples/c4a_script/tutorial/scripts/01-basic-interaction.c4a Normal file
View File

@@ -0,0 +1,18 @@
# Basic Page Interaction
# This script demonstrates basic C4A commands
# Navigate to the playground
GO http://127.0.0.1:8080/playground/
# Wait for page to load
WAIT `body` 2
# Handle cookie banner if present
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
# Close newsletter popup if it appears
WAIT 3
IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`
# Click the start tutorial button
CLICK `#start-tutorial`

27
docs/examples/c4a_script/tutorial/scripts/02-login-flow.c4a Normal file
View File

@@ -0,0 +1,27 @@
# Complete Login Flow
# Demonstrates form interaction and authentication
# Click login button
CLICK `#login-btn`
# Wait for login modal
WAIT `.login-form` 3
# Fill in credentials
CLICK `#email`
TYPE "demo@example.com"
CLICK `#password`
TYPE "demo123"
# Check remember me
IF (EXISTS `#remember-me`) THEN CLICK `#remember-me`
# Submit form
CLICK `button[type="submit"]`
# Wait for success
WAIT `.welcome-message` 5
# Verify login succeeded
IF (EXISTS `.user-info`) THEN EVAL `console.log('✅ Login successful!')`

32
docs/examples/c4a_script/tutorial/scripts/03-infinite-scroll.c4a Normal file
View File

@@ -0,0 +1,32 @@
# Infinite Scroll Product Loading
# Load all products using scroll automation
# Navigate to catalog
CLICK `#catalog-link`
WAIT `.product-grid` 3
# Switch to infinite scroll mode
CLICK `#infinite-scroll-btn`
# Define scroll procedure
PROC load_more_products
# Get current product count
EVAL `window.initialCount = document.querySelectorAll('.product-card').length`
# Scroll down
SCROLL DOWN 1000
WAIT 2
# Check if more products loaded
EVAL `
const newCount = document.querySelectorAll('.product-card').length;
console.log('Products loaded: ' + newCount);
window.moreLoaded = newCount > window.initialCount;
`
ENDPROC
# Load products until no more
REPEAT (load_more_products, `window.moreLoaded !== false`)
# Final count
EVAL `console.log('✅ Total products: ' + document.querySelectorAll('.product-card').length)`

41
docs/examples/c4a_script/tutorial/scripts/04-multi-step-form.c4a Normal file
View File

@@ -0,0 +1,41 @@
# Multi-step Form Wizard
# Complete a complex form with multiple steps
# Navigate to forms section
CLICK `a[href="#forms"]`
WAIT `#survey-form` 2
# Step 1: Basic Information
CLICK `#full-name`
TYPE "John Doe"
CLICK `#survey-email`
TYPE "john.doe@example.com"
# Go to next step
CLICK `.next-step`
WAIT 1
# Step 2: Select Interests
# Select multiple options
CLICK `#interests`
CLICK `option[value="tech"]`
CLICK `option[value="music"]`
CLICK `option[value="travel"]`
# Continue to final step
CLICK `.next-step`
WAIT 1
# Step 3: Review and Submit
# Verify we're on the last step
IF (EXISTS `#submit-survey`) THEN EVAL `console.log('📋 On final step')`
# Submit the form
CLICK `#submit-survey`
# Wait for success message
WAIT `.success-message` 5
# Verify submission
IF (EXISTS `.success-message`) THEN EVAL `console.log('✅ Survey submitted successfully!')`

82
docs/examples/c4a_script/tutorial/scripts/05-complex-workflow.c4a Normal file
View File

@@ -0,0 +1,82 @@
# Complete E-commerce Workflow
# Login, browse products, and interact with various elements
# Define reusable procedures
PROC handle_popups
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`
ENDPROC
PROC login_user
CLICK `#login-btn`
WAIT `.login-form` 2
CLICK `#email`
TYPE "demo@example.com"
CLICK `#password`
TYPE "demo123"
CLICK `button[type="submit"]`
WAIT `.welcome-message` 5
ENDPROC
PROC browse_products
# Go to catalog
CLICK `#catalog-link`
WAIT `.product-grid` 3
# Apply filters
CLICK `.collapsible`
WAIT 0.5
CLICK `input[type="checkbox"]`
# Load some products
SCROLL DOWN 500
WAIT 1
SCROLL DOWN 500
WAIT 1
ENDPROC
# Main workflow
GO http://127.0.0.1:8080/playground/
WAIT `body` 2
# Handle initial popups
handle_popups
# Login if not already
IF (NOT EXISTS `.user-info`) THEN login_user
# Browse products
browse_products
# Navigate to tabs demo
CLICK `a[href="#tabs"]`
WAIT `.tabs-container` 2
# Interact with tabs
CLICK `button[data-tab="reviews"]`
WAIT 1
# Load comments
IF (EXISTS `.load-comments`) THEN CLICK `.load-comments`
WAIT `.comments-section` 2
# Check specifications
CLICK `button[data-tab="specs"]`
WAIT 1
# Final navigation to data tables
CLICK `a[href="#data"]`
WAIT `.data-table` 2
# Search in table
CLICK `.search-input`
TYPE "User"
# Load more rows
CLICK `.load-more-rows`
WAIT 1
# Export data
CLICK `#export-btn`
EVAL `console.log('✅ Workflow completed successfully!')`

304
docs/examples/c4a_script/tutorial/server.py Normal file
View File

@@ -0,0 +1,304 @@
#!/usr/bin/env python3
"""
C4A-Script Tutorial Server
Serves the tutorial app and provides C4A compilation API
"""
import sys
import os
from pathlib import Path
from flask import Flask, render_template_string, request, jsonify, send_from_directory
from flask_cors import CORS
# Add parent directories to path to import crawl4ai
sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent))
try:
from crawl4ai.script import compile as c4a_compile
C4A_AVAILABLE = True
except ImportError:
print("⚠️ C4A compiler not available. Using mock compiler.")
C4A_AVAILABLE = False
app = Flask(__name__)
CORS(app)
# Serve static files
@app.route('/')
def index():
return send_from_directory('.', 'index.html')
@app.route('/assets/<path:path>')
def serve_assets(path):
return send_from_directory('assets', path)
@app.route('/playground/')
def playground():
return send_from_directory('playground', 'index.html')
@app.route('/playground/<path:path>')
def serve_playground(path):
return send_from_directory('playground', path)
# API endpoint for C4A compilation
@app.route('/api/compile', methods=['POST'])
def compile_endpoint():
try:
data = request.get_json()
script = data.get('script', '')
if not script:
return jsonify({
'success': False,
'error': {
'line': 1,
'column': 1,
'message': 'No script provided',
'suggestion': 'Write some C4A commands'
}
})
if C4A_AVAILABLE:
# Use real C4A compiler
result = c4a_compile(script)
if result.success:
return jsonify({
'success': True,
'jsCode': result.js_code,
'metadata': {
'lineCount': len(result.js_code),
'sourceLines': len(script.split('\n'))
}
})
else:
error = result.first_error
return jsonify({
'success': False,
'error': {
'line': error.line,
'column': error.column,
'message': error.message,
'suggestion': error.suggestions[0].message if error.suggestions else None,
'code': error.code,
'sourceLine': error.source_line
}
})
else:
# Use mock compiler for demo
result = mock_compile(script)
return jsonify(result)
except Exception as e:
return jsonify({
'success': False,
'error': {
'line': 1,
'column': 1,
'message': f'Server error: {str(e)}',
'suggestion': 'Check server logs'
}
}), 500
def mock_compile(script):
"""Simple mock compiler for demo when C4A is not available"""
lines = [line for line in script.split('\n') if line.strip() and not line.strip().startswith('#')]
js_code = []
for i, line in enumerate(lines):
line = line.strip()
try:
if line.startswith('GO '):
url = line[3:].strip()
# Handle relative URLs
if not url.startswith(('http://', 'https://')):
url = '/' + url.lstrip('/')
js_code.append(f"await page.goto('{url}');")
elif line.startswith('WAIT '):
parts = line[5:].strip().split(' ')
if parts[0].startswith('`'):
selector = parts[0].strip('`')
timeout = parts[1] if len(parts) > 1 else '5'
js_code.append(f"await page.waitForSelector('{selector}', {{ timeout: {timeout}000 }});")
else:
seconds = parts[0]
js_code.append(f"await page.waitForTimeout({seconds}000);")
elif line.startswith('CLICK '):
selector = line[6:].strip().strip('`')
js_code.append(f"await page.click('{selector}');")
elif line.startswith('TYPE '):
text = line[5:].strip().strip('"')
js_code.append(f"await page.keyboard.type('{text}');")
elif line.startswith('SCROLL '):
parts = line[7:].strip().split(' ')
direction = parts[0]
amount = parts[1] if len(parts) > 1 else '500'
if direction == 'DOWN':
js_code.append(f"await page.evaluate(() => window.scrollBy(0, {amount}));")
elif direction == 'UP':
js_code.append(f"await page.evaluate(() => window.scrollBy(0, -{amount}));")
elif line.startswith('IF '):
if 'THEN' not in line:
return {
'success': False,
'error': {
'line': i + 1,
'column': len(line),
'message': "Missing 'THEN' keyword after IF condition",
'suggestion': "Add 'THEN' after the condition",
'sourceLine': line
}
}
condition = line[3:line.index('THEN')].strip()
action = line[line.index('THEN') + 4:].strip()
if 'EXISTS' in condition:
selector_match = condition.split('`')
if len(selector_match) >= 2:
selector = selector_match[1]
action_selector = action.split('`')[1] if '`' in action else ''
js_code.append(
f"if (await page.$$('{selector}').length > 0) {{ "
f"await page.click('{action_selector}'); }}"
)
elif line.startswith('PRESS '):
key = line[6:].strip()
js_code.append(f"await page.keyboard.press('{key}');")
else:
# Unknown command
return {
'success': False,
'error': {
'line': i + 1,
'column': 1,
'message': f"Unknown command: {line.split()[0]}",
'suggestion': "Check command syntax",
'sourceLine': line
}
}
except Exception as e:
return {
'success': False,
'error': {
'line': i + 1,
'column': 1,
'message': f"Failed to parse: {str(e)}",
'suggestion': "Check syntax",
'sourceLine': line
}
}
return {
'success': True,
'jsCode': js_code,
'metadata': {
'lineCount': len(js_code),
'sourceLines': len(lines)
}
}
# Example scripts endpoint
@app.route('/api/examples')
def get_examples():
examples = [
{
'id': 'cookie-banner',
'name': 'Handle Cookie Banner',
'description': 'Accept cookies and close newsletter popup',
'script': '''# Handle cookie banner and newsletter
GO http://127.0.0.1:8080/playground/
WAIT `body` 2
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
IF (EXISTS `.newsletter-popup`) THEN CLICK `.close`'''
},
{
'id': 'login',
'name': 'Login Flow',
'description': 'Complete login with credentials',
'script': '''# Login to the site
CLICK `#login-btn`
WAIT `.login-form` 2
CLICK `#email`
TYPE "demo@example.com"
CLICK `#password`
TYPE "demo123"
IF (EXISTS `#remember-me`) THEN CLICK `#remember-me`
CLICK `button[type="submit"]`
WAIT `.welcome-message` 5'''
},
{
'id': 'infinite-scroll',
'name': 'Infinite Scroll',
'description': 'Load products with scrolling',
'script': '''# Navigate to catalog and scroll
CLICK `#catalog-link`
WAIT `.product-grid` 3
# Scroll multiple times to load products
SCROLL DOWN 1000
WAIT 1
SCROLL DOWN 1000
WAIT 1
SCROLL DOWN 1000'''
},
{
'id': 'form-wizard',
'name': 'Multi-step Form',
'description': 'Complete a multi-step survey',
'script': '''# Navigate to forms
CLICK `a[href="#forms"]`
WAIT `#survey-form` 2
# Step 1: Basic info
CLICK `#full-name`
TYPE "John Doe"
CLICK `#survey-email`
TYPE "john@example.com"
CLICK `.next-step`
WAIT 1
# Step 2: Preferences
CLICK `#interests`
CLICK `option[value="tech"]`
CLICK `option[value="music"]`
CLICK `.next-step`
WAIT 1
# Step 3: Submit
CLICK `#submit-survey`
WAIT `.success-message` 5'''
}
]
return jsonify(examples)
if __name__ == '__main__':
port = int(os.environ.get('PORT', 8000))
print(f"""
╔══════════════════════════════════════════════════════════╗
║ C4A-Script Interactive Tutorial Server ║
╠══════════════════════════════════════════════════════════╣
║ ║
║ Server running at: http://localhost:{port:<6} ║
║ ║
║ Features: ║
║ • C4A-Script compilation API ║
║ • Interactive playground ║
║ • Real-time execution visualization ║
║ ║
║ C4A Compiler: {'✓ Available' if C4A_AVAILABLE else '✗ Using mock compiler':<30} ║
║ ║
╚══════════════════════════════════════════════════════════╝
""")
app.run(host='0.0.0.0', port=port, debug=True)

69
docs/examples/c4a_script/tutorial/test_blockly.html Normal file
View File

@@ -0,0 +1,69 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Blockly Test</title>
<style>
body {
margin: 0;
padding: 20px;
background: #0e0e10;
color: #e0e0e0;
font-family: monospace;
}
#blocklyDiv {
height: 600px;
width: 100%;
border: 1px solid #2a2a2c;
}
#output {
margin-top: 20px;
padding: 15px;
background: #1a1a1b;
border: 1px solid #2a2a2c;
white-space: pre-wrap;
}
</style>
</head>
<body>
<h1>C4A-Script Blockly Test</h1>
<div id="blocklyDiv"></div>
<div id="output">
<h3>Generated C4A-Script:</h3>
<pre id="code-output"></pre>
</div>
<script src="https://unpkg.com/blockly/blockly.min.js"></script>
<script src="assets/c4a-blocks.js"></script>
<script>
// Simple test
const workspace = Blockly.inject('blocklyDiv', {
toolbox: `
<xml>
<category name="Test" colour="#1E88E5">
<block type="c4a_go"></block>
<block type="c4a_wait_time"></block>
<block type="c4a_click"></block>
</category>
</xml>
`,
theme: Blockly.Theme.defineTheme('dark', {
'base': Blockly.Themes.Classic,
'componentStyles': {
'workspaceBackgroundColour': '#0e0e10',
'toolboxBackgroundColour': '#1a1a1b',
'toolboxForegroundColour': '#e0e0e0',
'flyoutBackgroundColour': '#1a1a1b',
'flyoutForegroundColour': '#e0e0e0',
}
})
});
workspace.addChangeListener((event) => {
const code = Blockly.JavaScript.workspaceToCode(workspace);
document.getElementById('code-output').textContent = code;
});
</script>
</body>
</html>

2
docs/examples/extraction_strategies_examples.py
View File

@@ -12,7 +12,7 @@ import os
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
from crawl4ai import LLMConfig
from crawl4ai.extraction_strategy import (
from crawl4ai import (
LLMExtractionStrategy,
JsonCssExtractionStrategy,
JsonXPathExtractionStrategy,

376
docs/examples/link_head_extraction_example.py Normal file
View File

@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""
Link Head Extraction & Scoring Example
This example demonstrates Crawl4AI's advanced link analysis capabilities:
1. Basic link head extraction
2. Three-layer scoring system (intrinsic, contextual, total)
3. Pattern-based filtering
4. Multiple practical use cases
Requirements:
- crawl4ai installed
- Internet connection
Usage:
python link_head_extraction_example.py
"""
import asyncio
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
from crawl4ai.async_configs import LinkPreviewConfig
async def basic_link_head_extraction():
"""
Basic example: Extract head content from internal links with scoring
"""
print("🔗 Basic Link Head Extraction Example")
print("=" * 50)
config = CrawlerRunConfig(
# Enable link head extraction
link_preview_config=LinkPreviewConfig(
include_internal=True, # Process internal links
include_external=False, # Skip external links for this demo
max_links=5, # Limit to 5 links
concurrency=3, # Process 3 links simultaneously
timeout=10, # 10 second timeout per link
query="API documentation guide", # Query for relevance scoring
verbose=True # Show detailed progress
),
# Enable intrinsic link scoring
score_links=True,
only_text=True
)
async with AsyncWebCrawler() as crawler:
result = await crawler.arun("https://docs.python.org/3/", config=config)
if result.success:
print(f"\n✅ Successfully crawled: {result.url}")
internal_links = result.links.get("internal", [])
links_with_head = [link for link in internal_links
if link.get("head_data") is not None]
print(f"🧠 Links with head data: {len(links_with_head)}")
# Show detailed results
for i, link in enumerate(links_with_head[:3]):
print(f"\n📄 Link {i+1}: {link['href']}")
print(f" Text: '{link.get('text', 'No text')[:50]}...'")
# Show all three score types
intrinsic = link.get('intrinsic_score')
contextual = link.get('contextual_score')
total = link.get('total_score')
print(f" 📊 Scores:")
if intrinsic is not None:
print(f" • Intrinsic: {intrinsic:.2f}/10.0")
if contextual is not None:
print(f" • Contextual: {contextual:.3f}")
if total is not None:
print(f" • Total: {total:.3f}")
# Show head data
head_data = link.get("head_data", {})
if head_data:
title = head_data.get("title", "No title")
description = head_data.get("meta", {}).get("description", "")
print(f" 📰 Title: {title[:60]}...")
if description:
print(f" 📝 Description: {description[:80]}...")
else:
print(f"❌ Crawl failed: {result.error_message}")
async def research_assistant_example():
"""
Research Assistant: Find highly relevant documentation pages
"""
print("\n\n🔍 Research Assistant Example")
print("=" * 50)
config = CrawlerRunConfig(
link_preview_config=LinkPreviewConfig(
include_internal=True,
include_external=True,
include_patterns=["*/docs/*", "*/tutorial/*", "*/guide/*"],
exclude_patterns=["*/login*", "*/admin*"],
query="machine learning neural networks deep learning",
max_links=15,
score_threshold=0.4, # Only include high-relevance links
concurrency=8,
verbose=False # Clean output for this example
),
score_links=True
)
# Test with scikit-learn documentation
async with AsyncWebCrawler() as crawler:
result = await crawler.arun("https://scikit-learn.org/stable/", config=config)
if result.success:
print(f"✅ Analyzed: {result.url}")
all_links = result.links.get("internal", []) + result.links.get("external", [])
# Filter for high-scoring links
high_scoring_links = [link for link in all_links
if link.get("total_score", 0) > 0.6]
# Sort by total score (highest first)
high_scoring_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
print(f"\n🎯 Found {len(high_scoring_links)} highly relevant links:")
print(" (Showing top 5 by relevance score)")
for i, link in enumerate(high_scoring_links[:5]):
score = link.get("total_score", 0)
title = link.get("head_data", {}).get("title", "No title")
print(f"\n{i+1}. ⭐ {score:.3f} - {title[:70]}...")
print(f" 🔗 {link['href']}")
# Show score breakdown
intrinsic = link.get('intrinsic_score', 0)
contextual = link.get('contextual_score', 0)
print(f" 📊 Quality: {intrinsic:.1f}/10 | Relevance: {contextual:.3f}")
else:
print(f"❌ Research failed: {result.error_message}")
async def api_discovery_example():
"""
API Discovery: Find API endpoints and references
"""
print("\n\n🔧 API Discovery Example")
print("=" * 50)
config = CrawlerRunConfig(
link_preview_config=LinkPreviewConfig(
include_internal=True,
include_patterns=["*/api/*", "*/reference/*", "*/endpoint/*"],
exclude_patterns=["*/deprecated/*", "*/v1/*"], # Skip old versions
max_links=25,
concurrency=10,
timeout=8,
verbose=False
),
score_links=True
)
# Example with a documentation site that has API references
async with AsyncWebCrawler() as crawler:
result = await crawler.arun("https://httpbin.org/", config=config)
if result.success:
print(f"✅ Discovered APIs at: {result.url}")
api_links = result.links.get("internal", [])
# Categorize by detected content
endpoints = {"GET": [], "POST": [], "PUT": [], "DELETE": [], "OTHER": []}
for link in api_links:
if link.get("head_data"):
title = link.get("head_data", {}).get("title", "").upper()
text = link.get("text", "").upper()
# Simple categorization based on content
if "GET" in title or "GET" in text:
endpoints["GET"].append(link)
elif "POST" in title or "POST" in text:
endpoints["POST"].append(link)
elif "PUT" in title or "PUT" in text:
endpoints["PUT"].append(link)
elif "DELETE" in title or "DELETE" in text:
endpoints["DELETE"].append(link)
else:
endpoints["OTHER"].append(link)
# Display results
total_found = sum(len(links) for links in endpoints.values())
print(f"\n📡 Found {total_found} API-related links:")
for method, links in endpoints.items():
if links:
print(f"\n{method} Endpoints ({len(links)}):")
for link in links[:3]: # Show first 3 of each type
title = link.get("head_data", {}).get("title", "No title")
score = link.get("intrinsic_score", 0)
print(f" • [{score:.1f}] {title[:50]}...")
print(f" {link['href']}")
else:
print(f"❌ API discovery failed: {result.error_message}")
async def link_quality_analysis():
"""
Link Quality Analysis: Analyze website structure and link quality
"""
print("\n\n📊 Link Quality Analysis Example")
print("=" * 50)
config = CrawlerRunConfig(
link_preview_config=LinkPreviewConfig(
include_internal=True,
max_links=30, # Analyze more links for better statistics
concurrency=15,
timeout=6,
verbose=False
),
score_links=True
)
async with AsyncWebCrawler() as crawler:
# Test with a content-rich site
result = await crawler.arun("https://docs.python.org/3/", config=config)
if result.success:
print(f"✅ Analyzed: {result.url}")
links = result.links.get("internal", [])
# Extract intrinsic scores for analysis
scores = [link.get('intrinsic_score', 0) for link in links if link.get('intrinsic_score') is not None]
if scores:
avg_score = sum(scores) / len(scores)
high_quality = len([s for s in scores if s >= 7.0])
medium_quality = len([s for s in scores if 4.0 <= s < 7.0])
low_quality = len([s for s in scores if s < 4.0])
print(f"\n📈 Quality Analysis Results:")
print(f" 📊 Average Score: {avg_score:.2f}/10.0")
print(f" 🟢 High Quality (≥7.0): {high_quality} links")
print(f" 🟡 Medium Quality (4.0-6.9): {medium_quality} links")
print(f" 🔴 Low Quality (<4.0): {low_quality} links")
# Show best and worst links
scored_links = [(link, link.get('intrinsic_score', 0)) for link in links
if link.get('intrinsic_score') is not None]
scored_links.sort(key=lambda x: x[1], reverse=True)
print(f"\n🏆 Top 3 Quality Links:")
for i, (link, score) in enumerate(scored_links[:3]):
text = link.get('text', 'No text')[:40]
print(f" {i+1}. [{score:.1f}] {text}...")
print(f" {link['href']}")
print(f"\n⚠️ Bottom 3 Quality Links:")
for i, (link, score) in enumerate(scored_links[-3:]):
text = link.get('text', 'No text')[:40]
print(f" {i+1}. [{score:.1f}] {text}...")
print(f" {link['href']}")
else:
print("❌ No scoring data available")
else:
print(f"❌ Analysis failed: {result.error_message}")
async def pattern_filtering_example():
"""
Pattern Filtering: Demonstrate advanced filtering capabilities
"""
print("\n\n🎯 Pattern Filtering Example")
print("=" * 50)
# Example with multiple filtering strategies
filters = [
{
"name": "Documentation Only",
"config": LinkPreviewConfig(
include_internal=True,
max_links=10,
concurrency=5,
verbose=False,
include_patterns=["*/docs/*", "*/documentation/*"],
exclude_patterns=["*/api/*"]
)
},
{
"name": "API References Only",
"config": LinkPreviewConfig(
include_internal=True,
max_links=10,
concurrency=5,
verbose=False,
include_patterns=["*/api/*", "*/reference/*"],
exclude_patterns=["*/tutorial/*"]
)
},
{
"name": "Exclude Admin Areas",
"config": LinkPreviewConfig(
include_internal=True,
max_links=10,
concurrency=5,
verbose=False,
exclude_patterns=["*/admin/*", "*/login/*", "*/dashboard/*"]
)
}
]
async with AsyncWebCrawler() as crawler:
for filter_example in filters:
print(f"\n🔍 Testing: {filter_example['name']}")
config = CrawlerRunConfig(
link_preview_config=filter_example['config'],
score_links=True
)
result = await crawler.arun("https://docs.python.org/3/", config=config)
if result.success:
links = result.links.get("internal", [])
links_with_head = [link for link in links if link.get("head_data")]
print(f" 📊 Found {len(links_with_head)} matching links")
if links_with_head:
# Show sample matches
for link in links_with_head[:2]:
title = link.get("head_data", {}).get("title", "No title")
print(f" • {title[:50]}...")
print(f" {link['href']}")
else:
print(f" ❌ Failed: {result.error_message}")
async def main():
"""
Run all examples
"""
print("🚀 Crawl4AI Link Head Extraction Examples")
print("=" * 60)
print("This will demonstrate various link analysis capabilities.\n")
try:
# Run all examples
await basic_link_head_extraction()
await research_assistant_example()
await api_discovery_example()
await link_quality_analysis()
await pattern_filtering_example()
print("\n" + "=" * 60)
print("✨ All examples completed successfully!")
print("\nNext steps:")
print("1. Try modifying the queries and patterns above")
print("2. Test with your own websites")
print("3. Experiment with different score thresholds")
print("4. Check out the full documentation for more options")
except KeyboardInterrupt:
print("\n⏹️ Examples interrupted by user")
except Exception as e:
print(f"\n💥 Error running examples: {str(e)}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
asyncio.run(main())

72
docs/examples/llm_extraction_openai_pricing.py
View File

@@ -1,43 +1,55 @@
from crawl4ai import LLMConfig
from crawl4ai import AsyncWebCrawler, LLMExtractionStrategy
import asyncio
import os
import json
from pydantic import BaseModel, Field
url = "https://openai.com/api/pricing/"
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, LLMConfig, BrowserConfig, CacheMode
from crawl4ai.extraction_strategy import LLMExtractionStrategy
from typing import Dict
import os
class OpenAIModelFee(BaseModel):
model_name: str = Field(..., description="Name of the OpenAI model.")
input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
output_fee: str = Field(
..., description="Fee for output token for the OpenAI model."
output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
async def extract_structured_data_using_llm(provider: str, api_token: str = None, extra_headers: Dict[str, str] = None):
print(f"\n--- Extracting Structured Data with {provider} ---")
if api_token is None and provider != "ollama":
print(f"API token is required for {provider}. Skipping this example.")
return
browser_config = BrowserConfig(headless=True)
extra_args = {"temperature": 0, "top_p": 0.9, "max_tokens": 2000}
if extra_headers:
extra_args["extra_headers"] = extra_headers
crawler_config = CrawlerRunConfig(
cache_mode=CacheMode.BYPASS,
word_count_threshold=1,
page_timeout=80000,
extraction_strategy=LLMExtractionStrategy(
llm_config=LLMConfig(provider=provider, api_token=api_token),
schema=OpenAIModelFee.model_json_schema(),
extraction_type="schema",
instruction="""From the crawled content, extract all mentioned model names along with their fees for input and output tokens.
Do not miss any models in the entire content.""",
extra_args=extra_args,
),
)
async def main():
# Use AsyncWebCrawler
async with AsyncWebCrawler() as crawler:
async with AsyncWebCrawler(config=browser_config) as crawler:
result = await crawler.arun(
url=url,
word_count_threshold=1,
extraction_strategy=LLMExtractionStrategy(
# provider= "openai/gpt-4o", api_token = os.getenv('OPENAI_API_KEY'),
llm_config=LLMConfig(provider="groq/llama-3.1-70b-versatile", api_token=os.getenv("GROQ_API_KEY")),
schema=OpenAIModelFee.model_json_schema(),
extraction_type="schema",
instruction="From the crawled content, extract all mentioned model names along with their "
"fees for input and output tokens. Make sure not to miss anything in the entire content. "
"One extracted model JSON format should look like this: "
'{ "model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens" }',
),
url="https://openai.com/api/pricing/",
config=crawler_config
)
print("Success:", result.success)
model_fees = json.loads(result.extracted_content)
print(len(model_fees))
with open(".data/data.json", "w", encoding="utf-8") as f:
f.write(result.extracted_content)
print(result.extracted_content)
asyncio.run(main())
if __name__ == "__main__":
asyncio.run(
extract_structured_data_using_llm(
provider="openai/gpt-4o", api_token=os.getenv("OPENAI_API_KEY")
)
)

4
docs/examples/quickstart.ipynb
View File

@@ -518,7 +518,7 @@
}
],
"source": [
"from crawl4ai.extraction_strategy import LLMExtractionStrategy\n",
"from crawl4ai import LLMExtractionStrategy\n",
"from pydantic import BaseModel, Field\n",
"import os, json\n",
"\n",
@@ -594,7 +594,7 @@
}
],
"source": [
"from crawl4ai.extraction_strategy import CosineStrategy\n",
"from crawl4ai import CosineStrategy\n",
"\n",
"async def cosine_similarity_extraction():\n",
" async with AsyncWebCrawler() as crawler:\n",

4
docs/examples/quickstart.py
View File

@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field
from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
from crawl4ai.content_filter_strategy import PruningContentFilter
from crawl4ai.extraction_strategy import (
from crawl4ai import (
JsonCssExtractionStrategy,
LLMExtractionStrategy,
)
@@ -416,7 +416,7 @@ async def crawl_dynamic_content_pages_method_2():
async def cosine_similarity_extraction():
from crawl4ai.extraction_strategy import CosineStrategy
from crawl4ai import CosineStrategy
crawl_config = CrawlerRunConfig(
cache_mode=CacheMode.BYPASS,
extraction_strategy=CosineStrategy(

4
docs/examples/quickstart_examples_set_2.py
View File

@@ -16,7 +16,7 @@ from pydantic import BaseModel, Field
from crawl4ai import AsyncWebCrawler, CacheMode, BrowserConfig, CrawlerRunConfig
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
from crawl4ai.content_filter_strategy import PruningContentFilter
from crawl4ai.extraction_strategy import (
from crawl4ai import (
JsonCssExtractionStrategy,
LLMExtractionStrategy,
)
@@ -416,7 +416,7 @@ async def crawl_dynamic_content_pages_method_2():
async def cosine_similarity_extraction():
from crawl4ai.extraction_strategy import CosineStrategy
from crawl4ai import CosineStrategy
crawl_config = CrawlerRunConfig(
cache_mode=CacheMode.BYPASS,
extraction_strategy=CosineStrategy(

2
docs/examples/summarize_page.py
View File

@@ -2,7 +2,7 @@ import os
import json
from crawl4ai.web_crawler import WebCrawler
from crawl4ai.chunking_strategy import *
from crawl4ai.extraction_strategy import *
from crawl4ai import *
from crawl4ai.crawler_strategy import *
url = r"https://marketplace.visualstudio.com/items?itemName=Unclecode.groqopilot"

2
docs/examples/tutorial_v0.5.py
View File

@@ -18,7 +18,7 @@ from crawl4ai import RoundRobinProxyStrategy
from crawl4ai.content_filter_strategy import LLMContentFilter
from crawl4ai import DefaultMarkdownGenerator
from crawl4ai import LLMConfig
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
from crawl4ai import JsonCssExtractionStrategy
from crawl4ai.processors.pdf import PDFCrawlerStrategy, PDFContentScrapingStrategy
from pprint import pprint

1171
docs/examples/url_seeder/Crawl4AI_URL_Seeder_Tutorial.ipynb Normal file
View File

File diff suppressed because it is too large Load Diff

807
docs/examples/url_seeder/bbc_sport_research_assistant.py Normal file
View File

@@ -0,0 +1,807 @@
"""
BBC Sport Research Assistant Pipeline
=====================================
This example demonstrates how URLSeeder helps create an efficient research pipeline:
1. Discover all available URLs without crawling
2. Filter and rank them based on relevance
3. Crawl only the most relevant content
4. Generate comprehensive research insights
Pipeline Steps:
1. Get user query
2. Optionally enhance query using LLM
3. Use URLSeeder to discover and rank URLs
4. Crawl top K URLs with BM25 filtering
5. Generate detailed response with citations
Requirements:
- pip install crawl4ai
- pip install litellm
- export GEMINI_API_KEY="your-api-key"
Usage:
- Run normally: python bbc_sport_research_assistant.py
- Run test mode: python bbc_sport_research_assistant.py test
Note: AsyncUrlSeeder now uses context manager for automatic cleanup.
"""
import asyncio
import json
import os
import hashlib
import pickle
from typing import List, Dict, Optional, Tuple
from dataclasses import dataclass, asdict
from datetime import datetime
from pathlib import Path
# Rich for colored output
from rich.console import Console
from rich.text import Text
from rich.panel import Panel
from rich.table import Table
from rich.progress import Progress, SpinnerColumn, TextColumn
# Crawl4AI imports
from crawl4ai import (
AsyncWebCrawler,
BrowserConfig,
CrawlerRunConfig,
AsyncUrlSeeder,
SeedingConfig,
AsyncLogger
)
from crawl4ai.content_filter_strategy import PruningContentFilter
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
# LiteLLM for AI communication
import litellm
# Initialize Rich console
console = Console()
# Get the current directory where this script is located
SCRIPT_DIR = Path(__file__).parent.resolve()
# Cache configuration - relative to script directory
CACHE_DIR = SCRIPT_DIR / "temp_cache"
CACHE_DIR.mkdir(parents=True, exist_ok=True)
# Testing limits
TESTING_MODE = True
MAX_URLS_DISCOVERY = 100 if TESTING_MODE else 1000
MAX_URLS_TO_CRAWL = 5 if TESTING_MODE else 10
def get_cache_key(prefix: str, *args) -> str:
"""Generate cache key from prefix and arguments"""
content = f"{prefix}:{'|'.join(str(arg) for arg in args)}"
return hashlib.md5(content.encode()).hexdigest()
def load_from_cache(cache_key: str) -> Optional[any]:
"""Load data from cache if exists"""
cache_path = CACHE_DIR / f"{cache_key}.pkl"
if cache_path.exists():
with open(cache_path, 'rb') as f:
return pickle.load(f)
return None
def save_to_cache(cache_key: str, data: any) -> None:
"""Save data to cache"""
cache_path = CACHE_DIR / f"{cache_key}.pkl"
with open(cache_path, 'wb') as f:
pickle.dump(data, f)
@dataclass
class ResearchConfig:
"""Configuration for research pipeline"""
# Core settings
domain: str = "www.bbc.com/sport"
max_urls_discovery: int = 100
max_urls_to_crawl: int = 10
top_k_urls: int = 10
# Scoring and filtering
score_threshold: float = 0.1
scoring_method: str = "bm25"
# Processing options
use_llm_enhancement: bool = True
extract_head_metadata: bool = True
live_check: bool = True
force_refresh: bool = False
# Crawler settings
max_concurrent_crawls: int = 5
timeout: int = 30000
headless: bool = True
# Output settings
save_json: bool = True
save_markdown: bool = True
output_dir: str = None # Will be set in __post_init__
# Development settings
test_mode: bool = False
interactive_mode: bool = False
verbose: bool = True
def __post_init__(self):
"""Adjust settings based on test mode"""
if self.test_mode:
self.max_urls_discovery = 50
self.max_urls_to_crawl = 3
self.top_k_urls = 5
# Set default output directory relative to script location
if self.output_dir is None:
self.output_dir = str(SCRIPT_DIR / "research_results")
@dataclass
class ResearchQuery:
"""Container for research query and metadata"""
original_query: str
enhanced_query: Optional[str] = None
search_patterns: List[str] = None
timestamp: str = None
@dataclass
class ResearchResult:
"""Container for research results"""
query: ResearchQuery
discovered_urls: List[Dict]
crawled_content: List[Dict]
synthesis: str
citations: List[Dict]
metadata: Dict
async def get_user_query() -> str:
"""
Get research query from user input
"""
query = input("\n🔍 Enter your research query: ")
return query.strip()
async def enhance_query_with_llm(query: str) -> ResearchQuery:
"""
Use LLM to enhance the research query:
- Extract key terms
- Generate search patterns
- Identify related topics
"""
# Check cache
cache_key = get_cache_key("enhanced_query", query)
cached_result = load_from_cache(cache_key)
if cached_result:
console.print("[dim cyan]📦 Using cached enhanced query[/dim cyan]")
return cached_result
try:
response = await litellm.acompletion(
model="gemini/gemini-2.5-flash-preview-04-17",
messages=[{
"role": "user",
"content": f"""Given this research query: "{query}"
Extract:
1. Key terms and concepts (as a list)
2. Related search terms
3. A more specific/enhanced version of the query
Return as JSON:
{{
"key_terms": ["term1", "term2"],
"related_terms": ["related1", "related2"],
"enhanced_query": "enhanced version of query"
}}"""
}],
# reasoning_effort="low",
temperature=0.3,
response_format={"type": "json_object"}
)
data = json.loads(response.choices[0].message.content)
# Create search patterns
all_terms = data["key_terms"] + data["related_terms"]
patterns = [f"*{term.lower()}*" for term in all_terms]
result = ResearchQuery(
original_query=query,
enhanced_query=data["enhanced_query"],
search_patterns=patterns[:10], # Limit patterns
timestamp=datetime.now().isoformat()
)
# Cache the result
save_to_cache(cache_key, result)
return result
except Exception as e:
console.print(f"[yellow]⚠️ LLM enhancement failed: {e}[/yellow]")
# Fallback to simple tokenization
return ResearchQuery(
original_query=query,
enhanced_query=query,
search_patterns=tokenize_query_to_patterns(query),
timestamp=datetime.now().isoformat()
)
def tokenize_query_to_patterns(query: str) -> List[str]:
"""
Convert query into URL patterns for URLSeeder
Example: "AI startups funding" -> ["*ai*", "*startup*", "*funding*"]
"""
# Simple tokenization - split and create patterns
words = query.lower().split()
# Filter out common words
stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'that'}
keywords = [w for w in words if w not in stop_words and len(w) > 2]
# Create patterns
patterns = [f"*{keyword}*" for keyword in keywords]
return patterns[:8] # Limit to 8 patterns
async def discover_urls(domain: str, query: str, config: ResearchConfig) -> List[Dict]:
"""
Use URLSeeder to discover and rank URLs:
1. Fetch all URLs from domain
2. Filter by patterns
3. Extract metadata (titles, descriptions)
4. Rank by BM25 relevance score
5. Return top K URLs
"""
# Check cache
cache_key = get_cache_key("discovered_urls", domain, query, config.top_k_urls)
cached_result = load_from_cache(cache_key)
if cached_result and not config.force_refresh:
console.print("[dim cyan]📦 Using cached URL discovery[/dim cyan]")
return cached_result
console.print(f"\n[cyan]🔍 Discovering URLs from {domain}...[/cyan]")
# Initialize URL seeder with context manager
async with AsyncUrlSeeder(logger=AsyncLogger(verbose=config.verbose)) as seeder:
# Configure seeding
seeding_config = SeedingConfig(
source="sitemap+cc", # Use both sitemap and Common Crawl
extract_head=config.extract_head_metadata,
query=query,
scoring_method=config.scoring_method,
score_threshold=config.score_threshold,
max_urls=config.max_urls_discovery,
live_check=config.live_check,
force=config.force_refresh
)
try:
# Discover URLs
urls = await seeder.urls(domain, seeding_config)
# Sort by relevance score (descending)
sorted_urls = sorted(
urls,
key=lambda x: x.get('relevance_score', 0),
reverse=True
)
# Take top K
top_urls = sorted_urls[:config.top_k_urls]
console.print(f"[green]✅ Discovered {len(urls)} URLs, selected top {len(top_urls)}[/green]")
# Cache the result
save_to_cache(cache_key, top_urls)
return top_urls
except Exception as e:
console.print(f"[red]❌ URL discovery failed: {e}[/red]")
return []
async def crawl_selected_urls(urls: List[str], query: str, config: ResearchConfig) -> List[Dict]:
"""
Crawl selected URLs with content filtering:
- Use AsyncWebCrawler.arun_many()
- Apply content filter
- Generate clean markdown
"""
# Extract just URLs from the discovery results
url_list = [u['url'] for u in urls if 'url' in u][:config.max_urls_to_crawl]
if not url_list:
console.print("[red]❌ No URLs to crawl[/red]")
return []
console.print(f"\n[cyan]🕷️ Crawling {len(url_list)} URLs...[/cyan]")
# Check cache for each URL
crawled_results = []
urls_to_crawl = []
for url in url_list:
cache_key = get_cache_key("crawled_content", url, query)
cached_content = load_from_cache(cache_key)
if cached_content and not config.force_refresh:
crawled_results.append(cached_content)
else:
urls_to_crawl.append(url)
if urls_to_crawl:
console.print(f"[cyan]📥 Crawling {len(urls_to_crawl)} new URLs (cached: {len(crawled_results)})[/cyan]")
# Configure markdown generator with content filter
md_generator = DefaultMarkdownGenerator(
content_filter=PruningContentFilter(
threshold=0.48,
threshold_type="dynamic",
min_word_threshold=10
),
)
# Configure crawler
crawler_config = CrawlerRunConfig(
markdown_generator=md_generator,
exclude_external_links=True,
excluded_tags=['nav', 'header', 'footer', 'aside'],
)
# Create crawler with browser config
async with AsyncWebCrawler(
config=BrowserConfig(
headless=config.headless,
verbose=config.verbose
)
) as crawler:
# Crawl URLs
results = await crawler.arun_many(
urls_to_crawl,
config=crawler_config,
max_concurrent=config.max_concurrent_crawls
)
# Process results
for url, result in zip(urls_to_crawl, results):
if result.success:
content_data = {
'url': url,
'title': result.metadata.get('title', ''),
'markdown': result.markdown.fit_markdown or result.markdown.raw_markdown,
'raw_length': len(result.markdown.raw_markdown),
'fit_length': len(result.markdown.fit_markdown) if result.markdown.fit_markdown else len(result.markdown.raw_markdown),
'metadata': result.metadata
}
crawled_results.append(content_data)
# Cache the result
cache_key = get_cache_key("crawled_content", url, query)
save_to_cache(cache_key, content_data)
else:
console.print(f" [red]❌ Failed: {url[:50]}... - {result.error}[/red]")
console.print(f"[green]✅ Successfully crawled {len(crawled_results)} URLs[/green]")
return crawled_results
async def generate_research_synthesis(
query: str,
crawled_content: List[Dict]
) -> Tuple[str, List[Dict]]:
"""
Use LLM to synthesize research findings:
- Analyze all crawled content
- Generate comprehensive answer
- Extract citations and references
"""
if not crawled_content:
return "No content available for synthesis.", []
console.print("\n[cyan]🤖 Generating research synthesis...[/cyan]")
# Prepare content for LLM
content_sections = []
for i, content in enumerate(crawled_content, 1):
section = f"""
SOURCE {i}:
Title: {content['title']}
URL: {content['url']}
Content Preview:
{content['markdown'][:1500]}...
"""
content_sections.append(section)
combined_content = "\n---\n".join(content_sections)
try:
response = await litellm.acompletion(
model="gemini/gemini-2.5-flash-preview-04-17",
messages=[{
"role": "user",
"content": f"""Research Query: "{query}"
Based on the following sources, provide a comprehensive research synthesis.
{combined_content}
Please provide:
1. An executive summary (2-3 sentences)
2. Key findings (3-5 bullet points)
3. Detailed analysis (2-3 paragraphs)
4. Future implications or trends
Format your response with clear sections and cite sources using [Source N] notation.
Keep the total response under 800 words."""
}],
# reasoning_effort="medium",
temperature=0.7
)
synthesis = response.choices[0].message.content
# Extract citations from the synthesis
citations = []
for i, content in enumerate(crawled_content, 1):
if f"[Source {i}]" in synthesis or f"Source {i}" in synthesis:
citations.append({
'source_id': i,
'title': content['title'],
'url': content['url']
})
return synthesis, citations
except Exception as e:
console.print(f"[red]❌ Synthesis generation failed: {e}[/red]")
# Fallback to simple summary
summary = f"Research on '{query}' found {len(crawled_content)} relevant articles:\n\n"
for content in crawled_content[:3]:
summary += f"- {content['title']}\n {content['url']}\n\n"
return summary, []
def format_research_output(result: ResearchResult) -> str:
"""
Format the final research output with:
- Executive summary
- Key findings
- Detailed analysis
- Citations and sources
"""
output = []
output.append("\n" + "=" * 60)
output.append("🔬 RESEARCH RESULTS")
output.append("=" * 60)
# Query info
output.append(f"\n📋 Query: {result.query.original_query}")
if result.query.enhanced_query != result.query.original_query:
output.append(f" Enhanced: {result.query.enhanced_query}")
# Discovery stats
output.append(f"\n📊 Statistics:")
output.append(f" - URLs discovered: {len(result.discovered_urls)}")
output.append(f" - URLs crawled: {len(result.crawled_content)}")
output.append(f" - Processing time: {result.metadata.get('duration', 'N/A')}")
# Synthesis
output.append(f"\n📝 SYNTHESIS")
output.append("-" * 60)
output.append(result.synthesis)
# Citations
if result.citations:
output.append(f"\n📚 SOURCES")
output.append("-" * 60)
for citation in result.citations:
output.append(f"[{citation['source_id']}] {citation['title']}")
output.append(f" {citation['url']}")
return "\n".join(output)
async def save_research_results(result: ResearchResult, config: ResearchConfig) -> Tuple[str, str]:
"""
Save research results in JSON and Markdown formats
Returns:
Tuple of (json_path, markdown_path)
"""
# Create output directory
output_dir = Path(config.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# Generate filename based on query and timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
query_slug = result.query.original_query[:50].replace(" ", "_").replace("/", "_")
base_filename = f"{timestamp}_{query_slug}"
json_path = None
md_path = None
# Save JSON
if config.save_json:
json_path = output_dir / f"{base_filename}.json"
with open(json_path, 'w') as f:
json.dump(asdict(result), f, indent=2, default=str)
console.print(f"\n[green]💾 JSON saved: {json_path}[/green]")
# Save Markdown
if config.save_markdown:
md_path = output_dir / f"{base_filename}.md"
# Create formatted markdown
md_content = [
f"# Research Report: {result.query.original_query}",
f"\n**Generated on:** {result.metadata.get('timestamp', 'N/A')}",
f"\n**Domain:** {result.metadata.get('domain', 'N/A')}",
f"\n**Processing time:** {result.metadata.get('duration', 'N/A')}",
"\n---\n",
"## Query Information",
f"- **Original Query:** {result.query.original_query}",
f"- **Enhanced Query:** {result.query.enhanced_query or 'N/A'}",
f"- **Search Patterns:** {', '.join(result.query.search_patterns or [])}",
"\n## Statistics",
f"- **URLs Discovered:** {len(result.discovered_urls)}",
f"- **URLs Crawled:** {len(result.crawled_content)}",
f"- **Sources Cited:** {len(result.citations)}",
"\n## Research Synthesis\n",
result.synthesis,
"\n## Sources\n"
]
# Add citations
for citation in result.citations:
md_content.append(f"### [{citation['source_id']}] {citation['title']}")
md_content.append(f"- **URL:** [{citation['url']}]({citation['url']})")
md_content.append("")
# Add discovered URLs summary
md_content.extend([
"\n## Discovered URLs (Top 10)\n",
"| Score | URL | Title |",
"|-------|-----|-------|"
])
for url_data in result.discovered_urls[:10]:
score = url_data.get('relevance_score', 0)
url = url_data.get('url', '')
title = 'N/A'
if 'head_data' in url_data and url_data['head_data']:
title = url_data['head_data'].get('title', 'N/A')[:60] + '...'
md_content.append(f"| {score:.3f} | {url[:50]}... | {title} |")
# Write markdown
with open(md_path, 'w') as f:
f.write('\n'.join(md_content))
console.print(f"[green]📄 Markdown saved: {md_path}[/green]")
return str(json_path) if json_path else None, str(md_path) if md_path else None
async def wait_for_user(message: str = "\nPress Enter to continue..."):
"""Wait for user input in interactive mode"""
input(message)
async def research_pipeline(
query: str,
config: ResearchConfig
) -> ResearchResult:
"""
Main research pipeline orchestrator with configurable settings
"""
start_time = datetime.now()
# Display pipeline header
header = Panel(
f"[bold cyan]Research Pipeline[/bold cyan]\n\n"
f"[dim]Domain:[/dim] {config.domain}\n"
f"[dim]Mode:[/dim] {'Test' if config.test_mode else 'Production'}\n"
f"[dim]Interactive:[/dim] {'Yes' if config.interactive_mode else 'No'}",
title="🚀 Starting",
border_style="cyan"
)
console.print(header)
# Step 1: Enhance query (optional)
console.print(f"\n[bold cyan]📝 Step 1: Query Processing[/bold cyan]")
if config.interactive_mode:
await wait_for_user()
if config.use_llm_enhancement:
research_query = await enhance_query_with_llm(query)
else:
research_query = ResearchQuery(
original_query=query,
enhanced_query=query,
search_patterns=tokenize_query_to_patterns(query),
timestamp=datetime.now().isoformat()
)
console.print(f" [green]✅ Query ready:[/green] {research_query.enhanced_query or query}")
# Step 2: Discover URLs
console.print(f"\n[bold cyan]🔍 Step 2: URL Discovery[/bold cyan]")
if config.interactive_mode:
await wait_for_user()
discovered_urls = await discover_urls(
domain=config.domain,
query=research_query.enhanced_query or query,
config=config
)
if not discovered_urls:
return ResearchResult(
query=research_query,
discovered_urls=[],
crawled_content=[],
synthesis="No relevant URLs found for the given query.",
citations=[],
metadata={'duration': str(datetime.now() - start_time)}
)
console.print(f" [green]✅ Found {len(discovered_urls)} relevant URLs[/green]")
# Step 3: Crawl selected URLs
console.print(f"\n[bold cyan]🕷️ Step 3: Content Crawling[/bold cyan]")
if config.interactive_mode:
await wait_for_user()
crawled_content = await crawl_selected_urls(
urls=discovered_urls,
query=research_query.enhanced_query or query,
config=config
)
console.print(f" [green]✅ Successfully crawled {len(crawled_content)} pages[/green]")
# Step 4: Generate synthesis
console.print(f"\n[bold cyan]🤖 Step 4: Synthesis Generation[/bold cyan]")
if config.interactive_mode:
await wait_for_user()
synthesis, citations = await generate_research_synthesis(
query=research_query.enhanced_query or query,
crawled_content=crawled_content
)
console.print(f" [green]✅ Generated synthesis with {len(citations)} citations[/green]")
# Step 5: Create result
result = ResearchResult(
query=research_query,
discovered_urls=discovered_urls,
crawled_content=crawled_content,
synthesis=synthesis,
citations=citations,
metadata={
'duration': str(datetime.now() - start_time),
'domain': config.domain,
'timestamp': datetime.now().isoformat(),
'config': asdict(config)
}
)
duration = datetime.now() - start_time
console.print(f"\n[bold green]✅ Research completed in {duration}[/bold green]")
return result
async def main():
"""
Main entry point for the BBC Sport Research Assistant
"""
# Example queries
example_queries = [
"Premier League transfer news and rumors",
"Champions League match results and analysis",
"World Cup qualifying updates",
"Football injury reports and return dates",
"Tennis grand slam tournament results"
]
# Display header
console.print(Panel.fit(
"[bold cyan]BBC Sport Research Assistant[/bold cyan]\n\n"
"This tool demonstrates efficient research using URLSeeder:\n"
"[dim]• Discover all URLs without crawling\n"
"• Filter and rank by relevance\n"
"• Crawl only the most relevant content\n"
"• Generate AI-powered insights with citations[/dim]\n\n"
f"[dim]📁 Working directory: {SCRIPT_DIR}[/dim]",
title="🔬 Welcome",
border_style="cyan"
))
# Configuration options table
config_table = Table(title="\n⚙️ Configuration Options", show_header=False, box=None)
config_table.add_column(style="bold cyan", width=3)
config_table.add_column()
config_table.add_row("1", "Quick Test Mode (3 URLs, fast)")
config_table.add_row("2", "Standard Mode (10 URLs, balanced)")
config_table.add_row("3", "Comprehensive Mode (20 URLs, thorough)")
config_table.add_row("4", "Custom Configuration")
console.print(config_table)
config_choice = input("\nSelect configuration (1-4): ").strip()
# Create config based on choice
if config_choice == "1":
config = ResearchConfig(test_mode=True, interactive_mode=False)
elif config_choice == "2":
config = ResearchConfig(max_urls_to_crawl=10, top_k_urls=10)
elif config_choice == "3":
config = ResearchConfig(max_urls_to_crawl=20, top_k_urls=20, max_urls_discovery=200)
else:
# Custom configuration
config = ResearchConfig()
config.test_mode = input("\nTest mode? (y/n): ").lower() == 'y'
config.interactive_mode = input("Interactive mode (pause between steps)? (y/n): ").lower() == 'y'
config.use_llm_enhancement = input("Use AI to enhance queries? (y/n): ").lower() == 'y'
if not config.test_mode:
try:
config.max_urls_to_crawl = int(input("Max URLs to crawl (default 10): ") or "10")
config.top_k_urls = int(input("Top K URLs to select (default 10): ") or "10")
except ValueError:
console.print("[yellow]Using default values[/yellow]")
# Display example queries
query_table = Table(title="\n📋 Example Queries", show_header=False, box=None)
query_table.add_column(style="bold cyan", width=3)
query_table.add_column()
for i, q in enumerate(example_queries, 1):
query_table.add_row(str(i), q)
console.print(query_table)
query_input = input("\nSelect a query (1-5) or enter your own: ").strip()
if query_input.isdigit() and 1 <= int(query_input) <= len(example_queries):
query = example_queries[int(query_input) - 1]
else:
query = query_input if query_input else example_queries[0]
console.print(f"\n[bold cyan]📝 Selected Query:[/bold cyan] {query}")
# Run the research pipeline
result = await research_pipeline(query=query, config=config)
# Display results
formatted_output = format_research_output(result)
# print(formatted_output)
console.print(Panel.fit(
formatted_output,
title="🔬 Research Results",
border_style="green"
))
# Save results
if config.save_json or config.save_markdown:
json_path, md_path = await save_research_results(result, config)
# print(f"\n✅ Results saved successfully!")
if json_path:
console.print(f"[green]JSON saved at:[/green] {json_path}")
if md_path:
console.print(f"[green]Markdown saved at:[/green] {md_path}")
if __name__ == "__main__":
asyncio.run(main())

155
docs/examples/url_seeder/convert_tutorial_to_colab.py Normal file
View File

@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""
Convert Crawl4AI URL Seeder tutorial markdown to Colab notebook format
"""
import json
import re
from pathlib import Path
def parse_markdown_to_cells(markdown_content):
"""Parse markdown content and convert to notebook cells"""
cells = []
# Split content by cell markers
lines = markdown_content.split('\n')
# Extract the header content before first cell marker
header_lines = []
i = 0
while i < len(lines) and not lines[i].startswith('# cell'):
header_lines.append(lines[i])
i += 1
# Add header as markdown cell if it exists
if header_lines:
header_content = '\n'.join(header_lines).strip()
if header_content:
cells.append({
"cell_type": "markdown",
"metadata": {},
"source": header_content.split('\n')
})
# Process cells marked with # cell X type:Y
current_cell_content = []
current_cell_type = None
while i < len(lines):
line = lines[i]
# Check for cell marker
cell_match = re.match(r'^# cell (\d+) type:(markdown|code)$', line)
if cell_match:
# Save previous cell if exists
if current_cell_content and current_cell_type:
content = '\n'.join(current_cell_content).strip()
if content:
if current_cell_type == 'code':
cells.append({
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": content.split('\n')
})
else:
cells.append({
"cell_type": "markdown",
"metadata": {},
"source": content.split('\n')
})
# Start new cell
current_cell_type = cell_match.group(2)
current_cell_content = []
else:
# Add line to current cell
current_cell_content.append(line)
i += 1
# Add last cell if exists
if current_cell_content and current_cell_type:
content = '\n'.join(current_cell_content).strip()
if content:
if current_cell_type == 'code':
cells.append({
"cell_type": "code",
"execution_count": None,
"metadata": {},
"outputs": [],
"source": content.split('\n')
})
else:
cells.append({
"cell_type": "markdown",
"metadata": {},
"source": content.split('\n')
})
return cells
def create_colab_notebook(cells):
"""Create a Colab notebook structure"""
notebook = {
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "Crawl4AI_URL_Seeder_Tutorial.ipynb",
"provenance": [],
"collapsed_sections": [],
"toc_visible": True
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": cells
}
return notebook
def main():
# Read the markdown file
md_path = Path("tutorial_url_seeder.md")
if not md_path.exists():
print(f"Error: {md_path} not found!")
return
print(f"Reading {md_path}...")
with open(md_path, 'r', encoding='utf-8') as f:
markdown_content = f.read()
# Parse markdown to cells
print("Parsing markdown content...")
cells = parse_markdown_to_cells(markdown_content)
print(f"Created {len(cells)} cells")
# Create notebook
print("Creating Colab notebook...")
notebook = create_colab_notebook(cells)
# Save notebook
output_path = Path("Crawl4AI_URL_Seeder_Tutorial.ipynb")
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(notebook, f, indent=2, ensure_ascii=False)
print(f"✅ Successfully created {output_path}")
print(f" - Total cells: {len(cells)}")
print(f" - Markdown cells: {sum(1 for c in cells if c['cell_type'] == 'markdown')}")
print(f" - Code cells: {sum(1 for c in cells if c['cell_type'] == 'code')}")
if __name__ == "__main__":
main()

1035
docs/examples/url_seeder/tutorial_url_seeder.md Normal file
View File

File diff suppressed because it is too large Load Diff

263
docs/examples/url_seeder/url_seeder_demo.py Normal file
View File

@@ -0,0 +1,263 @@
"""
URL Seeder Demo - Interactive showcase of Crawl4AI's URL discovery capabilities
This demo shows:
1. Basic URL discovery from sitemaps and Common Crawl
2. Cache management and forced refresh
3. Live URL validation and metadata extraction
4. BM25 relevance scoring for intelligent filtering
5. Integration with AsyncWebCrawler for the complete pipeline
6. Multi-domain discovery across multiple sites
Note: The AsyncUrlSeeder now supports context manager protocol for automatic cleanup.
"""
import asyncio
import time
from datetime import datetime
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from rich.progress import Progress, SpinnerColumn, BarColumn, TimeElapsedColumn
from rich.prompt import Prompt, Confirm
from crawl4ai import (
AsyncWebCrawler,
CrawlerRunConfig,
AsyncUrlSeeder,
SeedingConfig
)
console = Console()
console.rule("[bold green]🌐 Crawl4AI URL Seeder: Interactive Demo")
DOMAIN = "crawl4ai.com"
# Utils
def print_head_info(head_data):
table = Table(title="<head> Metadata", expand=True)
table.add_column("Key", style="cyan", no_wrap=True)
table.add_column("Value", style="magenta")
if not head_data:
console.print("[yellow]No head data found.")
return
if head_data.get("title"):
table.add_row("title", head_data["title"])
if head_data.get("charset"):
table.add_row("charset", head_data["charset"])
for k, v in head_data.get("meta", {}).items():
table.add_row(f"meta:{k}", v)
for rel, items in head_data.get("link", {}).items():
for item in items:
table.add_row(f"link:{rel}", item.get("href", ""))
console.print(table)
async def section_1_basic_exploration(seed: AsyncUrlSeeder):
console.rule("[bold cyan]1. Basic Seeding")
cfg = SeedingConfig(source="cc+sitemap", pattern="*", verbose=True)
start_time = time.time()
with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
p.add_task(description="Fetching from Common Crawl + Sitemap...", total=None)
urls = await seed.urls(DOMAIN, cfg)
elapsed = time.time() - start_time
console.print(f"[green]✓ Fetched {len(urls)} URLs in {elapsed:.2f} seconds")
console.print(f"[dim] Speed: {len(urls)/elapsed:.0f} URLs/second[/dim]\n")
console.print("[bold]Sample URLs:[/bold]")
for u in urls[:5]:
console.print(f" • {u['url']}")
async def section_2_cache_demo(seed: AsyncUrlSeeder):
console.rule("[bold cyan]2. Caching Demonstration")
console.print("[yellow]Using `force=True` to bypass cache and fetch fresh data.[/yellow]")
cfg = SeedingConfig(source="cc", pattern="*crawl4ai.com/core/*", verbose=False, force = True)
await seed.urls(DOMAIN, cfg)
async def section_3_live_head(seed: AsyncUrlSeeder):
console.rule("[bold cyan]3. Live Check + Head Extraction")
cfg = SeedingConfig(
extract_head=True,
concurrency=10,
hits_per_sec=5,
pattern="*crawl4ai.com/*",
max_urls=10,
verbose=False,
)
urls = await seed.urls(DOMAIN, cfg)
valid = [u for u in urls if u["status"] == "valid"]
console.print(f"[green]Valid: {len(valid)} / {len(urls)}")
if valid:
print_head_info(valid[0]["head_data"])
async def section_4_bm25_scoring(seed: AsyncUrlSeeder):
console.rule("[bold cyan]4. BM25 Relevance Scoring")
console.print("[yellow]Using AI-powered relevance scoring to find the most relevant content[/yellow]")
query = "markdown generation extraction strategies"
cfg = SeedingConfig(
source="sitemap",
extract_head=True,
query=query,
scoring_method="bm25",
score_threshold=0.3, # Only URLs with >30% relevance
max_urls=20,
verbose=False
)
with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
p.add_task(description=f"Searching for: '{query}'", total=None)
urls = await seed.urls(DOMAIN, cfg)
console.print(f"[green]Found {len(urls)} relevant URLs (score > 0.3)")
# Show top results with scores
table = Table(title="Top 5 Most Relevant Pages", expand=True)
table.add_column("Score", style="cyan", width=8)
table.add_column("Title", style="magenta")
table.add_column("URL", style="blue", overflow="fold")
for url in urls[:5]:
score = f"{url['relevance_score']:.2f}"
title = url['head_data'].get('title', 'No title')[:60] + "..."
table.add_row(score, title, url['url'])
console.print(table)
async def section_5_keyword_filter_to_agent(seed: AsyncUrlSeeder):
console.rule("[bold cyan]5. Complete Pipeline: Discover → Filter → Crawl")
cfg = SeedingConfig(
extract_head=True,
concurrency=20,
hits_per_sec=10,
max_urls=10,
pattern="*crawl4ai.com/*",
force=True,
)
urls = await seed.urls(DOMAIN, cfg)
keywords = ["deep crawling", "markdown", "llm"]
selected = [u for u in urls if any(k in str(u["head_data"]).lower() for k in keywords)]
console.print(f"[cyan]Selected {len(selected)} URLs with relevant keywords:")
for u in selected[:10]:
console.print("•", u["url"])
console.print("\n[yellow]Passing above URLs to arun_many() LLM agent for crawling...")
async with AsyncWebCrawler(verbose=True) as crawler:
crawl_run_config = CrawlerRunConfig(
# Example crawl settings for these URLs:
only_text=True, # Just get text content
screenshot=False,
pdf=False,
word_count_threshold=50, # Only process pages with at least 50 words
stream=True,
verbose=False # Keep logs clean for arun_many in this demo
)
# Extract just the URLs from the selected results
urls_to_crawl = [u["url"] for u in selected]
# We'll stream results for large lists, but collect them here for demonstration
crawled_results_stream = await crawler.arun_many(urls_to_crawl, config=crawl_run_config)
final_crawled_data = []
async for result in crawled_results_stream:
final_crawled_data.append(result)
if len(final_crawled_data) % 5 == 0:
print(f" Processed {len(final_crawled_data)}/{len(urls_to_crawl)} URLs...")
print(f"\n Successfully crawled {len(final_crawled_data)} URLs.")
if final_crawled_data:
print("\n Example of a crawled result's URL and Markdown (first successful one):")
for result in final_crawled_data:
if result.success and result.markdown.raw_markdown:
print(f" URL: {result.url}")
print(f" Markdown snippet: {result.markdown.raw_markdown[:200]}...")
break
else:
print(" No successful crawls with markdown found.")
else:
print(" No successful crawls found.")
async def section_6_multi_domain(seed: AsyncUrlSeeder):
console.rule("[bold cyan]6. Multi-Domain Discovery")
console.print("[yellow]Discovering Python tutorials across multiple educational sites[/yellow]\n")
domains = ["docs.python.org", "realpython.com", "docs.crawl4ai.com"]
cfg = SeedingConfig(
source="sitemap",
extract_head=True,
query="python tutorial guide",
scoring_method="bm25",
score_threshold=0.2,
max_urls=5 # Per domain
)
start_time = time.time()
with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
task = p.add_task(description="Discovering across domains...", total=None)
results = await seed.many_urls(domains, cfg)
elapsed = time.time() - start_time
total_urls = sum(len(urls) for urls in results.values())
console.print(f"[green]✓ Found {total_urls} relevant URLs across {len(domains)} domains in {elapsed:.2f}s\n")
# Show results per domain
for domain, urls in results.items():
console.print(f"[bold]{domain}:[/bold] {len(urls)} relevant pages")
if urls:
top = urls[0]
console.print(f" Top result: [{top['relevance_score']:.2f}] {top['head_data'].get('title', 'No title')}")
async def main():
async with AsyncUrlSeeder() as seed:
# Interactive menu
sections = {
"1": ("Basic URL Discovery", section_1_basic_exploration),
"2": ("Cache Management Demo", section_2_cache_demo),
"3": ("Live Check & Metadata Extraction", section_3_live_head),
"4": ("BM25 Relevance Scoring", section_4_bm25_scoring),
"5": ("Complete Pipeline (Discover → Filter → Crawl)", section_5_keyword_filter_to_agent),
"6": ("Multi-Domain Discovery", section_6_multi_domain),
"7": ("Run All Demos", None)
}
console.print("\n[bold]Available Demos:[/bold]")
for key, (title, _) in sections.items():
console.print(f" {key}. {title}")
choice = Prompt.ask("\n[cyan]Which demo would you like to run?[/cyan]",
choices=list(sections.keys()),
default="7")
console.print()
if choice == "7":
# Run all demos
for key, (title, func) in sections.items():
if key != "7" and func:
await func(seed)
if key != "6": # Don't pause after the last demo
if not Confirm.ask("\n[yellow]Continue to next demo?[/yellow]", default=True):
break
console.print()
else:
# Run selected demo
_, func = sections[choice]
await func(seed)
console.rule("[bold green]Demo Complete ✔︎")
if __name__ == "__main__":
asyncio.run(main())

Some files were not shown because too many files have changed in this diff Show More