diff --git a/.gitignore b/.gitignore index 623af2eb..80600b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -297,3 +297,6 @@ scripts/ *.rdb *.ldb MEMORY.md + +# Handoff files +HANDOFF-*.md diff --git a/crawl4ai/browser_profiler.py b/crawl4ai/browser_profiler.py index d5fb8af1..fd26f389 100644 --- a/crawl4ai/browser_profiler.py +++ b/crawl4ai/browser_profiler.py @@ -421,16 +421,30 @@ class BrowserProfiler: ``` """ # Create default browser config if none provided + # IMPORTANT: We disable cookie encryption so profiles can be transferred + # between machines (e.g., local -> cloud). Without this, Chrome encrypts + # cookies with OS keychain which isn't portable. + portable_profile_args = [ + "--password-store=basic", # Linux: use basic store, not gnome-keyring + "--use-mock-keychain", # macOS: use mock keychain, not real one + ] + if browser_config is None: from .async_configs import BrowserConfig browser_config = BrowserConfig( browser_type="chromium", headless=False, # Must be visible for user interaction - verbose=True + verbose=True, + extra_args=portable_profile_args, ) else: # Ensure headless is False for user interaction browser_config.headless = False + # Add portable profile args + if browser_config.extra_args: + browser_config.extra_args.extend(portable_profile_args) + else: + browser_config.extra_args = portable_profile_args # Generate profile name if not provided if not profile_name: diff --git a/crawl4ai/cli.py b/crawl4ai/cli.py index 44dbb9a1..a4dd9720 100644 --- a/crawl4ai/cli.py +++ b/crawl4ai/cli.py @@ -1378,17 +1378,104 @@ def config_set_cmd(key: str, value: str): console.print(f"[green]Successfully set[/green] [cyan]{key}[/cyan] = [green]{display_value}[/green]") -@cli.command("profiles") -def profiles_cmd(): - """Manage browser profiles interactively +@cli.group("profiles", invoke_without_command=True) +@click.pass_context +def profiles_cmd(ctx): + """Manage browser profiles for authenticated crawling Launch an interactive browser profile manager where you can: - List all existing profiles - Create new profiles for authenticated browsing - Delete unused profiles + + Subcommands: + crwl profiles create - Create a new profile + crwl profiles list - List all profiles + crwl profiles delete - Delete a profile + + Or run without subcommand for interactive menu: + crwl profiles """ - # Run interactive profile manager - anyio.run(manage_profiles) + # If no subcommand provided, run interactive manager + if ctx.invoked_subcommand is None: + anyio.run(manage_profiles) + + +@profiles_cmd.command("create") +@click.argument("name") +def profiles_create_cmd(name: str): + """Create a new browser profile + + Opens a browser window for you to log in and set up your identity. + Press 'q' in the terminal when finished to save the profile. + + Example: + crwl profiles create github-auth + """ + profiler = BrowserProfiler() + console.print(Panel(f"[bold cyan]Creating Profile: {name}[/bold cyan]\n" + "A browser window will open for you to set up your identity.\n" + "Log in to sites, adjust settings, then press 'q' to save.", + border_style="cyan")) + + async def _create(): + try: + profile_path = await profiler.create_profile(name) + if profile_path: + console.print(f"[green]Profile successfully created at:[/green] {profile_path}") + else: + console.print("[red]Failed to create profile.[/red]") + sys.exit(1) + except Exception as e: + console.print(f"[red]Error creating profile: {str(e)}[/red]") + sys.exit(1) + + anyio.run(_create) + + +@profiles_cmd.command("list") +def profiles_list_cmd(): + """List all browser profiles + + Example: + crwl profiles list + """ + profiler = BrowserProfiler() + profiles = profiler.list_profiles() + display_profiles_table(profiles) + + +@profiles_cmd.command("delete") +@click.argument("name") +@click.option("--force", "-f", is_flag=True, help="Skip confirmation") +def profiles_delete_cmd(name: str, force: bool): + """Delete a browser profile + + Example: + crwl profiles delete old-profile + crwl profiles delete old-profile --force + """ + profiler = BrowserProfiler() + + # Find profile by name + profiles = profiler.list_profiles() + profile = next((p for p in profiles if p["name"] == name), None) + + if not profile: + console.print(f"[red]Profile not found:[/red] {name}") + sys.exit(1) + + if not force: + if not Confirm.ask(f"[yellow]Delete profile '{name}'?[/yellow]"): + console.print("[cyan]Cancelled.[/cyan]") + return + + try: + profiler.delete_profile(name) + console.print(f"[green]Profile '{name}' deleted successfully.[/green]") + except Exception as e: + console.print(f"[red]Error deleting profile: {str(e)}[/red]") + sys.exit(1) @cli.command("shrink")