Fix: Disable cookie encryption for portable profiles
- Add --password-store=basic and --use-mock-keychain flags when creating profiles to prevent OS keychain encryption of cookies - Without this, cookies are encrypted with machine-specific keys and profiles can't be transferred between machines (local -> cloud) Also adds direct CLI commands for profile management: - crwl profiles create <name> - crwl profiles list - crwl profiles delete <name> The interactive menu (crwl profiles) still works as before.
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -297,3 +297,6 @@ scripts/
|
|||||||
*.rdb
|
*.rdb
|
||||||
*.ldb
|
*.ldb
|
||||||
MEMORY.md
|
MEMORY.md
|
||||||
|
|
||||||
|
# Handoff files
|
||||||
|
HANDOFF-*.md
|
||||||
|
|||||||
@@ -421,16 +421,30 @@ class BrowserProfiler:
|
|||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
# Create default browser config if none provided
|
# Create default browser config if none provided
|
||||||
|
# IMPORTANT: We disable cookie encryption so profiles can be transferred
|
||||||
|
# between machines (e.g., local -> cloud). Without this, Chrome encrypts
|
||||||
|
# cookies with OS keychain which isn't portable.
|
||||||
|
portable_profile_args = [
|
||||||
|
"--password-store=basic", # Linux: use basic store, not gnome-keyring
|
||||||
|
"--use-mock-keychain", # macOS: use mock keychain, not real one
|
||||||
|
]
|
||||||
|
|
||||||
if browser_config is None:
|
if browser_config is None:
|
||||||
from .async_configs import BrowserConfig
|
from .async_configs import BrowserConfig
|
||||||
browser_config = BrowserConfig(
|
browser_config = BrowserConfig(
|
||||||
browser_type="chromium",
|
browser_type="chromium",
|
||||||
headless=False, # Must be visible for user interaction
|
headless=False, # Must be visible for user interaction
|
||||||
verbose=True
|
verbose=True,
|
||||||
|
extra_args=portable_profile_args,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# Ensure headless is False for user interaction
|
# Ensure headless is False for user interaction
|
||||||
browser_config.headless = False
|
browser_config.headless = False
|
||||||
|
# Add portable profile args
|
||||||
|
if browser_config.extra_args:
|
||||||
|
browser_config.extra_args.extend(portable_profile_args)
|
||||||
|
else:
|
||||||
|
browser_config.extra_args = portable_profile_args
|
||||||
|
|
||||||
# Generate profile name if not provided
|
# Generate profile name if not provided
|
||||||
if not profile_name:
|
if not profile_name:
|
||||||
|
|||||||
@@ -1378,17 +1378,104 @@ def config_set_cmd(key: str, value: str):
|
|||||||
|
|
||||||
console.print(f"[green]Successfully set[/green] [cyan]{key}[/cyan] = [green]{display_value}[/green]")
|
console.print(f"[green]Successfully set[/green] [cyan]{key}[/cyan] = [green]{display_value}[/green]")
|
||||||
|
|
||||||
@cli.command("profiles")
|
@cli.group("profiles", invoke_without_command=True)
|
||||||
def profiles_cmd():
|
@click.pass_context
|
||||||
"""Manage browser profiles interactively
|
def profiles_cmd(ctx):
|
||||||
|
"""Manage browser profiles for authenticated crawling
|
||||||
|
|
||||||
Launch an interactive browser profile manager where you can:
|
Launch an interactive browser profile manager where you can:
|
||||||
- List all existing profiles
|
- List all existing profiles
|
||||||
- Create new profiles for authenticated browsing
|
- Create new profiles for authenticated browsing
|
||||||
- Delete unused profiles
|
- Delete unused profiles
|
||||||
|
|
||||||
|
Subcommands:
|
||||||
|
crwl profiles create <name> - Create a new profile
|
||||||
|
crwl profiles list - List all profiles
|
||||||
|
crwl profiles delete <name> - Delete a profile
|
||||||
|
|
||||||
|
Or run without subcommand for interactive menu:
|
||||||
|
crwl profiles
|
||||||
"""
|
"""
|
||||||
# Run interactive profile manager
|
# If no subcommand provided, run interactive manager
|
||||||
anyio.run(manage_profiles)
|
if ctx.invoked_subcommand is None:
|
||||||
|
anyio.run(manage_profiles)
|
||||||
|
|
||||||
|
|
||||||
|
@profiles_cmd.command("create")
|
||||||
|
@click.argument("name")
|
||||||
|
def profiles_create_cmd(name: str):
|
||||||
|
"""Create a new browser profile
|
||||||
|
|
||||||
|
Opens a browser window for you to log in and set up your identity.
|
||||||
|
Press 'q' in the terminal when finished to save the profile.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
crwl profiles create github-auth
|
||||||
|
"""
|
||||||
|
profiler = BrowserProfiler()
|
||||||
|
console.print(Panel(f"[bold cyan]Creating Profile: {name}[/bold cyan]\n"
|
||||||
|
"A browser window will open for you to set up your identity.\n"
|
||||||
|
"Log in to sites, adjust settings, then press 'q' to save.",
|
||||||
|
border_style="cyan"))
|
||||||
|
|
||||||
|
async def _create():
|
||||||
|
try:
|
||||||
|
profile_path = await profiler.create_profile(name)
|
||||||
|
if profile_path:
|
||||||
|
console.print(f"[green]Profile successfully created at:[/green] {profile_path}")
|
||||||
|
else:
|
||||||
|
console.print("[red]Failed to create profile.[/red]")
|
||||||
|
sys.exit(1)
|
||||||
|
except Exception as e:
|
||||||
|
console.print(f"[red]Error creating profile: {str(e)}[/red]")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
anyio.run(_create)
|
||||||
|
|
||||||
|
|
||||||
|
@profiles_cmd.command("list")
|
||||||
|
def profiles_list_cmd():
|
||||||
|
"""List all browser profiles
|
||||||
|
|
||||||
|
Example:
|
||||||
|
crwl profiles list
|
||||||
|
"""
|
||||||
|
profiler = BrowserProfiler()
|
||||||
|
profiles = profiler.list_profiles()
|
||||||
|
display_profiles_table(profiles)
|
||||||
|
|
||||||
|
|
||||||
|
@profiles_cmd.command("delete")
|
||||||
|
@click.argument("name")
|
||||||
|
@click.option("--force", "-f", is_flag=True, help="Skip confirmation")
|
||||||
|
def profiles_delete_cmd(name: str, force: bool):
|
||||||
|
"""Delete a browser profile
|
||||||
|
|
||||||
|
Example:
|
||||||
|
crwl profiles delete old-profile
|
||||||
|
crwl profiles delete old-profile --force
|
||||||
|
"""
|
||||||
|
profiler = BrowserProfiler()
|
||||||
|
|
||||||
|
# Find profile by name
|
||||||
|
profiles = profiler.list_profiles()
|
||||||
|
profile = next((p for p in profiles if p["name"] == name), None)
|
||||||
|
|
||||||
|
if not profile:
|
||||||
|
console.print(f"[red]Profile not found:[/red] {name}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if not force:
|
||||||
|
if not Confirm.ask(f"[yellow]Delete profile '{name}'?[/yellow]"):
|
||||||
|
console.print("[cyan]Cancelled.[/cyan]")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
profiler.delete_profile(name)
|
||||||
|
console.print(f"[green]Profile '{name}' deleted successfully.[/green]")
|
||||||
|
except Exception as e:
|
||||||
|
console.print(f"[red]Error deleting profile: {str(e)}[/red]")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
@cli.command("shrink")
|
@cli.command("shrink")
|
||||||
|
|||||||
Reference in New Issue
Block a user