refactor(browser): reorganize browser strategies and improve Docker implementation
Reorganize browser strategy code into separate modules for better maintainability and separation of concerns. Improve Docker implementation with: - Add Alpine and Debian-based Dockerfiles for better container options - Enhance Docker registry to share configuration with BuiltinBrowserStrategy - Add CPU and memory limits to container configuration - Improve error handling and logging - Update documentation and examples BREAKING CHANGE: DockerConfig, DockerRegistry, and DockerUtils have been moved to new locations and their APIs have been updated.
This commit is contained in:
UncleCode
@@ -18,11 +18,20 @@ Key Features:
|
||||
|
||||
import asyncio
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import re
|
||||
import plotly.express as px
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, LXMLWebScrapingStrategy
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig,
|
||||
CacheMode,
|
||||
LXMLWebScrapingStrategy,
|
||||
)
|
||||
from crawl4ai import CrawlResult
|
||||
from typing import List
|
||||
from IPython.display import HTML
|
||||
|
||||
__current_dir__ = __file__.rsplit("/", 1)[0]
|
||||
|
||||
class CryptoAlphaGenerator:
|
||||
"""
|
||||
@@ -31,134 +40,319 @@ class CryptoAlphaGenerator:
|
||||
- Liquidity scores
|
||||
- Momentum-risk ratios
|
||||
- Machine learning-inspired trading signals
|
||||
|
||||
|
||||
Methods:
|
||||
analyze_tables(): Process raw tables into trading insights
|
||||
create_visuals(): Generate institutional-grade visualizations
|
||||
generate_insights(): Create plain English trading recommendations
|
||||
"""
|
||||
|
||||
|
||||
def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Convert crypto market data to machine-readable format
|
||||
Handles currency symbols, units (B=Billions), and percentage values
|
||||
Convert crypto market data to machine-readable format.
|
||||
Handles currency symbols, units (B=Billions), and percentage values.
|
||||
"""
|
||||
# Clean numeric columns
|
||||
df['Price'] = df['Price'].str.replace('[^\d.]', '', regex=True).astype(float)
|
||||
df['Market Cap'] = df['Market Cap'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
|
||||
df['Volume(24h)'] = df['Volume(24h)'].str.extract(r'\$([\d.]+)B')[0].astype(float) * 1e9
|
||||
# Make a copy to avoid SettingWithCopyWarning
|
||||
df = df.copy()
|
||||
|
||||
# Clean Price column (handle currency symbols)
|
||||
df["Price"] = df["Price"].astype(str).str.replace("[^\d.]", "", regex=True).astype(float)
|
||||
|
||||
# Handle Market Cap and Volume, considering both Billions and Trillions
|
||||
def convert_large_numbers(value):
|
||||
if pd.isna(value):
|
||||
return float('nan')
|
||||
value = str(value)
|
||||
multiplier = 1
|
||||
if 'B' in value:
|
||||
multiplier = 1e9
|
||||
elif 'T' in value:
|
||||
multiplier = 1e12
|
||||
# Handle cases where the value might already be numeric
|
||||
cleaned_value = re.sub(r"[^\d.]", "", value)
|
||||
return float(cleaned_value) * multiplier if cleaned_value else float('nan')
|
||||
|
||||
df["Market Cap"] = df["Market Cap"].apply(convert_large_numbers)
|
||||
df["Volume(24h)"] = df["Volume(24h)"].apply(convert_large_numbers)
|
||||
|
||||
# Convert percentages to decimal values
|
||||
for col in ['1h %', '24h %', '7d %']:
|
||||
df[col] = df[col].str.replace('%', '').astype(float) / 100
|
||||
|
||||
for col in ["1h %", "24h %", "7d %"]:
|
||||
if col in df.columns:
|
||||
# First ensure it's string, then clean
|
||||
df[col] = (
|
||||
df[col].astype(str)
|
||||
.str.replace("%", "")
|
||||
.str.replace(",", ".")
|
||||
.replace("nan", np.nan)
|
||||
)
|
||||
df[col] = pd.to_numeric(df[col], errors='coerce') / 100
|
||||
|
||||
return df
|
||||
|
||||
def calculate_metrics(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Compute advanced trading metrics used by quantitative funds:
|
||||
|
||||
|
||||
1. Volume/Market Cap Ratio - Measures liquidity efficiency
|
||||
(High ratio = Underestimated attention)
|
||||
|
||||
2. Volatility Score - Risk-adjusted momentum potential
|
||||
(High ratio = Underestimated attention, and small-cap = higher growth potential)
|
||||
|
||||
2. Volatility Score - Risk-adjusted momentum potential - Shows how stable is the trend
|
||||
(STD of 1h/24h/7d returns)
|
||||
|
||||
3. Momentum Score - Weighted average of returns
|
||||
|
||||
3. Momentum Score - Weighted average of returns - Shows how strong is the trend
|
||||
(1h:30% + 24h:50% + 7d:20%)
|
||||
|
||||
|
||||
4. Volume Anomaly - 3σ deviation detection
|
||||
(Flags potential insider activity)
|
||||
(Flags potential insider activity) - Unusual trading activity – Flags coins with volume spikes (potential insider buying or news).
|
||||
"""
|
||||
# Liquidity Metrics
|
||||
df['Volume/Market Cap Ratio'] = df['Volume(24h)'] / df['Market Cap']
|
||||
|
||||
df["Volume/Market Cap Ratio"] = df["Volume(24h)"] / df["Market Cap"]
|
||||
|
||||
# Risk Metrics
|
||||
df['Volatility Score'] = df[['1h %','24h %','7d %']].std(axis=1)
|
||||
|
||||
df["Volatility Score"] = df[["1h %", "24h %", "7d %"]].std(axis=1)
|
||||
|
||||
# Momentum Metrics
|
||||
df['Momentum Score'] = (df['1h %']*0.3 + df['24h %']*0.5 + df['7d %']*0.2)
|
||||
|
||||
df["Momentum Score"] = df["1h %"] * 0.3 + df["24h %"] * 0.5 + df["7d %"] * 0.2
|
||||
|
||||
# Anomaly Detection
|
||||
median_vol = df['Volume(24h)'].median()
|
||||
df['Volume Anomaly'] = df['Volume(24h)'] > 3 * median_vol
|
||||
|
||||
median_vol = df["Volume(24h)"].median()
|
||||
df["Volume Anomaly"] = df["Volume(24h)"] > 3 * median_vol
|
||||
|
||||
# Value Flags
|
||||
df['Undervalued Flag'] = (df['Market Cap'] < 1e9) & (df['Momentum Score'] > 0.05)
|
||||
df['Liquid Giant'] = (df['Volume/Market Cap Ratio'] > 0.15) & (df['Market Cap'] > 1e9)
|
||||
|
||||
# Undervalued Flag - Low market cap and high momentum
|
||||
# (High growth potential and low attention)
|
||||
df["Undervalued Flag"] = (df["Market Cap"] < 1e9) & (
|
||||
df["Momentum Score"] > 0.05
|
||||
)
|
||||
# Liquid Giant Flag - High volume/market cap ratio and large market cap
|
||||
# (High liquidity and large market cap = institutional interest)
|
||||
df["Liquid Giant"] = (df["Volume/Market Cap Ratio"] > 0.15) & (
|
||||
df["Market Cap"] > 1e9
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
def create_visuals(self, df: pd.DataFrame) -> dict:
|
||||
def generate_insights_simple(self, df: pd.DataFrame) -> str:
|
||||
"""
|
||||
Generate three institutional-grade visualizations:
|
||||
|
||||
1. 3D Market Map - X:Size, Y:Liquidity, Z:Momentum
|
||||
2. Liquidity Tree - Color:Volume Efficiency
|
||||
3. Momentum Leaderboard - Top sustainable movers
|
||||
Generates an ultra-actionable crypto trading report with:
|
||||
- Risk-tiered opportunities (High/Medium/Low)
|
||||
- Concrete examples for each trade type
|
||||
- Entry/exit strategies spelled out
|
||||
- Visual cues for quick scanning
|
||||
"""
|
||||
# 3D Market Overview
|
||||
fig1 = px.scatter_3d(
|
||||
df,
|
||||
x='Market Cap',
|
||||
y='Volume/Market Cap Ratio',
|
||||
z='Momentum Score',
|
||||
size='Volatility Score',
|
||||
color='Volume Anomaly',
|
||||
hover_name='Name',
|
||||
title='Smart Money Market Map: Spot Overlooked Opportunities',
|
||||
labels={'Market Cap': 'Size (Log $)', 'Volume/Market Cap Ratio': 'Liquidity Power'},
|
||||
log_x=True,
|
||||
template='plotly_dark'
|
||||
)
|
||||
|
||||
# Liquidity Efficiency Tree
|
||||
fig2 = px.treemap(
|
||||
df,
|
||||
path=['Name'],
|
||||
values='Market Cap',
|
||||
color='Volume/Market Cap Ratio',
|
||||
hover_data=['Momentum Score'],
|
||||
title='Liquidity Forest: Green = High Trading Efficiency',
|
||||
color_continuous_scale='RdYlGn'
|
||||
)
|
||||
|
||||
# Momentum Leaders
|
||||
fig3 = px.bar(
|
||||
df.sort_values('Momentum Score', ascending=False).head(10),
|
||||
x='Name',
|
||||
y='Momentum Score',
|
||||
color='Volatility Score',
|
||||
title='Sustainable Momentum Leaders (Low Volatility + High Growth)',
|
||||
text='7d %',
|
||||
template='plotly_dark'
|
||||
)
|
||||
|
||||
return {'market_map': fig1, 'liquidity_tree': fig2, 'momentum_leaders': fig3}
|
||||
report = [
|
||||
"🚀 **CRYPTO TRADING CHEAT SHEET** 🚀",
|
||||
"*Based on quantitative signals + hedge fund tactics*",
|
||||
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
]
|
||||
|
||||
# 1. HIGH-RISK: Undervalued Small-Caps (Momentum Plays)
|
||||
high_risk = df[df["Undervalued Flag"]].sort_values("Momentum Score", ascending=False)
|
||||
if not high_risk.empty:
|
||||
example_coin = high_risk.iloc[0]
|
||||
report.extend([
|
||||
"\n🔥 **HIGH-RISK: Rocket Fuel Small-Caps**",
|
||||
f"*Example Trade:* {example_coin['Name']} (Price: ${example_coin['Price']:.6f})",
|
||||
"📊 *Why?* Tiny market cap (<$1B) but STRONG momentum (+{:.0f}% last week)".format(example_coin['7d %']*100),
|
||||
"🎯 *Strategy:*",
|
||||
"1. Wait for 5-10% dip from recent high (${:.6f} → Buy under ${:.6f})".format(
|
||||
example_coin['Price'] / (1 - example_coin['24h %']), # Approx recent high
|
||||
example_coin['Price'] * 0.95
|
||||
),
|
||||
"2. Set stop-loss at -10% (${:.6f})".format(example_coin['Price'] * 0.90),
|
||||
"3. Take profit at +20% (${:.6f})".format(example_coin['Price'] * 1.20),
|
||||
"⚠️ *Risk Warning:* These can drop 30% fast! Never bet more than 5% of your portfolio.",
|
||||
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
])
|
||||
|
||||
# 2. MEDIUM-RISK: Liquid Giants (Swing Trades)
|
||||
medium_risk = df[df["Liquid Giant"]].sort_values("Volume/Market Cap Ratio", ascending=False)
|
||||
if not medium_risk.empty:
|
||||
example_coin = medium_risk.iloc[0]
|
||||
report.extend([
|
||||
"\n💎 **MEDIUM-RISK: Liquid Giants (Safe Swing Trades)**",
|
||||
f"*Example Trade:* {example_coin['Name']} (Market Cap: ${example_coin['Market Cap']/1e9:.1f}B)",
|
||||
"📊 *Why?* Huge volume (${:.1f}M/day) makes it easy to enter/exit".format(example_coin['Volume(24h)']/1e6),
|
||||
"🎯 *Strategy:*",
|
||||
"1. Buy when 24h volume > 15% of market cap (Current: {:.0f}%)".format(example_coin['Volume/Market Cap Ratio']*100),
|
||||
"2. Hold 1-4 weeks (Big coins trend longer)",
|
||||
"3. Exit when momentum drops below 5% (Current: {:.0f}%)".format(example_coin['Momentum Score']*100),
|
||||
"📉 *Pro Tip:* Watch Bitcoin's trend - if BTC drops 5%, these usually follow.",
|
||||
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
])
|
||||
|
||||
# 3. LOW-RISK: Stable Momentum (DCA Targets)
|
||||
low_risk = df[
|
||||
(df["Momentum Score"] > 0.05) &
|
||||
(df["Volatility Score"] < 0.03)
|
||||
].sort_values("Market Cap", ascending=False)
|
||||
if not low_risk.empty:
|
||||
example_coin = low_risk.iloc[0]
|
||||
report.extend([
|
||||
"\n🛡️ **LOW-RISK: Steady Climbers (DCA & Forget)**",
|
||||
f"*Example Trade:* {example_coin['Name']} (Volatility: {example_coin['Volatility Score']:.2f}/5)",
|
||||
"📊 *Why?* Rises steadily (+{:.0f}%/week) with LOW drama".format(example_coin['7d %']*100),
|
||||
"🎯 *Strategy:*",
|
||||
"1. Buy small amounts every Tuesday/Friday (DCA)",
|
||||
"2. Hold for 3+ months (Compound gains work best here)",
|
||||
"3. Sell 10% at every +25% milestone",
|
||||
"💰 *Best For:* Long-term investors who hate sleepless nights",
|
||||
"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
])
|
||||
|
||||
# Volume Spike Alerts
|
||||
anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False)
|
||||
if not anomalies.empty:
|
||||
example_coin = anomalies.iloc[0]
|
||||
report.extend([
|
||||
"\n🚨 **Volume Spike Alert (Possible News/Whale Action)**",
|
||||
f"*Coin:* {example_coin['Name']} (Volume: ${example_coin['Volume(24h)']/1e6:.1f}M, usual: ${example_coin['Volume(24h)']/3/1e6:.1f}M)",
|
||||
"🔍 *Check:* Twitter/CoinGecko for news before trading",
|
||||
"⚡ *If no news:* Could be insider buying - watch price action:",
|
||||
"- Break above today's high → Buy with tight stop-loss",
|
||||
"- Fade back down → Avoid (may be a fakeout)"
|
||||
])
|
||||
|
||||
# Pro Tip Footer
|
||||
report.append("\n✨ *Pro Tip:* Bookmark this report & check back in 24h to see if signals held up.")
|
||||
|
||||
return "\n".join(report)
|
||||
|
||||
def generate_insights(self, df: pd.DataFrame) -> str:
|
||||
"""
|
||||
Create plain English trading insights explaining:
|
||||
- Volume spikes and their implications
|
||||
- Risk-reward ratios of top movers
|
||||
- Liquidity warnings for large positions
|
||||
Generates a tactical trading report with:
|
||||
- Top 3 trades per risk level (High/Medium/Low)
|
||||
- Auto-calculated entry/exit prices
|
||||
- BTC chart toggle tip
|
||||
"""
|
||||
top_coin = df.sort_values('Momentum Score', ascending=False).iloc[0]
|
||||
anomaly_coins = df[df['Volume Anomaly']].sort_values('Volume(24h)', ascending=False)
|
||||
# Filter top candidates for each risk level
|
||||
high_risk = (
|
||||
df[df["Undervalued Flag"]]
|
||||
.sort_values("Momentum Score", ascending=False)
|
||||
.head(3)
|
||||
)
|
||||
medium_risk = (
|
||||
df[df["Liquid Giant"]]
|
||||
.sort_values("Volume/Market Cap Ratio", ascending=False)
|
||||
.head(3)
|
||||
)
|
||||
low_risk = (
|
||||
df[(df["Momentum Score"] > 0.05) & (df["Volatility Score"] < 0.03)]
|
||||
.sort_values("Momentum Score", ascending=False)
|
||||
.head(3)
|
||||
)
|
||||
|
||||
report = ["# 🎯 Crypto Trading Tactical Report (Top 3 Per Risk Tier)"]
|
||||
|
||||
report = f"""
|
||||
🚀 Top Alpha Opportunity: {top_coin['Name']}
|
||||
- Momentum Score: {top_coin['Momentum Score']:.2%} (Top 1%)
|
||||
- Risk-Reward Ratio: {top_coin['Momentum Score']/top_coin['Volatility Score']:.1f}
|
||||
- Liquidity Warning: {'✅ Safe' if top_coin['Liquid Giant'] else '⚠️ Thin Markets'}
|
||||
# 1. High-Risk Trades (Small-Cap Momentum)
|
||||
if not high_risk.empty:
|
||||
report.append("\n## 🔥 HIGH RISK: Small-Cap Rockets (5-50% Potential)")
|
||||
for i, coin in high_risk.iterrows():
|
||||
current_price = coin["Price"]
|
||||
entry = current_price * 0.95 # -5% dip
|
||||
stop_loss = current_price * 0.90 # -10%
|
||||
take_profit = current_price * 1.20 # +20%
|
||||
|
||||
report.append(
|
||||
f"\n### {coin['Name']} (Momentum: {coin['Momentum Score']:.1%})"
|
||||
f"\n- **Current Price:** ${current_price:.4f}"
|
||||
f"\n- **Entry:** < ${entry:.4f} (Wait for pullback)"
|
||||
f"\n- **Stop-Loss:** ${stop_loss:.4f} (-10%)"
|
||||
f"\n- **Target:** ${take_profit:.4f} (+20%)"
|
||||
f"\n- **Risk/Reward:** 1:2"
|
||||
f"\n- **Watch:** Volume spikes above {coin['Volume(24h)']/1e6:.1f}M"
|
||||
)
|
||||
|
||||
# 2. Medium-Risk Trades (Liquid Giants)
|
||||
if not medium_risk.empty:
|
||||
report.append("\n## 💎 MEDIUM RISK: Liquid Swing Trades (10-30% Potential)")
|
||||
for i, coin in medium_risk.iterrows():
|
||||
current_price = coin["Price"]
|
||||
entry = current_price * 0.98 # -2% dip
|
||||
stop_loss = current_price * 0.94 # -6%
|
||||
take_profit = current_price * 1.15 # +15%
|
||||
|
||||
report.append(
|
||||
f"\n### {coin['Name']} (Liquidity Score: {coin['Volume/Market Cap Ratio']:.1%})"
|
||||
f"\n- **Current Price:** ${current_price:.2f}"
|
||||
f"\n- **Entry:** < ${entry:.2f} (Buy slight dips)"
|
||||
f"\n- **Stop-Loss:** ${stop_loss:.2f} (-6%)"
|
||||
f"\n- **Target:** ${take_profit:.2f} (+15%)"
|
||||
f"\n- **Hold Time:** 1-3 weeks"
|
||||
f"\n- **Key Metric:** Volume/Cap > 15%"
|
||||
)
|
||||
|
||||
# 3. Low-Risk Trades (Stable Momentum)
|
||||
if not low_risk.empty:
|
||||
report.append("\n## 🛡️ LOW RISK: Steady Gainers (5-15% Potential)")
|
||||
for i, coin in low_risk.iterrows():
|
||||
current_price = coin["Price"]
|
||||
entry = current_price * 0.99 # -1% dip
|
||||
stop_loss = current_price * 0.97 # -3%
|
||||
take_profit = current_price * 1.10 # +10%
|
||||
|
||||
report.append(
|
||||
f"\n### {coin['Name']} (Stability Score: {1/coin['Volatility Score']:.1f}x)"
|
||||
f"\n- **Current Price:** ${current_price:.2f}"
|
||||
f"\n- **Entry:** < ${entry:.2f} (Safe zone)"
|
||||
f"\n- **Stop-Loss:** ${stop_loss:.2f} (-3%)"
|
||||
f"\n- **Target:** ${take_profit:.2f} (+10%)"
|
||||
f"\n- **DCA Suggestion:** 3 buys over 72 hours"
|
||||
)
|
||||
|
||||
# Volume Anomaly Alert
|
||||
anomalies = df[df["Volume Anomaly"]].sort_values("Volume(24h)", ascending=False).head(2)
|
||||
if not anomalies.empty:
|
||||
report.append("\n⚠️ **Volume Spike Alerts**")
|
||||
for i, coin in anomalies.iterrows():
|
||||
report.append(
|
||||
f"- {coin['Name']}: Volume {coin['Volume(24h)']/1e6:.1f}M "
|
||||
f"(3x normal) | Price moved: {coin['24h %']:.1%}"
|
||||
)
|
||||
|
||||
# Pro Tip
|
||||
report.append(
|
||||
"\n📊 **Chart Hack:** Hide BTC in visuals:\n"
|
||||
"```python\n"
|
||||
"# For 3D Map:\n"
|
||||
"fig.update_traces(visible=False, selector={'name':'Bitcoin'})\n"
|
||||
"# For Treemap:\n"
|
||||
"df = df[df['Name'] != 'Bitcoin']\n"
|
||||
"```"
|
||||
)
|
||||
|
||||
return "\n".join(report)
|
||||
|
||||
def create_visuals(self, df: pd.DataFrame) -> dict:
|
||||
"""Enhanced visuals with BTC toggle support"""
|
||||
# 3D Market Map (with BTC toggle hint)
|
||||
fig1 = px.scatter_3d(
|
||||
df,
|
||||
x="Market Cap",
|
||||
y="Volume/Market Cap Ratio",
|
||||
z="Momentum Score",
|
||||
color="Name", # Color by name to allow toggling
|
||||
hover_name="Name",
|
||||
title="Market Map (Toggle BTC in legend to focus on alts)",
|
||||
log_x=True
|
||||
)
|
||||
fig1.update_traces(
|
||||
marker=dict(size=df["Volatility Score"]*100 + 5) # Dynamic sizing
|
||||
)
|
||||
|
||||
🔥 Volume Spikes Detected ({len(anomaly_coins)} coins):
|
||||
{anomaly_coins[['Name', 'Volume(24h)']].head(3).to_markdown(index=False)}
|
||||
# Liquidity Tree (exclude BTC if too dominant)
|
||||
if df[df["Name"] == "BitcoinBTC"]["Market Cap"].values[0] > df["Market Cap"].median() * 10:
|
||||
df = df[df["Name"] != "BitcoinBTC"]
|
||||
|
||||
💡 Smart Money Tip: Coins with Volume/Cap > 15% and Momentum > 5%
|
||||
historically outperform by 22% weekly returns.
|
||||
"""
|
||||
return report
|
||||
fig2 = px.treemap(
|
||||
df,
|
||||
path=["Name"],
|
||||
values="Market Cap",
|
||||
color="Volume/Market Cap Ratio",
|
||||
title="Liquidity Tree (BTC auto-removed if dominant)"
|
||||
)
|
||||
|
||||
return {"market_map": fig1, "liquidity_tree": fig2}
|
||||
|
||||
async def main():
|
||||
"""
|
||||
@@ -171,60 +365,79 @@ async def main():
|
||||
"""
|
||||
# Configure browser with anti-detection features
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
stealth=True,
|
||||
block_resources=["image", "media"]
|
||||
headless=False,
|
||||
)
|
||||
|
||||
|
||||
# Initialize crawler with smart table detection
|
||||
crawler = AsyncWebCrawler(config=browser_config)
|
||||
await crawler.start()
|
||||
|
||||
|
||||
try:
|
||||
# Set up scraping parameters
|
||||
crawl_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
scraping_strategy=LXMLWebScrapingStrategy(
|
||||
table_score_threshold=8, # Strict table detection
|
||||
keep_data_attributes=True
|
||||
)
|
||||
table_score_threshold=8, # Strict table detection
|
||||
keep_data_attributes=True,
|
||||
scraping_strategy=LXMLWebScrapingStrategy(),
|
||||
scan_full_page=True,
|
||||
scroll_delay=0.2,
|
||||
)
|
||||
|
||||
# Execute market data extraction
|
||||
results: List[CrawlResult] = await crawler.arun(
|
||||
url='https://coinmarketcap.com/?page=1',
|
||||
config=crawl_config
|
||||
)
|
||||
|
||||
# Process results
|
||||
for result in results:
|
||||
if result.success and result.media['tables']:
|
||||
# Extract primary market table
|
||||
raw_df = pd.DataFrame(
|
||||
result.media['tables'][0]['rows'],
|
||||
columns=result.media['tables'][0]['headers']
|
||||
)
|
||||
|
||||
# Initialize analysis engine
|
||||
analyzer = CryptoAlphaGenerator()
|
||||
clean_df = analyzer.clean_data(raw_df)
|
||||
analyzed_df = analyzer.calculate_metrics(clean_df)
|
||||
|
||||
# Generate outputs
|
||||
visuals = analyzer.create_visuals(analyzed_df)
|
||||
insights = analyzer.generate_insights(analyzed_df)
|
||||
|
||||
# Save visualizations
|
||||
visuals['market_map'].write_html("market_map.html")
|
||||
visuals['liquidity_tree'].write_html("liquidity_tree.html")
|
||||
|
||||
# Display results
|
||||
print("🔑 Key Trading Insights:")
|
||||
print(insights)
|
||||
print("\n📊 Open 'market_map.html' for interactive analysis")
|
||||
|
||||
# # Execute market data extraction
|
||||
# results: List[CrawlResult] = await crawler.arun(
|
||||
# url="https://coinmarketcap.com/?page=1", config=crawl_config
|
||||
# )
|
||||
|
||||
# # Process results
|
||||
# raw_df = pd.DataFrame()
|
||||
# for result in results:
|
||||
# if result.success and result.media["tables"]:
|
||||
# # Extract primary market table
|
||||
# # DataFrame
|
||||
# raw_df = pd.DataFrame(
|
||||
# result.media["tables"][0]["rows"],
|
||||
# columns=result.media["tables"][0]["headers"],
|
||||
# )
|
||||
# break
|
||||
|
||||
|
||||
# This is for debugging only
|
||||
# ////// Remove this in production from here..
|
||||
# Save raw data for debugging
|
||||
# raw_df.to_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv", index=False)
|
||||
# print("🔍 Raw data saved to 'raw_crypto_data.csv'")
|
||||
|
||||
# Read from file for debugging
|
||||
raw_df = pd.read_csv(f"{__current_dir__}/tmp/raw_crypto_data.csv")
|
||||
# ////// ..to here
|
||||
|
||||
# Select top 20
|
||||
raw_df = raw_df.head(50)
|
||||
# Remove "Buy" from name
|
||||
raw_df["Name"] = raw_df["Name"].str.replace("Buy", "")
|
||||
|
||||
# Initialize analysis engine
|
||||
analyzer = CryptoAlphaGenerator()
|
||||
clean_df = analyzer.clean_data(raw_df)
|
||||
analyzed_df = analyzer.calculate_metrics(clean_df)
|
||||
|
||||
# Generate outputs
|
||||
visuals = analyzer.create_visuals(analyzed_df)
|
||||
insights = analyzer.generate_insights(analyzed_df)
|
||||
|
||||
# Save visualizations
|
||||
visuals["market_map"].write_html(f"{__current_dir__}/tmp/market_map.html")
|
||||
visuals["liquidity_tree"].write_html(f"{__current_dir__}/tmp/liquidity_tree.html")
|
||||
|
||||
# Display results
|
||||
print("🔑 Key Trading Insights:")
|
||||
print(insights)
|
||||
print("\n📊 Open 'market_map.html' for interactive analysis")
|
||||
print("\n📊 Open 'liquidity_tree.html' for interactive analysis")
|
||||
|
||||
finally:
|
||||
await crawler.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
asyncio.run(main())
|
||||
|
||||
Reference in New Issue
Block a user