feat(marketplace): Add Crawl4AI marketplace with secure configuration
- Implement marketplace frontend and admin dashboard - Add FastAPI backend with environment-based configuration - Use .env file for secrets management - Include data generation scripts - Add proper CORS configuration - Remove hardcoded password from admin login - Update gitignore for security
This commit is contained in:
unclecode
14
docs/md_v2/marketplace/backend/.env.example
Normal file
14
docs/md_v2/marketplace/backend/.env.example
Normal file
@@ -0,0 +1,14 @@
|
||||
# Marketplace Configuration
|
||||
# Copy this to .env and update with your values
|
||||
|
||||
# Admin password (required)
|
||||
MARKETPLACE_ADMIN_PASSWORD=change_this_password
|
||||
|
||||
# JWT secret key (required) - generate with: python3 -c "import secrets; print(secrets.token_urlsafe(32))"
|
||||
MARKETPLACE_JWT_SECRET=change_this_to_a_secure_random_key
|
||||
|
||||
# Database path (optional, defaults to ./marketplace.db)
|
||||
MARKETPLACE_DB_PATH=./marketplace.db
|
||||
|
||||
# Token expiry in hours (optional, defaults to 4)
|
||||
MARKETPLACE_TOKEN_EXPIRY=4
|
||||
59
docs/md_v2/marketplace/backend/config.py
Normal file
59
docs/md_v2/marketplace/backend/config.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Marketplace Configuration - Loads from .env file
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load .env file
|
||||
env_path = Path(__file__).parent / '.env'
|
||||
if not env_path.exists():
|
||||
print("\n❌ ERROR: No .env file found!")
|
||||
print("Please copy .env.example to .env and update with your values:")
|
||||
print(f" cp {Path(__file__).parent}/.env.example {Path(__file__).parent}/.env")
|
||||
print("\nThen edit .env with your secure values.")
|
||||
sys.exit(1)
|
||||
|
||||
load_dotenv(env_path)
|
||||
|
||||
# Required environment variables
|
||||
required_vars = ['MARKETPLACE_ADMIN_PASSWORD', 'MARKETPLACE_JWT_SECRET']
|
||||
missing_vars = [var for var in required_vars if not os.getenv(var)]
|
||||
|
||||
if missing_vars:
|
||||
print(f"\n❌ ERROR: Missing required environment variables: {', '.join(missing_vars)}")
|
||||
print("Please check your .env file and ensure all required variables are set.")
|
||||
sys.exit(1)
|
||||
|
||||
class Config:
|
||||
"""Configuration loaded from environment variables"""
|
||||
|
||||
# Admin authentication - hashed from password in .env
|
||||
ADMIN_PASSWORD_HASH = hashlib.sha256(
|
||||
os.getenv('MARKETPLACE_ADMIN_PASSWORD').encode()
|
||||
).hexdigest()
|
||||
|
||||
# JWT secret for token generation
|
||||
JWT_SECRET_KEY = os.getenv('MARKETPLACE_JWT_SECRET')
|
||||
|
||||
# Database path
|
||||
DATABASE_PATH = os.getenv('MARKETPLACE_DB_PATH', './marketplace.db')
|
||||
|
||||
# Token expiry in hours
|
||||
TOKEN_EXPIRY_HOURS = int(os.getenv('MARKETPLACE_TOKEN_EXPIRY', '4'))
|
||||
|
||||
# CORS origins - hardcoded as they don't contain secrets
|
||||
ALLOWED_ORIGINS = [
|
||||
"http://localhost:8000",
|
||||
"http://localhost:8080",
|
||||
"http://localhost:8100",
|
||||
"http://127.0.0.1:8000",
|
||||
"http://127.0.0.1:8080",
|
||||
"http://127.0.0.1:8100",
|
||||
"https://crawl4ai.com",
|
||||
"https://www.crawl4ai.com",
|
||||
"https://docs.crawl4ai.com",
|
||||
"https://market.crawl4ai.com"
|
||||
]
|
||||
117
docs/md_v2/marketplace/backend/database.py
Normal file
117
docs/md_v2/marketplace/backend/database.py
Normal file
@@ -0,0 +1,117 @@
|
||||
import sqlite3
|
||||
import yaml
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any
|
||||
|
||||
class DatabaseManager:
|
||||
def __init__(self, db_path=None, schema_path='schema.yaml'):
|
||||
self.schema = self._load_schema(schema_path)
|
||||
# Use provided path or fallback to schema default
|
||||
self.db_path = db_path or self.schema['database']['name']
|
||||
self.conn = None
|
||||
self._init_database()
|
||||
|
||||
def _load_schema(self, path: str) -> Dict:
|
||||
with open(path, 'r') as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
def _init_database(self):
|
||||
"""Auto-create/migrate database from schema"""
|
||||
self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
|
||||
self.conn.row_factory = sqlite3.Row
|
||||
|
||||
for table_name, table_def in self.schema['tables'].items():
|
||||
self._create_or_update_table(table_name, table_def['columns'])
|
||||
|
||||
def _create_or_update_table(self, table_name: str, columns: Dict):
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
# Check if table exists
|
||||
cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table_name,))
|
||||
table_exists = cursor.fetchone() is not None
|
||||
|
||||
if not table_exists:
|
||||
# Create table
|
||||
col_defs = []
|
||||
for col_name, col_spec in columns.items():
|
||||
col_def = f"{col_name} {col_spec['type']}"
|
||||
if col_spec.get('primary'):
|
||||
col_def += " PRIMARY KEY"
|
||||
if col_spec.get('autoincrement'):
|
||||
col_def += " AUTOINCREMENT"
|
||||
if col_spec.get('unique'):
|
||||
col_def += " UNIQUE"
|
||||
if col_spec.get('required'):
|
||||
col_def += " NOT NULL"
|
||||
if 'default' in col_spec:
|
||||
default = col_spec['default']
|
||||
if default == 'CURRENT_TIMESTAMP':
|
||||
col_def += f" DEFAULT {default}"
|
||||
elif isinstance(default, str):
|
||||
col_def += f" DEFAULT '{default}'"
|
||||
else:
|
||||
col_def += f" DEFAULT {default}"
|
||||
col_defs.append(col_def)
|
||||
|
||||
create_sql = f"CREATE TABLE {table_name} ({', '.join(col_defs)})"
|
||||
cursor.execute(create_sql)
|
||||
else:
|
||||
# Check for new columns and add them
|
||||
cursor.execute(f"PRAGMA table_info({table_name})")
|
||||
existing_columns = {row[1] for row in cursor.fetchall()}
|
||||
|
||||
for col_name, col_spec in columns.items():
|
||||
if col_name not in existing_columns:
|
||||
col_def = f"{col_spec['type']}"
|
||||
if 'default' in col_spec:
|
||||
default = col_spec['default']
|
||||
if default == 'CURRENT_TIMESTAMP':
|
||||
col_def += f" DEFAULT {default}"
|
||||
elif isinstance(default, str):
|
||||
col_def += f" DEFAULT '{default}'"
|
||||
else:
|
||||
col_def += f" DEFAULT {default}"
|
||||
|
||||
cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN {col_name} {col_def}")
|
||||
|
||||
self.conn.commit()
|
||||
|
||||
def get_all(self, table: str, limit: int = 100, offset: int = 0, where: str = None) -> List[Dict]:
|
||||
cursor = self.conn.cursor()
|
||||
query = f"SELECT * FROM {table}"
|
||||
if where:
|
||||
query += f" WHERE {where}"
|
||||
query += f" LIMIT {limit} OFFSET {offset}"
|
||||
|
||||
cursor.execute(query)
|
||||
rows = cursor.fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def search(self, query: str, tables: List[str] = None) -> Dict[str, List[Dict]]:
|
||||
if not tables:
|
||||
tables = list(self.schema['tables'].keys())
|
||||
|
||||
results = {}
|
||||
cursor = self.conn.cursor()
|
||||
|
||||
for table in tables:
|
||||
# Search in text columns
|
||||
columns = self.schema['tables'][table]['columns']
|
||||
text_cols = [col for col, spec in columns.items()
|
||||
if spec['type'] == 'TEXT' and col != 'id']
|
||||
|
||||
if text_cols:
|
||||
where_clause = ' OR '.join([f"{col} LIKE ?" for col in text_cols])
|
||||
params = [f'%{query}%'] * len(text_cols)
|
||||
|
||||
cursor.execute(f"SELECT * FROM {table} WHERE {where_clause} LIMIT 10", params)
|
||||
rows = cursor.fetchall()
|
||||
if rows:
|
||||
results[table] = [dict(row) for row in rows]
|
||||
|
||||
return results
|
||||
|
||||
def close(self):
|
||||
if self.conn:
|
||||
self.conn.close()
|
||||
267
docs/md_v2/marketplace/backend/dummy_data.py
Normal file
267
docs/md_v2/marketplace/backend/dummy_data.py
Normal file
@@ -0,0 +1,267 @@
|
||||
import sqlite3
|
||||
import json
|
||||
import random
|
||||
from datetime import datetime, timedelta
|
||||
from database import DatabaseManager
|
||||
|
||||
def generate_slug(text):
|
||||
return text.lower().replace(' ', '-').replace('&', 'and')
|
||||
|
||||
def generate_dummy_data():
|
||||
db = DatabaseManager()
|
||||
conn = db.conn
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Clear existing data
|
||||
for table in ['apps', 'articles', 'categories', 'sponsors']:
|
||||
cursor.execute(f"DELETE FROM {table}")
|
||||
|
||||
# Categories
|
||||
categories = [
|
||||
("Browser Automation", "⚙", "Tools for browser automation and control"),
|
||||
("Proxy Services", "🔒", "Proxy providers and rotation services"),
|
||||
("LLM Integration", "🤖", "AI/LLM tools and integrations"),
|
||||
("Data Processing", "📊", "Data extraction and processing tools"),
|
||||
("Cloud Infrastructure", "☁", "Cloud browser and computing services"),
|
||||
("Developer Tools", "🛠", "Development and testing utilities")
|
||||
]
|
||||
|
||||
for i, (name, icon, desc) in enumerate(categories):
|
||||
cursor.execute("""
|
||||
INSERT INTO categories (name, slug, icon, description, order_index)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (name, generate_slug(name), icon, desc, i))
|
||||
|
||||
# Apps with real Unsplash images
|
||||
apps_data = [
|
||||
# Browser Automation
|
||||
("Playwright Cloud", "Browser Automation", "Paid", True, True,
|
||||
"Scalable browser automation in the cloud with Playwright", "https://playwright.cloud",
|
||||
None, "$99/month starter", 4.8, 12500,
|
||||
"https://images.unsplash.com/photo-1633356122544-f134324a6cee?w=800&h=400&fit=crop"),
|
||||
|
||||
("Selenium Grid Hub", "Browser Automation", "Freemium", False, False,
|
||||
"Distributed Selenium grid for parallel testing", "https://seleniumhub.io",
|
||||
"https://github.com/seleniumhub/grid", "Free - $299/month", 4.2, 8400,
|
||||
"https://images.unsplash.com/photo-1555066931-4365d14bab8c?w=800&h=400&fit=crop"),
|
||||
|
||||
("Puppeteer Extra", "Browser Automation", "Open Source", True, False,
|
||||
"Enhanced Puppeteer with stealth plugins and more", "https://puppeteer-extra.dev",
|
||||
"https://github.com/berstend/puppeteer-extra", "Free", 4.6, 15200,
|
||||
"https://images.unsplash.com/photo-1461749280684-dccba630e2f6?w=800&h=400&fit=crop"),
|
||||
|
||||
# Proxy Services
|
||||
("BrightData", "Proxy Services", "Paid", True, True,
|
||||
"Premium proxy network with 72M+ IPs worldwide", "https://brightdata.com",
|
||||
None, "Starting $500/month", 4.7, 9800,
|
||||
"https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=800&h=400&fit=crop"),
|
||||
|
||||
("SmartProxy", "Proxy Services", "Paid", False, True,
|
||||
"Residential and datacenter proxies with rotation", "https://smartproxy.com",
|
||||
None, "Starting $75/month", 4.3, 7600,
|
||||
"https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=800&h=400&fit=crop"),
|
||||
|
||||
("ProxyMesh", "Proxy Services", "Freemium", False, False,
|
||||
"Rotating proxy servers with sticky sessions", "https://proxymesh.com",
|
||||
None, "$10-$50/month", 4.0, 4200,
|
||||
"https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"),
|
||||
|
||||
# LLM Integration
|
||||
("LangChain Crawl", "LLM Integration", "Open Source", True, False,
|
||||
"LangChain integration for Crawl4AI workflows", "https://langchain-crawl.dev",
|
||||
"https://github.com/langchain/crawl", "Free", 4.5, 18900,
|
||||
"https://images.unsplash.com/photo-1677442136019-21780ecad995?w=800&h=400&fit=crop"),
|
||||
|
||||
("GPT Scraper", "LLM Integration", "Freemium", False, False,
|
||||
"Extract structured data using GPT models", "https://gptscraper.ai",
|
||||
None, "Free - $99/month", 4.1, 5600,
|
||||
"https://images.unsplash.com/photo-1655720828018-edd2daec9349?w=800&h=400&fit=crop"),
|
||||
|
||||
("Claude Extract", "LLM Integration", "Paid", True, True,
|
||||
"Professional extraction using Claude AI", "https://claude-extract.com",
|
||||
None, "$199/month", 4.9, 3200,
|
||||
"https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=800&h=400&fit=crop"),
|
||||
|
||||
# Data Processing
|
||||
("DataMiner Pro", "Data Processing", "Paid", False, False,
|
||||
"Advanced data extraction and transformation", "https://dataminer.pro",
|
||||
None, "$149/month", 4.2, 6700,
|
||||
"https://images.unsplash.com/photo-1551288049-bebda4e38f71?w=800&h=400&fit=crop"),
|
||||
|
||||
("ScraperAPI", "Data Processing", "Freemium", True, True,
|
||||
"Simple API for web scraping with proxy rotation", "https://scraperapi.com",
|
||||
None, "Free - $299/month", 4.6, 22300,
|
||||
"https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=800&h=400&fit=crop"),
|
||||
|
||||
("Apify", "Data Processing", "Freemium", False, False,
|
||||
"Web scraping and automation platform", "https://apify.com",
|
||||
None, "$49-$499/month", 4.4, 14500,
|
||||
"https://images.unsplash.com/photo-1504639725590-34d0984388bd?w=800&h=400&fit=crop"),
|
||||
|
||||
# Cloud Infrastructure
|
||||
("BrowserCloud", "Cloud Infrastructure", "Paid", True, True,
|
||||
"Managed headless browsers in the cloud", "https://browsercloud.io",
|
||||
None, "$199/month", 4.5, 8900,
|
||||
"https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=800&h=400&fit=crop"),
|
||||
|
||||
("LambdaTest", "Cloud Infrastructure", "Freemium", False, False,
|
||||
"Cross-browser testing on cloud", "https://lambdatest.com",
|
||||
None, "Free - $99/month", 4.1, 11200,
|
||||
"https://images.unsplash.com/photo-1451187580459-43490279c0fa?w=800&h=400&fit=crop"),
|
||||
|
||||
("Browserless", "Cloud Infrastructure", "Freemium", True, False,
|
||||
"Headless browser automation API", "https://browserless.io",
|
||||
None, "$50-$500/month", 4.7, 19800,
|
||||
"https://images.unsplash.com/photo-1639762681485-074b7f938ba0?w=800&h=400&fit=crop"),
|
||||
|
||||
# Developer Tools
|
||||
("Crawl4AI VSCode", "Developer Tools", "Open Source", True, False,
|
||||
"VSCode extension for Crawl4AI development", "https://marketplace.visualstudio.com",
|
||||
"https://github.com/crawl4ai/vscode", "Free", 4.8, 34500,
|
||||
"https://images.unsplash.com/photo-1629654297299-c8506221ca97?w=800&h=400&fit=crop"),
|
||||
|
||||
("Postman Collection", "Developer Tools", "Open Source", False, False,
|
||||
"Postman collection for Crawl4AI API testing", "https://postman.com/crawl4ai",
|
||||
"https://github.com/crawl4ai/postman", "Free", 4.3, 7800,
|
||||
"https://images.unsplash.com/photo-1599507593499-a3f7d7d97667?w=800&h=400&fit=crop"),
|
||||
|
||||
("Debug Toolkit", "Developer Tools", "Open Source", False, False,
|
||||
"Debugging tools for crawler development", "https://debug.crawl4ai.com",
|
||||
"https://github.com/crawl4ai/debug", "Free", 4.0, 4300,
|
||||
"https://images.unsplash.com/photo-1515879218367-8466d910aaa4?w=800&h=400&fit=crop"),
|
||||
]
|
||||
|
||||
for name, category, type_, featured, sponsored, desc, url, github, pricing, rating, downloads, image in apps_data:
|
||||
screenshots = json.dumps([
|
||||
f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop",
|
||||
f"https://images.unsplash.com/photo-{random.randint(1500000000000, 1700000000000)}-{random.randint(1000000000000, 9999999999999)}?w=800&h=600&fit=crop"
|
||||
])
|
||||
cursor.execute("""
|
||||
INSERT INTO apps (name, slug, description, category, type, featured, sponsored,
|
||||
website_url, github_url, pricing, rating, downloads, image, screenshots, logo_url,
|
||||
integration_guide, contact_email, views)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (name, generate_slug(name), desc, category, type_, featured, sponsored,
|
||||
url, github, pricing, rating, downloads, image, screenshots,
|
||||
f"https://ui-avatars.com/api/?name={name}&background=50ffff&color=070708&size=128",
|
||||
f"# {name} Integration\n\n```python\nfrom crawl4ai import AsyncWebCrawler\n# Integration code coming soon...\n```",
|
||||
f"contact@{generate_slug(name)}.com",
|
||||
random.randint(100, 5000)))
|
||||
|
||||
# Articles with real images
|
||||
articles_data = [
|
||||
("Browser Automation Showdown: Playwright vs Puppeteer vs Selenium",
|
||||
"Review", "John Doe", ["Playwright Cloud", "Puppeteer Extra"],
|
||||
["browser-automation", "comparison", "2024"],
|
||||
"https://images.unsplash.com/photo-1587620962725-abab7fe55159?w=1200&h=630&fit=crop"),
|
||||
|
||||
("Top 5 Proxy Services for Web Scraping in 2024",
|
||||
"Comparison", "Jane Smith", ["BrightData", "SmartProxy", "ProxyMesh"],
|
||||
["proxy", "web-scraping", "guide"],
|
||||
"https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=1200&h=630&fit=crop"),
|
||||
|
||||
("Integrating LLMs with Crawl4AI: A Complete Guide",
|
||||
"Tutorial", "Crawl4AI Team", ["LangChain Crawl", "GPT Scraper", "Claude Extract"],
|
||||
["llm", "integration", "tutorial"],
|
||||
"https://images.unsplash.com/photo-1677442136019-21780ecad995?w=1200&h=630&fit=crop"),
|
||||
|
||||
("Building Scalable Crawlers with Cloud Infrastructure",
|
||||
"Tutorial", "Mike Johnson", ["BrowserCloud", "Browserless"],
|
||||
["cloud", "scalability", "architecture"],
|
||||
"https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=1200&h=630&fit=crop"),
|
||||
|
||||
("What's New in Crawl4AI Marketplace",
|
||||
"News", "Crawl4AI Team", [],
|
||||
["marketplace", "announcement", "news"],
|
||||
"https://images.unsplash.com/photo-1556075798-4825dfaaf498?w=1200&h=630&fit=crop"),
|
||||
|
||||
("Cost Analysis: Self-Hosted vs Cloud Browser Solutions",
|
||||
"Comparison", "Sarah Chen", ["BrowserCloud", "LambdaTest", "Browserless"],
|
||||
["cost", "cloud", "comparison"],
|
||||
"https://images.unsplash.com/photo-1554224155-8d04cb21cd6c?w=1200&h=630&fit=crop"),
|
||||
|
||||
("Getting Started with Browser Automation",
|
||||
"Tutorial", "Crawl4AI Team", ["Playwright Cloud", "Selenium Grid Hub"],
|
||||
["beginner", "tutorial", "automation"],
|
||||
"https://images.unsplash.com/photo-1498050108023-c5249f4df085?w=1200&h=630&fit=crop"),
|
||||
|
||||
("The Future of Web Scraping: AI-Powered Extraction",
|
||||
"News", "Dr. Alan Turing", ["Claude Extract", "GPT Scraper"],
|
||||
["ai", "future", "trends"],
|
||||
"https://images.unsplash.com/photo-1593720213428-28a5b9e94613?w=1200&h=630&fit=crop")
|
||||
]
|
||||
|
||||
for title, category, author, related_apps, tags, image in articles_data:
|
||||
# Get app IDs for related apps
|
||||
related_ids = []
|
||||
for app_name in related_apps:
|
||||
cursor.execute("SELECT id FROM apps WHERE name = ?", (app_name,))
|
||||
result = cursor.fetchone()
|
||||
if result:
|
||||
related_ids.append(result[0])
|
||||
|
||||
content = f"""# {title}
|
||||
|
||||
By {author} | {datetime.now().strftime('%B %d, %Y')}
|
||||
|
||||
## Introduction
|
||||
|
||||
This is a comprehensive article about {title.lower()}. Lorem ipsum dolor sit amet, consectetur adipiscing elit.
|
||||
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
|
||||
|
||||
## Key Points
|
||||
|
||||
- Important point about the topic
|
||||
- Another crucial insight
|
||||
- Technical details and specifications
|
||||
- Performance comparisons
|
||||
|
||||
## Conclusion
|
||||
|
||||
In summary, this article explored various aspects of the topic. Stay tuned for more updates!
|
||||
"""
|
||||
|
||||
cursor.execute("""
|
||||
INSERT INTO articles (title, slug, content, author, category, related_apps,
|
||||
featured_image, tags, views)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (title, generate_slug(title), content, author, category,
|
||||
json.dumps(related_ids), image, json.dumps(tags),
|
||||
random.randint(200, 10000)))
|
||||
|
||||
# Sponsors
|
||||
sponsors_data = [
|
||||
("BrightData", "Gold", "https://brightdata.com",
|
||||
"https://images.unsplash.com/photo-1558494949-ef010cbdcc31?w=728&h=90&fit=crop"),
|
||||
("ScraperAPI", "Gold", "https://scraperapi.com",
|
||||
"https://images.unsplash.com/photo-1460925895917-afdab827c52f?w=728&h=90&fit=crop"),
|
||||
("BrowserCloud", "Silver", "https://browsercloud.io",
|
||||
"https://images.unsplash.com/photo-1667372393119-3d4c48d07fc9?w=728&h=90&fit=crop"),
|
||||
("Claude Extract", "Silver", "https://claude-extract.com",
|
||||
"https://images.unsplash.com/photo-1686191128892-3b09ad503b4f?w=728&h=90&fit=crop"),
|
||||
("SmartProxy", "Bronze", "https://smartproxy.com",
|
||||
"https://images.unsplash.com/photo-1544197150-b99a580bb7a8?w=728&h=90&fit=crop")
|
||||
]
|
||||
|
||||
for company, tier, landing_url, banner in sponsors_data:
|
||||
start_date = datetime.now() - timedelta(days=random.randint(1, 30))
|
||||
end_date = datetime.now() + timedelta(days=random.randint(30, 180))
|
||||
|
||||
cursor.execute("""
|
||||
INSERT INTO sponsors (company_name, logo_url, tier, banner_url,
|
||||
landing_url, active, start_date, end_date)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (company,
|
||||
f"https://ui-avatars.com/api/?name={company}&background=09b5a5&color=fff&size=200",
|
||||
tier, banner, landing_url, 1,
|
||||
start_date.isoformat(), end_date.isoformat()))
|
||||
|
||||
conn.commit()
|
||||
print("✓ Dummy data generated successfully!")
|
||||
print(f" - {len(categories)} categories")
|
||||
print(f" - {len(apps_data)} apps")
|
||||
print(f" - {len(articles_data)} articles")
|
||||
print(f" - {len(sponsors_data)} sponsors")
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_dummy_data()
|
||||
5
docs/md_v2/marketplace/backend/requirements.txt
Normal file
5
docs/md_v2/marketplace/backend/requirements.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
fastapi
|
||||
uvicorn
|
||||
pyyaml
|
||||
python-multipart
|
||||
python-dotenv
|
||||
75
docs/md_v2/marketplace/backend/schema.yaml
Normal file
75
docs/md_v2/marketplace/backend/schema.yaml
Normal file
@@ -0,0 +1,75 @@
|
||||
database:
|
||||
name: marketplace.db
|
||||
|
||||
tables:
|
||||
apps:
|
||||
columns:
|
||||
id: {type: INTEGER, primary: true, autoincrement: true}
|
||||
name: {type: TEXT, required: true}
|
||||
slug: {type: TEXT, unique: true}
|
||||
description: {type: TEXT}
|
||||
long_description: {type: TEXT}
|
||||
logo_url: {type: TEXT}
|
||||
image: {type: TEXT}
|
||||
screenshots: {type: JSON, default: '[]'}
|
||||
category: {type: TEXT}
|
||||
type: {type: TEXT, default: 'Open Source'}
|
||||
status: {type: TEXT, default: 'Active'}
|
||||
website_url: {type: TEXT}
|
||||
github_url: {type: TEXT}
|
||||
demo_url: {type: TEXT}
|
||||
video_url: {type: TEXT}
|
||||
documentation_url: {type: TEXT}
|
||||
support_url: {type: TEXT}
|
||||
discord_url: {type: TEXT}
|
||||
pricing: {type: TEXT}
|
||||
rating: {type: REAL, default: 0.0}
|
||||
downloads: {type: INTEGER, default: 0}
|
||||
featured: {type: BOOLEAN, default: 0}
|
||||
sponsored: {type: BOOLEAN, default: 0}
|
||||
integration_guide: {type: TEXT}
|
||||
documentation: {type: TEXT}
|
||||
examples: {type: TEXT}
|
||||
installation_command: {type: TEXT}
|
||||
requirements: {type: TEXT}
|
||||
changelog: {type: TEXT}
|
||||
tags: {type: JSON, default: '[]'}
|
||||
added_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
|
||||
updated_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
|
||||
contact_email: {type: TEXT}
|
||||
views: {type: INTEGER, default: 0}
|
||||
|
||||
articles:
|
||||
columns:
|
||||
id: {type: INTEGER, primary: true, autoincrement: true}
|
||||
title: {type: TEXT, required: true}
|
||||
slug: {type: TEXT, unique: true}
|
||||
content: {type: TEXT}
|
||||
author: {type: TEXT, default: 'Crawl4AI Team'}
|
||||
category: {type: TEXT}
|
||||
related_apps: {type: JSON, default: '[]'}
|
||||
featured_image: {type: TEXT}
|
||||
published_date: {type: DATETIME, default: CURRENT_TIMESTAMP}
|
||||
tags: {type: JSON, default: '[]'}
|
||||
views: {type: INTEGER, default: 0}
|
||||
|
||||
categories:
|
||||
columns:
|
||||
id: {type: INTEGER, primary: true, autoincrement: true}
|
||||
name: {type: TEXT, unique: true}
|
||||
slug: {type: TEXT, unique: true}
|
||||
icon: {type: TEXT}
|
||||
description: {type: TEXT}
|
||||
order_index: {type: INTEGER, default: 0}
|
||||
|
||||
sponsors:
|
||||
columns:
|
||||
id: {type: INTEGER, primary: true, autoincrement: true}
|
||||
company_name: {type: TEXT, required: true}
|
||||
logo_url: {type: TEXT}
|
||||
tier: {type: TEXT, default: 'Bronze'}
|
||||
banner_url: {type: TEXT}
|
||||
landing_url: {type: TEXT}
|
||||
active: {type: BOOLEAN, default: 1}
|
||||
start_date: {type: DATETIME}
|
||||
end_date: {type: DATETIME}
|
||||
390
docs/md_v2/marketplace/backend/server.py
Normal file
390
docs/md_v2/marketplace/backend/server.py
Normal file
@@ -0,0 +1,390 @@
|
||||
from fastapi import FastAPI, HTTPException, Query, Depends, Body
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from typing import Optional, List, Dict, Any
|
||||
import json
|
||||
import hashlib
|
||||
import secrets
|
||||
from database import DatabaseManager
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Import configuration (will exit if .env not found or invalid)
|
||||
from config import Config
|
||||
|
||||
app = FastAPI(title="Crawl4AI Marketplace API")
|
||||
|
||||
# Security setup
|
||||
security = HTTPBearer()
|
||||
tokens = {} # In production, use Redis or database for token storage
|
||||
|
||||
# CORS configuration
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=Config.ALLOWED_ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
||||
allow_headers=["*"],
|
||||
max_age=3600
|
||||
)
|
||||
|
||||
# Initialize database with configurable path
|
||||
db = DatabaseManager(Config.DATABASE_PATH)
|
||||
|
||||
def json_response(data, cache_time=3600):
|
||||
"""Helper to return JSON with cache headers"""
|
||||
return JSONResponse(
|
||||
content=data,
|
||||
headers={
|
||||
"Cache-Control": f"public, max-age={cache_time}",
|
||||
"X-Content-Type-Options": "nosniff"
|
||||
}
|
||||
)
|
||||
|
||||
# ============= PUBLIC ENDPOINTS =============
|
||||
|
||||
@app.get("/api/apps")
|
||||
async def get_apps(
|
||||
category: Optional[str] = None,
|
||||
type: Optional[str] = None,
|
||||
featured: Optional[bool] = None,
|
||||
sponsored: Optional[bool] = None,
|
||||
limit: int = Query(default=20, le=10000),
|
||||
offset: int = Query(default=0)
|
||||
):
|
||||
"""Get apps with optional filters"""
|
||||
where_clauses = []
|
||||
if category:
|
||||
where_clauses.append(f"category = '{category}'")
|
||||
if type:
|
||||
where_clauses.append(f"type = '{type}'")
|
||||
if featured is not None:
|
||||
where_clauses.append(f"featured = {1 if featured else 0}")
|
||||
if sponsored is not None:
|
||||
where_clauses.append(f"sponsored = {1 if sponsored else 0}")
|
||||
|
||||
where = " AND ".join(where_clauses) if where_clauses else None
|
||||
apps = db.get_all('apps', limit=limit, offset=offset, where=where)
|
||||
|
||||
# Parse JSON fields
|
||||
for app in apps:
|
||||
if app.get('screenshots'):
|
||||
app['screenshots'] = json.loads(app['screenshots'])
|
||||
|
||||
return json_response(apps)
|
||||
|
||||
@app.get("/api/apps/{slug}")
|
||||
async def get_app(slug: str):
|
||||
"""Get single app by slug"""
|
||||
apps = db.get_all('apps', where=f"slug = '{slug}'", limit=1)
|
||||
if not apps:
|
||||
raise HTTPException(status_code=404, detail="App not found")
|
||||
|
||||
app = apps[0]
|
||||
if app.get('screenshots'):
|
||||
app['screenshots'] = json.loads(app['screenshots'])
|
||||
|
||||
return json_response(app)
|
||||
|
||||
@app.get("/api/articles")
|
||||
async def get_articles(
|
||||
category: Optional[str] = None,
|
||||
limit: int = Query(default=20, le=10000),
|
||||
offset: int = Query(default=0)
|
||||
):
|
||||
"""Get articles with optional category filter"""
|
||||
where = f"category = '{category}'" if category else None
|
||||
articles = db.get_all('articles', limit=limit, offset=offset, where=where)
|
||||
|
||||
# Parse JSON fields
|
||||
for article in articles:
|
||||
if article.get('related_apps'):
|
||||
article['related_apps'] = json.loads(article['related_apps'])
|
||||
if article.get('tags'):
|
||||
article['tags'] = json.loads(article['tags'])
|
||||
|
||||
return json_response(articles)
|
||||
|
||||
@app.get("/api/articles/{slug}")
|
||||
async def get_article(slug: str):
|
||||
"""Get single article by slug"""
|
||||
articles = db.get_all('articles', where=f"slug = '{slug}'", limit=1)
|
||||
if not articles:
|
||||
raise HTTPException(status_code=404, detail="Article not found")
|
||||
|
||||
article = articles[0]
|
||||
if article.get('related_apps'):
|
||||
article['related_apps'] = json.loads(article['related_apps'])
|
||||
if article.get('tags'):
|
||||
article['tags'] = json.loads(article['tags'])
|
||||
|
||||
return json_response(article)
|
||||
|
||||
@app.get("/api/categories")
|
||||
async def get_categories():
|
||||
"""Get all categories ordered by index"""
|
||||
categories = db.get_all('categories', limit=50)
|
||||
categories.sort(key=lambda x: x.get('order_index', 0))
|
||||
return json_response(categories, cache_time=7200)
|
||||
|
||||
@app.get("/api/sponsors")
|
||||
async def get_sponsors(active: Optional[bool] = True):
|
||||
"""Get sponsors, default active only"""
|
||||
where = f"active = {1 if active else 0}" if active is not None else None
|
||||
sponsors = db.get_all('sponsors', where=where, limit=20)
|
||||
|
||||
# Filter by date if active
|
||||
if active:
|
||||
now = datetime.now().isoformat()
|
||||
sponsors = [s for s in sponsors
|
||||
if (not s.get('start_date') or s['start_date'] <= now) and
|
||||
(not s.get('end_date') or s['end_date'] >= now)]
|
||||
|
||||
return json_response(sponsors)
|
||||
|
||||
@app.get("/api/search")
|
||||
async def search(q: str = Query(min_length=2)):
|
||||
"""Search across apps and articles"""
|
||||
if len(q) < 2:
|
||||
return json_response({})
|
||||
|
||||
results = db.search(q, tables=['apps', 'articles'])
|
||||
|
||||
# Parse JSON fields in results
|
||||
for table, items in results.items():
|
||||
for item in items:
|
||||
if table == 'apps' and item.get('screenshots'):
|
||||
item['screenshots'] = json.loads(item['screenshots'])
|
||||
elif table == 'articles':
|
||||
if item.get('related_apps'):
|
||||
item['related_apps'] = json.loads(item['related_apps'])
|
||||
if item.get('tags'):
|
||||
item['tags'] = json.loads(item['tags'])
|
||||
|
||||
return json_response(results, cache_time=1800)
|
||||
|
||||
@app.get("/api/stats")
|
||||
async def get_stats():
|
||||
"""Get marketplace statistics"""
|
||||
stats = {
|
||||
"total_apps": len(db.get_all('apps', limit=10000)),
|
||||
"total_articles": len(db.get_all('articles', limit=10000)),
|
||||
"total_categories": len(db.get_all('categories', limit=1000)),
|
||||
"active_sponsors": len(db.get_all('sponsors', where="active = 1", limit=1000))
|
||||
}
|
||||
return json_response(stats, cache_time=1800)
|
||||
|
||||
# ============= ADMIN AUTHENTICATION =============
|
||||
|
||||
def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
|
||||
"""Verify admin authentication token"""
|
||||
token = credentials.credentials
|
||||
if token not in tokens or tokens[token] < datetime.now():
|
||||
raise HTTPException(status_code=401, detail="Invalid or expired token")
|
||||
return token
|
||||
|
||||
@app.post("/api/admin/login")
|
||||
async def admin_login(password: str = Body(..., embed=True)):
|
||||
"""Admin login with password"""
|
||||
provided_hash = hashlib.sha256(password.encode()).hexdigest()
|
||||
|
||||
if provided_hash != Config.ADMIN_PASSWORD_HASH:
|
||||
# Log failed attempt in production
|
||||
print(f"Failed login attempt at {datetime.now()}")
|
||||
raise HTTPException(status_code=401, detail="Invalid password")
|
||||
|
||||
# Generate secure token
|
||||
token = secrets.token_urlsafe(32)
|
||||
tokens[token] = datetime.now() + timedelta(hours=Config.TOKEN_EXPIRY_HOURS)
|
||||
|
||||
return {
|
||||
"token": token,
|
||||
"expires_in": Config.TOKEN_EXPIRY_HOURS * 3600
|
||||
}
|
||||
|
||||
# ============= ADMIN ENDPOINTS =============
|
||||
|
||||
@app.get("/api/admin/stats", dependencies=[Depends(verify_token)])
|
||||
async def get_admin_stats():
|
||||
"""Get detailed admin statistics"""
|
||||
stats = {
|
||||
"apps": {
|
||||
"total": len(db.get_all('apps', limit=10000)),
|
||||
"featured": len(db.get_all('apps', where="featured = 1", limit=10000)),
|
||||
"sponsored": len(db.get_all('apps', where="sponsored = 1", limit=10000))
|
||||
},
|
||||
"articles": len(db.get_all('articles', limit=10000)),
|
||||
"categories": len(db.get_all('categories', limit=1000)),
|
||||
"sponsors": {
|
||||
"active": len(db.get_all('sponsors', where="active = 1", limit=1000)),
|
||||
"total": len(db.get_all('sponsors', limit=10000))
|
||||
},
|
||||
"total_views": sum(app.get('views', 0) for app in db.get_all('apps', limit=10000))
|
||||
}
|
||||
return stats
|
||||
|
||||
# Apps CRUD
|
||||
@app.post("/api/admin/apps", dependencies=[Depends(verify_token)])
|
||||
async def create_app(app_data: Dict[str, Any]):
|
||||
"""Create new app"""
|
||||
try:
|
||||
# Handle JSON fields
|
||||
for field in ['screenshots', 'tags']:
|
||||
if field in app_data and isinstance(app_data[field], list):
|
||||
app_data[field] = json.dumps(app_data[field])
|
||||
|
||||
cursor = db.conn.cursor()
|
||||
columns = ', '.join(app_data.keys())
|
||||
placeholders = ', '.join(['?' for _ in app_data])
|
||||
cursor.execute(f"INSERT INTO apps ({columns}) VALUES ({placeholders})",
|
||||
list(app_data.values()))
|
||||
db.conn.commit()
|
||||
return {"id": cursor.lastrowid, "message": "App created"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.put("/api/admin/apps/{app_id}", dependencies=[Depends(verify_token)])
|
||||
async def update_app(app_id: int, app_data: Dict[str, Any]):
|
||||
"""Update app"""
|
||||
try:
|
||||
# Handle JSON fields
|
||||
for field in ['screenshots', 'tags']:
|
||||
if field in app_data and isinstance(app_data[field], list):
|
||||
app_data[field] = json.dumps(app_data[field])
|
||||
|
||||
set_clause = ', '.join([f"{k} = ?" for k in app_data.keys()])
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute(f"UPDATE apps SET {set_clause} WHERE id = ?",
|
||||
list(app_data.values()) + [app_id])
|
||||
db.conn.commit()
|
||||
return {"message": "App updated"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.delete("/api/admin/apps/{app_id}", dependencies=[Depends(verify_token)])
|
||||
async def delete_app(app_id: int):
|
||||
"""Delete app"""
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute("DELETE FROM apps WHERE id = ?", (app_id,))
|
||||
db.conn.commit()
|
||||
return {"message": "App deleted"}
|
||||
|
||||
# Articles CRUD
|
||||
@app.post("/api/admin/articles", dependencies=[Depends(verify_token)])
|
||||
async def create_article(article_data: Dict[str, Any]):
|
||||
"""Create new article"""
|
||||
try:
|
||||
for field in ['related_apps', 'tags']:
|
||||
if field in article_data and isinstance(article_data[field], list):
|
||||
article_data[field] = json.dumps(article_data[field])
|
||||
|
||||
cursor = db.conn.cursor()
|
||||
columns = ', '.join(article_data.keys())
|
||||
placeholders = ', '.join(['?' for _ in article_data])
|
||||
cursor.execute(f"INSERT INTO articles ({columns}) VALUES ({placeholders})",
|
||||
list(article_data.values()))
|
||||
db.conn.commit()
|
||||
return {"id": cursor.lastrowid, "message": "Article created"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.put("/api/admin/articles/{article_id}", dependencies=[Depends(verify_token)])
|
||||
async def update_article(article_id: int, article_data: Dict[str, Any]):
|
||||
"""Update article"""
|
||||
try:
|
||||
for field in ['related_apps', 'tags']:
|
||||
if field in article_data and isinstance(article_data[field], list):
|
||||
article_data[field] = json.dumps(article_data[field])
|
||||
|
||||
set_clause = ', '.join([f"{k} = ?" for k in article_data.keys()])
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute(f"UPDATE articles SET {set_clause} WHERE id = ?",
|
||||
list(article_data.values()) + [article_id])
|
||||
db.conn.commit()
|
||||
return {"message": "Article updated"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.delete("/api/admin/articles/{article_id}", dependencies=[Depends(verify_token)])
|
||||
async def delete_article(article_id: int):
|
||||
"""Delete article"""
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute("DELETE FROM articles WHERE id = ?", (article_id,))
|
||||
db.conn.commit()
|
||||
return {"message": "Article deleted"}
|
||||
|
||||
# Categories CRUD
|
||||
@app.post("/api/admin/categories", dependencies=[Depends(verify_token)])
|
||||
async def create_category(category_data: Dict[str, Any]):
|
||||
"""Create new category"""
|
||||
try:
|
||||
cursor = db.conn.cursor()
|
||||
columns = ', '.join(category_data.keys())
|
||||
placeholders = ', '.join(['?' for _ in category_data])
|
||||
cursor.execute(f"INSERT INTO categories ({columns}) VALUES ({placeholders})",
|
||||
list(category_data.values()))
|
||||
db.conn.commit()
|
||||
return {"id": cursor.lastrowid, "message": "Category created"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.put("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)])
|
||||
async def update_category(cat_id: int, category_data: Dict[str, Any]):
|
||||
"""Update category"""
|
||||
try:
|
||||
set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()])
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?",
|
||||
list(category_data.values()) + [cat_id])
|
||||
db.conn.commit()
|
||||
return {"message": "Category updated"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
# Sponsors CRUD
|
||||
@app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)])
|
||||
async def create_sponsor(sponsor_data: Dict[str, Any]):
|
||||
"""Create new sponsor"""
|
||||
try:
|
||||
cursor = db.conn.cursor()
|
||||
columns = ', '.join(sponsor_data.keys())
|
||||
placeholders = ', '.join(['?' for _ in sponsor_data])
|
||||
cursor.execute(f"INSERT INTO sponsors ({columns}) VALUES ({placeholders})",
|
||||
list(sponsor_data.values()))
|
||||
db.conn.commit()
|
||||
return {"id": cursor.lastrowid, "message": "Sponsor created"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.put("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)])
|
||||
async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]):
|
||||
"""Update sponsor"""
|
||||
try:
|
||||
set_clause = ', '.join([f"{k} = ?" for k in sponsor_data.keys()])
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute(f"UPDATE sponsors SET {set_clause} WHERE id = ?",
|
||||
list(sponsor_data.values()) + [sponsor_id])
|
||||
db.conn.commit()
|
||||
return {"message": "Sponsor updated"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""API info"""
|
||||
return {
|
||||
"name": "Crawl4AI Marketplace API",
|
||||
"version": "1.0.0",
|
||||
"endpoints": [
|
||||
"/api/apps",
|
||||
"/api/articles",
|
||||
"/api/categories",
|
||||
"/api/sponsors",
|
||||
"/api/search?q=query",
|
||||
"/api/stats"
|
||||
]
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="127.0.0.1", port=8100)
|
||||
Reference in New Issue
Block a user