Compare commits
4 Commits
feature/do
...
docker/fix
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
05ec0535cd | ||
|
|
83aeb565ee | ||
|
|
c0f1865287 | ||
|
|
46ef1116c4 |
@@ -2,8 +2,8 @@
|
|||||||
import asyncio, json, hashlib, time, psutil
|
import asyncio, json, hashlib, time, psutil
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
from crawl4ai import AsyncWebCrawler, BrowserConfig, BrowserAdapter
|
||||||
from typing import Dict
|
from typing import Dict ,Optional
|
||||||
from utils import load_config
|
from utils import load_config
|
||||||
|
|
||||||
CONFIG = load_config()
|
CONFIG = load_config()
|
||||||
@@ -15,11 +15,22 @@ LOCK = asyncio.Lock()
|
|||||||
MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) # % RAM – refuse new browsers above this
|
MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) # % RAM – refuse new browsers above this
|
||||||
IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800) # close if unused for 30 min
|
IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800) # close if unused for 30 min
|
||||||
|
|
||||||
def _sig(cfg: BrowserConfig) -> str:
|
|
||||||
payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
|
def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
|
||||||
|
try:
|
||||||
|
config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
# Fallback to string representation if JSON serialization fails
|
||||||
|
config_payload = str(cfg.to_dict())
|
||||||
|
adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter"
|
||||||
|
payload = f"{config_payload}:{adapter_name}"
|
||||||
return hashlib.sha1(payload.encode()).hexdigest()
|
return hashlib.sha1(payload.encode()).hexdigest()
|
||||||
|
|
||||||
async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
|
|
||||||
|
async def get_crawler(
|
||||||
|
cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None
|
||||||
|
) -> AsyncWebCrawler:
|
||||||
|
sig = None
|
||||||
try:
|
try:
|
||||||
sig = _sig(cfg)
|
sig = _sig(cfg)
|
||||||
async with LOCK:
|
async with LOCK:
|
||||||
@@ -37,6 +48,7 @@ async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise RuntimeError(f"Failed to start browser: {e}")
|
raise RuntimeError(f"Failed to start browser: {e}")
|
||||||
finally:
|
finally:
|
||||||
|
if sig:
|
||||||
if sig in POOL:
|
if sig in POOL:
|
||||||
LAST_USED[sig] = time.time()
|
LAST_USED[sig] = time.time()
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -278,12 +278,12 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.tab-content {
|
.tab-content {
|
||||||
display: none;
|
display: none !important;
|
||||||
padding: 2rem;
|
padding: 2rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
.tab-content.active {
|
.tab-content.active {
|
||||||
display: block;
|
display: block !important;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Overview Layout */
|
/* Overview Layout */
|
||||||
|
|||||||
@@ -73,8 +73,8 @@
|
|||||||
<div class="tabs">
|
<div class="tabs">
|
||||||
<button class="tab-btn active" data-tab="overview">Overview</button>
|
<button class="tab-btn active" data-tab="overview">Overview</button>
|
||||||
<button class="tab-btn" data-tab="integration">Integration</button>
|
<button class="tab-btn" data-tab="integration">Integration</button>
|
||||||
<button class="tab-btn" data-tab="docs">Documentation</button>
|
<!-- <button class="tab-btn" data-tab="docs">Documentation</button>
|
||||||
<button class="tab-btn" data-tab="support">Support</button>
|
<button class="tab-btn" data-tab="support">Support</button> -->
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<section id="overview-tab" class="tab-content active">
|
<section id="overview-tab" class="tab-content active">
|
||||||
@@ -130,17 +130,15 @@
|
|||||||
|
|
||||||
<section id="integration-tab" class="tab-content">
|
<section id="integration-tab" class="tab-content">
|
||||||
<div class="integration-content" id="app-integration">
|
<div class="integration-content" id="app-integration">
|
||||||
<!-- Integration guide markdown content will be rendered here -->
|
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section id="docs-tab" class="tab-content">
|
<!-- <section id="docs-tab" class="tab-content">
|
||||||
<div class="docs-content" id="app-docs">
|
<div class="docs-content" id="app-docs">
|
||||||
<!-- Documentation markdown content will be rendered here -->
|
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section> -->
|
||||||
|
|
||||||
<section id="support-tab" class="tab-content">
|
<!-- <section id="support-tab" class="tab-content">
|
||||||
<div class="docs-content">
|
<div class="docs-content">
|
||||||
<h2>Support</h2>
|
<h2>Support</h2>
|
||||||
<div class="support-grid">
|
<div class="support-grid">
|
||||||
@@ -158,7 +156,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section> -->
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
</main>
|
</main>
|
||||||
|
|||||||
@@ -112,7 +112,7 @@ class AppDetailPage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Contact
|
// Contact
|
||||||
document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available';
|
document.getElementById('app-contact') && (document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available');
|
||||||
|
|
||||||
// Sidebar info
|
// Sidebar info
|
||||||
document.getElementById('sidebar-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
|
document.getElementById('sidebar-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
|
||||||
@@ -263,18 +263,27 @@ class AppDetailPage {
|
|||||||
setupEventListeners() {
|
setupEventListeners() {
|
||||||
// Tab switching
|
// Tab switching
|
||||||
const tabs = document.querySelectorAll('.tab-btn');
|
const tabs = document.querySelectorAll('.tab-btn');
|
||||||
|
|
||||||
tabs.forEach(tab => {
|
tabs.forEach(tab => {
|
||||||
tab.addEventListener('click', () => {
|
tab.addEventListener('click', () => {
|
||||||
// Update active tab
|
// Update active tab button
|
||||||
tabs.forEach(t => t.classList.remove('active'));
|
tabs.forEach(t => t.classList.remove('active'));
|
||||||
tab.classList.add('active');
|
tab.classList.add('active');
|
||||||
|
|
||||||
// Show corresponding content
|
// Show corresponding content
|
||||||
const tabName = tab.dataset.tab;
|
const tabName = tab.dataset.tab;
|
||||||
document.querySelectorAll('.tab-content').forEach(content => {
|
|
||||||
|
// Hide all tab contents
|
||||||
|
const allTabContents = document.querySelectorAll('.tab-content');
|
||||||
|
allTabContents.forEach(content => {
|
||||||
content.classList.remove('active');
|
content.classList.remove('active');
|
||||||
});
|
});
|
||||||
document.getElementById(`${tabName}-tab`).classList.add('active');
|
|
||||||
|
// Show the selected tab content
|
||||||
|
const targetTab = document.getElementById(`${tabName}-tab`);
|
||||||
|
if (targetTab) {
|
||||||
|
targetTab.classList.add('active');
|
||||||
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -471,13 +471,17 @@ async def delete_sponsor(sponsor_id: int):
|
|||||||
|
|
||||||
app.include_router(router)
|
app.include_router(router)
|
||||||
|
|
||||||
|
# Version info
|
||||||
|
VERSION = "1.1.0"
|
||||||
|
BUILD_DATE = "2025-10-26"
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
async def root():
|
async def root():
|
||||||
"""API info"""
|
"""API info"""
|
||||||
return {
|
return {
|
||||||
"name": "Crawl4AI Marketplace API",
|
"name": "Crawl4AI Marketplace API",
|
||||||
"version": "1.0.0",
|
"version": VERSION,
|
||||||
|
"build_date": BUILD_DATE,
|
||||||
"endpoints": [
|
"endpoints": [
|
||||||
"/marketplace/api/apps",
|
"/marketplace/api/apps",
|
||||||
"/marketplace/api/articles",
|
"/marketplace/api/articles",
|
||||||
|
|||||||
Reference in New Issue
Block a user