Compare commits
4 Commits
feature/do
...
docker/fix
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
05ec0535cd | ||
|
|
83aeb565ee | ||
|
|
c0f1865287 | ||
|
|
46ef1116c4 |
@@ -2,8 +2,8 @@
|
||||
import asyncio, json, hashlib, time, psutil
|
||||
from contextlib import suppress
|
||||
from typing import Dict
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||||
from typing import Dict
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, BrowserAdapter
|
||||
from typing import Dict ,Optional
|
||||
from utils import load_config
|
||||
|
||||
CONFIG = load_config()
|
||||
@@ -15,11 +15,22 @@ LOCK = asyncio.Lock()
|
||||
MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0) # % RAM – refuse new browsers above this
|
||||
IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 1800) # close if unused for 30 min
|
||||
|
||||
def _sig(cfg: BrowserConfig) -> str:
|
||||
payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
|
||||
|
||||
def _sig(cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None) -> str:
|
||||
try:
|
||||
config_payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",", ":"))
|
||||
except (TypeError, ValueError):
|
||||
# Fallback to string representation if JSON serialization fails
|
||||
config_payload = str(cfg.to_dict())
|
||||
adapter_name = adapter.__class__.__name__ if adapter else "PlaywrightAdapter"
|
||||
payload = f"{config_payload}:{adapter_name}"
|
||||
return hashlib.sha1(payload.encode()).hexdigest()
|
||||
|
||||
async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
|
||||
|
||||
async def get_crawler(
|
||||
cfg: BrowserConfig, adapter: Optional[BrowserAdapter] = None
|
||||
) -> AsyncWebCrawler:
|
||||
sig = None
|
||||
try:
|
||||
sig = _sig(cfg)
|
||||
async with LOCK:
|
||||
@@ -37,12 +48,13 @@ async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to start browser: {e}")
|
||||
finally:
|
||||
if sig in POOL:
|
||||
LAST_USED[sig] = time.time()
|
||||
else:
|
||||
# If we failed to start the browser, we should remove it from the pool
|
||||
POOL.pop(sig, None)
|
||||
LAST_USED.pop(sig, None)
|
||||
if sig:
|
||||
if sig in POOL:
|
||||
LAST_USED[sig] = time.time()
|
||||
else:
|
||||
# If we failed to start the browser, we should remove it from the pool
|
||||
POOL.pop(sig, None)
|
||||
LAST_USED.pop(sig, None)
|
||||
# If we failed to start the browser, we should remove it from the pool
|
||||
async def close_all():
|
||||
async with LOCK:
|
||||
|
||||
@@ -278,12 +278,12 @@
|
||||
}
|
||||
|
||||
.tab-content {
|
||||
display: none;
|
||||
display: none !important;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
.tab-content.active {
|
||||
display: block;
|
||||
display: block !important;
|
||||
}
|
||||
|
||||
/* Overview Layout */
|
||||
|
||||
@@ -73,8 +73,8 @@
|
||||
<div class="tabs">
|
||||
<button class="tab-btn active" data-tab="overview">Overview</button>
|
||||
<button class="tab-btn" data-tab="integration">Integration</button>
|
||||
<button class="tab-btn" data-tab="docs">Documentation</button>
|
||||
<button class="tab-btn" data-tab="support">Support</button>
|
||||
<!-- <button class="tab-btn" data-tab="docs">Documentation</button>
|
||||
<button class="tab-btn" data-tab="support">Support</button> -->
|
||||
</div>
|
||||
|
||||
<section id="overview-tab" class="tab-content active">
|
||||
@@ -130,17 +130,15 @@
|
||||
|
||||
<section id="integration-tab" class="tab-content">
|
||||
<div class="integration-content" id="app-integration">
|
||||
<!-- Integration guide markdown content will be rendered here -->
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="docs-tab" class="tab-content">
|
||||
<!-- <section id="docs-tab" class="tab-content">
|
||||
<div class="docs-content" id="app-docs">
|
||||
<!-- Documentation markdown content will be rendered here -->
|
||||
</div>
|
||||
</section>
|
||||
</section> -->
|
||||
|
||||
<section id="support-tab" class="tab-content">
|
||||
<!-- <section id="support-tab" class="tab-content">
|
||||
<div class="docs-content">
|
||||
<h2>Support</h2>
|
||||
<div class="support-grid">
|
||||
@@ -158,7 +156,7 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</section> -->
|
||||
</div>
|
||||
|
||||
</main>
|
||||
|
||||
@@ -112,7 +112,7 @@ class AppDetailPage {
|
||||
}
|
||||
|
||||
// Contact
|
||||
document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available';
|
||||
document.getElementById('app-contact') && (document.getElementById('app-contact').textContent = this.appData.contact_email || 'Not available');
|
||||
|
||||
// Sidebar info
|
||||
document.getElementById('sidebar-downloads').textContent = this.formatNumber(this.appData.downloads || 0);
|
||||
@@ -263,18 +263,27 @@ class AppDetailPage {
|
||||
setupEventListeners() {
|
||||
// Tab switching
|
||||
const tabs = document.querySelectorAll('.tab-btn');
|
||||
|
||||
tabs.forEach(tab => {
|
||||
tab.addEventListener('click', () => {
|
||||
// Update active tab
|
||||
// Update active tab button
|
||||
tabs.forEach(t => t.classList.remove('active'));
|
||||
tab.classList.add('active');
|
||||
|
||||
// Show corresponding content
|
||||
const tabName = tab.dataset.tab;
|
||||
document.querySelectorAll('.tab-content').forEach(content => {
|
||||
|
||||
// Hide all tab contents
|
||||
const allTabContents = document.querySelectorAll('.tab-content');
|
||||
allTabContents.forEach(content => {
|
||||
content.classList.remove('active');
|
||||
});
|
||||
document.getElementById(`${tabName}-tab`).classList.add('active');
|
||||
|
||||
// Show the selected tab content
|
||||
const targetTab = document.getElementById(`${tabName}-tab`);
|
||||
if (targetTab) {
|
||||
targetTab.classList.add('active');
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -471,13 +471,17 @@ async def delete_sponsor(sponsor_id: int):
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
# Version info
|
||||
VERSION = "1.1.0"
|
||||
BUILD_DATE = "2025-10-26"
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""API info"""
|
||||
return {
|
||||
"name": "Crawl4AI Marketplace API",
|
||||
"version": "1.0.0",
|
||||
"version": VERSION,
|
||||
"build_date": BUILD_DATE,
|
||||
"endpoints": [
|
||||
"/marketplace/api/apps",
|
||||
"/marketplace/api/articles",
|
||||
|
||||
Reference in New Issue
Block a user