From 70af81d9d7945432628981d6c32e977dafb3e3d7 Mon Sep 17 00:00:00 2001 From: ntohidi Date: Tue, 30 Sep 2025 11:54:21 +0800 Subject: [PATCH 1/5] refactor(release): remove memory management section for cleaner documentation. ref #1443 --- docs/blog/release-v0.7.4.md | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/docs/blog/release-v0.7.4.md b/docs/blog/release-v0.7.4.md index d9a57845..72cfe3ae 100644 --- a/docs/blog/release-v0.7.4.md +++ b/docs/blog/release-v0.7.4.md @@ -10,7 +10,6 @@ Today I'm releasing Crawl4AI v0.7.4β€”the Intelligent Table Extraction & Perform - **πŸš€ LLMTableExtraction**: Revolutionary table extraction with intelligent chunking for massive tables - **⚑ Enhanced Concurrency**: True concurrency improvements for fast-completing tasks in batch operations -- **🧹 Memory Management Refactor**: Streamlined memory utilities and better resource management - **πŸ”§ Browser Manager Fixes**: Resolved race conditions in concurrent page creation - **⌨️ Cross-Platform Browser Profiler**: Improved keyboard handling and quit mechanisms - **πŸ”— Advanced URL Processing**: Better handling of raw URLs and base tag link resolution @@ -158,40 +157,6 @@ async with AsyncWebCrawler() as crawler: - **Monitoring Systems**: Faster health checks and status page monitoring - **Data Aggregation**: Improved performance for real-time data collection -## 🧹 Memory Management Refactor: Cleaner Architecture - -**The Problem:** Memory utilities were scattered and difficult to maintain, with potential import conflicts and unclear organization. - -**My Solution:** I consolidated all memory-related utilities into the main `utils.py` module, creating a cleaner, more maintainable architecture. - -### Improved Memory Handling - -```python -# All memory utilities now consolidated -from crawl4ai.utils import get_true_memory_usage_percent, MemoryMonitor - -# Enhanced memory monitoring -monitor = MemoryMonitor() -monitor.start_monitoring() - -async with AsyncWebCrawler() as crawler: - # Memory-efficient batch processing - results = await crawler.arun_many(large_url_list) - - # Get accurate memory metrics - memory_usage = get_true_memory_usage_percent() - memory_report = monitor.get_report() - - print(f"Memory efficiency: {memory_report['efficiency']:.1f}%") - print(f"Peak usage: {memory_report['peak_mb']:.1f} MB") -``` - -**Expected Real-World Impact:** -- **Production Stability**: More reliable memory tracking and management -- **Code Maintainability**: Cleaner architecture for easier debugging -- **Import Clarity**: Resolved potential conflicts and import issues -- **Developer Experience**: Simpler API for memory monitoring - ## πŸ”§ Critical Stability Fixes ### Browser Manager Race Condition Resolution From 5145d42df7539c2c0743494f50cd0c11b8b1cef7 Mon Sep 17 00:00:00 2001 From: unclecode Date: Fri, 3 Oct 2025 20:09:48 +0800 Subject: [PATCH 2/5] fix(docs): hide copy menu on non-markdown pages --- docs/md_v2/assets/page_actions.js | 301 ++++++++++++++++++------------ 1 file changed, 186 insertions(+), 115 deletions(-) diff --git a/docs/md_v2/assets/page_actions.js b/docs/md_v2/assets/page_actions.js index 92893730..984a2c09 100644 --- a/docs/md_v2/assets/page_actions.js +++ b/docs/md_v2/assets/page_actions.js @@ -7,9 +7,12 @@ document.addEventListener('DOMContentLoaded', () => { githubRepo: 'unclecode/crawl4ai', githubBranch: 'main', docsPath: 'docs/md_v2', - excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/'], // Don't show on app pages + excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/', '/core/ask-ai/'], // Don't show on app pages }; + let cachedMarkdown = null; + let cachedMarkdownPath = null; + // Check if we should show the button on this page function shouldShowButton() { const currentPath = window.location.pathname; @@ -19,6 +22,17 @@ document.addEventListener('DOMContentLoaded', () => { return false; } + // Don't show on 404 pages + if (document.title && document.title.toLowerCase().includes('404')) { + return false; + } + + // Require mkdocs main content container + const mainContent = document.getElementById('terminal-mkdocs-main-content'); + if (!mainContent) { + return false; + } + // Don't show on excluded paths (apps) for (const excludePath of config.excludePaths) { if (currentPath.includes(excludePath)) { @@ -53,6 +67,56 @@ document.addEventListener('DOMContentLoaded', () => { return `${path}.md`; } + async function loadMarkdownContent() { + const mdPath = getCurrentMarkdownPath(); + + if (!mdPath) { + throw new Error('Invalid markdown path'); + } + + const rawUrl = getGithubRawUrl(); + const response = await fetch(rawUrl); + + if (!response.ok) { + throw new Error(`Failed to fetch markdown: ${response.status}`); + } + + const markdown = await response.text(); + cachedMarkdown = markdown; + cachedMarkdownPath = mdPath; + return markdown; + } + + async function ensureMarkdownCached() { + const mdPath = getCurrentMarkdownPath(); + + if (!mdPath) { + return false; + } + + if (cachedMarkdown && cachedMarkdownPath === mdPath) { + return true; + } + + try { + await loadMarkdownContent(); + return true; + } catch (error) { + console.warn('Page Actions: Markdown not available for this page.', error); + cachedMarkdown = null; + cachedMarkdownPath = null; + return false; + } + } + + async function getMarkdownContent() { + const available = await ensureMarkdownCached(); + if (!available) { + throw new Error('Markdown not available for this page.'); + } + return cachedMarkdown; + } + // Get GitHub raw URL for current page function getGithubRawUrl() { const mdPath = getCurrentMarkdownPath(); @@ -180,19 +244,11 @@ document.addEventListener('DOMContentLoaded', () => { // Copy markdown to clipboard async function copyMarkdownToClipboard(link) { - const rawUrl = getGithubRawUrl(); - // Add loading state link.classList.add('loading'); try { - const response = await fetch(rawUrl); - - if (!response.ok) { - throw new Error(`Failed to fetch markdown: ${response.status}`); - } - - const markdown = await response.text(); + const markdown = await getMarkdownContent(); // Copy to clipboard await navigator.clipboard.writeText(markdown); @@ -221,126 +277,141 @@ document.addEventListener('DOMContentLoaded', () => { window.open(githubUrl, '_blank', 'noopener,noreferrer'); } - // Initialize - const { button, dropdown, overlay } = createPageActionsUI(); - - // Event listeners - button.addEventListener('click', (e) => { - e.stopPropagation(); - toggleDropdown(button, dropdown, overlay); - }); - - overlay.addEventListener('click', () => { - closeDropdown(button, dropdown, overlay); - }); - - // Copy markdown action - document.getElementById('action-copy-markdown').addEventListener('click', async (e) => { - e.preventDefault(); - e.stopPropagation(); - await copyMarkdownToClipboard(e.currentTarget); - }); - - // View markdown action - document.getElementById('action-view-markdown').addEventListener('click', (e) => { - e.preventDefault(); - e.stopPropagation(); - viewMarkdown(); - closeDropdown(button, dropdown, overlay); - }); - - // Ask AI action (disabled for now) - document.getElementById('action-ask-ai').addEventListener('click', (e) => { - e.preventDefault(); - e.stopPropagation(); - // Future: Integrate with Ask AI feature - // For now, do nothing (disabled state) - }); - - // Close on ESC key - document.addEventListener('keydown', (e) => { - if (e.key === 'Escape' && dropdown.classList.contains('active')) { - closeDropdown(button, dropdown, overlay); + (async () => { + if (!shouldShowButton()) { + return; } - }); - // Close when clicking outside - document.addEventListener('click', (e) => { - if (!dropdown.contains(e.target) && !button.contains(e.target)) { - closeDropdown(button, dropdown, overlay); + const markdownAvailable = await ensureMarkdownCached(); + if (!markdownAvailable) { + return; } - }); - // Prevent dropdown from closing when clicking inside - dropdown.addEventListener('click', (e) => { - // Only stop propagation if not clicking on a link - if (!e.target.closest('.page-action-link')) { + const ui = createPageActionsUI(); + if (!ui) { + return; + } + + const { button, dropdown, overlay } = ui; + + // Event listeners + button.addEventListener('click', (e) => { e.stopPropagation(); - } - }); - - // Close dropdown on link click (except for copy which handles itself) - dropdown.querySelectorAll('.page-action-link:not(#action-copy-markdown)').forEach(link => { - link.addEventListener('click', () => { - if (!link.classList.contains('disabled')) { - setTimeout(() => { - closeDropdown(button, dropdown, overlay); - }, 100); - } + toggleDropdown(button, dropdown, overlay); }); - }); - // Handle window resize - let resizeTimer; - window.addEventListener('resize', () => { - clearTimeout(resizeTimer); - resizeTimer = setTimeout(() => { - // Close dropdown on resize to prevent positioning issues - if (dropdown.classList.contains('active')) { + overlay.addEventListener('click', () => { + closeDropdown(button, dropdown, overlay); + }); + + // Copy markdown action + document.getElementById('action-copy-markdown').addEventListener('click', async (e) => { + e.preventDefault(); + e.stopPropagation(); + await copyMarkdownToClipboard(e.currentTarget); + }); + + // View markdown action + document.getElementById('action-view-markdown').addEventListener('click', (e) => { + e.preventDefault(); + e.stopPropagation(); + viewMarkdown(); + closeDropdown(button, dropdown, overlay); + }); + + // Ask AI action (disabled for now) + document.getElementById('action-ask-ai').addEventListener('click', (e) => { + e.preventDefault(); + e.stopPropagation(); + // Future: Integrate with Ask AI feature + // For now, do nothing (disabled state) + }); + + // Close on ESC key + document.addEventListener('keydown', (e) => { + if (e.key === 'Escape' && dropdown.classList.contains('active')) { closeDropdown(button, dropdown, overlay); } - }, 250); - }); + }); - // Accessibility: Focus management - button.addEventListener('keydown', (e) => { - if (e.key === 'Enter' || e.key === ' ') { - e.preventDefault(); - toggleDropdown(button, dropdown, overlay); + // Close when clicking outside + document.addEventListener('click', (e) => { + if (!dropdown.contains(e.target) && !button.contains(e.target)) { + closeDropdown(button, dropdown, overlay); + } + }); - // Focus first menu item when opening - if (dropdown.classList.contains('active')) { - const firstLink = dropdown.querySelector('.page-action-link:not(.disabled)'); - if (firstLink) { - setTimeout(() => firstLink.focus(), 100); + // Prevent dropdown from closing when clicking inside + dropdown.addEventListener('click', (e) => { + // Only stop propagation if not clicking on a link + if (!e.target.closest('.page-action-link')) { + e.stopPropagation(); + } + }); + + // Close dropdown on link click (except for copy which handles itself) + dropdown.querySelectorAll('.page-action-link:not(#action-copy-markdown)').forEach(link => { + link.addEventListener('click', () => { + if (!link.classList.contains('disabled')) { + setTimeout(() => { + closeDropdown(button, dropdown, overlay); + }, 100); + } + }); + }); + + // Handle window resize + let resizeTimer; + window.addEventListener('resize', () => { + clearTimeout(resizeTimer); + resizeTimer = setTimeout(() => { + // Close dropdown on resize to prevent positioning issues + if (dropdown.classList.contains('active')) { + closeDropdown(button, dropdown, overlay); + } + }, 250); + }); + + // Accessibility: Focus management + button.addEventListener('keydown', (e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + toggleDropdown(button, dropdown, overlay); + + // Focus first menu item when opening + if (dropdown.classList.contains('active')) { + const firstLink = dropdown.querySelector('.page-action-link:not(.disabled)'); + if (firstLink) { + setTimeout(() => firstLink.focus(), 100); + } } } - } - }); + }); - // Arrow key navigation within menu - dropdown.addEventListener('keydown', (e) => { - if (!dropdown.classList.contains('active')) return; + // Arrow key navigation within menu + dropdown.addEventListener('keydown', (e) => { + if (!dropdown.classList.contains('active')) return; - const links = Array.from(dropdown.querySelectorAll('.page-action-link:not(.disabled)')); - const currentIndex = links.indexOf(document.activeElement); + const links = Array.from(dropdown.querySelectorAll('.page-action-link:not(.disabled)')); + const currentIndex = links.indexOf(document.activeElement); - if (e.key === 'ArrowDown') { - e.preventDefault(); - const nextIndex = (currentIndex + 1) % links.length; - links[nextIndex].focus(); - } else if (e.key === 'ArrowUp') { - e.preventDefault(); - const prevIndex = (currentIndex - 1 + links.length) % links.length; - links[prevIndex].focus(); - } else if (e.key === 'Home') { - e.preventDefault(); - links[0].focus(); - } else if (e.key === 'End') { - e.preventDefault(); - links[links.length - 1].focus(); - } - }); + if (e.key === 'ArrowDown') { + e.preventDefault(); + const nextIndex = (currentIndex + 1) % links.length; + links[nextIndex].focus(); + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + const prevIndex = (currentIndex - 1 + links.length) % links.length; + links[prevIndex].focus(); + } else if (e.key === 'Home') { + e.preventDefault(); + links[0].focus(); + } else if (e.key === 'End') { + e.preventDefault(); + links[links.length - 1].focus(); + } + }); - console.log('Page Actions initialized for:', getCurrentMarkdownPath()); + console.log('Page Actions initialized for:', getCurrentMarkdownPath()); + })(); }); \ No newline at end of file From 8c622777181bdc612c13904f9d35ef08814fd4c0 Mon Sep 17 00:00:00 2001 From: unclecode Date: Mon, 6 Oct 2025 20:58:35 +0800 Subject: [PATCH 3/5] feat(marketplace): add sponsor logo uploads Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- docs/md_v2/marketplace/admin/admin.css | 109 +++++++++++++ docs/md_v2/marketplace/admin/admin.js | 143 ++++++++++++++++-- docs/md_v2/marketplace/backend/server.py | 46 +++++- .../marketplace/backend/uploads/.gitignore | 2 + docs/md_v2/marketplace/marketplace.css | 15 ++ docs/md_v2/marketplace/marketplace.js | 19 ++- 6 files changed, 321 insertions(+), 13 deletions(-) create mode 100644 docs/md_v2/marketplace/backend/uploads/.gitignore diff --git a/docs/md_v2/marketplace/admin/admin.css b/docs/md_v2/marketplace/admin/admin.css index 7296a801..66b975a9 100644 --- a/docs/md_v2/marketplace/admin/admin.css +++ b/docs/md_v2/marketplace/admin/admin.css @@ -431,6 +431,16 @@ gap: 0.5rem; } +.table-logo { + width: 48px; + height: 48px; + object-fit: contain; + border-radius: 6px; + background: var(--bg-tertiary); + border: 1px solid var(--border-color); + padding: 4px; +} + .btn-edit, .btn-delete, .btn-duplicate { padding: 0.25rem 0.5rem; background: transparent; @@ -585,6 +595,105 @@ cursor: pointer; } +.sponsor-form { + grid-template-columns: 200px repeat(2, minmax(220px, 1fr)); + align-items: flex-start; + grid-auto-flow: dense; +} + +.sponsor-logo-group { + grid-row: span 3; + display: flex; + flex-direction: column; + gap: 0.75rem; +} + +.span-two { + grid-column: span 2; +} + +.logo-upload { + position: relative; + width: 180px; +} + +.image-preview { + width: 180px; + height: 180px; + border: 1px dashed var(--border-color); + border-radius: 12px; + display: flex; + align-items: center; + justify-content: center; + background: var(--bg-tertiary); + overflow: hidden; +} + +.image-preview.empty { + color: var(--text-secondary); + font-size: 0.75rem; + text-align: center; + padding: 0.75rem; +} + +.image-preview img { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +.upload-btn { + position: absolute; + left: 50%; + bottom: 12px; + transform: translateX(-50%); + padding: 0.35rem 1rem; + background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal)); + border: none; + border-radius: 999px; + color: var(--bg-dark); + font-size: 0.75rem; + font-weight: 600; + cursor: pointer; + box-shadow: 0 6px 18px rgba(80, 255, 255, 0.25); +} + +.upload-btn:hover { + box-shadow: 0 8px 22px rgba(80, 255, 255, 0.35); +} + +.logo-upload input[type="file"] { + display: none; +} + +.upload-hint { + font-size: 0.75rem; + color: var(--text-secondary); + margin: 0; +} + +@media (max-width: 960px) { + .sponsor-form { + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + } + + .sponsor-logo-group { + grid-column: 1 / -1; + grid-row: auto; + flex-direction: row; + align-items: center; + gap: 1.5rem; + } + + .logo-upload { + width: 160px; + } + + .span-two { + grid-column: 1 / -1; + } +} + /* Rich Text Editor */ .editor-toolbar { display: flex; diff --git a/docs/md_v2/marketplace/admin/admin.js b/docs/md_v2/marketplace/admin/admin.js index 258858da..adb31d0b 100644 --- a/docs/md_v2/marketplace/admin/admin.js +++ b/docs/md_v2/marketplace/admin/admin.js @@ -1,5 +1,21 @@ // Admin Dashboard - Smart & Powerful -const API_BASE = '/api'; +const { API_BASE, API_ORIGIN } = (() => { + const { hostname, port } = window.location; + if ((hostname === 'localhost' || hostname === '127.0.0.1') && port === '8000') { + const origin = 'http://127.0.0.1:8100'; + return { API_BASE: `${origin}/api`, API_ORIGIN: origin }; + } + return { API_BASE: '/api', API_ORIGIN: '' }; +})(); + +const resolveAssetUrl = (path) => { + if (!path) return ''; + if (/^https?:\/\//i.test(path)) return path; + if (path.startsWith('/') && API_ORIGIN) { + return `${API_ORIGIN}${path}`; + } + return path; +}; class AdminDashboard { constructor() { @@ -144,13 +160,19 @@ class AdminDashboard { } async apiCall(endpoint, options = {}) { + const isFormData = options.body instanceof FormData; + const headers = { + 'Authorization': `Bearer ${this.token}`, + ...options.headers + }; + + if (!isFormData && !headers['Content-Type']) { + headers['Content-Type'] = 'application/json'; + } + const response = await fetch(`${API_BASE}${endpoint}`, { ...options, - headers: { - 'Authorization': `Bearer ${this.token}`, - 'Content-Type': 'application/json', - ...options.headers - } + headers }); if (response.status === 401) { @@ -189,7 +211,10 @@ class AdminDashboard { } async loadSponsors() { - this.data.sponsors = await this.apiCall('/sponsors'); + const cacheBuster = Date.now(); + this.data.sponsors = await this.apiCall(`/sponsors?limit=100&_=${cacheBuster}`, { + cache: 'no-store' + }); this.renderSponsorsTable(this.data.sponsors); } @@ -314,6 +339,7 @@ class AdminDashboard { ID + Logo Company Tier Start @@ -326,6 +352,7 @@ class AdminDashboard { ${sponsors.map(sponsor => ` ${sponsor.id} + ${sponsor.logo_url ? `` : '-'} ${sponsor.company_name} ${sponsor.tier} ${new Date(sponsor.start_date).toLocaleDateString()} @@ -389,6 +416,10 @@ class AdminDashboard { modal.classList.remove('hidden'); modal.dataset.type = type; + + if (type === 'sponsors') { + this.setupLogoUploadHandlers(); + } } getAppForm(app) { @@ -524,9 +555,22 @@ class AdminDashboard { } getSponsorForm(sponsor) { + const existingFile = sponsor?.logo_url ? sponsor.logo_url.split('/').pop().split('?')[0] : ''; return ` -
-
+ - + \ No newline at end of file diff --git a/docs/md_v2/marketplace/backend/server.py b/docs/md_v2/marketplace/backend/server.py index 77540020..0f177386 100644 --- a/docs/md_v2/marketplace/backend/server.py +++ b/docs/md_v2/marketplace/backend/server.py @@ -7,6 +7,7 @@ from typing import Optional, Dict, Any import json import hashlib import secrets +import re from pathlib import Path from database import DatabaseManager from datetime import datetime, timedelta @@ -58,6 +59,29 @@ def json_response(data, cache_time=3600): } ) + +def to_int(value, default=0): + """Coerce incoming values to integers, falling back to default.""" + if value is None: + return default + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return default + + match = re.match(r"^-?\d+", stripped) + if match: + try: + return int(match.group()) + except ValueError: + return default + return default + # ============= PUBLIC ENDPOINTS ============= @app.get("/api/apps") @@ -141,6 +165,8 @@ async def get_article(slug: str): async def get_categories(): """Get all categories ordered by index""" categories = db.get_all('categories', limit=50) + for category in categories: + category['order_index'] = to_int(category.get('order_index'), 0) categories.sort(key=lambda x: x.get('order_index', 0)) return json_response(categories, cache_time=7200) @@ -360,6 +386,9 @@ async def delete_article(article_id: int): async def create_category(category_data: Dict[str, Any]): """Create new category""" try: + category_data = dict(category_data) + category_data['order_index'] = to_int(category_data.get('order_index'), 0) + cursor = db.conn.cursor() columns = ', '.join(category_data.keys()) placeholders = ', '.join(['?' for _ in category_data]) @@ -374,6 +403,10 @@ async def create_category(category_data: Dict[str, Any]): async def update_category(cat_id: int, category_data: Dict[str, Any]): """Update category""" try: + category_data = dict(category_data) + if 'order_index' in category_data: + category_data['order_index'] = to_int(category_data.get('order_index'), 0) + set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()]) cursor = db.conn.cursor() cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?", @@ -383,6 +416,18 @@ async def update_category(cat_id: int, category_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) + +@app.delete("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)]) +async def delete_category(cat_id: int): + """Delete category""" + try: + cursor = db.conn.cursor() + cursor.execute("DELETE FROM categories WHERE id = ?", (cat_id,)) + db.conn.commit() + return {"message": "Category deleted"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + # Sponsors CRUD @app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)]) async def create_sponsor(sponsor_data: Dict[str, Any]): @@ -411,6 +456,18 @@ async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]): except Exception as e: raise HTTPException(status_code=400, detail=str(e)) + +@app.delete("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)]) +async def delete_sponsor(sponsor_id: int): + """Delete sponsor""" + try: + cursor = db.conn.cursor() + cursor.execute("DELETE FROM sponsors WHERE id = ?", (sponsor_id,)) + db.conn.commit() + return {"message": "Sponsor deleted"} + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + @app.get("/") async def root(): """API info"""