Merge branch 'develop' into release/v0.7.5

This commit is contained in:
ntohidi
2025-10-09 12:53:39 +08:00
11 changed files with 626 additions and 187 deletions

View File

@@ -10,7 +10,6 @@ Today I'm releasing Crawl4AI v0.7.4—the Intelligent Table Extraction & Perform
- **🚀 LLMTableExtraction**: Revolutionary table extraction with intelligent chunking for massive tables - **🚀 LLMTableExtraction**: Revolutionary table extraction with intelligent chunking for massive tables
- **⚡ Enhanced Concurrency**: True concurrency improvements for fast-completing tasks in batch operations - **⚡ Enhanced Concurrency**: True concurrency improvements for fast-completing tasks in batch operations
- **🧹 Memory Management Refactor**: Streamlined memory utilities and better resource management
- **🔧 Browser Manager Fixes**: Resolved race conditions in concurrent page creation - **🔧 Browser Manager Fixes**: Resolved race conditions in concurrent page creation
- **⌨️ Cross-Platform Browser Profiler**: Improved keyboard handling and quit mechanisms - **⌨️ Cross-Platform Browser Profiler**: Improved keyboard handling and quit mechanisms
- **🔗 Advanced URL Processing**: Better handling of raw URLs and base tag link resolution - **🔗 Advanced URL Processing**: Better handling of raw URLs and base tag link resolution
@@ -158,40 +157,6 @@ async with AsyncWebCrawler() as crawler:
- **Monitoring Systems**: Faster health checks and status page monitoring - **Monitoring Systems**: Faster health checks and status page monitoring
- **Data Aggregation**: Improved performance for real-time data collection - **Data Aggregation**: Improved performance for real-time data collection
## 🧹 Memory Management Refactor: Cleaner Architecture
**The Problem:** Memory utilities were scattered and difficult to maintain, with potential import conflicts and unclear organization.
**My Solution:** I consolidated all memory-related utilities into the main `utils.py` module, creating a cleaner, more maintainable architecture.
### Improved Memory Handling
```python
# All memory utilities now consolidated
from crawl4ai.utils import get_true_memory_usage_percent, MemoryMonitor
# Enhanced memory monitoring
monitor = MemoryMonitor()
monitor.start_monitoring()
async with AsyncWebCrawler() as crawler:
# Memory-efficient batch processing
results = await crawler.arun_many(large_url_list)
# Get accurate memory metrics
memory_usage = get_true_memory_usage_percent()
memory_report = monitor.get_report()
print(f"Memory efficiency: {memory_report['efficiency']:.1f}%")
print(f"Peak usage: {memory_report['peak_mb']:.1f} MB")
```
**Expected Real-World Impact:**
- **Production Stability**: More reliable memory tracking and management
- **Code Maintainability**: Cleaner architecture for easier debugging
- **Import Clarity**: Resolved potential conflicts and import issues
- **Developer Experience**: Simpler API for memory monitoring
## 🔧 Critical Stability Fixes ## 🔧 Critical Stability Fixes
### Browser Manager Race Condition Resolution ### Browser Manager Race Condition Resolution

View File

@@ -201,18 +201,6 @@ ul>li.page-action-item::after{
} }
/* Badge */ /* Badge */
.page-action-badge {
display: inline-block;
background: #f59e0b;
color: #070708;
padding: 0.125rem 0.5rem;
border-radius: 12px;
font-size: 0.65rem;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
}
/* External link indicator */ /* External link indicator */
.page-action-external::after { .page-action-external::after {
content: '→'; content: '→';

View File

@@ -7,9 +7,12 @@ document.addEventListener('DOMContentLoaded', () => {
githubRepo: 'unclecode/crawl4ai', githubRepo: 'unclecode/crawl4ai',
githubBranch: 'main', githubBranch: 'main',
docsPath: 'docs/md_v2', docsPath: 'docs/md_v2',
excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/'], // Don't show on app pages excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/', '/core/ask-ai/'], // Don't show on app pages
}; };
let cachedMarkdown = null;
let cachedMarkdownPath = null;
// Check if we should show the button on this page // Check if we should show the button on this page
function shouldShowButton() { function shouldShowButton() {
const currentPath = window.location.pathname; const currentPath = window.location.pathname;
@@ -19,6 +22,17 @@ document.addEventListener('DOMContentLoaded', () => {
return false; return false;
} }
// Don't show on 404 pages
if (document.title && document.title.toLowerCase().includes('404')) {
return false;
}
// Require mkdocs main content container
const mainContent = document.getElementById('terminal-mkdocs-main-content');
if (!mainContent) {
return false;
}
// Don't show on excluded paths (apps) // Don't show on excluded paths (apps)
for (const excludePath of config.excludePaths) { for (const excludePath of config.excludePaths) {
if (currentPath.includes(excludePath)) { if (currentPath.includes(excludePath)) {
@@ -53,6 +67,56 @@ document.addEventListener('DOMContentLoaded', () => {
return `${path}.md`; return `${path}.md`;
} }
async function loadMarkdownContent() {
const mdPath = getCurrentMarkdownPath();
if (!mdPath) {
throw new Error('Invalid markdown path');
}
const rawUrl = getGithubRawUrl();
const response = await fetch(rawUrl);
if (!response.ok) {
throw new Error(`Failed to fetch markdown: ${response.status}`);
}
const markdown = await response.text();
cachedMarkdown = markdown;
cachedMarkdownPath = mdPath;
return markdown;
}
async function ensureMarkdownCached() {
const mdPath = getCurrentMarkdownPath();
if (!mdPath) {
return false;
}
if (cachedMarkdown && cachedMarkdownPath === mdPath) {
return true;
}
try {
await loadMarkdownContent();
return true;
} catch (error) {
console.warn('Page Actions: Markdown not available for this page.', error);
cachedMarkdown = null;
cachedMarkdownPath = null;
return false;
}
}
async function getMarkdownContent() {
const available = await ensureMarkdownCached();
if (!available) {
throw new Error('Markdown not available for this page.');
}
return cachedMarkdown;
}
// Get GitHub raw URL for current page // Get GitHub raw URL for current page
function getGithubRawUrl() { function getGithubRawUrl() {
const mdPath = getCurrentMarkdownPath(); const mdPath = getCurrentMarkdownPath();
@@ -112,13 +176,11 @@ document.addEventListener('DOMContentLoaded', () => {
</li> </li>
<div class="page-actions-divider"></div> <div class="page-actions-divider"></div>
<li class="page-action-item"> <li class="page-action-item">
<a href="#" class="page-action-link disabled" id="action-ask-ai" role="menuitem"> <a href="#" class="page-action-link page-action-external" id="action-open-chatgpt" role="menuitem">
<span class="page-action-icon icon-ai"></span> <span class="page-action-icon icon-ai"></span>
<span class="page-action-text"> <span class="page-action-text">
<span class="page-action-label">Ask AI about page</span> <span class="page-action-label">Open in ChatGPT</span>
<span class="page-action-description"> <span class="page-action-description">Ask questions about this page</span>
<span class="page-action-badge">Coming Soon</span>
</span>
</span> </span>
</a> </a>
</li> </li>
@@ -180,19 +242,11 @@ document.addEventListener('DOMContentLoaded', () => {
// Copy markdown to clipboard // Copy markdown to clipboard
async function copyMarkdownToClipboard(link) { async function copyMarkdownToClipboard(link) {
const rawUrl = getGithubRawUrl();
// Add loading state // Add loading state
link.classList.add('loading'); link.classList.add('loading');
try { try {
const response = await fetch(rawUrl); const markdown = await getMarkdownContent();
if (!response.ok) {
throw new Error(`Failed to fetch markdown: ${response.status}`);
}
const markdown = await response.text();
// Copy to clipboard // Copy to clipboard
await navigator.clipboard.writeText(markdown); await navigator.clipboard.writeText(markdown);
@@ -221,8 +275,34 @@ document.addEventListener('DOMContentLoaded', () => {
window.open(githubUrl, '_blank', 'noopener,noreferrer'); window.open(githubUrl, '_blank', 'noopener,noreferrer');
} }
// Initialize function getCurrentPageUrl() {
const { button, dropdown, overlay } = createPageActionsUI(); const { href } = window.location;
return href.split('#')[0];
}
function openChatGPT() {
const pageUrl = getCurrentPageUrl();
const prompt = encodeURIComponent(`Read ${pageUrl} so I can ask questions about it.`);
const chatUrl = `https://chatgpt.com/?hint=search&prompt=${prompt}`;
window.open(chatUrl, '_blank', 'noopener,noreferrer');
}
(async () => {
if (!shouldShowButton()) {
return;
}
const markdownAvailable = await ensureMarkdownCached();
if (!markdownAvailable) {
return;
}
const ui = createPageActionsUI();
if (!ui) {
return;
}
const { button, dropdown, overlay } = ui;
// Event listeners // Event listeners
button.addEventListener('click', (e) => { button.addEventListener('click', (e) => {
@@ -249,12 +329,12 @@ document.addEventListener('DOMContentLoaded', () => {
closeDropdown(button, dropdown, overlay); closeDropdown(button, dropdown, overlay);
}); });
// Ask AI action (disabled for now) // Open in ChatGPT action
document.getElementById('action-ask-ai').addEventListener('click', (e) => { document.getElementById('action-open-chatgpt').addEventListener('click', (e) => {
e.preventDefault(); e.preventDefault();
e.stopPropagation(); e.stopPropagation();
// Future: Integrate with Ask AI feature openChatGPT();
// For now, do nothing (disabled state) closeDropdown(button, dropdown, overlay);
}); });
// Close on ESC key // Close on ESC key
@@ -343,4 +423,5 @@ document.addEventListener('DOMContentLoaded', () => {
}); });
console.log('Page Actions initialized for:', getCurrentMarkdownPath()); console.log('Page Actions initialized for:', getCurrentMarkdownPath());
})();
}); });

View File

@@ -431,6 +431,16 @@
gap: 0.5rem; gap: 0.5rem;
} }
.table-logo {
width: 48px;
height: 48px;
object-fit: contain;
border-radius: 6px;
background: var(--bg-tertiary);
border: 1px solid var(--border-color);
padding: 4px;
}
.btn-edit, .btn-delete, .btn-duplicate { .btn-edit, .btn-delete, .btn-duplicate {
padding: 0.25rem 0.5rem; padding: 0.25rem 0.5rem;
background: transparent; background: transparent;
@@ -585,6 +595,105 @@
cursor: pointer; cursor: pointer;
} }
.sponsor-form {
grid-template-columns: 200px repeat(2, minmax(220px, 1fr));
align-items: flex-start;
grid-auto-flow: dense;
}
.sponsor-logo-group {
grid-row: span 3;
display: flex;
flex-direction: column;
gap: 0.75rem;
}
.span-two {
grid-column: span 2;
}
.logo-upload {
position: relative;
width: 180px;
}
.image-preview {
width: 180px;
height: 180px;
border: 1px dashed var(--border-color);
border-radius: 12px;
display: flex;
align-items: center;
justify-content: center;
background: var(--bg-tertiary);
overflow: hidden;
}
.image-preview.empty {
color: var(--text-secondary);
font-size: 0.75rem;
text-align: center;
padding: 0.75rem;
}
.image-preview img {
max-width: 100%;
max-height: 100%;
object-fit: contain;
}
.upload-btn {
position: absolute;
left: 50%;
bottom: 12px;
transform: translateX(-50%);
padding: 0.35rem 1rem;
background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
border: none;
border-radius: 999px;
color: var(--bg-dark);
font-size: 0.75rem;
font-weight: 600;
cursor: pointer;
box-shadow: 0 6px 18px rgba(80, 255, 255, 0.25);
}
.upload-btn:hover {
box-shadow: 0 8px 22px rgba(80, 255, 255, 0.35);
}
.logo-upload input[type="file"] {
display: none;
}
.upload-hint {
font-size: 0.75rem;
color: var(--text-secondary);
margin: 0;
}
@media (max-width: 960px) {
.sponsor-form {
grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
}
.sponsor-logo-group {
grid-column: 1 / -1;
grid-row: auto;
flex-direction: row;
align-items: center;
gap: 1.5rem;
}
.logo-upload {
width: 160px;
}
.span-two {
grid-column: 1 / -1;
}
}
/* Rich Text Editor */ /* Rich Text Editor */
.editor-toolbar { .editor-toolbar {
display: flex; display: flex;

View File

@@ -1,5 +1,49 @@
// Admin Dashboard - Smart & Powerful // Admin Dashboard - Smart & Powerful
const API_BASE = '/api'; const { API_BASE, API_ORIGIN } = (() => {
const cleanOrigin = (value) => value ? value.replace(/\/$/, '') : '';
const params = new URLSearchParams(window.location.search);
const overrideParam = cleanOrigin(params.get('api_origin'));
let storedOverride = '';
try {
storedOverride = cleanOrigin(localStorage.getItem('marketplace_api_origin'));
} catch (error) {
storedOverride = '';
}
let origin = overrideParam || storedOverride;
if (overrideParam && overrideParam !== storedOverride) {
try {
localStorage.setItem('marketplace_api_origin', overrideParam);
} catch (error) {
// ignore storage errors (private mode, etc.)
}
}
const { protocol, hostname, port } = window.location;
const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
if (!origin && isLocalHost && port !== '8100') {
origin = `${protocol}//127.0.0.1:8100`;
}
if (origin) {
const normalized = cleanOrigin(origin);
return { API_BASE: `${normalized}/api`, API_ORIGIN: normalized };
}
return { API_BASE: '/api', API_ORIGIN: '' };
})();
const resolveAssetUrl = (path) => {
if (!path) return '';
if (/^https?:\/\//i.test(path)) return path;
if (path.startsWith('/') && API_ORIGIN) {
return `${API_ORIGIN}${path}`;
}
return path;
};
class AdminDashboard { class AdminDashboard {
constructor() { constructor() {
@@ -144,13 +188,19 @@ class AdminDashboard {
} }
async apiCall(endpoint, options = {}) { async apiCall(endpoint, options = {}) {
const isFormData = options.body instanceof FormData;
const headers = {
'Authorization': `Bearer ${this.token}`,
...options.headers
};
if (!isFormData && !headers['Content-Type']) {
headers['Content-Type'] = 'application/json';
}
const response = await fetch(`${API_BASE}${endpoint}`, { const response = await fetch(`${API_BASE}${endpoint}`, {
...options, ...options,
headers: { headers
'Authorization': `Bearer ${this.token}`,
'Content-Type': 'application/json',
...options.headers
}
}); });
if (response.status === 401) { if (response.status === 401) {
@@ -163,7 +213,9 @@ class AdminDashboard {
} }
async loadStats() { async loadStats() {
const stats = await this.apiCall('/admin/stats'); const stats = await this.apiCall(`/admin/stats?_=${Date.now()}`, {
cache: 'no-store'
});
document.getElementById('stat-apps').textContent = stats.apps.total; document.getElementById('stat-apps').textContent = stats.apps.total;
document.getElementById('stat-featured').textContent = stats.apps.featured; document.getElementById('stat-featured').textContent = stats.apps.featured;
@@ -174,22 +226,32 @@ class AdminDashboard {
} }
async loadApps() { async loadApps() {
this.data.apps = await this.apiCall('/apps?limit=100'); this.data.apps = await this.apiCall(`/apps?limit=100&_=${Date.now()}`, {
cache: 'no-store'
});
this.renderAppsTable(this.data.apps); this.renderAppsTable(this.data.apps);
} }
async loadArticles() { async loadArticles() {
this.data.articles = await this.apiCall('/articles?limit=100'); this.data.articles = await this.apiCall(`/articles?limit=100&_=${Date.now()}`, {
cache: 'no-store'
});
this.renderArticlesTable(this.data.articles); this.renderArticlesTable(this.data.articles);
} }
async loadCategories() { async loadCategories() {
this.data.categories = await this.apiCall('/categories'); const cacheBuster = Date.now();
this.data.categories = await this.apiCall(`/categories?_=${cacheBuster}`, {
cache: 'no-store'
});
this.renderCategoriesTable(this.data.categories); this.renderCategoriesTable(this.data.categories);
} }
async loadSponsors() { async loadSponsors() {
this.data.sponsors = await this.apiCall('/sponsors'); const cacheBuster = Date.now();
this.data.sponsors = await this.apiCall(`/sponsors?limit=100&_=${cacheBuster}`, {
cache: 'no-store'
});
this.renderSponsorsTable(this.data.sponsors); this.renderSponsorsTable(this.data.sponsors);
} }
@@ -314,6 +376,7 @@ class AdminDashboard {
<thead> <thead>
<tr> <tr>
<th>ID</th> <th>ID</th>
<th>Logo</th>
<th>Company</th> <th>Company</th>
<th>Tier</th> <th>Tier</th>
<th>Start</th> <th>Start</th>
@@ -326,6 +389,7 @@ class AdminDashboard {
${sponsors.map(sponsor => ` ${sponsors.map(sponsor => `
<tr> <tr>
<td>${sponsor.id}</td> <td>${sponsor.id}</td>
<td>${sponsor.logo_url ? `<img class="table-logo" src="${resolveAssetUrl(sponsor.logo_url)}" alt="${sponsor.company_name} logo">` : '-'}</td>
<td>${sponsor.company_name}</td> <td>${sponsor.company_name}</td>
<td>${sponsor.tier}</td> <td>${sponsor.tier}</td>
<td>${new Date(sponsor.start_date).toLocaleDateString()}</td> <td>${new Date(sponsor.start_date).toLocaleDateString()}</td>
@@ -389,6 +453,10 @@ class AdminDashboard {
modal.classList.remove('hidden'); modal.classList.remove('hidden');
modal.dataset.type = type; modal.dataset.type = type;
if (type === 'sponsors') {
this.setupLogoUploadHandlers();
}
} }
getAppForm(app) { getAppForm(app) {
@@ -524,9 +592,22 @@ class AdminDashboard {
} }
getSponsorForm(sponsor) { getSponsorForm(sponsor) {
const existingFile = sponsor?.logo_url ? sponsor.logo_url.split('/').pop().split('?')[0] : '';
return ` return `
<div class="form-grid"> <div class="form-grid sponsor-form">
<div class="form-group"> <div class="form-group sponsor-logo-group">
<label>Logo</label>
<input type="hidden" id="form-logo-url" value="${sponsor?.logo_url || ''}">
<div class="logo-upload">
<div class="image-preview ${sponsor?.logo_url ? '' : 'empty'}" id="form-logo-preview">
${sponsor?.logo_url ? `<img src="${resolveAssetUrl(sponsor.logo_url)}" alt="Logo preview">` : '<span>No logo uploaded</span>'}
</div>
<button type="button" class="upload-btn" id="form-logo-button">Upload Logo</button>
<input type="file" id="form-logo-file" accept="image/png,image/jpeg,image/webp,image/svg+xml" hidden>
</div>
<p class="upload-hint" id="form-logo-filename">${existingFile ? `Current: ${existingFile}` : 'No file selected'}</p>
</div>
<div class="form-group span-two">
<label>Company Name *</label> <label>Company Name *</label>
<input type="text" id="form-name" value="${sponsor?.company_name || ''}" required> <input type="text" id="form-name" value="${sponsor?.company_name || ''}" required>
</div> </div>
@@ -567,9 +648,30 @@ class AdminDashboard {
async saveItem() { async saveItem() {
const modal = document.getElementById('form-modal'); const modal = document.getElementById('form-modal');
const type = modal.dataset.type; const type = modal.dataset.type;
const data = this.collectFormData(type);
try { try {
if (type === 'sponsors') {
const fileInput = document.getElementById('form-logo-file');
if (fileInput && fileInput.files && fileInput.files[0]) {
const formData = new FormData();
formData.append('file', fileInput.files[0]);
formData.append('folder', 'sponsors');
const uploadResponse = await this.apiCall('/admin/upload-image', {
method: 'POST',
body: formData
});
if (!uploadResponse.url) {
throw new Error('Image upload failed');
}
document.getElementById('form-logo-url').value = uploadResponse.url;
}
}
const data = this.collectFormData(type);
if (this.editingItem) { if (this.editingItem) {
await this.apiCall(`/admin/${type}/${this.editingItem.id}`, { await this.apiCall(`/admin/${type}/${this.editingItem.id}`, {
method: 'PUT', method: 'PUT',
@@ -599,8 +701,10 @@ class AdminDashboard {
data.description = document.getElementById('form-description').value; data.description = document.getElementById('form-description').value;
data.category = document.getElementById('form-category').value; data.category = document.getElementById('form-category').value;
data.type = document.getElementById('form-type').value; data.type = document.getElementById('form-type').value;
data.rating = parseFloat(document.getElementById('form-rating').value); const rating = parseFloat(document.getElementById('form-rating').value);
data.downloads = parseInt(document.getElementById('form-downloads').value); const downloads = parseInt(document.getElementById('form-downloads').value, 10);
data.rating = Number.isFinite(rating) ? rating : 0;
data.downloads = Number.isFinite(downloads) ? downloads : 0;
data.image = document.getElementById('form-image').value; data.image = document.getElementById('form-image').value;
data.website_url = document.getElementById('form-website').value; data.website_url = document.getElementById('form-website').value;
data.github_url = document.getElementById('form-github').value; data.github_url = document.getElementById('form-github').value;
@@ -621,9 +725,11 @@ class AdminDashboard {
data.slug = this.generateSlug(data.name); data.slug = this.generateSlug(data.name);
data.icon = document.getElementById('form-icon').value; data.icon = document.getElementById('form-icon').value;
data.description = document.getElementById('form-description').value; data.description = document.getElementById('form-description').value;
data.order_index = parseInt(document.getElementById('form-order').value); const orderIndex = parseInt(document.getElementById('form-order').value, 10);
data.order_index = Number.isFinite(orderIndex) ? orderIndex : 0;
} else if (type === 'sponsors') { } else if (type === 'sponsors') {
data.company_name = document.getElementById('form-name').value; data.company_name = document.getElementById('form-name').value;
data.logo_url = document.getElementById('form-logo-url').value;
data.tier = document.getElementById('form-tier').value; data.tier = document.getElementById('form-tier').value;
data.landing_url = document.getElementById('form-landing').value; data.landing_url = document.getElementById('form-landing').value;
data.banner_url = document.getElementById('form-banner').value; data.banner_url = document.getElementById('form-banner').value;
@@ -635,6 +741,63 @@ class AdminDashboard {
return data; return data;
} }
setupLogoUploadHandlers() {
const fileInput = document.getElementById('form-logo-file');
const preview = document.getElementById('form-logo-preview');
const logoUrlInput = document.getElementById('form-logo-url');
const trigger = document.getElementById('form-logo-button');
const fileNameEl = document.getElementById('form-logo-filename');
if (!fileInput || !preview || !logoUrlInput) return;
const setFileName = (text) => {
if (fileNameEl) {
fileNameEl.textContent = text;
}
};
const setEmptyState = () => {
preview.innerHTML = '<span>No logo uploaded</span>';
preview.classList.add('empty');
setFileName('No file selected');
};
const setExistingState = () => {
if (logoUrlInput.value) {
const existingFile = logoUrlInput.value.split('/').pop().split('?')[0];
preview.innerHTML = `<img src="${resolveAssetUrl(logoUrlInput.value)}" alt="Logo preview">`;
preview.classList.remove('empty');
setFileName(existingFile ? `Current: ${existingFile}` : 'Current logo');
} else {
setEmptyState();
}
};
setExistingState();
if (trigger) {
trigger.onclick = () => fileInput.click();
}
fileInput.addEventListener('change', (event) => {
const file = event.target.files && event.target.files[0];
if (!file) {
setExistingState();
return;
}
setFileName(file.name);
const reader = new FileReader();
reader.onload = () => {
preview.innerHTML = `<img src="${reader.result}" alt="Logo preview">`;
preview.classList.remove('empty');
};
reader.readAsDataURL(file);
});
}
async deleteItem(type, id) { async deleteItem(type, id) {
if (!confirm(`Are you sure you want to delete this ${type.slice(0, -1)}?`)) return; if (!confirm(`Are you sure you want to delete this ${type.slice(0, -1)}?`)) return;

View File

@@ -210,6 +210,6 @@
</div> </div>
</div> </div>
<script src="admin.js?v=1759327900"></script> <script src="admin.js?v=1759334000"></script>
</body> </body>
</html> </html>

View File

@@ -1,11 +1,14 @@
from fastapi import FastAPI, HTTPException, Query, Depends, Body from fastapi import FastAPI, HTTPException, Query, Depends, Body, UploadFile, File, Form
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from typing import Optional, List, Dict, Any from typing import Optional, Dict, Any
import json import json
import hashlib import hashlib
import secrets import secrets
import re
from pathlib import Path
from database import DatabaseManager from database import DatabaseManager
from datetime import datetime, timedelta from datetime import datetime, timedelta
@@ -31,6 +34,21 @@ app.add_middleware(
# Initialize database with configurable path # Initialize database with configurable path
db = DatabaseManager(Config.DATABASE_PATH) db = DatabaseManager(Config.DATABASE_PATH)
BASE_DIR = Path(__file__).parent
UPLOAD_ROOT = BASE_DIR / "uploads"
UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
app.mount("/uploads", StaticFiles(directory=UPLOAD_ROOT), name="uploads")
ALLOWED_IMAGE_TYPES = {
"image/png": ".png",
"image/jpeg": ".jpg",
"image/webp": ".webp",
"image/svg+xml": ".svg"
}
ALLOWED_UPLOAD_FOLDERS = {"sponsors"}
MAX_UPLOAD_SIZE = 2 * 1024 * 1024 # 2 MB
def json_response(data, cache_time=3600): def json_response(data, cache_time=3600):
"""Helper to return JSON with cache headers""" """Helper to return JSON with cache headers"""
return JSONResponse( return JSONResponse(
@@ -41,6 +59,29 @@ def json_response(data, cache_time=3600):
} }
) )
def to_int(value, default=0):
"""Coerce incoming values to integers, falling back to default."""
if value is None:
return default
if isinstance(value, bool):
return int(value)
if isinstance(value, (int, float)):
return int(value)
if isinstance(value, str):
stripped = value.strip()
if not stripped:
return default
match = re.match(r"^-?\d+", stripped)
if match:
try:
return int(match.group())
except ValueError:
return default
return default
# ============= PUBLIC ENDPOINTS ============= # ============= PUBLIC ENDPOINTS =============
@app.get("/api/apps") @app.get("/api/apps")
@@ -124,6 +165,8 @@ async def get_article(slug: str):
async def get_categories(): async def get_categories():
"""Get all categories ordered by index""" """Get all categories ordered by index"""
categories = db.get_all('categories', limit=50) categories = db.get_all('categories', limit=50)
for category in categories:
category['order_index'] = to_int(category.get('order_index'), 0)
categories.sort(key=lambda x: x.get('order_index', 0)) categories.sort(key=lambda x: x.get('order_index', 0))
return json_response(categories, cache_time=7200) return json_response(categories, cache_time=7200)
@@ -183,6 +226,31 @@ def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
raise HTTPException(status_code=401, detail="Invalid or expired token") raise HTTPException(status_code=401, detail="Invalid or expired token")
return token return token
@app.post("/api/admin/upload-image", dependencies=[Depends(verify_token)])
async def upload_image(file: UploadFile = File(...), folder: str = Form("sponsors")):
"""Upload image files for admin assets"""
folder = (folder or "").strip().lower()
if folder not in ALLOWED_UPLOAD_FOLDERS:
raise HTTPException(status_code=400, detail="Invalid upload folder")
if file.content_type not in ALLOWED_IMAGE_TYPES:
raise HTTPException(status_code=400, detail="Unsupported file type")
contents = await file.read()
if len(contents) > MAX_UPLOAD_SIZE:
raise HTTPException(status_code=400, detail="File too large (max 2MB)")
extension = ALLOWED_IMAGE_TYPES[file.content_type]
filename = f"{datetime.now().strftime('%Y%m%d%H%M%S')}_{secrets.token_hex(8)}{extension}"
target_dir = UPLOAD_ROOT / folder
target_dir.mkdir(parents=True, exist_ok=True)
target_path = target_dir / filename
target_path.write_bytes(contents)
return {"url": f"/uploads/{folder}/{filename}"}
@app.post("/api/admin/login") @app.post("/api/admin/login")
async def admin_login(password: str = Body(..., embed=True)): async def admin_login(password: str = Body(..., embed=True)):
"""Admin login with password""" """Admin login with password"""
@@ -318,6 +386,9 @@ async def delete_article(article_id: int):
async def create_category(category_data: Dict[str, Any]): async def create_category(category_data: Dict[str, Any]):
"""Create new category""" """Create new category"""
try: try:
category_data = dict(category_data)
category_data['order_index'] = to_int(category_data.get('order_index'), 0)
cursor = db.conn.cursor() cursor = db.conn.cursor()
columns = ', '.join(category_data.keys()) columns = ', '.join(category_data.keys())
placeholders = ', '.join(['?' for _ in category_data]) placeholders = ', '.join(['?' for _ in category_data])
@@ -332,6 +403,10 @@ async def create_category(category_data: Dict[str, Any]):
async def update_category(cat_id: int, category_data: Dict[str, Any]): async def update_category(cat_id: int, category_data: Dict[str, Any]):
"""Update category""" """Update category"""
try: try:
category_data = dict(category_data)
if 'order_index' in category_data:
category_data['order_index'] = to_int(category_data.get('order_index'), 0)
set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()]) set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()])
cursor = db.conn.cursor() cursor = db.conn.cursor()
cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?", cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?",
@@ -341,6 +416,18 @@ async def update_category(cat_id: int, category_data: Dict[str, Any]):
except Exception as e: except Exception as e:
raise HTTPException(status_code=400, detail=str(e)) raise HTTPException(status_code=400, detail=str(e))
@app.delete("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)])
async def delete_category(cat_id: int):
"""Delete category"""
try:
cursor = db.conn.cursor()
cursor.execute("DELETE FROM categories WHERE id = ?", (cat_id,))
db.conn.commit()
return {"message": "Category deleted"}
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
# Sponsors CRUD # Sponsors CRUD
@app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)]) @app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)])
async def create_sponsor(sponsor_data: Dict[str, Any]): async def create_sponsor(sponsor_data: Dict[str, Any]):
@@ -369,6 +456,18 @@ async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]):
except Exception as e: except Exception as e:
raise HTTPException(status_code=400, detail=str(e)) raise HTTPException(status_code=400, detail=str(e))
@app.delete("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)])
async def delete_sponsor(sponsor_id: int):
"""Delete sponsor"""
try:
cursor = db.conn.cursor()
cursor.execute("DELETE FROM sponsors WHERE id = ?", (sponsor_id,))
db.conn.commit()
return {"message": "Sponsor deleted"}
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
@app.get("/") @app.get("/")
async def root(): async def root():
"""API info""" """API info"""

View File

@@ -0,0 +1,2 @@
*
!.gitignore

View File

@@ -410,6 +410,21 @@ a:hover {
border: 1px solid var(--border-color); border: 1px solid var(--border-color);
} }
.sponsor-logo {
display: flex;
align-items: center;
justify-content: center;
height: 60px;
margin-bottom: 0.75rem;
}
.sponsor-logo img {
max-height: 60px;
max-width: 100%;
width: auto;
object-fit: contain;
}
.sponsor-card h4 { .sponsor-card h4 {
color: var(--accent-pink); color: var(--accent-pink);
margin-bottom: 0.5rem; margin-bottom: 0.5rem;

View File

@@ -1,5 +1,21 @@
// Marketplace JS - Magazine Layout // Marketplace JS - Magazine Layout
const API_BASE = '/api'; const { API_BASE, API_ORIGIN } = (() => {
const { hostname, port } = window.location;
if ((hostname === 'localhost' || hostname === '127.0.0.1') && port === '8000') {
const origin = 'http://127.0.0.1:8100';
return { API_BASE: `${origin}/api`, API_ORIGIN: origin };
}
return { API_BASE: '/api', API_ORIGIN: '' };
})();
const resolveAssetUrl = (path) => {
if (!path) return '';
if (/^https?:\/\//i.test(path)) return path;
if (path.startsWith('/') && API_ORIGIN) {
return `${API_ORIGIN}${path}`;
}
return path;
};
const CACHE_TTL = 3600000; // 1 hour in ms const CACHE_TTL = 3600000; // 1 hour in ms
class MarketplaceCache { class MarketplaceCache {
@@ -204,6 +220,7 @@ class MarketplaceUI {
const container = document.getElementById('sponsored-content'); const container = document.getElementById('sponsored-content');
container.innerHTML = sponsors.slice(0, 5).map(sponsor => ` container.innerHTML = sponsors.slice(0, 5).map(sponsor => `
<div class="sponsor-card"> <div class="sponsor-card">
${sponsor.logo_url ? `<div class="sponsor-logo"><img src="${resolveAssetUrl(sponsor.logo_url)}" alt="${sponsor.company_name} logo"></div>` : ''}
<h4>${sponsor.company_name}</h4> <h4>${sponsor.company_name}</h4>
<p>${sponsor.tier} Sponsor - Premium Solutions</p> <p>${sponsor.tier} Sponsor - Premium Solutions</p>
<a href="${sponsor.landing_url}" target="_blank">Learn More →</a> <a href="${sponsor.landing_url}" target="_blank">Learn More →</a>

View File

@@ -115,4 +115,4 @@ extra_javascript:
- assets/copy_code.js - assets/copy_code.js
- assets/floating_ask_ai_button.js - assets/floating_ask_ai_button.js
- assets/mobile_menu.js - assets/mobile_menu.js
- assets/page_actions.js - assets/page_actions.js?v=20251006