Merge branch 'develop' into release/v0.7.5
This commit is contained in:
@@ -10,7 +10,6 @@ Today I'm releasing Crawl4AI v0.7.4—the Intelligent Table Extraction & Perform
|
||||
|
||||
- **🚀 LLMTableExtraction**: Revolutionary table extraction with intelligent chunking for massive tables
|
||||
- **⚡ Enhanced Concurrency**: True concurrency improvements for fast-completing tasks in batch operations
|
||||
- **🧹 Memory Management Refactor**: Streamlined memory utilities and better resource management
|
||||
- **🔧 Browser Manager Fixes**: Resolved race conditions in concurrent page creation
|
||||
- **⌨️ Cross-Platform Browser Profiler**: Improved keyboard handling and quit mechanisms
|
||||
- **🔗 Advanced URL Processing**: Better handling of raw URLs and base tag link resolution
|
||||
@@ -158,40 +157,6 @@ async with AsyncWebCrawler() as crawler:
|
||||
- **Monitoring Systems**: Faster health checks and status page monitoring
|
||||
- **Data Aggregation**: Improved performance for real-time data collection
|
||||
|
||||
## 🧹 Memory Management Refactor: Cleaner Architecture
|
||||
|
||||
**The Problem:** Memory utilities were scattered and difficult to maintain, with potential import conflicts and unclear organization.
|
||||
|
||||
**My Solution:** I consolidated all memory-related utilities into the main `utils.py` module, creating a cleaner, more maintainable architecture.
|
||||
|
||||
### Improved Memory Handling
|
||||
|
||||
```python
|
||||
# All memory utilities now consolidated
|
||||
from crawl4ai.utils import get_true_memory_usage_percent, MemoryMonitor
|
||||
|
||||
# Enhanced memory monitoring
|
||||
monitor = MemoryMonitor()
|
||||
monitor.start_monitoring()
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
# Memory-efficient batch processing
|
||||
results = await crawler.arun_many(large_url_list)
|
||||
|
||||
# Get accurate memory metrics
|
||||
memory_usage = get_true_memory_usage_percent()
|
||||
memory_report = monitor.get_report()
|
||||
|
||||
print(f"Memory efficiency: {memory_report['efficiency']:.1f}%")
|
||||
print(f"Peak usage: {memory_report['peak_mb']:.1f} MB")
|
||||
```
|
||||
|
||||
**Expected Real-World Impact:**
|
||||
- **Production Stability**: More reliable memory tracking and management
|
||||
- **Code Maintainability**: Cleaner architecture for easier debugging
|
||||
- **Import Clarity**: Resolved potential conflicts and import issues
|
||||
- **Developer Experience**: Simpler API for memory monitoring
|
||||
|
||||
## 🔧 Critical Stability Fixes
|
||||
|
||||
### Browser Manager Race Condition Resolution
|
||||
|
||||
@@ -201,18 +201,6 @@ ul>li.page-action-item::after{
|
||||
}
|
||||
|
||||
/* Badge */
|
||||
.page-action-badge {
|
||||
display: inline-block;
|
||||
background: #f59e0b;
|
||||
color: #070708;
|
||||
padding: 0.125rem 0.5rem;
|
||||
border-radius: 12px;
|
||||
font-size: 0.65rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.05em;
|
||||
}
|
||||
|
||||
/* External link indicator */
|
||||
.page-action-external::after {
|
||||
content: '→';
|
||||
|
||||
@@ -7,9 +7,12 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
githubRepo: 'unclecode/crawl4ai',
|
||||
githubBranch: 'main',
|
||||
docsPath: 'docs/md_v2',
|
||||
excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/'], // Don't show on app pages
|
||||
excludePaths: ['/apps/c4a-script/', '/apps/llmtxt/', '/apps/crawl4ai-assistant/', '/core/ask-ai/'], // Don't show on app pages
|
||||
};
|
||||
|
||||
let cachedMarkdown = null;
|
||||
let cachedMarkdownPath = null;
|
||||
|
||||
// Check if we should show the button on this page
|
||||
function shouldShowButton() {
|
||||
const currentPath = window.location.pathname;
|
||||
@@ -19,6 +22,17 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Don't show on 404 pages
|
||||
if (document.title && document.title.toLowerCase().includes('404')) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Require mkdocs main content container
|
||||
const mainContent = document.getElementById('terminal-mkdocs-main-content');
|
||||
if (!mainContent) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Don't show on excluded paths (apps)
|
||||
for (const excludePath of config.excludePaths) {
|
||||
if (currentPath.includes(excludePath)) {
|
||||
@@ -53,6 +67,56 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
return `${path}.md`;
|
||||
}
|
||||
|
||||
async function loadMarkdownContent() {
|
||||
const mdPath = getCurrentMarkdownPath();
|
||||
|
||||
if (!mdPath) {
|
||||
throw new Error('Invalid markdown path');
|
||||
}
|
||||
|
||||
const rawUrl = getGithubRawUrl();
|
||||
const response = await fetch(rawUrl);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch markdown: ${response.status}`);
|
||||
}
|
||||
|
||||
const markdown = await response.text();
|
||||
cachedMarkdown = markdown;
|
||||
cachedMarkdownPath = mdPath;
|
||||
return markdown;
|
||||
}
|
||||
|
||||
async function ensureMarkdownCached() {
|
||||
const mdPath = getCurrentMarkdownPath();
|
||||
|
||||
if (!mdPath) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (cachedMarkdown && cachedMarkdownPath === mdPath) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
await loadMarkdownContent();
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.warn('Page Actions: Markdown not available for this page.', error);
|
||||
cachedMarkdown = null;
|
||||
cachedMarkdownPath = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function getMarkdownContent() {
|
||||
const available = await ensureMarkdownCached();
|
||||
if (!available) {
|
||||
throw new Error('Markdown not available for this page.');
|
||||
}
|
||||
return cachedMarkdown;
|
||||
}
|
||||
|
||||
// Get GitHub raw URL for current page
|
||||
function getGithubRawUrl() {
|
||||
const mdPath = getCurrentMarkdownPath();
|
||||
@@ -112,13 +176,11 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
</li>
|
||||
<div class="page-actions-divider"></div>
|
||||
<li class="page-action-item">
|
||||
<a href="#" class="page-action-link disabled" id="action-ask-ai" role="menuitem">
|
||||
<a href="#" class="page-action-link page-action-external" id="action-open-chatgpt" role="menuitem">
|
||||
<span class="page-action-icon icon-ai"></span>
|
||||
<span class="page-action-text">
|
||||
<span class="page-action-label">Ask AI about page</span>
|
||||
<span class="page-action-description">
|
||||
<span class="page-action-badge">Coming Soon</span>
|
||||
</span>
|
||||
<span class="page-action-label">Open in ChatGPT</span>
|
||||
<span class="page-action-description">Ask questions about this page</span>
|
||||
</span>
|
||||
</a>
|
||||
</li>
|
||||
@@ -180,19 +242,11 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
|
||||
// Copy markdown to clipboard
|
||||
async function copyMarkdownToClipboard(link) {
|
||||
const rawUrl = getGithubRawUrl();
|
||||
|
||||
// Add loading state
|
||||
link.classList.add('loading');
|
||||
|
||||
try {
|
||||
const response = await fetch(rawUrl);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch markdown: ${response.status}`);
|
||||
}
|
||||
|
||||
const markdown = await response.text();
|
||||
const markdown = await getMarkdownContent();
|
||||
|
||||
// Copy to clipboard
|
||||
await navigator.clipboard.writeText(markdown);
|
||||
@@ -221,126 +275,153 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
window.open(githubUrl, '_blank', 'noopener,noreferrer');
|
||||
}
|
||||
|
||||
// Initialize
|
||||
const { button, dropdown, overlay } = createPageActionsUI();
|
||||
function getCurrentPageUrl() {
|
||||
const { href } = window.location;
|
||||
return href.split('#')[0];
|
||||
}
|
||||
|
||||
// Event listeners
|
||||
button.addEventListener('click', (e) => {
|
||||
e.stopPropagation();
|
||||
toggleDropdown(button, dropdown, overlay);
|
||||
});
|
||||
function openChatGPT() {
|
||||
const pageUrl = getCurrentPageUrl();
|
||||
const prompt = encodeURIComponent(`Read ${pageUrl} so I can ask questions about it.`);
|
||||
const chatUrl = `https://chatgpt.com/?hint=search&prompt=${prompt}`;
|
||||
window.open(chatUrl, '_blank', 'noopener,noreferrer');
|
||||
}
|
||||
|
||||
overlay.addEventListener('click', () => {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
});
|
||||
|
||||
// Copy markdown action
|
||||
document.getElementById('action-copy-markdown').addEventListener('click', async (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
await copyMarkdownToClipboard(e.currentTarget);
|
||||
});
|
||||
|
||||
// View markdown action
|
||||
document.getElementById('action-view-markdown').addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
viewMarkdown();
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
});
|
||||
|
||||
// Ask AI action (disabled for now)
|
||||
document.getElementById('action-ask-ai').addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
// Future: Integrate with Ask AI feature
|
||||
// For now, do nothing (disabled state)
|
||||
});
|
||||
|
||||
// Close on ESC key
|
||||
document.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Escape' && dropdown.classList.contains('active')) {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
(async () => {
|
||||
if (!shouldShowButton()) {
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
// Close when clicking outside
|
||||
document.addEventListener('click', (e) => {
|
||||
if (!dropdown.contains(e.target) && !button.contains(e.target)) {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
const markdownAvailable = await ensureMarkdownCached();
|
||||
if (!markdownAvailable) {
|
||||
return;
|
||||
}
|
||||
});
|
||||
|
||||
// Prevent dropdown from closing when clicking inside
|
||||
dropdown.addEventListener('click', (e) => {
|
||||
// Only stop propagation if not clicking on a link
|
||||
if (!e.target.closest('.page-action-link')) {
|
||||
const ui = createPageActionsUI();
|
||||
if (!ui) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { button, dropdown, overlay } = ui;
|
||||
|
||||
// Event listeners
|
||||
button.addEventListener('click', (e) => {
|
||||
e.stopPropagation();
|
||||
}
|
||||
});
|
||||
|
||||
// Close dropdown on link click (except for copy which handles itself)
|
||||
dropdown.querySelectorAll('.page-action-link:not(#action-copy-markdown)').forEach(link => {
|
||||
link.addEventListener('click', () => {
|
||||
if (!link.classList.contains('disabled')) {
|
||||
setTimeout(() => {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
}, 100);
|
||||
}
|
||||
toggleDropdown(button, dropdown, overlay);
|
||||
});
|
||||
});
|
||||
|
||||
// Handle window resize
|
||||
let resizeTimer;
|
||||
window.addEventListener('resize', () => {
|
||||
clearTimeout(resizeTimer);
|
||||
resizeTimer = setTimeout(() => {
|
||||
// Close dropdown on resize to prevent positioning issues
|
||||
if (dropdown.classList.contains('active')) {
|
||||
overlay.addEventListener('click', () => {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
});
|
||||
|
||||
// Copy markdown action
|
||||
document.getElementById('action-copy-markdown').addEventListener('click', async (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
await copyMarkdownToClipboard(e.currentTarget);
|
||||
});
|
||||
|
||||
// View markdown action
|
||||
document.getElementById('action-view-markdown').addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
viewMarkdown();
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
});
|
||||
|
||||
// Open in ChatGPT action
|
||||
document.getElementById('action-open-chatgpt').addEventListener('click', (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
openChatGPT();
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
});
|
||||
|
||||
// Close on ESC key
|
||||
document.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Escape' && dropdown.classList.contains('active')) {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
}
|
||||
}, 250);
|
||||
});
|
||||
});
|
||||
|
||||
// Accessibility: Focus management
|
||||
button.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault();
|
||||
toggleDropdown(button, dropdown, overlay);
|
||||
// Close when clicking outside
|
||||
document.addEventListener('click', (e) => {
|
||||
if (!dropdown.contains(e.target) && !button.contains(e.target)) {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
}
|
||||
});
|
||||
|
||||
// Focus first menu item when opening
|
||||
if (dropdown.classList.contains('active')) {
|
||||
const firstLink = dropdown.querySelector('.page-action-link:not(.disabled)');
|
||||
if (firstLink) {
|
||||
setTimeout(() => firstLink.focus(), 100);
|
||||
// Prevent dropdown from closing when clicking inside
|
||||
dropdown.addEventListener('click', (e) => {
|
||||
// Only stop propagation if not clicking on a link
|
||||
if (!e.target.closest('.page-action-link')) {
|
||||
e.stopPropagation();
|
||||
}
|
||||
});
|
||||
|
||||
// Close dropdown on link click (except for copy which handles itself)
|
||||
dropdown.querySelectorAll('.page-action-link:not(#action-copy-markdown)').forEach(link => {
|
||||
link.addEventListener('click', () => {
|
||||
if (!link.classList.contains('disabled')) {
|
||||
setTimeout(() => {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
}, 100);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Handle window resize
|
||||
let resizeTimer;
|
||||
window.addEventListener('resize', () => {
|
||||
clearTimeout(resizeTimer);
|
||||
resizeTimer = setTimeout(() => {
|
||||
// Close dropdown on resize to prevent positioning issues
|
||||
if (dropdown.classList.contains('active')) {
|
||||
closeDropdown(button, dropdown, overlay);
|
||||
}
|
||||
}, 250);
|
||||
});
|
||||
|
||||
// Accessibility: Focus management
|
||||
button.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter' || e.key === ' ') {
|
||||
e.preventDefault();
|
||||
toggleDropdown(button, dropdown, overlay);
|
||||
|
||||
// Focus first menu item when opening
|
||||
if (dropdown.classList.contains('active')) {
|
||||
const firstLink = dropdown.querySelector('.page-action-link:not(.disabled)');
|
||||
if (firstLink) {
|
||||
setTimeout(() => firstLink.focus(), 100);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Arrow key navigation within menu
|
||||
dropdown.addEventListener('keydown', (e) => {
|
||||
if (!dropdown.classList.contains('active')) return;
|
||||
// Arrow key navigation within menu
|
||||
dropdown.addEventListener('keydown', (e) => {
|
||||
if (!dropdown.classList.contains('active')) return;
|
||||
|
||||
const links = Array.from(dropdown.querySelectorAll('.page-action-link:not(.disabled)'));
|
||||
const currentIndex = links.indexOf(document.activeElement);
|
||||
const links = Array.from(dropdown.querySelectorAll('.page-action-link:not(.disabled)'));
|
||||
const currentIndex = links.indexOf(document.activeElement);
|
||||
|
||||
if (e.key === 'ArrowDown') {
|
||||
e.preventDefault();
|
||||
const nextIndex = (currentIndex + 1) % links.length;
|
||||
links[nextIndex].focus();
|
||||
} else if (e.key === 'ArrowUp') {
|
||||
e.preventDefault();
|
||||
const prevIndex = (currentIndex - 1 + links.length) % links.length;
|
||||
links[prevIndex].focus();
|
||||
} else if (e.key === 'Home') {
|
||||
e.preventDefault();
|
||||
links[0].focus();
|
||||
} else if (e.key === 'End') {
|
||||
e.preventDefault();
|
||||
links[links.length - 1].focus();
|
||||
}
|
||||
});
|
||||
if (e.key === 'ArrowDown') {
|
||||
e.preventDefault();
|
||||
const nextIndex = (currentIndex + 1) % links.length;
|
||||
links[nextIndex].focus();
|
||||
} else if (e.key === 'ArrowUp') {
|
||||
e.preventDefault();
|
||||
const prevIndex = (currentIndex - 1 + links.length) % links.length;
|
||||
links[prevIndex].focus();
|
||||
} else if (e.key === 'Home') {
|
||||
e.preventDefault();
|
||||
links[0].focus();
|
||||
} else if (e.key === 'End') {
|
||||
e.preventDefault();
|
||||
links[links.length - 1].focus();
|
||||
}
|
||||
});
|
||||
|
||||
console.log('Page Actions initialized for:', getCurrentMarkdownPath());
|
||||
console.log('Page Actions initialized for:', getCurrentMarkdownPath());
|
||||
})();
|
||||
});
|
||||
@@ -431,6 +431,16 @@
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.table-logo {
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
object-fit: contain;
|
||||
border-radius: 6px;
|
||||
background: var(--bg-tertiary);
|
||||
border: 1px solid var(--border-color);
|
||||
padding: 4px;
|
||||
}
|
||||
|
||||
.btn-edit, .btn-delete, .btn-duplicate {
|
||||
padding: 0.25rem 0.5rem;
|
||||
background: transparent;
|
||||
@@ -585,6 +595,105 @@
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.sponsor-form {
|
||||
grid-template-columns: 200px repeat(2, minmax(220px, 1fr));
|
||||
align-items: flex-start;
|
||||
grid-auto-flow: dense;
|
||||
}
|
||||
|
||||
.sponsor-logo-group {
|
||||
grid-row: span 3;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.span-two {
|
||||
grid-column: span 2;
|
||||
}
|
||||
|
||||
.logo-upload {
|
||||
position: relative;
|
||||
width: 180px;
|
||||
}
|
||||
|
||||
.image-preview {
|
||||
width: 180px;
|
||||
height: 180px;
|
||||
border: 1px dashed var(--border-color);
|
||||
border-radius: 12px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: var(--bg-tertiary);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.image-preview.empty {
|
||||
color: var(--text-secondary);
|
||||
font-size: 0.75rem;
|
||||
text-align: center;
|
||||
padding: 0.75rem;
|
||||
}
|
||||
|
||||
.image-preview img {
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.upload-btn {
|
||||
position: absolute;
|
||||
left: 50%;
|
||||
bottom: 12px;
|
||||
transform: translateX(-50%);
|
||||
padding: 0.35rem 1rem;
|
||||
background: linear-gradient(135deg, var(--primary-cyan), var(--primary-teal));
|
||||
border: none;
|
||||
border-radius: 999px;
|
||||
color: var(--bg-dark);
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
box-shadow: 0 6px 18px rgba(80, 255, 255, 0.25);
|
||||
}
|
||||
|
||||
.upload-btn:hover {
|
||||
box-shadow: 0 8px 22px rgba(80, 255, 255, 0.35);
|
||||
}
|
||||
|
||||
.logo-upload input[type="file"] {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.upload-hint {
|
||||
font-size: 0.75rem;
|
||||
color: var(--text-secondary);
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
@media (max-width: 960px) {
|
||||
.sponsor-form {
|
||||
grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
|
||||
}
|
||||
|
||||
.sponsor-logo-group {
|
||||
grid-column: 1 / -1;
|
||||
grid-row: auto;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.logo-upload {
|
||||
width: 160px;
|
||||
}
|
||||
|
||||
.span-two {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Rich Text Editor */
|
||||
.editor-toolbar {
|
||||
display: flex;
|
||||
|
||||
@@ -1,5 +1,49 @@
|
||||
// Admin Dashboard - Smart & Powerful
|
||||
const API_BASE = '/api';
|
||||
const { API_BASE, API_ORIGIN } = (() => {
|
||||
const cleanOrigin = (value) => value ? value.replace(/\/$/, '') : '';
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
const overrideParam = cleanOrigin(params.get('api_origin'));
|
||||
|
||||
let storedOverride = '';
|
||||
try {
|
||||
storedOverride = cleanOrigin(localStorage.getItem('marketplace_api_origin'));
|
||||
} catch (error) {
|
||||
storedOverride = '';
|
||||
}
|
||||
|
||||
let origin = overrideParam || storedOverride;
|
||||
|
||||
if (overrideParam && overrideParam !== storedOverride) {
|
||||
try {
|
||||
localStorage.setItem('marketplace_api_origin', overrideParam);
|
||||
} catch (error) {
|
||||
// ignore storage errors (private mode, etc.)
|
||||
}
|
||||
}
|
||||
|
||||
const { protocol, hostname, port } = window.location;
|
||||
const isLocalHost = ['localhost', '127.0.0.1', '0.0.0.0'].includes(hostname);
|
||||
|
||||
if (!origin && isLocalHost && port !== '8100') {
|
||||
origin = `${protocol}//127.0.0.1:8100`;
|
||||
}
|
||||
|
||||
if (origin) {
|
||||
const normalized = cleanOrigin(origin);
|
||||
return { API_BASE: `${normalized}/api`, API_ORIGIN: normalized };
|
||||
}
|
||||
|
||||
return { API_BASE: '/api', API_ORIGIN: '' };
|
||||
})();
|
||||
|
||||
const resolveAssetUrl = (path) => {
|
||||
if (!path) return '';
|
||||
if (/^https?:\/\//i.test(path)) return path;
|
||||
if (path.startsWith('/') && API_ORIGIN) {
|
||||
return `${API_ORIGIN}${path}`;
|
||||
}
|
||||
return path;
|
||||
};
|
||||
|
||||
class AdminDashboard {
|
||||
constructor() {
|
||||
@@ -144,13 +188,19 @@ class AdminDashboard {
|
||||
}
|
||||
|
||||
async apiCall(endpoint, options = {}) {
|
||||
const isFormData = options.body instanceof FormData;
|
||||
const headers = {
|
||||
'Authorization': `Bearer ${this.token}`,
|
||||
...options.headers
|
||||
};
|
||||
|
||||
if (!isFormData && !headers['Content-Type']) {
|
||||
headers['Content-Type'] = 'application/json';
|
||||
}
|
||||
|
||||
const response = await fetch(`${API_BASE}${endpoint}`, {
|
||||
...options,
|
||||
headers: {
|
||||
'Authorization': `Bearer ${this.token}`,
|
||||
'Content-Type': 'application/json',
|
||||
...options.headers
|
||||
}
|
||||
headers
|
||||
});
|
||||
|
||||
if (response.status === 401) {
|
||||
@@ -163,7 +213,9 @@ class AdminDashboard {
|
||||
}
|
||||
|
||||
async loadStats() {
|
||||
const stats = await this.apiCall('/admin/stats');
|
||||
const stats = await this.apiCall(`/admin/stats?_=${Date.now()}`, {
|
||||
cache: 'no-store'
|
||||
});
|
||||
|
||||
document.getElementById('stat-apps').textContent = stats.apps.total;
|
||||
document.getElementById('stat-featured').textContent = stats.apps.featured;
|
||||
@@ -174,22 +226,32 @@ class AdminDashboard {
|
||||
}
|
||||
|
||||
async loadApps() {
|
||||
this.data.apps = await this.apiCall('/apps?limit=100');
|
||||
this.data.apps = await this.apiCall(`/apps?limit=100&_=${Date.now()}`, {
|
||||
cache: 'no-store'
|
||||
});
|
||||
this.renderAppsTable(this.data.apps);
|
||||
}
|
||||
|
||||
async loadArticles() {
|
||||
this.data.articles = await this.apiCall('/articles?limit=100');
|
||||
this.data.articles = await this.apiCall(`/articles?limit=100&_=${Date.now()}`, {
|
||||
cache: 'no-store'
|
||||
});
|
||||
this.renderArticlesTable(this.data.articles);
|
||||
}
|
||||
|
||||
async loadCategories() {
|
||||
this.data.categories = await this.apiCall('/categories');
|
||||
const cacheBuster = Date.now();
|
||||
this.data.categories = await this.apiCall(`/categories?_=${cacheBuster}`, {
|
||||
cache: 'no-store'
|
||||
});
|
||||
this.renderCategoriesTable(this.data.categories);
|
||||
}
|
||||
|
||||
async loadSponsors() {
|
||||
this.data.sponsors = await this.apiCall('/sponsors');
|
||||
const cacheBuster = Date.now();
|
||||
this.data.sponsors = await this.apiCall(`/sponsors?limit=100&_=${cacheBuster}`, {
|
||||
cache: 'no-store'
|
||||
});
|
||||
this.renderSponsorsTable(this.data.sponsors);
|
||||
}
|
||||
|
||||
@@ -314,6 +376,7 @@ class AdminDashboard {
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ID</th>
|
||||
<th>Logo</th>
|
||||
<th>Company</th>
|
||||
<th>Tier</th>
|
||||
<th>Start</th>
|
||||
@@ -326,6 +389,7 @@ class AdminDashboard {
|
||||
${sponsors.map(sponsor => `
|
||||
<tr>
|
||||
<td>${sponsor.id}</td>
|
||||
<td>${sponsor.logo_url ? `<img class="table-logo" src="${resolveAssetUrl(sponsor.logo_url)}" alt="${sponsor.company_name} logo">` : '-'}</td>
|
||||
<td>${sponsor.company_name}</td>
|
||||
<td>${sponsor.tier}</td>
|
||||
<td>${new Date(sponsor.start_date).toLocaleDateString()}</td>
|
||||
@@ -389,6 +453,10 @@ class AdminDashboard {
|
||||
|
||||
modal.classList.remove('hidden');
|
||||
modal.dataset.type = type;
|
||||
|
||||
if (type === 'sponsors') {
|
||||
this.setupLogoUploadHandlers();
|
||||
}
|
||||
}
|
||||
|
||||
getAppForm(app) {
|
||||
@@ -524,9 +592,22 @@ class AdminDashboard {
|
||||
}
|
||||
|
||||
getSponsorForm(sponsor) {
|
||||
const existingFile = sponsor?.logo_url ? sponsor.logo_url.split('/').pop().split('?')[0] : '';
|
||||
return `
|
||||
<div class="form-grid">
|
||||
<div class="form-group">
|
||||
<div class="form-grid sponsor-form">
|
||||
<div class="form-group sponsor-logo-group">
|
||||
<label>Logo</label>
|
||||
<input type="hidden" id="form-logo-url" value="${sponsor?.logo_url || ''}">
|
||||
<div class="logo-upload">
|
||||
<div class="image-preview ${sponsor?.logo_url ? '' : 'empty'}" id="form-logo-preview">
|
||||
${sponsor?.logo_url ? `<img src="${resolveAssetUrl(sponsor.logo_url)}" alt="Logo preview">` : '<span>No logo uploaded</span>'}
|
||||
</div>
|
||||
<button type="button" class="upload-btn" id="form-logo-button">Upload Logo</button>
|
||||
<input type="file" id="form-logo-file" accept="image/png,image/jpeg,image/webp,image/svg+xml" hidden>
|
||||
</div>
|
||||
<p class="upload-hint" id="form-logo-filename">${existingFile ? `Current: ${existingFile}` : 'No file selected'}</p>
|
||||
</div>
|
||||
<div class="form-group span-two">
|
||||
<label>Company Name *</label>
|
||||
<input type="text" id="form-name" value="${sponsor?.company_name || ''}" required>
|
||||
</div>
|
||||
@@ -567,9 +648,30 @@ class AdminDashboard {
|
||||
async saveItem() {
|
||||
const modal = document.getElementById('form-modal');
|
||||
const type = modal.dataset.type;
|
||||
const data = this.collectFormData(type);
|
||||
|
||||
try {
|
||||
if (type === 'sponsors') {
|
||||
const fileInput = document.getElementById('form-logo-file');
|
||||
if (fileInput && fileInput.files && fileInput.files[0]) {
|
||||
const formData = new FormData();
|
||||
formData.append('file', fileInput.files[0]);
|
||||
formData.append('folder', 'sponsors');
|
||||
|
||||
const uploadResponse = await this.apiCall('/admin/upload-image', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
|
||||
if (!uploadResponse.url) {
|
||||
throw new Error('Image upload failed');
|
||||
}
|
||||
|
||||
document.getElementById('form-logo-url').value = uploadResponse.url;
|
||||
}
|
||||
}
|
||||
|
||||
const data = this.collectFormData(type);
|
||||
|
||||
if (this.editingItem) {
|
||||
await this.apiCall(`/admin/${type}/${this.editingItem.id}`, {
|
||||
method: 'PUT',
|
||||
@@ -599,8 +701,10 @@ class AdminDashboard {
|
||||
data.description = document.getElementById('form-description').value;
|
||||
data.category = document.getElementById('form-category').value;
|
||||
data.type = document.getElementById('form-type').value;
|
||||
data.rating = parseFloat(document.getElementById('form-rating').value);
|
||||
data.downloads = parseInt(document.getElementById('form-downloads').value);
|
||||
const rating = parseFloat(document.getElementById('form-rating').value);
|
||||
const downloads = parseInt(document.getElementById('form-downloads').value, 10);
|
||||
data.rating = Number.isFinite(rating) ? rating : 0;
|
||||
data.downloads = Number.isFinite(downloads) ? downloads : 0;
|
||||
data.image = document.getElementById('form-image').value;
|
||||
data.website_url = document.getElementById('form-website').value;
|
||||
data.github_url = document.getElementById('form-github').value;
|
||||
@@ -621,9 +725,11 @@ class AdminDashboard {
|
||||
data.slug = this.generateSlug(data.name);
|
||||
data.icon = document.getElementById('form-icon').value;
|
||||
data.description = document.getElementById('form-description').value;
|
||||
data.order_index = parseInt(document.getElementById('form-order').value);
|
||||
const orderIndex = parseInt(document.getElementById('form-order').value, 10);
|
||||
data.order_index = Number.isFinite(orderIndex) ? orderIndex : 0;
|
||||
} else if (type === 'sponsors') {
|
||||
data.company_name = document.getElementById('form-name').value;
|
||||
data.logo_url = document.getElementById('form-logo-url').value;
|
||||
data.tier = document.getElementById('form-tier').value;
|
||||
data.landing_url = document.getElementById('form-landing').value;
|
||||
data.banner_url = document.getElementById('form-banner').value;
|
||||
@@ -635,6 +741,63 @@ class AdminDashboard {
|
||||
return data;
|
||||
}
|
||||
|
||||
setupLogoUploadHandlers() {
|
||||
const fileInput = document.getElementById('form-logo-file');
|
||||
const preview = document.getElementById('form-logo-preview');
|
||||
const logoUrlInput = document.getElementById('form-logo-url');
|
||||
const trigger = document.getElementById('form-logo-button');
|
||||
const fileNameEl = document.getElementById('form-logo-filename');
|
||||
|
||||
if (!fileInput || !preview || !logoUrlInput) return;
|
||||
|
||||
const setFileName = (text) => {
|
||||
if (fileNameEl) {
|
||||
fileNameEl.textContent = text;
|
||||
}
|
||||
};
|
||||
|
||||
const setEmptyState = () => {
|
||||
preview.innerHTML = '<span>No logo uploaded</span>';
|
||||
preview.classList.add('empty');
|
||||
setFileName('No file selected');
|
||||
};
|
||||
|
||||
const setExistingState = () => {
|
||||
if (logoUrlInput.value) {
|
||||
const existingFile = logoUrlInput.value.split('/').pop().split('?')[0];
|
||||
preview.innerHTML = `<img src="${resolveAssetUrl(logoUrlInput.value)}" alt="Logo preview">`;
|
||||
preview.classList.remove('empty');
|
||||
setFileName(existingFile ? `Current: ${existingFile}` : 'Current logo');
|
||||
} else {
|
||||
setEmptyState();
|
||||
}
|
||||
};
|
||||
|
||||
setExistingState();
|
||||
|
||||
if (trigger) {
|
||||
trigger.onclick = () => fileInput.click();
|
||||
}
|
||||
|
||||
fileInput.addEventListener('change', (event) => {
|
||||
const file = event.target.files && event.target.files[0];
|
||||
|
||||
if (!file) {
|
||||
setExistingState();
|
||||
return;
|
||||
}
|
||||
|
||||
setFileName(file.name);
|
||||
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
preview.innerHTML = `<img src="${reader.result}" alt="Logo preview">`;
|
||||
preview.classList.remove('empty');
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
async deleteItem(type, id) {
|
||||
if (!confirm(`Are you sure you want to delete this ${type.slice(0, -1)}?`)) return;
|
||||
|
||||
|
||||
@@ -210,6 +210,6 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="admin.js?v=1759327900"></script>
|
||||
<script src="admin.js?v=1759334000"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,11 +1,14 @@
|
||||
from fastapi import FastAPI, HTTPException, Query, Depends, Body
|
||||
from fastapi import FastAPI, HTTPException, Query, Depends, Body, UploadFile, File, Form
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from typing import Optional, List, Dict, Any
|
||||
from typing import Optional, Dict, Any
|
||||
import json
|
||||
import hashlib
|
||||
import secrets
|
||||
import re
|
||||
from pathlib import Path
|
||||
from database import DatabaseManager
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
@@ -31,6 +34,21 @@ app.add_middleware(
|
||||
# Initialize database with configurable path
|
||||
db = DatabaseManager(Config.DATABASE_PATH)
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
UPLOAD_ROOT = BASE_DIR / "uploads"
|
||||
UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
app.mount("/uploads", StaticFiles(directory=UPLOAD_ROOT), name="uploads")
|
||||
|
||||
ALLOWED_IMAGE_TYPES = {
|
||||
"image/png": ".png",
|
||||
"image/jpeg": ".jpg",
|
||||
"image/webp": ".webp",
|
||||
"image/svg+xml": ".svg"
|
||||
}
|
||||
ALLOWED_UPLOAD_FOLDERS = {"sponsors"}
|
||||
MAX_UPLOAD_SIZE = 2 * 1024 * 1024 # 2 MB
|
||||
|
||||
def json_response(data, cache_time=3600):
|
||||
"""Helper to return JSON with cache headers"""
|
||||
return JSONResponse(
|
||||
@@ -41,6 +59,29 @@ def json_response(data, cache_time=3600):
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def to_int(value, default=0):
|
||||
"""Coerce incoming values to integers, falling back to default."""
|
||||
if value is None:
|
||||
return default
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
|
||||
if isinstance(value, str):
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
return default
|
||||
|
||||
match = re.match(r"^-?\d+", stripped)
|
||||
if match:
|
||||
try:
|
||||
return int(match.group())
|
||||
except ValueError:
|
||||
return default
|
||||
return default
|
||||
|
||||
# ============= PUBLIC ENDPOINTS =============
|
||||
|
||||
@app.get("/api/apps")
|
||||
@@ -124,6 +165,8 @@ async def get_article(slug: str):
|
||||
async def get_categories():
|
||||
"""Get all categories ordered by index"""
|
||||
categories = db.get_all('categories', limit=50)
|
||||
for category in categories:
|
||||
category['order_index'] = to_int(category.get('order_index'), 0)
|
||||
categories.sort(key=lambda x: x.get('order_index', 0))
|
||||
return json_response(categories, cache_time=7200)
|
||||
|
||||
@@ -183,6 +226,31 @@ def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
|
||||
raise HTTPException(status_code=401, detail="Invalid or expired token")
|
||||
return token
|
||||
|
||||
|
||||
@app.post("/api/admin/upload-image", dependencies=[Depends(verify_token)])
|
||||
async def upload_image(file: UploadFile = File(...), folder: str = Form("sponsors")):
|
||||
"""Upload image files for admin assets"""
|
||||
folder = (folder or "").strip().lower()
|
||||
if folder not in ALLOWED_UPLOAD_FOLDERS:
|
||||
raise HTTPException(status_code=400, detail="Invalid upload folder")
|
||||
|
||||
if file.content_type not in ALLOWED_IMAGE_TYPES:
|
||||
raise HTTPException(status_code=400, detail="Unsupported file type")
|
||||
|
||||
contents = await file.read()
|
||||
if len(contents) > MAX_UPLOAD_SIZE:
|
||||
raise HTTPException(status_code=400, detail="File too large (max 2MB)")
|
||||
|
||||
extension = ALLOWED_IMAGE_TYPES[file.content_type]
|
||||
filename = f"{datetime.now().strftime('%Y%m%d%H%M%S')}_{secrets.token_hex(8)}{extension}"
|
||||
|
||||
target_dir = UPLOAD_ROOT / folder
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
target_path = target_dir / filename
|
||||
target_path.write_bytes(contents)
|
||||
|
||||
return {"url": f"/uploads/{folder}/{filename}"}
|
||||
|
||||
@app.post("/api/admin/login")
|
||||
async def admin_login(password: str = Body(..., embed=True)):
|
||||
"""Admin login with password"""
|
||||
@@ -318,6 +386,9 @@ async def delete_article(article_id: int):
|
||||
async def create_category(category_data: Dict[str, Any]):
|
||||
"""Create new category"""
|
||||
try:
|
||||
category_data = dict(category_data)
|
||||
category_data['order_index'] = to_int(category_data.get('order_index'), 0)
|
||||
|
||||
cursor = db.conn.cursor()
|
||||
columns = ', '.join(category_data.keys())
|
||||
placeholders = ', '.join(['?' for _ in category_data])
|
||||
@@ -332,6 +403,10 @@ async def create_category(category_data: Dict[str, Any]):
|
||||
async def update_category(cat_id: int, category_data: Dict[str, Any]):
|
||||
"""Update category"""
|
||||
try:
|
||||
category_data = dict(category_data)
|
||||
if 'order_index' in category_data:
|
||||
category_data['order_index'] = to_int(category_data.get('order_index'), 0)
|
||||
|
||||
set_clause = ', '.join([f"{k} = ?" for k in category_data.keys()])
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute(f"UPDATE categories SET {set_clause} WHERE id = ?",
|
||||
@@ -341,6 +416,18 @@ async def update_category(cat_id: int, category_data: Dict[str, Any]):
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@app.delete("/api/admin/categories/{cat_id}", dependencies=[Depends(verify_token)])
|
||||
async def delete_category(cat_id: int):
|
||||
"""Delete category"""
|
||||
try:
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute("DELETE FROM categories WHERE id = ?", (cat_id,))
|
||||
db.conn.commit()
|
||||
return {"message": "Category deleted"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
# Sponsors CRUD
|
||||
@app.post("/api/admin/sponsors", dependencies=[Depends(verify_token)])
|
||||
async def create_sponsor(sponsor_data: Dict[str, Any]):
|
||||
@@ -369,6 +456,18 @@ async def update_sponsor(sponsor_id: int, sponsor_data: Dict[str, Any]):
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
|
||||
@app.delete("/api/admin/sponsors/{sponsor_id}", dependencies=[Depends(verify_token)])
|
||||
async def delete_sponsor(sponsor_id: int):
|
||||
"""Delete sponsor"""
|
||||
try:
|
||||
cursor = db.conn.cursor()
|
||||
cursor.execute("DELETE FROM sponsors WHERE id = ?", (sponsor_id,))
|
||||
db.conn.commit()
|
||||
return {"message": "Sponsor deleted"}
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
|
||||
@app.get("/")
|
||||
async def root():
|
||||
"""API info"""
|
||||
|
||||
2
docs/md_v2/marketplace/backend/uploads/.gitignore
vendored
Normal file
2
docs/md_v2/marketplace/backend/uploads/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
@@ -410,6 +410,21 @@ a:hover {
|
||||
border: 1px solid var(--border-color);
|
||||
}
|
||||
|
||||
.sponsor-logo {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 60px;
|
||||
margin-bottom: 0.75rem;
|
||||
}
|
||||
|
||||
.sponsor-logo img {
|
||||
max-height: 60px;
|
||||
max-width: 100%;
|
||||
width: auto;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.sponsor-card h4 {
|
||||
color: var(--accent-pink);
|
||||
margin-bottom: 0.5rem;
|
||||
|
||||
@@ -1,5 +1,21 @@
|
||||
// Marketplace JS - Magazine Layout
|
||||
const API_BASE = '/api';
|
||||
const { API_BASE, API_ORIGIN } = (() => {
|
||||
const { hostname, port } = window.location;
|
||||
if ((hostname === 'localhost' || hostname === '127.0.0.1') && port === '8000') {
|
||||
const origin = 'http://127.0.0.1:8100';
|
||||
return { API_BASE: `${origin}/api`, API_ORIGIN: origin };
|
||||
}
|
||||
return { API_BASE: '/api', API_ORIGIN: '' };
|
||||
})();
|
||||
|
||||
const resolveAssetUrl = (path) => {
|
||||
if (!path) return '';
|
||||
if (/^https?:\/\//i.test(path)) return path;
|
||||
if (path.startsWith('/') && API_ORIGIN) {
|
||||
return `${API_ORIGIN}${path}`;
|
||||
}
|
||||
return path;
|
||||
};
|
||||
const CACHE_TTL = 3600000; // 1 hour in ms
|
||||
|
||||
class MarketplaceCache {
|
||||
@@ -204,6 +220,7 @@ class MarketplaceUI {
|
||||
const container = document.getElementById('sponsored-content');
|
||||
container.innerHTML = sponsors.slice(0, 5).map(sponsor => `
|
||||
<div class="sponsor-card">
|
||||
${sponsor.logo_url ? `<div class="sponsor-logo"><img src="${resolveAssetUrl(sponsor.logo_url)}" alt="${sponsor.company_name} logo"></div>` : ''}
|
||||
<h4>${sponsor.company_name}</h4>
|
||||
<p>${sponsor.tier} Sponsor - Premium Solutions</p>
|
||||
<a href="${sponsor.landing_url}" target="_blank">Learn More →</a>
|
||||
|
||||
@@ -115,4 +115,4 @@ extra_javascript:
|
||||
- assets/copy_code.js
|
||||
- assets/floating_ask_ai_button.js
|
||||
- assets/mobile_menu.js
|
||||
- assets/page_actions.js
|
||||
- assets/page_actions.js?v=20251006
|
||||
Reference in New Issue
Block a user