From c56974cf5996302deb80a489163258607ec3cfde Mon Sep 17 00:00:00 2001 From: UncleCode Date: Mon, 14 Apr 2025 20:46:32 +0800 Subject: [PATCH] feat(docs): enhance documentation UI with ToC and GitHub stats Add new features to documentation UI: - Add table of contents with scroll spy functionality - Add GitHub repository statistics badge - Implement new centered layout system with fixed sidebar - Add conditional Playwright installation based on CRAWL4AI_MODE Breaking changes: None --- crawl4ai/install.py | 19 +- docs/md_v2/assets/github_stats.js | 119 ++++++++++++ docs/md_v2/assets/layout.css | 297 ++++++++++++++++++++++++++++++ docs/md_v2/assets/styles.css | 13 +- docs/md_v2/assets/toc.js | 144 +++++++++++++++ mkdocs.yml | 5 +- 6 files changed, 593 insertions(+), 4 deletions(-) create mode 100644 docs/md_v2/assets/github_stats.js create mode 100644 docs/md_v2/assets/layout.css create mode 100644 docs/md_v2/assets/toc.js diff --git a/crawl4ai/install.py b/crawl4ai/install.py index c0c3ab0d..b2fcca78 100644 --- a/crawl4ai/install.py +++ b/crawl4ai/install.py @@ -40,10 +40,25 @@ def setup_home_directory(): f.write("") def post_install(): - """Run all post-installation tasks""" + """ + Run all post-installation tasks. + Checks CRAWL4AI_MODE environment variable. If set to 'api', + skips Playwright browser installation. + """ logger.info("Running post-installation setup...", tag="INIT") setup_home_directory() - install_playwright() + + # Check environment variable to conditionally skip Playwright install + run_mode = os.getenv('CRAWL4AI_MODE') + if run_mode == 'api': + logger.warning( + "CRAWL4AI_MODE=api detected. Skipping Playwright browser installation.", + tag="SETUP" + ) + else: + # Proceed with installation only if mode is not 'api' + install_playwright() + run_migration() # TODO: Will be added in the future # setup_builtin_browser() diff --git a/docs/md_v2/assets/github_stats.js b/docs/md_v2/assets/github_stats.js new file mode 100644 index 00000000..a48b3de1 --- /dev/null +++ b/docs/md_v2/assets/github_stats.js @@ -0,0 +1,119 @@ +// ==== File: assets/github_stats.js ==== + +document.addEventListener('DOMContentLoaded', async () => { + // --- Configuration --- + const targetHeaderSelector = '.terminal .container:first-child'; // Selector for your header container + const insertBeforeSelector = '.terminal-nav'; // Selector for the element to insert the badge BEFORE (e.g., the main nav) + // Or set to null to append at the end of the header. + + // --- Find elements --- + const headerContainer = document.querySelector(targetHeaderSelector); + if (!headerContainer) { + console.warn('GitHub Stats: Header container not found with selector:', targetHeaderSelector); + return; + } + + const repoLinkElement = headerContainer.querySelector('a[href*="github.com/"]'); // Find the existing GitHub link + let repoUrl = 'https://github.com/unclecode/crawl4ai'; + // if (repoLinkElement) { + // repoUrl = repoLinkElement.href; + // } else { + // // Fallback: Try finding from config (requires template injection - harder) + // // Or hardcode if necessary, but reading from the link is better. + // console.warn('GitHub Stats: GitHub repo link not found in header.'); + // // Try to get repo_url from mkdocs config if available globally (less likely) + // // repoUrl = window.mkdocs_config?.repo_url; // Requires setting this variable + // // if (!repoUrl) return; // Exit if still no URL + // return; // Exit for now if link isn't found + // } + + + // --- Extract Repo Owner/Name --- + let owner = ''; + let repo = ''; + try { + const url = new URL(repoUrl); + const pathParts = url.pathname.split('/').filter(part => part.length > 0); + if (pathParts.length >= 2) { + owner = pathParts[0]; + repo = pathParts[1]; + } + } catch (e) { + console.error('GitHub Stats: Could not parse repository URL:', repoUrl, e); + return; + } + + if (!owner || !repo) { + console.warn('GitHub Stats: Could not extract owner/repo from URL:', repoUrl); + return; + } + + // --- Get Version (Attempt to extract from site title) --- + let version = ''; + const siteTitleElement = headerContainer.querySelector('.terminal-title, .site-title'); // Adjust selector based on theme's title element + // Example title: "Crawl4AI Documentation (v0.5.x)" + if (siteTitleElement) { + const match = siteTitleElement.textContent.match(/\((v?[^)]+)\)/); // Look for text in parentheses starting with 'v' (optional) + if (match && match[1]) { + version = match[1].trim(); + } + } + if (!version) { + console.info('GitHub Stats: Could not extract version from title. You might need to adjust the selector or regex.'); + // You could fallback to config.extra.version if injected into JS + // version = window.mkdocs_config?.extra?.version || 'N/A'; + } + + + // --- Fetch GitHub API Data --- + let stars = '...'; + let forks = '...'; + try { + const apiUrl = `https://api.github.com/repos/${owner}/${repo}`; + const response = await fetch(apiUrl); + + if (response.ok) { + const data = await response.json(); + // Format large numbers (optional) + stars = data.stargazers_count > 1000 ? `${(data.stargazers_count / 1000).toFixed(1)}k` : data.stargazers_count; + forks = data.forks_count > 1000 ? `${(data.forks_count / 1000).toFixed(1)}k` : data.forks_count; + } else { + console.warn(`GitHub Stats: API request failed with status ${response.status}. Rate limit exceeded?`); + stars = 'N/A'; + forks = 'N/A'; + } + } catch (error) { + console.error('GitHub Stats: Error fetching repository data:', error); + stars = 'N/A'; + forks = 'N/A'; + } + + // --- Create Badge HTML --- + const badgeContainer = document.createElement('div'); + badgeContainer.className = 'github-stats-badge'; + + // Use innerHTML for simplicity, including potential icons (requires FontAwesome or similar) + // Ensure your theme loads FontAwesome or add it yourself if you want icons. + badgeContainer.innerHTML = ` + + + + ${owner}/${repo} + ${version ? ` ${version}` : ''} + ${stars} + ${forks} + + `; + + // --- Inject Badge into Header --- + const insertBeforeElement = insertBeforeSelector ? headerContainer.querySelector(insertBeforeSelector) : null; + if (insertBeforeElement) { + // headerContainer.insertBefore(badgeContainer, insertBeforeElement); + headerContainer.querySelector(insertBeforeSelector).appendChild(badgeContainer); + } else { + headerContainer.appendChild(badgeContainer); + } + + console.info('GitHub Stats: Badge added to header.'); + +}); \ No newline at end of file diff --git a/docs/md_v2/assets/layout.css b/docs/md_v2/assets/layout.css new file mode 100644 index 00000000..db5fac55 --- /dev/null +++ b/docs/md_v2/assets/layout.css @@ -0,0 +1,297 @@ +/* ==== File: assets/layout.css (Non-Fluid Centered Layout) ==== */ + +:root { + --header-height: 55px; /* Adjust if needed */ + --sidebar-width: 280px; /* Adjust if needed */ + --toc-width: 340px; /* As specified */ + --content-max-width: 90em; /* Max width for the centered content */ + --layout-transition-speed: 0.2s; + --global-space: 10px; +} + +/* --- Basic Setup --- */ +html { + scroll-behavior: smooth; + scroll-padding-top: calc(var(--header-height) + 15px); + box-sizing: border-box; +} +*, *:before, *:after { + box-sizing: inherit; +} + +body { + padding-top: 0; + padding-bottom: 0; + background-color: var(--background-color); + color: var(--font-color); + /* Prevents horizontal scrollbars during transitions */ + overflow-x: hidden; +} + +/* --- Fixed Header --- */ +/* Full width, fixed header */ +.terminal .container:first-child { /* Assuming this targets the header container */ + position: fixed; + top: 0; + left: 0; + right: 0; + height: var(--header-height); + background-color: var(--background-color); + z-index: 1000; + border-bottom: 1px solid var(--progress-bar-background); + max-width: none; /* Override any container max-width */ + padding: 0 calc(var(--global-space) * 2); +} + +/* --- Main Layout Container (Below Header) --- */ +/* This container just provides space for the fixed header */ +.container:has(.terminal-mkdocs-main-grid) { + margin: 0 auto; + padding: 0; + padding-top: var(--header-height); /* Space for fixed header */ +} + +/* --- Flex Container: Grid holding content and toc (CENTERED) --- */ +/* THIS is the main centered block */ +.terminal-mkdocs-main-grid { + display: flex; + align-items: flex-start; + /* Enforce max-width and center */ + max-width: var(--content-max-width); + margin-left: auto; + margin-right: auto; + position: relative; + /* Apply side padding within the centered block */ + padding-left: calc(var(--global-space) * 2); + padding-right: calc(var(--global-space) * 2); + /* Add margin-left to clear the fixed sidebar */ + margin-left: var(--sidebar-width); +} + +/* --- 1. Fixed Left Sidebar (Viewport Relative) --- */ +#terminal-mkdocs-side-panel { + position: fixed; + top: var(--header-height); + left: max(0px, calc((100vw - var(--content-max-width)) / 2)); + bottom: 0; + width: var(--sidebar-width); + background-color: var(--background-color); + border-right: 1px solid var(--progress-bar-background); + overflow-y: auto; + z-index: 900; + padding: 1em calc(var(--global-space) * 2); + padding-bottom: 2em; + /* transition: left var(--layout-transition-speed) ease-in-out; */ +} + +/* --- 2. Main Content Area (Within Centered Grid) --- */ +#terminal-mkdocs-main-content { + flex-grow: 1; + flex-shrink: 1; + min-width: 0; /* Flexbox shrink fix */ + + /* No left/right margins needed here - handled by parent grid */ + margin-left: 0; + margin-right: 0; + + /* Internal Padding */ + padding: 1.5em 2em; + + position: relative; + z-index: 1; +} + +/* --- 3. Right Table of Contents (Sticky, Within Centered Grid) --- */ +#toc-sidebar { + flex-basis: var(--toc-width); + flex-shrink: 0; + width: var(--toc-width); + + position: sticky; /* Sticks within the centered grid */ + top: var(--header-height); + align-self: stretch; + height: calc(100vh - var(--header-height)); + overflow-y: auto; + + padding: 1.5em 1em; + font-size: 0.85em; + border-left: 1px solid var(--progress-bar-background); + z-index: 800; + /* display: none; /* JS handles */ +} + +/* (ToC link styles remain the same) */ +#toc-sidebar h4 { margin-top: 0; margin-bottom: 1em; font-size: 1.1em; color: var(--secondary-color); padding-left: 0.8em; } +#toc-sidebar ul { list-style: none; padding: 0; margin: 0; } +#toc-sidebar ul li a { display: block; padding: 0.3em 0; color: var(--secondary-color); text-decoration: none; border-left: 3px solid transparent; padding-left: 0.8em; transition: all 0.1s ease-in-out; line-height: 1.4; word-break: break-word; } +#toc-sidebar ul li.toc-level-3 a { padding-left: 1.8em; } +#toc-sidebar ul li.toc-level-4 a { padding-left: 2.8em; } +#toc-sidebar ul li a:hover { color: var(--font-color); background-color: rgba(255, 255, 255, 0.05); } +#toc-sidebar ul li a.active { color: var(--primary-color); border-left-color: var(--primary-color); background-color: rgba(80, 255, 255, 0.08); } + + +/* --- Footer Styling (Respects Centered Layout) --- */ +footer { + background-color: var(--code-bg-color); + color: var(--secondary-color); + position: relative; + z-index: 10; + margin-top: 2em; + + /* Apply margin-left to clear the fixed sidebar */ + margin-left: var(--sidebar-width); + + /* Constrain width relative to the centered grid it follows */ + max-width: calc(var(--content-max-width) - var(--sidebar-width)); + margin-right: auto; /* Keep it left-aligned within the space next to sidebar */ + + /* Use padding consistent with the grid */ + padding: 2em calc(var(--global-space) * 2); +} + +/* Adjust footer grid if needed */ +.terminal-mkdocs-footer-grid { + display: grid; + grid-template-columns: 1fr auto; + gap: 1em; + align-items: center; +} + +/* ========================================================================== + RESPONSIVENESS (Adapting the Non-Fluid Layout) + ========================================================================== */ + +/* --- Medium screens: Hide ToC --- */ +@media screen and (max-width: 1200px) { + #toc-sidebar { + display: none; + } + + .terminal-mkdocs-main-grid { + /* Grid adjusts automatically as ToC is removed */ + /* Ensure grid padding remains */ + padding-left: calc(var(--global-space) * 2); + padding-right: calc(var(--global-space) * 2); + } + + #terminal-mkdocs-main-content { + /* Content area naturally expands */ + } + + footer { + /* Footer still respects the left sidebar and overall max width */ + margin-left: var(--sidebar-width); + max-width: calc(var(--content-max-width) - var(--sidebar-width)); + /* Padding remains consistent */ + padding-left: calc(var(--global-space) * 2); + padding-right: calc(var(--global-space) * 2); + } +} + +/* --- Small screens: Hide left sidebar, full width content & footer --- */ +@media screen and (max-width: 768px) { + + #terminal-mkdocs-side-panel { + left: calc(-1 * var(--sidebar-width)); + z-index: 1100; + box-shadow: 2px 0 10px rgba(0,0,0,0.3); + } + #terminal-mkdocs-side-panel.sidebar-visible { + left: 0; + } + + .terminal-mkdocs-main-grid { + /* Grid now takes full width (minus body padding) */ + margin-left: 0; /* Override sidebar margin */ + margin-right: 0; /* Override auto margin */ + max-width: 100%; /* Allow full width */ + padding-left: var(--global-space); /* Reduce padding */ + padding-right: var(--global-space); + } + + #terminal-mkdocs-main-content { + padding: 1.5em 1em; /* Adjust internal padding */ + } + + footer { + margin-left: 0; /* Full width footer */ + max-width: 100%; /* Allow full width */ + padding: 2em 1em; /* Adjust internal padding */ + } + + .terminal-mkdocs-footer-grid { + grid-template-columns: 1fr; /* Stack footer items */ + text-align: center; + gap: 0.5em; + } + /* Remember JS for toggle button & overlay */ +} + + +/* ==== GitHub Stats Badge Styling ==== */ + +.github-stats-badge { + display: inline-block; /* Or flex if needed */ + margin-left: 2em; /* Adjust spacing */ + vertical-align: middle; /* Align with other header items */ + font-size: 0.9em; /* Slightly smaller font */ +} + +.github-stats-badge a { + color: var(--secondary-color); /* Use secondary color */ + text-decoration: none; + display: flex; /* Use flex for alignment */ + align-items: center; + gap: 0.8em; /* Space between items */ + padding: 0.2em 0.5em; + border: 1px solid var(--progress-bar-background); /* Subtle border */ + border-radius: 4px; + transition: color 0.2s, background-color 0.2s; +} + +.github-stats-badge a:hover { + color: var(--font-color); /* Brighter color on hover */ + background-color: var(--progress-bar-background); /* Subtle background on hover */ +} + +.github-stats-badge .repo-name { + color: var(--font-color); /* Make repo name stand out slightly */ + font-weight: 500; /* Optional bolder weight */ +} + +.github-stats-badge .stat { + /* Styles for individual stats (version, stars, forks) */ + white-space: nowrap; /* Prevent wrapping */ +} + +.github-stats-badge .stat i { + /* Optional: Style for FontAwesome icons */ + margin-right: 0.3em; + color: var(--secondary-dimmed-color); /* Dimmer color for icons */ +} + + +/* Adjust positioning relative to search/nav if needed */ +/* Example: If search is floated right */ +/* .terminal-nav { float: left; } */ +/* .github-stats-badge { float: left; } */ +/* #mkdocs-search-query { float: right; } */ + +/* --- Responsive adjustments --- */ +@media screen and (max-width: 900px) { /* Example breakpoint */ + .github-stats-badge .repo-name { + display: none; /* Hide full repo name on smaller screens */ + } + .github-stats-badge { + margin-left: 1em; + } + .github-stats-badge a { + gap: 0.5em; + } +} +@media screen and (max-width: 768px) { + /* Further hide or simplify on mobile if needed */ + .github-stats-badge { + display: none; /* Example: Hide completely on smallest screens */ + } +} \ No newline at end of file diff --git a/docs/md_v2/assets/styles.css b/docs/md_v2/assets/styles.css index 8ee8cbb1..751aabb7 100644 --- a/docs/md_v2/assets/styles.css +++ b/docs/md_v2/assets/styles.css @@ -50,8 +50,17 @@ --display-h1-decoration: none; --display-h1-decoration: none; + + --header-height: 65px; /* Adjust based on your actual header height */ + --sidebar-width: 280px; /* Adjust based on your desired sidebar width */ + --toc-width: 240px; /* Adjust based on your desired ToC width */ + --layout-transition-speed: 0.2s; /* For potential future animations */ + + --page-width : 90em; /* Adjust based on your design */ } + + /* body { background-color: var(--background-color); color: var(--font-color); @@ -256,4 +265,6 @@ div.badges a { } div.badges a > img { width: auto; -} \ No newline at end of file +} + + diff --git a/docs/md_v2/assets/toc.js b/docs/md_v2/assets/toc.js new file mode 100644 index 00000000..8dad06b2 --- /dev/null +++ b/docs/md_v2/assets/toc.js @@ -0,0 +1,144 @@ +// ==== File: assets/toc.js ==== + +document.addEventListener('DOMContentLoaded', () => { + const mainContent = document.getElementById('terminal-mkdocs-main-content'); + const tocContainer = document.getElementById('toc-sidebar'); + const mainGrid = document.querySelector('.terminal-mkdocs-main-grid'); // Get the flex container + + if (!mainContent) { + console.warn("TOC Generator: Main content area '#terminal-mkdocs-main-content' not found."); + return; + } + + // --- Create ToC container if it doesn't exist --- + let tocElement = tocContainer; + if (!tocElement) { + if (!mainGrid) { + console.warn("TOC Generator: Flex container '.terminal-mkdocs-main-grid' not found to append ToC."); + return; + } + tocElement = document.createElement('aside'); + tocElement.id = 'toc-sidebar'; + tocElement.style.display = 'none'; // Keep hidden initially + // Append it as the last child of the flex grid + mainGrid.appendChild(tocElement); + console.info("TOC Generator: Created '#toc-sidebar' element."); + } + + // --- Find Headings (h2, h3, h4 are common for ToC) --- + const headings = mainContent.querySelectorAll('h2, h3, h4'); + if (headings.length === 0) { + console.info("TOC Generator: No headings found on this page. ToC not generated."); + tocElement.style.display = 'none'; // Ensure it's hidden + return; + } + + // --- Generate ToC List --- + const tocList = document.createElement('ul'); + const observerTargets = []; // Store headings for IntersectionObserver + + headings.forEach((heading, index) => { + // Ensure heading has an ID for linking + if (!heading.id) { + // Create a simple slug-like ID + heading.id = `toc-heading-${index}-${heading.textContent.toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9-]/g, '')}`; + } + + const listItem = document.createElement('li'); + const link = document.createElement('a'); + + link.href = `#${heading.id}`; + link.textContent = heading.textContent; + + // Add class for styling based on heading level + const level = parseInt(heading.tagName.substring(1), 10); // Get 2, 3, or 4 + listItem.classList.add(`toc-level-${level}`); + + listItem.appendChild(link); + tocList.appendChild(listItem); + observerTargets.push(heading); // Add to observer list + }); + + // --- Populate and Show ToC --- + // Optional: Add a title + const tocTitle = document.createElement('h4'); + tocTitle.textContent = 'On this page'; // Customize title if needed + + tocElement.innerHTML = ''; // Clear previous content if any + tocElement.appendChild(tocTitle); + tocElement.appendChild(tocList); + tocElement.style.display = ''; // Show the ToC container + + console.info(`TOC Generator: Generated ToC with ${headings.length} items.`); + + // --- Scroll Spy using Intersection Observer --- + const tocLinks = tocElement.querySelectorAll('a'); + let activeLink = null; // Keep track of the current active link + + const observerOptions = { + // Observe changes relative to the viewport, offset by the header height + // Negative top margin pushes the intersection trigger point down + // Negative bottom margin ensures elements low on the screen can trigger before they exit + rootMargin: `-${getComputedStyle(document.documentElement).getPropertyValue('--header-height').trim()} 0px -60% 0px`, + threshold: 0 // Trigger as soon as any part enters/exits the boundary + }; + + const observerCallback = (entries) => { + let topmostVisibleHeading = null; + + entries.forEach(entry => { + const link = tocElement.querySelector(`a[href="#${entry.target.id}"]`); + if (!link) return; + + // Check if the heading is intersecting (partially or fully visible within rootMargin) + if (entry.isIntersecting) { + // Among visible headings, find the one closest to the top edge (within the rootMargin) + if (!topmostVisibleHeading || entry.boundingClientRect.top < topmostVisibleHeading.boundingClientRect.top) { + topmostVisibleHeading = entry.target; + } + } + }); + + // If we found a topmost visible heading, activate its link + if (topmostVisibleHeading) { + const newActiveLink = tocElement.querySelector(`a[href="#${topmostVisibleHeading.id}"]`); + if (newActiveLink && newActiveLink !== activeLink) { + // Remove active class from previous link + if (activeLink) { + activeLink.classList.remove('active'); + activeLink.parentElement.classList.remove('active-parent'); // Optional parent styling + } + // Add active class to the new link + newActiveLink.classList.add('active'); + newActiveLink.parentElement.classList.add('active-parent'); // Optional parent styling + activeLink = newActiveLink; + + // Optional: Scroll the ToC sidebar to keep the active link visible + // newActiveLink.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); + } + } + // If no headings are intersecting (scrolled past the last one?), maybe deactivate all + // Or keep the last one active - depends on desired behavior. Current logic keeps last active. + }; + + const observer = new IntersectionObserver(observerCallback, observerOptions); + + // Observe all target headings + observerTargets.forEach(heading => observer.observe(heading)); + + // Initial check in case a heading is already in view on load + // (Requires slight delay for accurate layout calculation) + setTimeout(() => { + observerCallback(observer.takeRecords()); // Process initial state + }, 100); + + // move footer and the hr before footer to the end of the main content + const footer = document.querySelector('footer'); + const hr = footer.previousElementSibling; + if (hr && hr.tagName === 'HR') { + mainContent.appendChild(hr); + } + mainContent.appendChild(footer); + console.info("TOC Generator: Footer moved to the end of the main content."); + +}); \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 82b2fa02..1c7be7a3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,6 +76,7 @@ extra: version: !ENV [CRAWL4AI_VERSION, 'development'] extra_css: + - assets/layout.css - assets/styles.css - assets/highlight.css - assets/dmvendor.css @@ -83,4 +84,6 @@ extra_css: extra_javascript: - assets/highlight.min.js - assets/highlight_init.js - - https://buttons.github.io/buttons.js \ No newline at end of file + - https://buttons.github.io/buttons.js + - assets/toc.js + - assets/github_stats.js \ No newline at end of file