ca0336af9e
This commit adds error handling for rate limit exceeded in the form submission process. If the server returns a 429 status code, the client will display an error message indicating the rate limit has been exceeded and provide information on when the user can try again. This improves the user experience by providing clear feedback and guidance when rate limits are reached.
231 lines
10 KiB
Markdown
231 lines
10 KiB
Markdown
# Interactive Demo for Crowler
|
|
<div id="demo">
|
|
<form id="crawlForm" class="terminal-form">
|
|
<fieldset>
|
|
<legend>Enter URL and Options</legend>
|
|
<div class="form-group">
|
|
<label for="url">Enter URL:</label>
|
|
<input type="text" id="url" name="url" required>
|
|
</div>
|
|
<div class="form-group">
|
|
<label for="screenshot">Get Screenshot:</label>
|
|
<input type="checkbox" id="screenshot" name="screenshot">
|
|
</div>
|
|
<div class="form-group">
|
|
<button class="btn btn-default" type="submit">Submit</button>
|
|
</div>
|
|
|
|
</fieldset>
|
|
</form>
|
|
|
|
<div id="loading" class="loading-message">
|
|
<div class="terminal-alert terminal-alert-primary">Loading... Please wait.</div>
|
|
</div>
|
|
|
|
<section id="response" class="response-section">
|
|
<h2>Response</h2>
|
|
<div class="tabs">
|
|
<ul class="tab-list">
|
|
<li class="tab-item" onclick="showTab('markdown')">Markdown</li>
|
|
<li class="tab-item" onclick="showTab('cleanedHtml')">Cleaned HTML</li>
|
|
<li class="tab-item" onclick="showTab('media')">Media</li>
|
|
<li class="tab-item" onclick="showTab('extractedContent')">Extracted Content</li>
|
|
<li class="tab-item" onclick="showTab('screenshot')">Screenshot</li>
|
|
<li class="tab-item" onclick="showTab('pythonCode')">Python Code</li>
|
|
</ul>
|
|
<div class="tab-content" id="tab-markdown">
|
|
<header>
|
|
<div>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="copyToClipboard('markdownContent')">Copy</button>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="downloadContent('markdownContent', 'markdown.md')">Download</button>
|
|
</div>
|
|
</header>
|
|
<pre><code id="markdownContent" class="language-markdown hljs"></code></pre>
|
|
</div>
|
|
|
|
<div class="tab-content" id="tab-cleanedHtml" style="display: none;">
|
|
<header >
|
|
<div>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="copyToClipboard('cleanedHtmlContent')">Copy</button>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="downloadContent('cleanedHtmlContent', 'cleaned.html')">Download</button>
|
|
</div>
|
|
</header>
|
|
<pre><code id="cleanedHtmlContent" class="language-html hljs"></code></pre>
|
|
</div>
|
|
|
|
<div class="tab-content" id="tab-media" style="display: none;">
|
|
<header >
|
|
<div>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="copyToClipboard('mediaContent')">Copy</button>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="downloadContent('mediaContent', 'media.json')">Download</button>
|
|
</div>
|
|
</header>
|
|
<pre><code id="mediaContent" class="language-json hljs"></code></pre>
|
|
</div>
|
|
|
|
<div class="tab-content" id="tab-extractedContent" style="display: none;">
|
|
<header >
|
|
<div>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="copyToClipboard('extractedContentContent')">Copy</button>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="downloadContent('extractedContentContent', 'extracted_content.json')">Download</button>
|
|
</div>
|
|
</header>
|
|
<pre><code id="extractedContentContent" class="language-json hljs"></code></pre>
|
|
</div>
|
|
|
|
<div class="tab-content" id="tab-screenshot" style="display: none;">
|
|
<header >
|
|
<div>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="downloadImage('screenshotContent', 'screenshot.png')">Download</button>
|
|
</div>
|
|
</header>
|
|
<pre><img id="screenshotContent" /></pre>
|
|
</div>
|
|
|
|
<div class="tab-content" id="tab-pythonCode" style="display: none;">
|
|
<header >
|
|
<div>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="copyToClipboard('pythonCode')">Copy</button>
|
|
<button class="btn btn-default btn-ghost btn-sm" onclick="downloadContent('pythonCode', 'example.py')">Download</button>
|
|
</div>
|
|
</header>
|
|
<pre><code id="pythonCode" class="language-python hljs"></code></pre>
|
|
</div>
|
|
</div>
|
|
</section>
|
|
|
|
<div id="error" class="error-message" style="display: none; margin-top:1em;">
|
|
<div class="terminal-alert terminal-alert-error"></div>
|
|
</div>
|
|
|
|
<script>
|
|
function showTab(tabId) {
|
|
const tabs = document.querySelectorAll('.tab-content');
|
|
tabs.forEach(tab => tab.style.display = 'none');
|
|
document.getElementById(`tab-${tabId}`).style.display = 'block';
|
|
}
|
|
|
|
function redo(codeBlock, codeText){
|
|
codeBlock.classList.remove('hljs');
|
|
codeBlock.removeAttribute('data-highlighted');
|
|
|
|
// Set new code and re-highlight
|
|
codeBlock.textContent = codeText;
|
|
hljs.highlightBlock(codeBlock);
|
|
}
|
|
|
|
function copyToClipboard(elementId) {
|
|
const content = document.getElementById(elementId).textContent;
|
|
navigator.clipboard.writeText(content).then(() => {
|
|
alert('Copied to clipboard');
|
|
});
|
|
}
|
|
|
|
function downloadContent(elementId, filename) {
|
|
const content = document.getElementById(elementId).textContent;
|
|
const blob = new Blob([content], { type: 'text/plain' });
|
|
const url = window.URL.createObjectURL(blob);
|
|
const a = document.createElement('a');
|
|
a.style.display = 'none';
|
|
a.href = url;
|
|
a.download = filename;
|
|
document.body.appendChild(a);
|
|
a.click();
|
|
window.URL.revokeObjectURL(url);
|
|
document.body.removeChild(a);
|
|
}
|
|
|
|
function downloadImage(elementId, filename) {
|
|
const content = document.getElementById(elementId).src;
|
|
const a = document.createElement('a');
|
|
a.style.display = 'none';
|
|
a.href = content;
|
|
a.download = filename;
|
|
document.body.appendChild(a);
|
|
a.click();
|
|
document.body.removeChild(a);
|
|
}
|
|
|
|
document.getElementById('crawlForm').addEventListener('submit', function(event) {
|
|
event.preventDefault();
|
|
document.getElementById('loading').style.display = 'block';
|
|
document.getElementById('response').style.display = 'none';
|
|
|
|
const url = document.getElementById('url').value;
|
|
const screenshot = document.getElementById('screenshot').checked;
|
|
const data = {
|
|
urls: [url],
|
|
bypass_cache: false,
|
|
word_count_threshold: 5,
|
|
screenshot: screenshot
|
|
};
|
|
|
|
fetch('/crawl', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json'
|
|
},
|
|
body: JSON.stringify(data)
|
|
})
|
|
.then(response => {
|
|
if (!response.ok) {
|
|
if (response.status === 429) {
|
|
return response.json().then(err => {
|
|
throw Object.assign(new Error('Rate limit exceeded'), { status: 429, details: err });
|
|
});
|
|
}
|
|
throw new Error('Network response was not ok');
|
|
}
|
|
return response.json();
|
|
})
|
|
.then(data => {
|
|
data = data.results[0]; // Only one URL is requested
|
|
document.getElementById('loading').style.display = 'none';
|
|
document.getElementById('response').style.display = 'block';
|
|
redo(document.getElementById('markdownContent'), data.markdown);
|
|
redo(document.getElementById('cleanedHtmlContent'), data.cleaned_html);
|
|
redo(document.getElementById('mediaContent'), JSON.stringify(data.media, null, 2));
|
|
redo(document.getElementById('extractedContentContent'), data.extracted_content);
|
|
if (screenshot) {
|
|
document.getElementById('screenshotContent').src = `data:image/png;base64,${data.screenshot}`;
|
|
}
|
|
const pythonCode = `
|
|
from crawl4ai.web_crawler import WebCrawler
|
|
|
|
crawler = WebCrawler()
|
|
crawler.warmup()
|
|
|
|
result = crawler.run(
|
|
url='${url}',
|
|
screenshot=${screenshot}
|
|
)
|
|
print(result)
|
|
`;
|
|
redo(document.getElementById('pythonCode'), pythonCode);
|
|
document.getElementById('error').style.display = 'none';
|
|
})
|
|
.catch(error => {
|
|
document.getElementById('loading').style.display = 'none';
|
|
document.getElementById('error').style.display = 'block';
|
|
let errorMessage = 'An unexpected error occurred. Please try again later.';
|
|
|
|
if (error.status === 429) {
|
|
const details = error.details;
|
|
if (details.retry_after) {
|
|
errorMessage = `Rate limit exceeded. Please wait ${parseFloat(details.retry_after).toFixed(1)} seconds before trying again.`;
|
|
} else if (details.reset_at) {
|
|
const resetTime = new Date(details.reset_at);
|
|
const waitTime = Math.ceil((resetTime - new Date()) / 1000);
|
|
errorMessage = `Rate limit exceeded. Please try again after ${waitTime} seconds.`;
|
|
} else {
|
|
errorMessage = `Rate limit exceeded. Please try again later.`;
|
|
}
|
|
} else if (error.message) {
|
|
errorMessage = error.message;
|
|
}
|
|
|
|
document.querySelector('#error .terminal-alert').textContent = errorMessage;
|
|
});
|
|
});
|
|
</script>
|
|
</div> |