Merge branch 'main' of https://github.com/unclecode/crawl4ai

docs/md/demo.md

@@ -14,6 +14,7 @@
             <div class="form-group">
                 <button class="btn btn-default" type="submit">Submit</button>
             </div>
+
         </fieldset>
     </form>
 
@@ -93,6 +94,10 @@
         </div>
     </section>
 
+    <div id="error" class="error-message" style="display: none; margin-top:1em;">
+        <div class="terminal-alert terminal-alert-error"></div>
+    </div>
+
     <script>
         function showTab(tabId) {
             const tabs = document.querySelectorAll('.tab-content');
@@ -162,7 +167,17 @@
                 },
                 body: JSON.stringify(data)
             })
-            .then(response => response.json())
+            .then(response => {
+                if (!response.ok) {
+                    if (response.status === 429) {
+                        return response.json().then(err => { 
+                            throw Object.assign(new Error('Rate limit exceeded'), { status: 429, details: err });
+                        });
+                    }
+                    throw new Error('Network response was not ok');
+                }
+                return response.json();
+            })
             .then(data => {
                 data = data.results[0]; // Only one URL is requested
                 document.getElementById('loading').style.display = 'none';
@@ -187,11 +202,29 @@ result = crawler.run(
 print(result)
                 `;
                 redo(document.getElementById('pythonCode'), pythonCode);
+                document.getElementById('error').style.display = 'none';
             })
             .catch(error => {
                 document.getElementById('loading').style.display = 'none';
-                document.getElementById('response').style.display = 'block';
-                document.getElementById('markdownContent').textContent = 'Error: ' + error;
+                document.getElementById('error').style.display = 'block';
+                let errorMessage = 'An unexpected error occurred. Please try again later.';
+                
+                if (error.status === 429) {
+                    const details = error.details;
+                    if (details.retry_after) {
+                        errorMessage = `Rate limit exceeded. Please wait ${parseFloat(details.retry_after).toFixed(1)} seconds before trying again.`;
+                    } else if (details.reset_at) {
+                        const resetTime = new Date(details.reset_at);
+                        const waitTime = Math.ceil((resetTime - new Date()) / 1000);
+                        errorMessage = `Rate limit exceeded. Please try again after ${waitTime} seconds.`;
+                    } else {
+                        errorMessage = `Rate limit exceeded. Please try again later.`;
+                    }
+                } else if (error.message) {
+                    errorMessage = error.message;
+                }
+                
+                document.querySelector('#error .terminal-alert').textContent = errorMessage;
             });
         });
     </script>

main.py

@@ -51,6 +51,8 @@ app.state.limiter = limiter
 
 # Dictionary to store last request times for each client
 last_request_times = {}
+last_rate_limit = {}
+
 
 def get_rate_limit():
     limit = os.environ.get('ACCESS_PER_MIN', "5")
@@ -58,15 +60,18 @@ def get_rate_limit():
 
 # Custom rate limit exceeded handler
 async def custom_rate_limit_exceeded_handler(request: Request, exc: RateLimitExceeded) -> JSONResponse:
-    try_after = last_request_times.get(request.client.host, 0) + 10 - time.time()
-    reset_at = time.time() + try_after
+    if request.client.host not in last_rate_limit or time.time() - last_rate_limit[request.client.host] > 60:
+        last_rate_limit[request.client.host] = time.time()
+    retry_after = 60 - (time.time() - last_rate_limit[request.client.host])
+    reset_at = time.time() + retry_after
     return JSONResponse(
         status_code=429,
         content={
             "detail": "Rate limit exceeded",
             "limit": str(exc.limit.limit),
-            "reset_at": reset_at,
-            "message": f"You have exceeded the rate limit of {exc.limit.limit}. Please try again after {try_after} seconds."
+            "retry_after": retry_after,
+            'reset_at': reset_at,
+            "message": f"You have exceeded the rate limit of {exc.limit.limit}."
         }
     )
     
@@ -95,7 +100,8 @@ class RateLimitMiddleware(BaseHTTPMiddleware):
                         content={
                             "detail": "Too many requests",
                             "message": "Rate limit exceeded. Please wait 10 seconds between requests.",
-                            "retry_after": max(0, SPAN - time_since_last_request)
+                            "retry_after": max(0, SPAN - time_since_last_request),
+                            "reset_at": current_time + max(0, SPAN - time_since_last_request),
                         }
                     )