#!/usr/bin/env python3
"""
Test script for VirtualScrollConfig with the /crawl API endpoint
"""
import requests
import json
def test_virtual_scroll_api():
"""Test the /crawl endpoint with VirtualScrollConfig"""
# Create a simple HTML page with virtual scroll for testing
test_html = '''
Virtual Scroll Test
Item 1
Item 2
Item 3
Item 4
Item 5
'''
# Save the HTML to a temporary file and serve it
import tempfile
import os
import http.server
import socketserver
import threading
import time
# Create temporary HTML file
with tempfile.NamedTemporaryFile(mode='w', suffix='.html', delete=False) as f:
f.write(test_html)
temp_file = f.name
# Start local server
os.chdir(os.path.dirname(temp_file))
port = 8080
class QuietHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def log_message(self, format, *args):
pass # Suppress log messages
try:
with socketserver.TCPServer(("", port), QuietHTTPRequestHandler) as httpd:
server_thread = threading.Thread(target=httpd.serve_forever)
server_thread.daemon = True
server_thread.start()
time.sleep(0.5) # Give server time to start
# Now test the API
url = f"http://crawl4ai.com/examples/assets/virtual_scroll_twitter_like.html"
payload = {
"urls": [url],
"browser_config": {
"type": "BrowserConfig",
"params": {
"headless": True,
"viewport_width": 1920,
"viewport_height": 1080
}
},
"crawler_config": {
"type": "CrawlerRunConfig",
"params": {
"virtual_scroll_config": {
"type": "VirtualScrollConfig",
"params": {
"container_selector": "#container",
"scroll_count": 3,
"scroll_by": "container_height",
"wait_after_scroll": 0.5
}
},
"cache_mode": "bypass",
"extraction_strategy": {
"type": "NoExtractionStrategy",
"params": {}
}
}
}
}
print("Testing VirtualScrollConfig with /crawl endpoint...")
print(f"Test URL: {url}")
print("Payload:")
print(json.dumps(payload, indent=2))
response = requests.post(
"http://localhost:11234/crawl",
json=payload,
headers={"Content-Type": "application/json"}
)
print(f"\nResponse Status: {response.status_code}")
if response.status_code == 200:
result = response.json()
print("✅ Success! VirtualScrollConfig is working.")
print(f"Content length: {len(result[0]['content']['raw_content'])} characters")
# Check if virtual scroll captured more content
if "Item 10" in result[0]['content']['raw_content']:
print("✅ Virtual scroll successfully captured additional content!")
else:
print("⚠️ Virtual scroll may not have worked as expected")
# Print a snippet of the content
content_preview = result[0]['content']['raw_content'][:500] + "..."
print(f"\nContent preview:\n{content_preview}")
else:
print(f"❌ Error: {response.status_code}")
print(f"Response: {response.text}")
except Exception as e:
print(f"❌ Test failed with error: {e}")
finally:
# Cleanup
try:
os.unlink(temp_file)
except:
pass
if __name__ == "__main__":
test_virtual_scroll_api()