feat: enhance image processing capabilities

- Enhanced image processing with srcset support and validation checks for better image selection.
This commit is contained in:
UncleCode
2024-11-22 16:00:17 +08:00
parent dbb751c8f0
commit 006bee4a5a
3 changed files with 172 additions and 11 deletions

34
crawl4ai/tools.py Normal file
View File

@@ -0,0 +1,34 @@
import time
import cProfile
import pstats
from functools import wraps
def profile_and_time(func):
@wraps(func)
def wrapper(self, *args, **kwargs):
# Start timer
start_time = time.perf_counter()
# Setup profiler
profiler = cProfile.Profile()
profiler.enable()
# Run function
result = func(self, *args, **kwargs)
# Stop profiler
profiler.disable()
# Calculate elapsed time
elapsed_time = time.perf_counter() - start_time
# Print timing
print(f"[PROFILER] Scraping completed in {elapsed_time:.2f} seconds")
# Print profiling stats
stats = pstats.Stats(profiler)
stats.sort_stats('cumulative') # Sort by cumulative time
stats.print_stats(20) # Print top 20 time-consuming functions
return result
return wrapper