fix: prevent session closing after each request to maintain connection pool. Fixes: https://github.com/unclecode/crawl4ai/issues/867

crawl4ai/async_crawler_strategy.py

@@ -1702,15 +1702,6 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
     async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
         await self.close()
 
-    @contextlib.asynccontextmanager
-    async def _session_context(self):
-        try:
-            if not self._session:
-                await self.start()
-            yield self._session
-        finally:
-            await self.close()
-
     def set_hook(self, hook_type: str, hook_func: Callable) -> None:
         if hook_type in self.hooks:
             self.hooks[hook_type] = partial(self._execute_hook, hook_type, hook_func)
@@ -1787,7 +1778,9 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
         url: str, 
         config: CrawlerRunConfig
     ) -> AsyncCrawlResponse:
-        async with self._session_context() as session:
+        if not self._session or self._session.closed:
+            await self.start()
+        
         timeout = ClientTimeout(
             total=config.page_timeout or self.DEFAULT_TIMEOUT,
             connect=10,
@@ -1814,7 +1807,7 @@ class AsyncHTTPCrawlerStrategy(AsyncCrawlerStrategy):
         await self.hooks['before_request'](url, request_kwargs)
 
         try:
-                async with session.request(self.browser_config.method, url, **request_kwargs) as response:
+            async with self._session.request(self.browser_config.method, url, **request_kwargs) as response:
                 content = memoryview(await response.read())
                 
                 if not (200 <= response.status < 300):