From 673f996f5ff0d2e3dfac2b3298ea71146d8422d3 Mon Sep 17 00:00:00 2001 From: saidsurucu Date: Mon, 21 Jul 2025 22:38:52 +0300 Subject: [PATCH] fix httpx efficiency --- mcp_server_main.py | 169 ++++++++++++++++++-------------- uyusmazlik_mcp_module/client.py | 85 +++++++++------- 2 files changed, 141 insertions(+), 113 deletions(-) diff --git a/mcp_server_main.py b/mcp_server_main.py index 6088551..0c89c3f 100644 --- a/mcp_server_main.py +++ b/mcp_server_main.py @@ -348,6 +348,19 @@ from fastmcp import FastMCP # Placeholder app for decorators - will be replaced in create_app() after all tools are defined app = FastMCP("Yargı MCP Server Placeholder") +# --- Shared HTTP Client for Health Checks --- +shared_health_check_client = None + +def get_or_create_health_check_client(): + """Get or create shared httpx client for health checks.""" + global shared_health_check_client + if shared_health_check_client is None: + shared_health_check_client = httpx.AsyncClient( + timeout=30.0, + verify=False + ) + return shared_health_check_client + # --- API Client Instances --- yargitay_client_instance = YargitayOfficialApiClient() danistay_client_instance = DanistayApiClient() @@ -1417,6 +1430,14 @@ def perform_cleanup(): ] async def close_all_clients_async(): tasks = [] + + # Close shared health check client first + global shared_health_check_client + if shared_health_check_client: + logger.info("Scheduling close for shared health check client") + tasks.append(shared_health_check_client.aclose()) + + # Close all module clients for client_instance in clients_to_close: if client_instance and hasattr(client_instance, 'close_client_session') and callable(client_instance.close_client_session): logger.info(f"Scheduling close for client session: {client_instance.__class__.__name__}") @@ -1467,50 +1488,49 @@ async def check_government_servers_health() -> Dict[str, Any]: } } - async with httpx.AsyncClient( - headers={ - "Accept": "*/*", - "Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7", - "Connection": "keep-alive", - "Content-Type": "application/json; charset=UTF-8", - "Origin": "https://karararama.yargitay.gov.tr", - "Referer": "https://karararama.yargitay.gov.tr/", - "Sec-Fetch-Dest": "empty", - "Sec-Fetch-Mode": "cors", - "Sec-Fetch-Site": "same-origin", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36", - "X-Requested-With": "XMLHttpRequest" - }, - timeout=30.0, - verify=False - ) as client: - response = await client.post( - "https://karararama.yargitay.gov.tr/aramalist", - json=yargitay_payload - ) + client = get_or_create_health_check_client() + headers = { + "Accept": "*/*", + "Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7", + "Connection": "keep-alive", + "Content-Type": "application/json; charset=UTF-8", + "Origin": "https://karararama.yargitay.gov.tr", + "Referer": "https://karararama.yargitay.gov.tr/", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36", + "X-Requested-With": "XMLHttpRequest" + } + + response = await client.post( + "https://karararama.yargitay.gov.tr/aramalist", + json=yargitay_payload, + headers=headers + ) + + if response.status_code == 200: + response_data = response.json() + records_total = response_data.get("data", {}).get("recordsTotal", 0) - if response.status_code == 200: - response_data = response.json() - records_total = response_data.get("data", {}).get("recordsTotal", 0) - - if records_total > 0: - health_results["yargitay"] = { - "status": "healthy", - "response_time_ms": response.elapsed.total_seconds() * 1000 - } - else: - health_results["yargitay"] = { - "status": "unhealthy", - "reason": "recordsTotal is 0 or missing", - "response_time_ms": response.elapsed.total_seconds() * 1000 - } + if records_total > 0: + health_results["yargitay"] = { + "status": "healthy", + "response_time_ms": response.elapsed.total_seconds() * 1000 + } else: health_results["yargitay"] = { "status": "unhealthy", - "reason": f"HTTP {response.status_code}", + "reason": "recordsTotal is 0 or missing", "response_time_ms": response.elapsed.total_seconds() * 1000 } - + else: + health_results["yargitay"] = { + "status": "unhealthy", + "reason": f"HTTP {response.status_code}", + "response_time_ms": response.elapsed.total_seconds() * 1000 + } + except Exception as e: health_results["yargitay"] = { "status": "unhealthy", @@ -1532,50 +1552,49 @@ async def check_government_servers_health() -> Dict[str, Any]: "paging": True } - async with httpx.AsyncClient( - headers={ - "Content-Type": "application/json", - "Accept": "application/json", - "User-Agent": "Mozilla/5.0 Health Check" - }, - timeout=30.0, - verify=False - ) as client: - response = await client.post( - "https://bedesten.adalet.gov.tr/emsal-karar/searchDocuments", - json=bedesten_payload - ) - - if response.status_code == 200: - response_data = response.json() - logger.debug(f"Bedesten API response: {response_data}") - if response_data and isinstance(response_data, dict): - data_section = response_data.get("data") - if data_section and isinstance(data_section, dict): - total_found = data_section.get("total", 0) - else: - total_found = 0 + client = get_or_create_health_check_client() + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "Mozilla/5.0 Health Check" + } + + response = await client.post( + "https://bedesten.adalet.gov.tr/emsal-karar/searchDocuments", + json=bedesten_payload, + headers=headers + ) + + if response.status_code == 200: + response_data = response.json() + logger.debug(f"Bedesten API response: {response_data}") + if response_data and isinstance(response_data, dict): + data_section = response_data.get("data") + if data_section and isinstance(data_section, dict): + total_found = data_section.get("total", 0) else: total_found = 0 - - if total_found > 0: - health_results["bedesten"] = { - "status": "healthy", - "response_time_ms": response.elapsed.total_seconds() * 1000 - } - else: - health_results["bedesten"] = { - "status": "unhealthy", - "reason": "total is 0 or missing in data field", - "response_time_ms": response.elapsed.total_seconds() * 1000 - } + else: + total_found = 0 + + if total_found > 0: + health_results["bedesten"] = { + "status": "healthy", + "response_time_ms": response.elapsed.total_seconds() * 1000 + } else: health_results["bedesten"] = { "status": "unhealthy", - "reason": f"HTTP {response.status_code}", + "reason": "total is 0 or missing in data field", "response_time_ms": response.elapsed.total_seconds() * 1000 } - + else: + health_results["bedesten"] = { + "status": "unhealthy", + "reason": f"HTTP {response.status_code}", + "response_time_ms": response.elapsed.total_seconds() * 1000 + } + except Exception as e: health_results["bedesten"] = { "status": "unhealthy", diff --git a/uyusmazlik_mcp_module/client.py b/uyusmazlik_mcp_module/client.py index 5c1b669..5dfb47d 100644 --- a/uyusmazlik_mcp_module/client.py +++ b/uyusmazlik_mcp_module/client.py @@ -1,7 +1,6 @@ # uyusmazlik_mcp_module/client.py import httpx -import aiohttp from bs4 import BeautifulSoup from typing import Dict, Any, List, Optional, Union, Tuple import logging @@ -9,7 +8,7 @@ import html import re import io from markitdown import MarkItDown -from urllib.parse import urljoin, urlencode # urlencode for aiohttp form data +from urllib.parse import urljoin from .models import ( UyusmazlikSearchRequest, @@ -56,17 +55,21 @@ class UyusmazlikApiClient: # Individual documents are fetched by their full URLs obtained from search results. def __init__(self, request_timeout: float = 30.0): - self.request_timeout = request_timeout # Store timeout for aiohttp and httpx - # Headers for aiohttp search. httpx for docs will create its own. - self.default_aiohttp_search_headers = { - "Accept": "*/*", # Mimicking browser headers provided by user - "Accept-Encoding": "gzip, deflate, br, zstd", - "Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7", - "X-Requested-With": "XMLHttpRequest", - "Origin": self.BASE_URL, - "Referer": self.BASE_URL + "/", - - } + self.request_timeout = request_timeout + # Create shared httpx client for all requests + self.http_client = httpx.AsyncClient( + base_url=self.BASE_URL, + headers={ + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate, br, zstd", + "Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7", + "X-Requested-With": "XMLHttpRequest", + "Origin": self.BASE_URL, + "Referer": self.BASE_URL + "/", + }, + timeout=request_timeout, + verify=False + ) async def search_decisions( @@ -107,32 +110,36 @@ class UyusmazlikApiClient: add_to_form_data("Hepsi", params.hepsi) add_to_form_data("Herhangibirisi", params.herhangi_birisi) add_to_form_data("NotHepsi", params.not_hepsi) - # X-Requested-With is handled by default_aiohttp_search_headers - search_url = urljoin(self.BASE_URL, self.SEARCH_ENDPOINT) - # For aiohttp, data for application/x-www-form-urlencoded should be a dict or str. - # Using urlencode for list of tuples. - encoded_form_payload = urlencode(form_data_list, encoding='UTF-8') + # Convert form data to dict for httpx + form_data_dict = {} + for key, value in form_data_list: + if key in form_data_dict: + # Handle multiple values (like KararSonucuList) + if not isinstance(form_data_dict[key], list): + form_data_dict[key] = [form_data_dict[key]] + form_data_dict[key].append(value) + else: + form_data_dict[key] = value - logger.info(f"UyusmazlikApiClient (aiohttp): Performing search to {search_url} with form_data: {encoded_form_payload}") + logger.info(f"UyusmazlikApiClient (httpx): Performing search to {self.SEARCH_ENDPOINT} with form_data: {form_data_dict}") - html_content = "" - aiohttp_headers = self.default_aiohttp_search_headers.copy() - aiohttp_headers["Content-Type"] = "application/x-www-form-urlencoded; charset=UTF-8" - try: - # Create a new session for each call for simplicity with aiohttp here - async with aiohttp.ClientSession(headers=aiohttp_headers) as session: - async with session.post(search_url, data=encoded_form_payload, timeout=self.request_timeout) as response: - response.raise_for_status() # Raises ClientResponseError for 400-599 - html_content = await response.text(encoding='utf-8') # Ensure correct encoding - logger.debug("UyusmazlikApiClient (aiohttp): Received HTML response for search.") + # Use shared httpx client + response = await self.http_client.post( + self.SEARCH_ENDPOINT, + data=form_data_dict, + headers={"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"} + ) + response.raise_for_status() + html_content = response.text + logger.debug("UyusmazlikApiClient (httpx): Received HTML response for search.") - except aiohttp.ClientError as e: - logger.error(f"UyusmazlikApiClient (aiohttp): HTTP client error during search: {e}") + except httpx.HTTPError as e: + logger.error(f"UyusmazlikApiClient (httpx): HTTP client error during search: {e}") raise # Re-raise to be handled by the MCP tool except Exception as e: - logger.error(f"UyusmazlikApiClient (aiohttp): Error processing search request: {e}") + logger.error(f"UyusmazlikApiClient (httpx): Error processing search request: {e}") raise # --- HTML Parsing (remains the same as previous version) --- @@ -215,10 +222,8 @@ class UyusmazlikApiClient: """ logger.info(f"UyusmazlikApiClient (httpx for docs): Fetching Uyuşmazlık document for Markdown from URL: {document_url}") try: - # Using a new httpx.AsyncClient instance for this GET request for simplicity - async with httpx.AsyncClient(verify=False, timeout=self.request_timeout) as doc_fetch_client: - - get_response = await doc_fetch_client.get(document_url, headers={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}) + # Use the existing shared http_client instead of creating a new one + get_response = await self.http_client.get(document_url, headers={"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}) get_response.raise_for_status() html_content_from_api = get_response.text @@ -236,5 +241,9 @@ class UyusmazlikApiClient: raise async def close_client_session(self): - - logger.info("UyusmazlikApiClient: No persistent client session from __init__ to close.") \ No newline at end of file + """Close the shared httpx client session.""" + if hasattr(self, 'http_client') and self.http_client: + await self.http_client.aclose() + logger.info("UyusmazlikApiClient: HTTP client session closed.") + else: + logger.info("UyusmazlikApiClient: No HTTP client session to close.") \ No newline at end of file