diff --git a/anayasa_mcp_module/bireysel_client.py b/anayasa_mcp_module/bireysel_client.py index 3d01787..676c76b 100644 --- a/anayasa_mcp_module/bireysel_client.py +++ b/anayasa_mcp_module/bireysel_client.py @@ -1,6 +1,7 @@ # anayasa_mcp_module/bireysel_client.py # This client is for Bireysel Başvuru: https://kararlarbilgibankasi.anayasa.gov.tr +import asyncio import httpx from bs4 import BeautifulSoup, Tag from typing import Dict, Any, List, Optional, Tuple @@ -302,7 +303,7 @@ class AnayasaBireyselBasvuruApiClient: elif "Karar Tarihi" in key and not karar_tarihi_from_page: karar_tarihi_from_page = value elif "Resmi Gazete Tarih / Sayı" in key: resmi_gazete_info_from_page = value - full_markdown_content = self._convert_html_to_markdown_bireysel(html_content_from_api) + full_markdown_content = await asyncio.to_thread(self._convert_html_to_markdown_bireysel, html_content_from_api) if not full_markdown_content: return AnayasaBireyselBasvuruDocumentMarkdown( diff --git a/anayasa_mcp_module/client.py b/anayasa_mcp_module/client.py index 9e15a48..22aec73 100644 --- a/anayasa_mcp_module/client.py +++ b/anayasa_mcp_module/client.py @@ -1,6 +1,7 @@ # anayasa_mcp_module/client.py # This client is for Norm Denetimi: https://normkararlarbilgibankasi.anayasa.gov.tr +import asyncio import httpx from bs4 import BeautifulSoup from typing import Dict, Any, List, Optional, Tuple @@ -309,7 +310,7 @@ class AnayasaMahkemesiApiClient: official_gazette_from_page = rg_text_content.replace("Resmî Gazete tarih ve sayısı:", "").replace("Resmi Gazete tarih/sayı:", "").strip() - full_markdown_content = self._convert_html_to_markdown_norm_denetimi(html_content_from_api) + full_markdown_content = await asyncio.to_thread(self._convert_html_to_markdown_norm_denetimi, html_content_from_api) if not full_markdown_content: return AnayasaDocumentMarkdown( diff --git a/bddk_mcp_module/client.py b/bddk_mcp_module/client.py index 158e082..4b9cc5a 100644 --- a/bddk_mcp_module/client.py +++ b/bddk_mcp_module/client.py @@ -1,5 +1,6 @@ # bddk_mcp_module/client.py +import asyncio import httpx from typing import List, Optional, Dict, Any import logging @@ -210,14 +211,19 @@ class BddkApiClient: # Convert to Markdown based on content type if "pdf" in content_type: - # Handle PDF documents + # Handle PDF documents. markitdown is sync; offload to thread + # so PDF parsing doesn't block the event-loop / other requests. pdf_stream = io.BytesIO(response.content) - result = self.markitdown.convert_stream(pdf_stream, file_extension=".pdf") + result = await asyncio.to_thread( + self.markitdown.convert_stream, pdf_stream, file_extension=".pdf" + ) markdown_content = result.text_content else: - # Handle HTML documents + # Handle HTML documents (sync conversion offloaded to thread) html_stream = io.BytesIO(response.content) - result = self.markitdown.convert_stream(html_stream, file_extension=".html") + result = await asyncio.to_thread( + self.markitdown.convert_stream, html_stream, file_extension=".html" + ) markdown_content = result.text_content # Clean up the markdown content diff --git a/bedesten_mcp_module/client.py b/bedesten_mcp_module/client.py index c076739..ff9d84e 100644 --- a/bedesten_mcp_module/client.py +++ b/bedesten_mcp_module/client.py @@ -21,6 +21,19 @@ from .enums import get_full_birim_adi logger = logging.getLogger(__name__) +class BedestenRateLimited(Exception): + """Raised when the local rate-limit bucket would block longer than allowed. + + Carries the suggested retry-after (seconds) so callers can surface a + structured 429-style response to the MCP client instead of silently + blocking the event-loop slot for the full bucket-pause window. + """ + + def __init__(self, retry_after: float) -> None: + self.retry_after = retry_after + super().__init__(f"local bucket would block {retry_after:.1f}s") + + class _TokenBucket: """Asyncio token bucket with explicit back-pressure. @@ -40,7 +53,12 @@ class _TokenBucket: self._not_before = 0.0 self._lock = asyncio.Lock() - async def acquire(self) -> None: + async def acquire(self, max_wait: Optional[float] = None) -> None: + """Acquire one token. If ``max_wait`` is set and the next wait would + exceed it, raise :class:`BedestenRateLimited` immediately instead of + sleeping — keeps a single rate-limited request from holding the + worker-slot for the full bucket-pause window (up to ~30s on 429).""" + deadline = (time.monotonic() + max_wait) if max_wait is not None else None while True: async with self._lock: now = time.monotonic() @@ -56,6 +74,10 @@ class _TokenBucket: self._tokens -= 1.0 return wait_s = (1.0 - self._tokens) / self.refill_per_s + if deadline is not None: + remaining = deadline - time.monotonic() + if wait_s > remaining: + raise BedestenRateLimited(retry_after=wait_s) await asyncio.sleep(wait_s) def penalize_until(self, monotonic_deadline: float) -> None: @@ -78,8 +100,11 @@ class BedestenApiClient: # 3.5s spacing (no burst, ~14% safety margin). Override via env: # BEDESTEN_RATE_CAPACITY (default 1) # BEDESTEN_RATE_REFILL_S (default 3.5; seconds per token) + # BEDESTEN_RATE_MAX_WAIT_S (default 8.0; max seconds to wait in the + # local bucket before returning a structured 429 to the caller) _DEFAULT_CAPACITY = int(os.getenv("BEDESTEN_RATE_CAPACITY", "1")) _DEFAULT_REFILL_S = float(os.getenv("BEDESTEN_RATE_REFILL_S", "3.5")) + _DEFAULT_MAX_WAIT_S = float(os.getenv("BEDESTEN_RATE_MAX_WAIT_S", "8.0")) def __init__(self, request_timeout: float = 60.0): self.http_client = httpx.AsyncClient( @@ -137,7 +162,7 @@ class BedestenApiClient: if not request_dict["data"]["birimAdi"]: # Remove if empty string del request_dict["data"]["birimAdi"] - await self._bucket.acquire() + await self._bucket.acquire(max_wait=self._DEFAULT_MAX_WAIT_S) response = await self.http_client.post( self.SEARCH_ENDPOINT, json=request_dict @@ -171,7 +196,7 @@ class BedestenApiClient: ) # Get document - await self._bucket.acquire() + await self._bucket.acquire(max_wait=self._DEFAULT_MAX_WAIT_S) response = await self.http_client.post( self.DOCUMENT_ENDPOINT, json=doc_request.model_dump() @@ -202,12 +227,20 @@ class BedestenApiClient: logger.info(f"BedestenApiClient: Document mime type: {mime_type}") - # Convert to markdown based on mime type + # Convert to markdown based on mime type. markitdown is sync and + # PDF parsing in particular can block the event-loop for seconds, + # which on a single-worker uvicorn deployment stalls every other + # in-flight MCP request and new TLS handshakes. Offload to a + # thread so the event-loop stays responsive. if mime_type == "text/html": html_content = content_bytes.decode('utf-8') - markdown_content = self._convert_html_to_markdown(html_content) + markdown_content = await asyncio.to_thread( + self._convert_html_to_markdown, html_content + ) elif mime_type == "application/pdf": - markdown_content = self._convert_pdf_to_markdown(content_bytes) + markdown_content = await asyncio.to_thread( + self._convert_pdf_to_markdown, content_bytes + ) else: logger.warning(f"Unsupported mime type: {mime_type}") markdown_content = f"Unsupported content type: {mime_type}. Unable to convert to markdown." diff --git a/danistay_mcp_module/client.py b/danistay_mcp_module/client.py index 0a713ff..6e17972 100644 --- a/danistay_mcp_module/client.py +++ b/danistay_mcp_module/client.py @@ -1,7 +1,8 @@ # danistay_mcp_module/client.py +import asyncio import httpx -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup from typing import Dict, Any, List, Optional import logging import html @@ -170,7 +171,7 @@ class DanistayApiClient: source_url=source_url ) - markdown_content = self._convert_html_to_markdown_danistay(html_content_from_api) + markdown_content = await asyncio.to_thread(self._convert_html_to_markdown_danistay, html_content_from_api) return DanistayDocumentMarkdown( id=id, diff --git a/emsal_mcp_module/client.py b/emsal_mcp_module/client.py index 235d515..1efd49d 100644 --- a/emsal_mcp_module/client.py +++ b/emsal_mcp_module/client.py @@ -1,5 +1,6 @@ # emsal_mcp_module/client.py +import asyncio import httpx # from bs4 import BeautifulSoup # Uncomment if needed for advanced HTML pre-processing from typing import Dict, Any, List, Optional @@ -153,7 +154,7 @@ class EmsalApiClient: logger.warning(f"EmsalApiClient: Received empty or non-string HTML in 'data' field for Emsal ID {id}.") return EmsalDocumentMarkdown(id=id, markdown_content=None, source_url=source_url) - markdown_content = self._clean_html_and_convert_to_markdown_emsal(html_content_from_api) + markdown_content = await asyncio.to_thread(self._clean_html_and_convert_to_markdown_emsal, html_content_from_api) return EmsalDocumentMarkdown( id=id, diff --git a/gib_mcp_module/client.py b/gib_mcp_module/client.py index bb55ce5..6f93e76 100644 --- a/gib_mcp_module/client.py +++ b/gib_mcp_module/client.py @@ -1,5 +1,6 @@ # gib_mcp_module/client.py +import asyncio import httpx import io import logging @@ -302,7 +303,7 @@ class GibApiClient: item = content[0] if isinstance(content[0], dict) else {} description_html = item.get("description") or "" - markdown_body = self._convert_html_to_markdown(description_html) or "" + markdown_body = (await asyncio.to_thread(self._convert_html_to_markdown, description_html)) or "" header_block = self._build_header_block(item) if header_block and markdown_body: diff --git a/kik_mcp_module/client_v2.py b/kik_mcp_module/client_v2.py index 62f5e8a..24c80b5 100644 --- a/kik_mcp_module/client_v2.py +++ b/kik_mcp_module/client_v2.py @@ -1,5 +1,6 @@ # kik_mcp_module/client_v2.py +import asyncio import httpx import logging import uuid @@ -439,7 +440,9 @@ class KikV2ApiClient: html_bytes = html_content.encode('utf-8') html_stream = BytesIO(html_bytes) - result = md.convert_stream(html_stream, file_extension=".html") + # markitdown is sync; offload to thread so HTML parsing doesn't + # block the event-loop / other in-flight MCP requests. + result = await asyncio.to_thread(md.convert_stream, html_stream, file_extension=".html") markdown_content = result.text_content return KikV2DocumentMarkdown( diff --git a/kvkk_mcp_module/client.py b/kvkk_mcp_module/client.py index ab26bb8..325d5ed 100644 --- a/kvkk_mcp_module/client.py +++ b/kvkk_mcp_module/client.py @@ -1,5 +1,6 @@ # kvkk_mcp_module/client.py +import asyncio import httpx from bs4 import BeautifulSoup from typing import List, Optional, Dict, Any @@ -291,7 +292,7 @@ class KvkkApiClient: # Convert HTML content to Markdown full_markdown_content = None if extracted_data["html_content"]: - full_markdown_content = self._convert_html_to_markdown(extracted_data["html_content"]) + full_markdown_content = await asyncio.to_thread(self._convert_html_to_markdown, extracted_data["html_content"]) if not full_markdown_content: return KvkkDocumentMarkdown( diff --git a/mcp_server_main.py b/mcp_server_main.py index 3c6a7a1..8fb8e80 100644 --- a/mcp_server_main.py +++ b/mcp_server_main.py @@ -258,7 +258,7 @@ def create_app(): # --- Module Imports --- from yargitay_mcp_module.client import YargitayOfficialApiClient -from bedesten_mcp_module.client import BedestenApiClient +from bedesten_mcp_module.client import BedestenApiClient, BedestenRateLimited from bedesten_mcp_module.models import ( BedestenSearchRequest, BedestenSearchData, BedestenDocumentMarkdown, BedestenCourtTypeEnum @@ -1182,6 +1182,26 @@ For best results, use exact phrases with quotes for legal terms."""), "page_size": pageSize, "searched_courts": court_types } + except BedestenRateLimited as e: + retry_after = f"{e.retry_after:.1f}" + logger.warning(f"Bedesten local rate-limit bucket full for search; retry-after={retry_after}s") + return { + "decisions": [], + "total_records": 0, + "requested_page": pageNumber, + "page_size": pageSize, + "searched_courts": court_types, + "error": "rate_limit_exceeded", + "status_code": 429, + "retry_after": retry_after, + "message": ( + "Bedesten istemci tarafı eşzamanlılık sınırına ulaşıldı " + "(yerel token-bucket dolu). Lütfen kısa bir süre bekleyip " + "aramayı tekrar deneyin. Alternatif olarak yargi-cli aracını " + "kullanabilirsiniz: https://github.com/saidsurucu/yargi-cli " + "(kullanım talimatları için repo README'sini okuyun)." + ), + } except httpx.HTTPStatusError as e: if e.response.status_code == 429: retry_after = e.response.headers.get("Retry-After", "") @@ -1227,6 +1247,23 @@ async def get_bedesten_document_markdown( try: return await bedesten_client_instance.get_document_as_markdown(documentId) + except BedestenRateLimited as e: + retry_after = f"{e.retry_after:.1f}" + logger.warning(f"Bedesten local rate-limit bucket full for document {documentId}; retry-after={retry_after}s") + message = ( + "Bedesten istemci tarafı eşzamanlılık sınırına ulaşıldı " + "(yerel token-bucket dolu). Lütfen kısa bir süre bekleyip " + "belgeyi tekrar talep edin. Alternatif olarak yargi-cli aracını " + "kullanabilirsiniz: https://github.com/saidsurucu/yargi-cli " + "(kullanım talimatları için repo README'sini okuyun). " + f"Retry-After: {retry_after}" + ) + return BedestenDocumentMarkdown( + documentId=documentId, + markdown_content=f"ERROR (rate_limit_exceeded, HTTP 429): {message}", + source_url=f"https://mevzuat.adalet.gov.tr/ictihat/{documentId}", + mime_type=None, + ) except httpx.HTTPStatusError as e: if e.response.status_code == 429: retry_after = e.response.headers.get("Retry-After", "") diff --git a/rekabet_mcp_module/client.py b/rekabet_mcp_module/client.py index 3fc5638..b70e39b 100644 --- a/rekabet_mcp_module/client.py +++ b/rekabet_mcp_module/client.py @@ -1,5 +1,6 @@ # rekabet_mcp_module/client.py +import asyncio import httpx from bs4 import BeautifulSoup from typing import List, Optional, Tuple, Dict, Any @@ -353,7 +354,7 @@ class RekabetKurumuApiClient: total_pdf_pages = total_pdf_pages_from_extraction if single_page_pdf_bytes: - markdown_for_requested_page = self._convert_pdf_bytes_to_markdown(single_page_pdf_bytes, str(pdf_url_to_report or full_landing_page_url)) + markdown_for_requested_page = await asyncio.to_thread(self._convert_pdf_bytes_to_markdown, single_page_pdf_bytes, str(pdf_url_to_report or full_landing_page_url)) if not markdown_for_requested_page: error_message = (error_message or "") + f"; Could not convert page {page_number} of PDF to Markdown." elif total_pdf_pages > 0 : diff --git a/sayistay_mcp_module/client.py b/sayistay_mcp_module/client.py index 5b48e8b..43e63ab 100644 --- a/sayistay_mcp_module/client.py +++ b/sayistay_mcp_module/client.py @@ -1,5 +1,6 @@ # sayistay_mcp_module/client.py +import asyncio import httpx import re from bs4 import BeautifulSoup @@ -657,7 +658,7 @@ class SayistayApiClient: ) # Convert HTML to Markdown using existing method - markdown_content = self._convert_html_to_markdown(html_content) + markdown_content = await asyncio.to_thread(self._convert_html_to_markdown, html_content) if markdown_content and "Error converting HTML content" not in markdown_content: logger.info(f"Successfully retrieved and converted document {decision_id} to Markdown") diff --git a/sigorta_tahkim_mcp_module/client.py b/sigorta_tahkim_mcp_module/client.py index 984139f..d7b1c62 100644 --- a/sigorta_tahkim_mcp_module/client.py +++ b/sigorta_tahkim_mcp_module/client.py @@ -1,5 +1,6 @@ # sigorta_tahkim_mcp_module/client.py +import asyncio import httpx from typing import Optional import logging @@ -202,7 +203,11 @@ class SigortaTahkimApiClient: response.raise_for_status() pdf_stream = io.BytesIO(response.content) - result = self.markitdown.convert_stream(pdf_stream, file_extension=".pdf") + # markitdown is sync; offload to thread so PDF parsing doesn't block + # the event-loop / other in-flight MCP requests. + result = await asyncio.to_thread( + self.markitdown.convert_stream, pdf_stream, file_extension=".pdf" + ) return result.text_content.strip(), pdf_url def _split_into_decisions(self, markdown_content: str) -> list[tuple[str, str]]: diff --git a/uyusmazlik_mcp_module/client.py b/uyusmazlik_mcp_module/client.py index 19022b2..ca2ffd0 100644 --- a/uyusmazlik_mcp_module/client.py +++ b/uyusmazlik_mcp_module/client.py @@ -1,8 +1,9 @@ # uyusmazlik_mcp_module/client.py -import httpx +import asyncio +import httpx from bs4 import BeautifulSoup -from typing import Dict, Any, List, Optional, Union, Tuple +from typing import Dict, Any, List, Optional, Union, Tuple import logging import html import re @@ -232,7 +233,7 @@ class UyusmazlikApiClient: logger.warning(f"UyusmazlikApiClient: Received empty or non-string HTML from URL {document_url}.") return UyusmazlikDocumentMarkdown(source_url=document_url, markdown_content=None) - markdown_content = self._convert_html_to_markdown_uyusmazlik(html_content_from_api) + markdown_content = await asyncio.to_thread(self._convert_html_to_markdown_uyusmazlik, html_content_from_api) return UyusmazlikDocumentMarkdown(source_url=document_url, markdown_content=markdown_content) except httpx.RequestError as e: logger.error(f"UyusmazlikApiClient (httpx for docs): HTTP error fetching Uyuşmazlık document from {document_url}: {e}") diff --git a/yargitay_mcp_module/client.py b/yargitay_mcp_module/client.py index ef0ef7e..545f2d8 100644 --- a/yargitay_mcp_module/client.py +++ b/yargitay_mcp_module/client.py @@ -1,5 +1,6 @@ # yargitay_mcp_module/client.py +import asyncio import httpx from bs4 import BeautifulSoup # Still needed for pre-processing HTML before markitdown from typing import Dict, Any, List, Optional @@ -159,7 +160,7 @@ class YargitayOfficialApiClient: logger.error(f"YargitayOfficialApiClient: 'data' field in API response is not a string or not found (ID: {id}).") raise ValueError("Expected HTML content not found in API response's 'data' field.") - markdown_content = self._convert_html_to_markdown(html_content_from_api) + markdown_content = await asyncio.to_thread(self._convert_html_to_markdown, html_content_from_api) return YargitayDocumentMarkdown( id=id,