From c3bc9e17eb42a0e154c4e49ea7c6624e40e123dc Mon Sep 17 00:00:00 2001 From: saidsurucu Date: Sat, 19 Jul 2025 17:11:33 +0300 Subject: [PATCH] add bddk module --- bddk_mcp_module/__init__.py | 17 +++ bddk_mcp_module/client.py | 247 ++++++++++++++++++++++++++++++++++++ bddk_mcp_module/models.py | 43 +++++++ mcp_server_main.py | 110 +++++++++++++++- 4 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 bddk_mcp_module/__init__.py create mode 100644 bddk_mcp_module/client.py create mode 100644 bddk_mcp_module/models.py diff --git a/bddk_mcp_module/__init__.py b/bddk_mcp_module/__init__.py new file mode 100644 index 0000000..7a2195d --- /dev/null +++ b/bddk_mcp_module/__init__.py @@ -0,0 +1,17 @@ +# bddk_mcp_module/__init__.py + +from .client import BddkApiClient +from .models import ( + BddkSearchRequest, + BddkDecisionSummary, + BddkSearchResult, + BddkDocumentMarkdown +) + +__all__ = [ + "BddkApiClient", + "BddkSearchRequest", + "BddkDecisionSummary", + "BddkSearchResult", + "BddkDocumentMarkdown" +] \ No newline at end of file diff --git a/bddk_mcp_module/client.py b/bddk_mcp_module/client.py new file mode 100644 index 0000000..158e082 --- /dev/null +++ b/bddk_mcp_module/client.py @@ -0,0 +1,247 @@ +# bddk_mcp_module/client.py + +import httpx +from typing import List, Optional, Dict, Any +import logging +import os +import re +import io +import math +from urllib.parse import urlparse +from markitdown import MarkItDown + +from .models import ( + BddkSearchRequest, + BddkDecisionSummary, + BddkSearchResult, + BddkDocumentMarkdown +) + +logger = logging.getLogger(__name__) +if not logger.hasHandlers(): + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + +class BddkApiClient: + """ + API client for searching and retrieving BDDK (Banking Regulation Authority) decisions + using Tavily Search API for discovery and direct HTTP requests for content retrieval. + """ + + TAVILY_API_URL = "https://api.tavily.com/search" + BDDK_BASE_URL = "https://www.bddk.org.tr" + DOCUMENT_URL_TEMPLATE = "https://www.bddk.org.tr/Mevzuat/DokumanGetir/{document_id}" + DOCUMENT_MARKDOWN_CHUNK_SIZE = 5000 # Character limit per page + + def __init__(self, request_timeout: float = 60.0): + """Initialize the BDDK API client.""" + self.tavily_api_key = os.getenv("TAVILY_API_KEY") + if not self.tavily_api_key: + # Fallback to development token + self.tavily_api_key = "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" + logger.info("Using fallback Tavily API token (development token)") + else: + logger.info("Using Tavily API key from environment variable") + + self.http_client = httpx.AsyncClient( + headers={ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" + }, + timeout=httpx.Timeout(request_timeout) + ) + self.markitdown = MarkItDown() + + async def close_client_session(self): + """Close the HTTP client session.""" + await self.http_client.aclose() + logger.info("BddkApiClient: HTTP client session closed.") + + def _extract_document_id(self, url: str) -> Optional[str]: + """Extract document ID from BDDK URL.""" + # Primary pattern: https://www.bddk.org.tr/Mevzuat/DokumanGetir/310 + match = re.search(r'/DokumanGetir/(\d+)', url) + if match: + return match.group(1) + + # Alternative patterns for different BDDK URL formats + # Pattern: /Liste/55 -> use as document ID + match = re.search(r'/Liste/(\d+)', url) + if match: + return match.group(1) + + # Pattern: /EkGetir/13?ekId=381 -> use ekId as document ID + match = re.search(r'ekId=(\d+)', url) + if match: + return match.group(1) + + return None + + async def search_decisions( + self, + request: BddkSearchRequest + ) -> BddkSearchResult: + """ + Search for BDDK decisions using Tavily API. + + Args: + request: Search request parameters + + Returns: + BddkSearchResult with matching decisions + """ + try: + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.tavily_api_key}" + } + + # Tavily API request - enhanced for BDDK decision documents + query = f"{request.keywords} \"Karar Sayısı\"" + payload = { + "query": query, + "country": "turkey", + "include_domains": ["https://www.bddk.org.tr/Mevzuat/DokumanGetir"], + "max_results": request.pageSize, + "search_depth": "advanced" + } + + # Calculate offset for pagination + if request.page > 1: + # Tavily doesn't have direct pagination, so we'll need to handle this + # For now, we'll just return empty for pages > 1 + logger.warning(f"Tavily API doesn't support pagination. Page {request.page} requested.") + + response = await self.http_client.post( + self.TAVILY_API_URL, + json=payload, + headers=headers + ) + response.raise_for_status() + + data = response.json() + + # Log raw Tavily response for debugging + logger.info(f"Tavily returned {len(data.get('results', []))} results") + + # Convert Tavily results to our format + decisions = [] + for result in data.get("results", []): + # Extract document ID from URL + url = result.get("url", "") + logger.debug(f"Processing URL: {url}") + doc_id = self._extract_document_id(url) + if doc_id: + decision = BddkDecisionSummary( + title=result.get("title", "").replace("[PDF] ", "").strip(), + document_id=doc_id, + content=result.get("content", "")[:500] # Limit content length + ) + decisions.append(decision) + logger.debug(f"Added decision: {decision.title} (ID: {doc_id})") + else: + logger.warning(f"Could not extract document ID from URL: {url}") + + return BddkSearchResult( + decisions=decisions, + total_results=len(data.get("results", [])), + page=request.page, + pageSize=request.pageSize + ) + + except httpx.HTTPStatusError as e: + logger.error(f"HTTP error searching BDDK decisions: {e}") + if e.response.status_code == 401: + raise Exception("Tavily API authentication failed. Check API key.") + raise Exception(f"Failed to search BDDK decisions: {str(e)}") + except Exception as e: + logger.error(f"Error searching BDDK decisions: {e}") + raise Exception(f"Failed to search BDDK decisions: {str(e)}") + + async def get_document_markdown( + self, + document_id: str, + page_number: int = 1 + ) -> BddkDocumentMarkdown: + """ + Retrieve a BDDK document and convert it to Markdown format. + + Args: + document_id: BDDK document ID (e.g., '310') + page_number: Page number for paginated content (1-indexed) + + Returns: + BddkDocumentMarkdown with paginated content + """ + try: + # Try different URL patterns for BDDK documents + potential_urls = [ + f"https://www.bddk.org.tr/Mevzuat/DokumanGetir/{document_id}", + f"https://www.bddk.org.tr/Mevzuat/Liste/{document_id}", + f"https://www.bddk.org.tr/KurumHakkinda/EkGetir/13?ekId={document_id}", + f"https://www.bddk.org.tr/KurumHakkinda/EkGetir/5?ekId={document_id}" + ] + + document_url = None + response = None + + # Try each URL pattern until one works + for url in potential_urls: + try: + logger.info(f"Trying BDDK document URL: {url}") + response = await self.http_client.get( + url, + follow_redirects=True + ) + response.raise_for_status() + document_url = url + break + except httpx.HTTPStatusError: + continue + + if not response or not document_url: + raise Exception(f"Could not find document with ID {document_id}") + + logger.info(f"Successfully fetched BDDK document from: {document_url}") + + # Determine content type + content_type = response.headers.get("content-type", "").lower() + + # Convert to Markdown based on content type + if "pdf" in content_type: + # Handle PDF documents + pdf_stream = io.BytesIO(response.content) + result = self.markitdown.convert_stream(pdf_stream, file_extension=".pdf") + markdown_content = result.text_content + else: + # Handle HTML documents + html_stream = io.BytesIO(response.content) + result = self.markitdown.convert_stream(html_stream, file_extension=".html") + markdown_content = result.text_content + + # Clean up the markdown content + markdown_content = markdown_content.strip() + + # Calculate pagination + total_length = len(markdown_content) + total_pages = math.ceil(total_length / self.DOCUMENT_MARKDOWN_CHUNK_SIZE) + + # Extract the requested page + start_idx = (page_number - 1) * self.DOCUMENT_MARKDOWN_CHUNK_SIZE + end_idx = start_idx + self.DOCUMENT_MARKDOWN_CHUNK_SIZE + page_content = markdown_content[start_idx:end_idx] + + return BddkDocumentMarkdown( + document_id=document_id, + markdown_content=page_content, + page_number=page_number, + total_pages=total_pages + ) + + except httpx.HTTPStatusError as e: + logger.error(f"HTTP error fetching BDDK document {document_id}: {e}") + raise Exception(f"Failed to fetch BDDK document: {str(e)}") + except Exception as e: + logger.error(f"Error processing BDDK document {document_id}: {e}") + raise Exception(f"Failed to process BDDK document: {str(e)}") \ No newline at end of file diff --git a/bddk_mcp_module/models.py b/bddk_mcp_module/models.py new file mode 100644 index 0000000..c945f26 --- /dev/null +++ b/bddk_mcp_module/models.py @@ -0,0 +1,43 @@ +# bddk_mcp_module/models.py + +from pydantic import BaseModel, Field +from typing import List, Optional + +class BddkSearchRequest(BaseModel): + """ + Request model for searching BDDK decisions via Tavily API. + + BDDK (Bankacılık Düzenleme ve Denetleme Kurumu) is Turkey's Banking + Regulation and Supervision Agency responsible for banking licenses, + electronic money institutions, and financial regulations. + """ + keywords: str = Field(..., description="Search keywords in Turkish") + page: int = Field(1, ge=1, description="Page number (1-indexed)") + pageSize: int = Field(10, ge=1, le=50, description="Results per page (1-50)") + +class BddkDecisionSummary(BaseModel): + """Summary of a BDDK decision from search results.""" + title: str = Field(..., description="Decision title") + document_id: str = Field(..., description="BDDK document ID (e.g., '310')") + content: str = Field(..., description="Decision summary/excerpt") + +class BddkSearchResult(BaseModel): + """Response model for BDDK decision search results.""" + decisions: List[BddkDecisionSummary] = Field( + default_factory=list, + description="List of matching BDDK decisions" + ) + total_results: int = Field(0, description="Total number of results") + page: int = Field(1, description="Current page number") + pageSize: int = Field(10, description="Results per page") + +class BddkDocumentMarkdown(BaseModel): + """ + BDDK decision document converted to Markdown format. + + Supports paginated content for long documents (5000 chars per page). + """ + document_id: str = Field(..., description="BDDK document ID") + markdown_content: str = Field("", description="Document content in Markdown") + page_number: int = Field(1, description="Current page number") + total_pages: int = Field(1, description="Total number of pages") \ No newline at end of file diff --git a/mcp_server_main.py b/mcp_server_main.py index 15ee90d..a2dffa5 100644 --- a/mcp_server_main.py +++ b/mcp_server_main.py @@ -313,6 +313,14 @@ from kvkk_mcp_module.models import ( KvkkDocumentMarkdown ) +# BDDK Module Imports +from bddk_mcp_module.client import BddkApiClient +from bddk_mcp_module.models import ( + BddkSearchRequest, + BddkSearchResult, + BddkDocumentMarkdown +) + app = create_app() @@ -1059,6 +1067,7 @@ bedesten_client_instance = BedestenApiClient() sayistay_client_instance = SayistayApiClient() sayistay_unified_client_instance = SayistayUnifiedClient() kvkk_client_instance = KvkkApiClient() +bddk_client_instance = BddkApiClient() KARAR_TURU_ADI_TO_GUID_ENUM_MAP = { @@ -2091,7 +2100,8 @@ def perform_cleanup(): globals().get('bedesten_client_instance'), globals().get('sayistay_client_instance'), globals().get('sayistay_unified_client_instance'), - globals().get('kvkk_client_instance') + globals().get('kvkk_client_instance'), + globals().get('bddk_client_instance') ] async def close_all_clients_async(): tasks = [] @@ -2378,6 +2388,104 @@ async def get_kvkk_document_markdown( error_message=f"Error retrieving KVKK document: {str(e)}" ) +# --- MCP Tools for BDDK (Banking Regulation Authority) --- +@app.tool( + description="Search BDDK banking regulation decisions", + annotations={ + "readOnlyHint": True, + "openWorldHint": True, + "idempotentHint": True + } +) +async def search_bddk_decisions( + keywords: str = Field(..., description="Search keywords in Turkish"), + page: int = Field(1, ge=1, description="Page number") + # pageSize: int = Field(10, ge=1, le=50, description="Results per page") +) -> dict: + """Search BDDK banking regulation and supervision decisions.""" + logger.info(f"BDDK search tool called with keywords: {keywords}, page: {page}") + + pageSize = 10 # Default value + + try: + search_request = BddkSearchRequest( + keywords=keywords, + page=page, + pageSize=pageSize + ) + + result = await bddk_client_instance.search_decisions(search_request) + logger.info(f"BDDK search completed. Found {len(result.decisions)} decisions on page {page}") + + return { + "decisions": [ + { + "title": dec.title, + "document_id": dec.document_id, + "content": dec.content + } + for dec in result.decisions + ], + "total_results": result.total_results, + "page": result.page, + "pageSize": result.pageSize + } + + except Exception as e: + logger.exception(f"Error searching BDDK decisions: {e}") + return { + "decisions": [], + "total_results": 0, + "page": page, + "pageSize": pageSize, + "error": str(e) + } + +@app.tool( + description="Get BDDK decision document as Markdown", + annotations={ + "readOnlyHint": True, + "openWorldHint": False, + "idempotentHint": True + } +) +async def get_bddk_document_markdown( + document_id: str = Field(..., description="BDDK document ID (e.g., '310')"), + page_number: int = Field(1, ge=1, description="Page number") +) -> dict: + """Retrieve BDDK decision document in Markdown format.""" + logger.info(f"BDDK document retrieval tool called for ID: {document_id}, page: {page_number}") + + if not document_id or not document_id.strip(): + return { + "document_id": document_id, + "markdown_content": "", + "page_number": page_number, + "total_pages": 0, + "error": "Document ID is required" + } + + try: + result = await bddk_client_instance.get_document_markdown(document_id, page_number) + logger.info(f"BDDK document retrieved successfully. Page {result.page_number}/{result.total_pages}") + + return { + "document_id": result.document_id, + "markdown_content": result.markdown_content, + "page_number": result.page_number, + "total_pages": result.total_pages + } + + except Exception as e: + logger.exception(f"Error retrieving BDDK document: {e}") + return { + "document_id": document_id, + "markdown_content": "", + "page_number": page_number, + "total_pages": 0, + "error": str(e) + } + # --- ChatGPT Deep Research Compatible Tools --- def get_preview_text(markdown_content: str, skip_chars: int = 100, preview_chars: int = 200) -> str: