diff --git a/.gitignore b/.gitignore index 30ae4ef..fec666d 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,4 @@ hello.py *.html test_kik_client.py +debug_rekabet_arama.py diff --git a/mcp_server_main.py b/mcp_server_main.py index 9942b0f..51e3fdd 100644 --- a/mcp_server_main.py +++ b/mcp_server_main.py @@ -73,6 +73,14 @@ from kik_mcp_module.models import ( KikDocumentMarkdown ) +from rekabet_mcp_module.client import RekabetKurumuApiClient +from rekabet_mcp_module.models import ( + RekabetKurumuSearchRequest, + RekabetSearchResult, + RekabetDocument, + RekabetKararTuruGuidEnum +) + app = FastMCP( name="YargiMCP", @@ -88,6 +96,17 @@ uyusmazlik_client_instance = UyusmazlikApiClient() anayasa_norm_client_instance = AnayasaMahkemesiApiClient() anayasa_bireysel_client_instance = AnayasaBireyselBasvuruApiClient() kik_client_instance = KikApiClient() +rekabet_client_instance = RekabetKurumuApiClient() + + +KARAR_TURU_ADI_TO_GUID_ENUM_MAP = { + "": RekabetKararTuruGuidEnum.TUMU, + "Birleşme ve Devralma": RekabetKararTuruGuidEnum.BIRLESME_DEVRALMA, + "Diğer": RekabetKararTuruGuidEnum.DIGER, + "Menfi Tespit ve Muafiyet": RekabetKararTuruGuidEnum.MENFI_TESPIT_MUAFIYET, + "Özelleştirme": RekabetKararTuruGuidEnum.OZELLESTIRME, + "Rekabet İhlali": RekabetKararTuruGuidEnum.REKABET_IHLALI, +} # --- MCP Tools for Yargitay --- @app.tool() @@ -664,6 +683,78 @@ async def get_kik_document_markdown( total_pages=1, is_paginated=False ) +@app.tool() +async def search_rekabet_kurumu_decisions( + sayfaAdi: Optional[str] = Field(None, description="Search in decision title (Başlık)."), + YayinlanmaTarihi: Optional[str] = Field(None, description="Publication date (Yayım Tarihi), e.g., DD.MM.YYYY."), + PdfText: Optional[str] = Field( + None, + description='Search in decision text (Metin). For an exact phrase match, enclose the phrase in double quotes (e.g., "\\"vertical agreement\\" competition). The website indicates that using "" provides more precise results for phrases.' + ), + KararTuru: Literal[ + "", + "Birleşme ve Devralma", + "Diğer", + "Menfi Tespit ve Muafiyet", + "Özelleştirme", + "Rekabet İhlali" + ] = Field("", description="Decision type (Karar Türü). Leave empty for 'All'. Options: '', 'Birleşme ve Devralma', 'Diğer', 'Menfi Tespit ve Muafiyet', 'Özelleştirme', 'Rekabet İhlali'."), + KararSayisi: Optional[str] = Field(None, description="Decision number (Karar Sayısı)."), + KararTarihi: Optional[str] = Field(None, description="Decision date (Karar Tarihi), e.g., DD.MM.YYYY."), + page: int = Field(1, ge=1, description="Page number to fetch for the results list.") +) -> RekabetSearchResult: + """ + Searches decisions of the Turkish Competition Authority (Rekabet Kurumu). + For an exact phrase search in the 'PdfText' field, enclose the phrase in double quotes. + Example for PdfText: "\\"tender process\\" consultancy" + """ + + karar_turu_guid_enum = KARAR_TURU_ADI_TO_GUID_ENUM_MAP.get(KararTuru) + + try: + if karar_turu_guid_enum is None: + logger.warning(f"Invalid user-provided KararTuru: '{KararTuru}'. Defaulting to TUMU (all).") + karar_turu_guid_enum = RekabetKararTuruGuidEnum.TUMU + except Exception as e_map: + logger.error(f"Error mapping KararTuru '{KararTuru}': {e_map}. Defaulting to TUMU.") + karar_turu_guid_enum = RekabetKararTuruGuidEnum.TUMU + + search_query = RekabetKurumuSearchRequest( + sayfaAdi=sayfaAdi, + YayinlanmaTarihi=YayinlanmaTarihi, + PdfText=PdfText, + KararTuruID=karar_turu_guid_enum, + KararSayisi=KararSayisi, + KararTarihi=KararTarihi, + page=page + ) + logger.info(f"Tool 'search_rekabet_kurumu_decisions' called. Query: {search_query.model_dump_json(exclude_none=True, indent=2)}") + try: + # rekabet_client_instance'ın tanımlı olduğunu varsayıyoruz + return await rekabet_client_instance.search_decisions(search_query) + except Exception as e: + logger.exception("Error in tool 'search_rekabet_kurumu_decisions'.") + return RekabetSearchResult(decisions=[], retrieved_page_number=page, total_records_found=0, total_pages=0) + +@app.tool() +async def get_rekabet_kurumu_document( + karar_id: str = Field(..., description="GUID (kararId) of the Rekabet Kurumu decision. This ID is obtained from search results."), + page_number: Optional[int] = Field(1, ge=1, description="Requested page number for the Markdown content converted from PDF (1-indexed). Default is 1.") +) -> RekabetDocument: + """ + Retrieves information for a specific Turkish Competition Authority (Rekabet Kurumu) decision + (landing page metadata, PDF link) and its PDF content converted to paginated Markdown. + """ + logger.info(f"Tool 'get_rekabet_kurumu_document' called. Karar ID: {karar_id}, Markdown Page: {page_number}") + + current_page_to_fetch = page_number if page_number is not None and page_number >= 1 else 1 + + try: + # rekabet_client_instance'ın tanımlı olduğunu varsayıyoruz + return await rekabet_client_instance.get_decision_document(karar_id, page_number=current_page_to_fetch) + except Exception as e: + logger.exception(f"Error in tool 'get_rekabet_kurumu_document'. Karar ID: {karar_id}") + raise # --- Application Shutdown Handling --- def perform_cleanup(): @@ -683,7 +774,8 @@ def perform_cleanup(): globals().get('uyusmazlik_client_instance'), globals().get('anayasa_norm_client_instance'), globals().get('anayasa_bireysel_client_instance'), - globals().get('kik_client_instance') + globals().get('kik_client_instance'), + globals().get('rekabet_client_instance') ] async def close_all_clients_async(): tasks = [] diff --git a/pyproject.toml b/pyproject.toml index 262128a..f8031f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,11 +7,12 @@ requires-python = ">=3.11" dependencies = [ "beautifulsoup4>=4.13.4", "httpx>=0.28.1", - "markitdown>=0.1.1", + "markitdown[pdf]>=0.1.1", "pydantic>=2.11.4", "aiohttp>=3.11.18", "playwright>=1.52.0", "fastmcp>=2.5.1", + "pypdf>=5.5.0", ] [project.scripts] diff --git a/rekabet_mcp_module/__init__.py b/rekabet_mcp_module/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rekabet_mcp_module/client.py b/rekabet_mcp_module/client.py new file mode 100644 index 0000000..df84b70 --- /dev/null +++ b/rekabet_mcp_module/client.py @@ -0,0 +1,407 @@ +# rekabet_mcp_module/client.py + +import httpx +from bs4 import BeautifulSoup +from typing import List, Optional, Tuple, Dict, Any +import logging +import html +import re +import io # For io.BytesIO +from urllib.parse import urlencode, urljoin, quote, parse_qs, urlparse +from markitdown import MarkItDown +import math + +# pypdf for PDF processing (lighter alternative to PyMuPDF) +from pypdf import PdfReader, PdfWriter # PyPDF2'nin devamı niteliğindeki pypdf + +from .models import ( + RekabetKurumuSearchRequest, + RekabetDecisionSummary, + RekabetSearchResult, + RekabetDocument, + RekabetKararTuruGuidEnum +) +from pydantic import HttpUrl # Ensure HttpUrl is imported from pydantic + +logger = logging.getLogger(__name__) +if not logger.hasHandlers(): # Pragma: no cover + logging.basicConfig( + level=logging.INFO, # Varsayılan log seviyesi + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + # Debug betiğinde daha detaylı loglama için seviye ayrıca ayarlanabilir. + +class RekabetKurumuApiClient: + BASE_URL = "https://www.rekabet.gov.tr" + SEARCH_PATH = "/tr/Kararlar" + DECISION_LANDING_PATH_TEMPLATE = "/Karar" + # PDF sayfa bazlı Markdown döndürüldüğü için bu sabit artık doğrudan kullanılmıyor. + # DOCUMENT_MARKDOWN_CHUNK_SIZE = 5000 + + def __init__(self, request_timeout: float = 60.0): + self.http_client = httpx.AsyncClient( + base_url=self.BASE_URL, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + }, + timeout=request_timeout, + verify=True, + follow_redirects=True + ) + + def _build_search_query_params(self, params: RekabetKurumuSearchRequest) -> List[Tuple[str, str]]: + query_params: List[Tuple[str, str]] = [] + query_params.append(("sayfaAdi", params.sayfaAdi if params.sayfaAdi is not None else "")) + query_params.append(("YayinlanmaTarihi", params.YayinlanmaTarihi if params.YayinlanmaTarihi is not None else "")) + query_params.append(("PdfText", params.PdfText if params.PdfText is not None else "")) + + karar_turu_id_value = "" + if params.KararTuruID is not None: + karar_turu_id_value = params.KararTuruID.value + query_params.append(("KararTuruID", karar_turu_id_value)) + + query_params.append(("KararSayisi", params.KararSayisi if params.KararSayisi is not None else "")) + query_params.append(("KararTarihi", params.KararTarihi if params.KararTarihi is not None else "")) + + if params.page and params.page > 1: + query_params.append(("page", str(params.page))) + + return query_params + + async def search_decisions(self, params: RekabetKurumuSearchRequest) -> RekabetSearchResult: + request_path = self.SEARCH_PATH + final_query_params = self._build_search_query_params(params) + logger.info(f"RekabetKurumuApiClient: Performing search. Path: {request_path}, Parameters: {final_query_params}") + + try: + response = await self.http_client.get(request_path, params=final_query_params) + response.raise_for_status() + html_content = response.text + except httpx.RequestError as e: + logger.error(f"RekabetKurumuApiClient: HTTP request error during search: {e}") + raise + + soup = BeautifulSoup(html_content, 'html.parser') + processed_decisions: List[RekabetDecisionSummary] = [] + total_records: Optional[int] = None + total_pages: Optional[int] = None + + pagination_div = soup.find("div", class_="yazi01") + if pagination_div: + text_content = pagination_div.get_text(separator=" ", strip=True) + total_match = re.search(r"Toplam\s*:\s*(\d+)", text_content) + if total_match: + try: + total_records = int(total_match.group(1)) + logger.debug(f"Total records found from pagination: {total_records}") + except ValueError: + logger.warning(f"Could not convert 'Toplam' value to int: {total_match.group(1)}") + else: + logger.warning("'Toplam :' string not found in pagination section.") + + results_per_page_assumed = 10 + if total_records is not None: + calculated_total_pages = math.ceil(total_records / results_per_page_assumed) + total_pages = calculated_total_pages if calculated_total_pages > 0 else (1 if total_records > 0 else 0) + logger.debug(f"Calculated total pages: {total_pages}") + + if total_pages is None: # Fallback if total_records couldn't be parsed + last_page_link = pagination_div.select_one("li.PagedList-skipToLast a") + if last_page_link and last_page_link.has_attr('href'): + qs = parse_qs(urlparse(last_page_link['href']).query) + if 'page' in qs and qs['page']: + try: + total_pages = int(qs['page'][0]) + logger.debug(f"Total pages found from 'Last >>' link: {total_pages}") + except ValueError: + logger.warning(f"Could not convert page value from 'Last >>' link to int: {qs['page'][0]}") + elif total_records == 0 : total_pages = 0 # If no records, 0 pages + elif total_records is not None and total_records > 0 : total_pages = 1 # If records exist but no last page link (e.g. single page) + else: logger.warning("'Last >>' link not found in pagination section.") + + decision_tables_container = soup.find("div", id="kararList") + if not decision_tables_container: + logger.warning("`div#kararList` (decision list container) not found. HTML structure might have changed or no decisions on this page.") + else: + decision_tables = decision_tables_container.find_all("table", class_="equalDivide") + logger.info(f"Found {len(decision_tables)} 'table' elements with class='equalDivide' for parsing.") + + if not decision_tables and total_records is not None and total_records > 0 : + logger.warning(f"Page indicates {total_records} records but no decision tables found with class='equalDivide'.") + + for idx, table in enumerate(decision_tables): + logger.debug(f"Processing table {idx + 1}...") + try: + rows = table.find_all("tr") + if len(rows) != 3: + logger.warning(f"Table {idx + 1} has an unexpected number of rows ({len(rows)} instead of 3). Skipping. HTML snippet:\n{table.prettify()[:500]}") + continue + + # Row 1: Publication Date, Decision Number, Related Cases Link + td_elements_r1 = rows[0].find_all("td") + pub_date = td_elements_r1[0].get_text(strip=True) if len(td_elements_r1) > 0 else None + dec_num = td_elements_r1[1].get_text(strip=True) if len(td_elements_r1) > 1 else None + + related_cases_link_tag = td_elements_r1[2].find("a", href=True) if len(td_elements_r1) > 2 else None + related_cases_url_str: Optional[str] = None + karar_id_from_related: Optional[str] = None + if related_cases_link_tag and related_cases_link_tag.has_attr('href'): + related_cases_url_str = urljoin(self.BASE_URL, related_cases_link_tag['href']) + qs_related = parse_qs(urlparse(related_cases_link_tag['href']).query) + if 'kararId' in qs_related and qs_related['kararId']: + karar_id_from_related = qs_related['kararId'][0] + + # Row 2: Decision Date, Decision Type + td_elements_r2 = rows[1].find_all("td") + dec_date = td_elements_r2[0].get_text(strip=True) if len(td_elements_r2) > 0 else None + dec_type_text = td_elements_r2[1].get_text(strip=True) if len(td_elements_r2) > 1 else None + + # Row 3: Title and Main Decision Link + title_cell = rows[2].find("td", colspan="5") + decision_link_tag = title_cell.find("a", href=True) if title_cell else None + + title_text: Optional[str] = None + decision_landing_url_str: Optional[str] = None + karar_id_from_main_link: Optional[str] = None + + if decision_link_tag and decision_link_tag.has_attr('href'): + title_text = decision_link_tag.get_text(strip=True) + href_val = decision_link_tag['href'] + if href_val.startswith(self.DECISION_LANDING_PATH_TEMPLATE + "?kararId="): # Ensure it's a decision link + decision_landing_url_str = urljoin(self.BASE_URL, href_val) + qs_main = parse_qs(urlparse(href_val).query) + if 'kararId' in qs_main and qs_main['kararId']: + karar_id_from_main_link = qs_main['kararId'][0] + else: + logger.warning(f"Table {idx+1} decision link has unexpected format: {href_val}") + else: + logger.warning(f"Table {idx+1} could not find title/decision link tag.") + + current_karar_id = karar_id_from_main_link or karar_id_from_related + + if not current_karar_id: + logger.warning(f"Table {idx+1} Karar ID not found. Skipping. Title (if any): {title_text}") + continue + + # Convert string URLs to HttpUrl for the model + final_decision_url = HttpUrl(decision_landing_url_str) if decision_landing_url_str else None + final_related_cases_url = HttpUrl(related_cases_url_str) if related_cases_url_str else None + + processed_decisions.append(RekabetDecisionSummary( + publication_date=pub_date, decision_number=dec_num, decision_date=dec_date, + decision_type_text=dec_type_text, title=title_text, + decision_url=final_decision_url, + karar_id=current_karar_id, + related_cases_url=final_related_cases_url + )) + logger.debug(f"Table {idx+1} parsed successfully: Karar ID '{current_karar_id}', Title '{title_text[:50] if title_text else 'N/A'}...'") + + except Exception as e: + logger.warning(f"RekabetKurumuApiClient: Error parsing decision summary {idx+1}: {e}. Problematic Table HTML:\n{table.prettify()}", exc_info=True) + continue + + return RekabetSearchResult( + decisions=processed_decisions, total_records_found=total_records, + retrieved_page_number=params.page, total_pages=total_pages if total_pages is not None else 0 + ) + + async def _extract_pdf_url_and_landing_page_metadata(self, karar_id: str, landing_page_html: str, landing_page_url: str) -> Dict[str, Any]: + soup = BeautifulSoup(landing_page_html, 'html.parser') + data: Dict[str, Any] = { + "pdf_url": None, + "title_on_landing_page": soup.title.string.strip() if soup.title and soup.title.string else f"Rekabet Kurumu Kararı {karar_id}", + } + # This part needs to be robust and specific to Rekabet Kurumu's landing page structure. + # Look for common patterns: direct links, download buttons, embedded viewers. + pdf_anchor = soup.find("a", href=re.compile(r"\.pdf(\?|$)", re.IGNORECASE)) # Basic PDF link + if not pdf_anchor: # Try other common patterns if the basic one fails + # Example: Look for links with specific text or class + pdf_anchor = soup.find("a", string=re.compile(r"karar metni|pdf indir", re.IGNORECASE)) + + if pdf_anchor and pdf_anchor.has_attr('href'): + pdf_path = pdf_anchor['href'] + data["pdf_url"] = urljoin(landing_page_url, pdf_path) + logger.info(f"PDF link found on landing page (): {data['pdf_url']}") + else: + iframe_pdf = soup.find("iframe", src=re.compile(r"\.pdf(\?|$)", re.IGNORECASE)) + if iframe_pdf and iframe_pdf.has_attr('src'): + pdf_path = iframe_pdf['src'] + data["pdf_url"] = urljoin(landing_page_url, pdf_path) + logger.info(f"PDF link found on landing page (