add kvkk module, several bug fix

This commit is contained in:
saidsurucu
2025-07-11 23:50:56 +03:00
parent 83f54a86a8
commit cb318faeba
14 changed files with 738 additions and 147 deletions
+11 -12
View File
@@ -7,8 +7,7 @@ from typing import Dict, Any, List, Optional, Tuple
import logging
import html
import re
import tempfile
import os
import io
from urllib.parse import urlencode, urljoin, quote
from markitdown import MarkItDown
import math # For math.ceil for pagination
@@ -230,23 +229,23 @@ class AnayasaBireyselBasvuruApiClient:
html_input_for_markdown = processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
# Ensure the content is wrapped in basic HTML structure if it's not already
if not html_input_for_markdown.strip().lower().startswith(("<html", "<!doctype")):
tmp_file.write(f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>")
html_content = f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>"
else:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
html_content = html_input_for_markdown
conversion_result = md_converter.convert(temp_file_path)
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
except Exception as e:
logger.error(f"AnayasaBireyselBasvuruApiClient: MarkItDown conversion error: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(
+19 -22
View File
@@ -7,8 +7,7 @@ from typing import Dict, Any, List, Optional, Tuple
import logging
import html
import re
import tempfile
import os
import io
from urllib.parse import urlencode, urljoin, quote
from markitdown import MarkItDown
import math # For math.ceil for pagination
@@ -82,6 +81,13 @@ class AnayasaMahkemesiApiClient:
if params.has_dissenting_opinion and params.has_dissenting_opinion.value and params.has_dissenting_opinion.value != "ALL": query_params.append(("KarsiOy", params.has_dissenting_opinion.value))
if params.has_different_reasoning and params.has_different_reasoning.value and params.has_different_reasoning.value != "ALL": query_params.append(("FarkliGerekce", params.has_different_reasoning.value))
# Add pagination and sorting parameters as query params instead of URL path
if params.results_per_page and params.results_per_page != 10:
query_params.append(("SatirSayisi", str(params.results_per_page)))
if params.sort_by_criteria and params.sort_by_criteria != "KararTarihi":
query_params.append(("Siralama", params.sort_by_criteria))
if params.page_to_fetch and params.page_to_fetch > 1:
query_params.append(("page", str(params.page_to_fetch)))
return query_params
@@ -90,16 +96,8 @@ class AnayasaMahkemesiApiClient:
self,
params: AnayasaNormDenetimiSearchRequest
) -> AnayasaSearchResult:
path_segments = []
if params.results_per_page and params.results_per_page != 10: # Default is 10
path_segments.append(f"SatirSayisi/{params.results_per_page}")
if params.sort_by_criteria and params.sort_by_criteria != "KararTarihi": # Default is KararTarihi
# Ensure correct quoting for criteria that might have Turkish chars or spaces
path_segments.append(f"Siralama/{quote(params.sort_by_criteria)}")
path_segments.append(self.SEARCH_PATH_SEGMENT)
request_path = "/" + "/".join(path_segments)
# Use simple /Ara endpoint - the complex path structure seems to cause 404s
request_path = f"/{self.SEARCH_PATH_SEGMENT}"
final_query_params = self._build_search_query_params_for_aym(params)
logger.info(f"AnayasaMahkemesiApiClient: Performing Norm Denetimi search. Path: {request_path}, Params: {final_query_params}")
@@ -222,24 +220,23 @@ class AnayasaMahkemesiApiClient:
html_input_for_markdown = str(body_tag) if body_tag else processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
# Ensure the content is wrapped in basic HTML structure if it's not already
if not html_input_for_markdown.strip().lower().startswith(("<html", "<!doctype")):
tmp_file.write(f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>")
html_content = f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>"
else:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
html_content = html_input_for_markdown
conversion_result = md_converter.convert(temp_file_path)
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
except Exception as e:
logger.error(f"AnayasaMahkemesiApiClient: MarkItDown conversion error: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(
+12 -27
View File
@@ -5,8 +5,7 @@ import base64
from typing import Optional
import logging
from markitdown import MarkItDown
import tempfile
import os
import io
from .models import (
BedestenSearchRequest, BedestenSearchResponse,
@@ -125,17 +124,14 @@ class BedestenApiClient:
if not html_content:
return None
temp_file_path = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
# Write HTML to temp file
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp:
tmp.write(html_content)
temp_file_path = tmp.name
# Convert
result = md_converter.convert(temp_file_path)
result = md_converter.convert(html_stream)
markdown_content = result.text_content
logger.info("Successfully converted HTML to Markdown")
@@ -144,27 +140,19 @@ class BedestenApiClient:
except Exception as e:
logger.error(f"Error converting HTML to Markdown: {e}")
return f"Error converting HTML content: {str(e)}"
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
def _convert_pdf_to_markdown(self, pdf_bytes: bytes) -> Optional[str]:
"""Convert PDF to Markdown using MarkItDown"""
if not pdf_bytes:
return None
temp_file_path = None
try:
# MarkItDown supports PDF with markitdown[pdf]
# Create BytesIO stream from PDF bytes
pdf_stream = io.BytesIO(pdf_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
# Write PDF to temp file
with tempfile.NamedTemporaryFile(mode="wb", delete=False, suffix=".pdf") as tmp:
tmp.write(pdf_bytes)
temp_file_path = tmp.name
# Convert
result = md_converter.convert(temp_file_path)
result = md_converter.convert(pdf_stream)
markdown_content = result.text_content
logger.info("Successfully converted PDF to Markdown")
@@ -173,9 +161,6 @@ class BedestenApiClient:
except Exception as e:
logger.error(f"Error converting PDF to Markdown: {e}")
return f"Error converting PDF content: {str(e)}. The document may be corrupted or in an unsupported format."
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
async def close_client_session(self):
"""Close HTTP client session"""
+10 -14
View File
@@ -6,8 +6,7 @@ from typing import Dict, Any, List, Optional
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from .models import (
@@ -124,31 +123,28 @@ class DanistayApiClient:
html_input_for_markdown = processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown() # Basic conversion
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_input_for_markdown.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
tmp_file.write(html_input_for_markdown) # Write the full HTML string
temp_file_path = tmp_file.name
conversion_result = md_converter.convert(temp_file_path)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
logger.info("DanistayApiClient: HTML to Markdown conversion successful.")
except Exception as e:
logger.error(f"DanistayApiClient: Error during MarkItDown HTML to Markdown conversion: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(self, id: str) -> DanistayDocumentMarkdown:
"""
Retrieves a specific Danıştay decision by ID and returns its content as Markdown.
The /getDokuman endpoint for Danıştay returns direct HTML.
The /getDokuman endpoint for Danıştay requires arananKelime parameter.
"""
document_api_url = f"{self.DOCUMENT_ENDPOINT}?id={id}"
# Add required arananKelime parameter - using empty string as minimum requirement
document_api_url = f"{self.DOCUMENT_ENDPOINT}?id={id}&arananKelime="
source_url = f"{self.BASE_URL}{document_api_url}"
logger.info(f"DanistayApiClient: Fetching Danistay document for Markdown (ID: {id}) from {source_url}")
+1 -1
View File
@@ -97,7 +97,7 @@ class DanistayApiResponseInnerData(BaseModel):
class DanistayApiResponse(BaseModel):
"""Model for the complete search response from the Danistay API."""
data: DanistayApiResponseInnerData
data: Optional[DanistayApiResponseInnerData] = Field(None, description="Response data, can be null when no results found")
metadata: Optional[Dict[str, Any]] = Field(None, description="Optional metadata (Meta Veri) from API.")
class DanistayDocumentMarkdown(BaseModel):
+7 -12
View File
@@ -6,8 +6,7 @@ from typing import Dict, Any, List, Optional
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from .models import (
@@ -114,22 +113,18 @@ class EmsalApiClient:
html_input_for_markdown = content
markdown_text = None
temp_file_path = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_input_for_markdown.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
conversion_result = md_converter.convert(temp_file_path)
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
logger.info("EmsalApiClient: HTML to Markdown conversion successful.")
except Exception as e:
logger.error(f"EmsalApiClient: Error during MarkItDown HTML to Markdown conversion for Emsal: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
+9 -10
View File
@@ -18,7 +18,7 @@ import html as html_parser
from markitdown import MarkItDown
import os
import math
import tempfile
import io
from .models import (
KikSearchRequest,
@@ -207,9 +207,7 @@ class KikApiClient:
try:
async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout):
if action_is_search_button_click:
await page.locator(search_button_selector).click()
else:
# Use __doPostBack for both search and pagination for consistency
await page.evaluate(f"javascript:__doPostBack('{event_target_for_submit}','')")
except PlaywrightTimeoutError:
await page.wait_for_timeout(2000)
@@ -251,16 +249,17 @@ class KikApiClient:
# ... (öncekiyle aynı) ...
if not html_fragment: return None
cleaned_html = self._clean_html_for_markdown(html_fragment)
markdown_output = None; temp_file_path = None
markdown_output = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = cleaned_html.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown(enable_plugins=True, remove_alt_whitespace=True, keep_underline=True)
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_html_file:
tmp_html_file.write(cleaned_html); temp_file_path = tmp_html_file.name
markdown_output = md_converter.convert(temp_file_path).text_content
markdown_output = md_converter.convert(html_stream).text_content
if markdown_output: markdown_output = re.sub(r'\n{3,}', '\n\n', markdown_output).strip()
except Exception as e: logger.error(f"MarkItDown conversion error: {e}", exc_info=True)
finally:
if temp_file_path and os.path.exists(temp_file_path): os.remove(temp_file_path)
return markdown_output
+1
View File
@@ -0,0 +1 @@
# kvkk_mcp_module/__init__.py
+372
View File
@@ -0,0 +1,372 @@
# kvkk_mcp_module/client.py
import httpx
from bs4 import BeautifulSoup
from typing import List, Optional, Dict, Any
import logging
import os
import re
import io
import math
from urllib.parse import urljoin, urlparse, parse_qs
from markitdown import MarkItDown
from pydantic import HttpUrl
from .models import (
KvkkSearchRequest,
KvkkDecisionSummary,
KvkkSearchResult,
KvkkDocumentMarkdown
)
logger = logging.getLogger(__name__)
if not logger.hasHandlers():
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
class KvkkApiClient:
"""
API client for searching and retrieving KVKK (Personal Data Protection Authority) decisions
using Brave Search API for discovery and direct HTTP requests for content retrieval.
"""
BRAVE_API_URL = "https://api.search.brave.com/res/v1/web/search"
KVKK_BASE_URL = "https://www.kvkk.gov.tr"
DOCUMENT_MARKDOWN_CHUNK_SIZE = 5000 # Character limit per page
def __init__(self, request_timeout: float = 60.0):
"""Initialize the KVKK API client."""
self.brave_api_token = os.getenv("BRAVE_API_TOKEN")
if not self.brave_api_token:
# Fallback to provided free token
self.brave_api_token = "BSAuaRKB-dvSDSQxIN0ft1p2k6N82Kq"
logger.info("Using fallback Brave API token (limited free token)")
else:
logger.info("Using Brave API token from environment variable")
self.http_client = httpx.AsyncClient(
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
},
timeout=request_timeout,
verify=True,
follow_redirects=True
)
def _construct_search_query(self, keywords: str) -> str:
"""Construct the search query for Brave API."""
base_query = 'site:kvkk.gov.tr "karar özeti"'
if keywords.strip():
return f"{base_query} {keywords.strip()}"
return base_query
def _extract_decision_id_from_url(self, url: str) -> Optional[str]:
"""Extract decision ID from KVKK decision URL."""
try:
# Example URL: https://www.kvkk.gov.tr/Icerik/7288/2021-1303
parsed_url = urlparse(url)
path_parts = parsed_url.path.strip('/').split('/')
if len(path_parts) >= 3 and path_parts[0] == 'Icerik':
# Extract the decision ID from the path
decision_id = '/'.join(path_parts[1:]) # e.g., "7288/2021-1303"
return decision_id
except Exception as e:
logger.debug(f"Could not extract decision ID from URL {url}: {e}")
return None
def _extract_decision_metadata_from_title(self, title: str) -> Dict[str, Optional[str]]:
"""Extract decision metadata from title string."""
metadata = {
"decision_date": None,
"decision_number": None
}
if not title:
return metadata
# Extract decision date (DD/MM/YYYY format)
date_match = re.search(r'(\d{1,2}/\d{1,2}/\d{4})', title)
if date_match:
metadata["decision_date"] = date_match.group(1)
# Extract decision number (YYYY/XXXX format)
number_match = re.search(r'(\d{4}/\d+)', title)
if number_match:
metadata["decision_number"] = number_match.group(1)
return metadata
async def search_decisions(self, params: KvkkSearchRequest) -> KvkkSearchResult:
"""Search for KVKK decisions using Brave API."""
search_query = self._construct_search_query(params.keywords)
logger.info(f"KvkkApiClient: Searching with query: {search_query}")
try:
# Calculate offset for pagination
offset = (params.page - 1) * params.pageSize
response = await self.http_client.get(
self.BRAVE_API_URL,
headers={
"Accept": "application/json",
"Accept-Encoding": "gzip",
"x-subscription-token": self.brave_api_token
},
params={
"q": search_query,
"country": "TR",
"search_lang": "tr",
"ui_lang": "tr-TR",
"offset": offset,
"count": params.pageSize
}
)
response.raise_for_status()
data = response.json()
# Extract search results
decisions = []
web_results = data.get("web", {}).get("results", [])
for result in web_results:
title = result.get("title", "")
url = result.get("url", "")
description = result.get("description", "")
# Extract metadata from title
metadata = self._extract_decision_metadata_from_title(title)
# Extract decision ID from URL
decision_id = self._extract_decision_id_from_url(url)
decision = KvkkDecisionSummary(
title=title,
url=HttpUrl(url) if url else None,
description=description,
decision_id=decision_id,
publication_date=metadata.get("decision_date"),
decision_number=metadata.get("decision_number")
)
decisions.append(decision)
# Get total results if available
total_results = None
query_info = data.get("query", {})
if "total_results" in query_info:
total_results = query_info["total_results"]
return KvkkSearchResult(
decisions=decisions,
total_results=total_results,
page=params.page,
pageSize=params.pageSize,
query=search_query
)
except httpx.RequestError as e:
logger.error(f"KvkkApiClient: HTTP request error during search: {e}")
return KvkkSearchResult(
decisions=[],
total_results=0,
page=params.page,
pageSize=params.pageSize,
query=search_query
)
except Exception as e:
logger.error(f"KvkkApiClient: Unexpected error during search: {e}")
return KvkkSearchResult(
decisions=[],
total_results=0,
page=params.page,
pageSize=params.pageSize,
query=search_query
)
def _extract_decision_content_from_html(self, html: str, url: str) -> Dict[str, Any]:
"""Extract decision content from KVKK decision page HTML."""
try:
soup = BeautifulSoup(html, 'html.parser')
# Extract title
title = None
title_element = soup.find('h3', class_='blog-post-title')
if title_element:
title = title_element.get_text(strip=True)
elif soup.title:
title = soup.title.get_text(strip=True)
# Extract decision content from the main content div
content_div = soup.find('div', class_='blog-post-inner')
if not content_div:
# Fallback to other possible content containers
content_div = soup.find('div', style='text-align:justify;')
if not content_div:
logger.warning(f"Could not find decision content div in {url}")
return {
"title": title,
"decision_date": None,
"decision_number": None,
"subject_summary": None,
"html_content": None
}
# Extract decision metadata from table
decision_date = None
decision_number = None
subject_summary = None
table = content_div.find('table')
if table:
rows = table.find_all('tr')
for row in rows:
cells = row.find_all('td')
if len(cells) >= 3:
field_name = cells[0].get_text(strip=True)
field_value = cells[2].get_text(strip=True)
if 'Karar Tarihi' in field_name:
decision_date = field_value
elif 'Karar No' in field_name:
decision_number = field_value
elif 'Konu Özeti' in field_name:
subject_summary = field_value
return {
"title": title,
"decision_date": decision_date,
"decision_number": decision_number,
"subject_summary": subject_summary,
"html_content": str(content_div)
}
except Exception as e:
logger.error(f"Error extracting content from HTML for {url}: {e}")
return {
"title": None,
"decision_date": None,
"decision_number": None,
"subject_summary": None,
"html_content": None
}
def _convert_html_to_markdown(self, html_content: str) -> Optional[str]:
"""Convert HTML content to Markdown using MarkItDown with BytesIO to avoid filename length issues."""
if not html_content:
return None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown(enable_plugins=False)
result = md_converter.convert(html_stream)
return result.text_content
except Exception as e:
logger.error(f"Error converting HTML to Markdown: {e}")
return None
async def get_decision_document(self, decision_url: str, page_number: int = 1) -> KvkkDocumentMarkdown:
"""Retrieve and convert a KVKK decision document to paginated Markdown."""
logger.info(f"KvkkApiClient: Getting decision document from: {decision_url}, page: {page_number}")
try:
# Fetch the decision page
response = await self.http_client.get(decision_url)
response.raise_for_status()
# Extract content from HTML
extracted_data = self._extract_decision_content_from_html(response.text, decision_url)
# Convert HTML content to Markdown
full_markdown_content = None
if extracted_data["html_content"]:
full_markdown_content = self._convert_html_to_markdown(extracted_data["html_content"])
if not full_markdown_content:
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=extracted_data["title"],
decision_date=extracted_data["decision_date"],
decision_number=extracted_data["decision_number"],
subject_summary=extracted_data["subject_summary"],
markdown_chunk=None,
current_page=page_number,
total_pages=0,
is_paginated=False,
error_message="Could not convert document content to Markdown"
)
# Calculate pagination
content_length = len(full_markdown_content)
total_pages = math.ceil(content_length / self.DOCUMENT_MARKDOWN_CHUNK_SIZE)
if total_pages == 0:
total_pages = 1
# Clamp page number to valid range
current_page_clamped = max(1, min(page_number, total_pages))
# Extract the requested chunk
start_index = (current_page_clamped - 1) * self.DOCUMENT_MARKDOWN_CHUNK_SIZE
end_index = start_index + self.DOCUMENT_MARKDOWN_CHUNK_SIZE
markdown_chunk = full_markdown_content[start_index:end_index]
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=extracted_data["title"],
decision_date=extracted_data["decision_date"],
decision_number=extracted_data["decision_number"],
subject_summary=extracted_data["subject_summary"],
markdown_chunk=markdown_chunk,
current_page=current_page_clamped,
total_pages=total_pages,
is_paginated=(total_pages > 1),
error_message=None
)
except httpx.HTTPStatusError as e:
error_msg = f"HTTP error {e.response.status_code} when fetching decision document"
logger.error(f"KvkkApiClient: {error_msg}")
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number,
total_pages=0,
is_paginated=False,
error_message=error_msg
)
except Exception as e:
error_msg = f"Unexpected error when fetching decision document: {str(e)}"
logger.error(f"KvkkApiClient: {error_msg}")
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number,
total_pages=0,
is_paginated=False,
error_message=error_msg
)
async def close_client_session(self):
"""Close the HTTP client session."""
if hasattr(self, 'http_client') and self.http_client and not self.http_client.is_closed:
await self.http_client.aclose()
logger.info("KvkkApiClient: HTTP client session closed.")
+49
View File
@@ -0,0 +1,49 @@
# kvkk_mcp_module/models.py
from pydantic import BaseModel, Field, HttpUrl
from typing import List, Optional, Any
class KvkkSearchRequest(BaseModel):
"""Model for KVKK (Personal Data Protection Authority) search request via Brave API."""
keywords: str = Field(..., description="""
Keywords to search for in KVKK decisions.
The search will automatically include 'site:kvkk.gov.tr "karar özeti"' to target KVKK decision summaries.
Examples: "açık rıza", "veri güvenliği", "kişisel veri işleme"
""")
page: int = Field(1, ge=1, le=50, description="Page number for search results (1-50).")
pageSize: int = Field(10, ge=1, le=20, description="Number of results per page (1-20).")
class KvkkDecisionSummary(BaseModel):
"""Model for a single KVKK decision summary from Brave search results."""
title: Optional[str] = Field(None, description="Decision title from search results.")
url: Optional[HttpUrl] = Field(None, description="URL to the KVKK decision page.")
description: Optional[str] = Field(None, description="Brief description or snippet from search results.")
decision_id: Optional[str] = Field(None, description="Extracted decision ID from URL (e.g., Icerik/7288/2021-1303).")
publication_date: Optional[str] = Field(None, description="Publication date if extractable from title or description.")
decision_number: Optional[str] = Field(None, description="Decision number if extractable from title or description.")
class KvkkSearchResult(BaseModel):
"""Model for the overall search result for KVKK decisions."""
decisions: List[KvkkDecisionSummary] = Field(default_factory=list, description="List of KVKK decisions found.")
total_results: Optional[int] = Field(None, description="Total number of results available (if provided by Brave API).")
page: int = Field(1, description="Current page number of results.")
pageSize: int = Field(10, description="Number of results per page.")
query: Optional[str] = Field(None, description="The actual search query sent to Brave API.")
class KvkkDocumentMarkdown(BaseModel):
"""Model for KVKK decision document content converted to paginated Markdown."""
source_url: HttpUrl = Field(description="URL of the original KVKK decision page.")
title: Optional[str] = Field(None, description="Title of the KVKK decision.")
decision_date: Optional[str] = Field(None, description="Decision date (Karar Tarihi).")
decision_number: Optional[str] = Field(None, description="Decision number (Karar No).")
subject_summary: Optional[str] = Field(None, description="Subject summary (Konu Özeti).")
markdown_chunk: Optional[str] = Field(None, description="A 5,000 character chunk of the Markdown content.")
current_page: int = Field(description="The current page number of the markdown chunk (1-indexed).")
total_pages: int = Field(description="Total number of pages for the full markdown content.")
is_paginated: bool = Field(description="True if the full markdown content is split into multiple pages.")
error_message: Optional[str] = Field(None, description="Error message if document retrieval or conversion failed.")
class Config:
json_encoders = {
HttpUrl: str
}
+219 -4
View File
@@ -4,7 +4,7 @@ import atexit
import logging
import os
from pydantic import HttpUrl, Field
from typing import Optional, Dict, List, Literal, Any
from typing import Optional, Dict, List, Literal, Any, Union
import urllib.parse
# --- Logging Configuration Start ---
@@ -101,6 +101,14 @@ from sayistay_mcp_module.models import (
)
from sayistay_mcp_module.enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
# KVKK Module Imports
from kvkk_mcp_module.client import KvkkApiClient
from kvkk_mcp_module.models import (
KvkkSearchRequest,
KvkkSearchResult,
KvkkDocumentMarkdown
)
app = create_app()
@@ -115,6 +123,7 @@ kik_client_instance = KikApiClient()
rekabet_client_instance = RekabetKurumuApiClient()
bedesten_client_instance = BedestenApiClient()
sayistay_client_instance = SayistayApiClient()
kvkk_client_instance = KvkkApiClient()
KARAR_TURU_ADI_TO_GUID_ENUM_MAP = {
@@ -1095,7 +1104,7 @@ async def search_anayasa_bireysel_basvuru_report(
)
async def get_anayasa_bireysel_basvuru_document_markdown(
document_url_path: str = Field(..., description="The URL path (e.g., /BB/YYYY/NNNN) of the AYM Bireysel Başvuru decision from kararlarbilgibankasi.anayasa.gov.tr."),
page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters).")
page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters). Accepts int.")
) -> AnayasaBireyselBasvuruDocumentMarkdown:
"""
Retrieves the full text of a Constitutional Court individual application decision in paginated Markdown format.
@@ -1133,7 +1142,14 @@ async def get_anayasa_bireysel_basvuru_document_markdown(
logger.info(f"Tool 'get_anayasa_bireysel_basvuru_document_markdown' called for URL path: {document_url_path}, Page: {page_number}")
if not document_url_path or not document_url_path.strip() or not document_url_path.startswith("/BB/"):
raise ValueError("Document URL path (e.g., /BB/YYYY/NNNN) is required for Anayasa Bireysel Başvuru document retrieval.")
current_page_to_fetch = page_number if page_number is not None and page_number >= 1 else 1
# Handle both int and string page_number inputs
try:
current_page_to_fetch = int(page_number) if page_number is not None else 1
if current_page_to_fetch < 1:
current_page_to_fetch = 1
except (ValueError, TypeError):
current_page_to_fetch = 1
try:
return await anayasa_bireysel_client_instance.get_decision_document_as_markdown(document_url_path, page_number=current_page_to_fetch)
except Exception as e:
@@ -2442,7 +2458,8 @@ def perform_cleanup():
globals().get('kik_client_instance'),
globals().get('rekabet_client_instance'),
globals().get('bedesten_client_instance'),
globals().get('sayistay_client_instance')
globals().get('sayistay_client_instance'),
globals().get('kvkk_client_instance')
]
async def close_all_clients_async():
tasks = []
@@ -2471,6 +2488,204 @@ def perform_cleanup():
atexit.register(perform_cleanup)
# --- MCP Tools for KVKK ---
@app.tool(
description="Search KVKK (Personal Data Protection Authority) decisions using Brave Search API with advanced filtering and Turkish language support. KVKK is Turkey's data protection authority enforcing personal data protection laws equivalent to GDPR",
annotations={
"readOnlyHint": True,
"openWorldHint": True,
"idempotentHint": True
}
)
async def search_kvkk_decisions(
keywords: str = Field(..., description="""
Keywords to search for in KVKK decisions. The search automatically targets KVKK decision summaries.
Search Tips:
• Use Turkish legal terms: "açık rıza" (explicit consent), "veri güvenliği" (data security)
• Combine relevant terms: "kişisel veri işleme" (personal data processing)
• Use specific concepts: "GDPR", "veri ihlali" (data breach), "aydınlatma yükümlülüğü"
Examples:
"açık rıza" - Explicit consent decisions
"veri güvenliği" - Data security cases
"kişisel veri işleme" - Personal data processing
"GDPR uyum" - GDPR compliance
"veri ihlali bildirimi" - Data breach notifications
"""),
page: int = Field(1, ge=1, le=50, description="Page number for results (1-50)."),
pageSize: int = Field(10, ge=1, le=20, description="Number of results per page (1-20).")
) -> KvkkSearchResult:
"""
Searches KVKK (Personal Data Protection Authority) decisions using Brave Search API.
KVKK is Turkey's data protection authority, equivalent to European Data Protection Authorities.
It enforces the Turkish Personal Data Protection Law (KVKK - Kişisel Verilerin Korunması Kanunu)
which is Turkey's GDPR-equivalent legislation.
Key Features:
• Brave Search API integration for comprehensive coverage
• Turkish language search with automatic site targeting
• Decision summaries with metadata extraction
• Pagination support for large result sets
• URL-based decision identification
Search Coverage:
• Administrative fines and penalties
• Data processing compliance decisions
• Data breach notification requirements
• Consent and transparency obligations
• International data transfer decisions
• Data subject rights enforcement
Use Cases:
• Research Turkish data protection precedents
• Analyze KVKK enforcement patterns
• Find specific data protection decisions
• Study compliance requirements and penalties
• Compare with GDPR implementation
Returns structured data with decision titles, URLs, descriptions, and extractable metadata
including decision dates and numbers where available.
"""
logger.info(f"KVKK search tool called with keywords: {keywords}")
search_request = KvkkSearchRequest(
keywords=keywords,
page=page,
pageSize=pageSize
)
try:
result = await kvkk_client_instance.search_decisions(search_request)
logger.info(f"KVKK search completed. Found {len(result.decisions)} decisions on page {page}")
return result
except Exception as e:
logger.exception(f"Error in KVKK search: {e}")
# Return empty result on error
return KvkkSearchResult(
decisions=[],
total_results=0,
page=page,
pageSize=pageSize,
query=keywords
)
@app.tool(
description="Retrieve the full text content of a KVKK decision document converted to Markdown format with metadata extraction and proper legal document formatting",
annotations={
"readOnlyHint": True,
"openWorldHint": False,
"idempotentHint": True
}
)
async def get_kvkk_document_markdown(
decision_url: str = Field(..., description="""
URL of the KVKK decision document to retrieve.
Expected URL format:
• Full KVKK decision page URL (e.g., https://www.kvkk.gov.tr/Icerik/7288/2021-1303)
• URL must point to a valid KVKK decision page
• URLs are typically obtained from search_kvkk_decisions results
Examples:
• https://www.kvkk.gov.tr/Icerik/7288/2021-1303
• https://www.kvkk.gov.tr/Icerik/8043/2023-1356
Note: The URL should be a complete KVKK decision page URL, not just a decision ID.
"""),
page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters). Accepts int.")
) -> KvkkDocumentMarkdown:
"""
Retrieves the full text of a KVKK decision document in paginated Markdown format.
This tool fetches complete KVKK decision content from the official KVKK website
and converts it to clean, readable Markdown format. Content is paginated into
5,000-character chunks for easier processing.
Input Requirements:
• decision_url: Complete KVKK decision page URL from search_kvkk_decisions results
• page_number: Page number for pagination (1-indexed, default: 1)
Output Format:
• Clean Markdown text with proper KVKK decision formatting
• Pagination information (current_page, total_pages, is_paginated)
• Decision metadata (title, date, number, subject summary)
Content Processing:
• Fetches HTML content from KVKK decision pages
• Extracts decision metadata (date, number, subject summary)
• Converts legal document content to properly formatted Markdown
• Preserves document structure and important formatting
• Removes navigation elements and website artifacts
Use Cases:
• Reading full KVKK decision texts with proper formatting
• Legal analysis of personal data protection decisions
• Content analysis and case summarization
• Citation extraction and legal reference building
Returns structured document with paginated Markdown content and extracted metadata.
"""
logger.info(f"KVKK document retrieval tool called for URL: {decision_url}")
# Handle page_number type conversion (Union[int, str] -> int)
if isinstance(page_number, str):
try:
page_number = int(page_number)
except ValueError:
logger.warning(f"Invalid page_number string '{page_number}', defaulting to 1")
page_number = 1
if not decision_url or not decision_url.strip():
return KvkkDocumentMarkdown(
source_url=HttpUrl("https://www.kvkk.gov.tr"),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number or 1,
total_pages=0,
is_paginated=False,
error_message="Decision URL is required and cannot be empty."
)
try:
# Validate URL format
if not decision_url.startswith("https://www.kvkk.gov.tr/"):
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number or 1,
total_pages=0,
is_paginated=False,
error_message="Invalid KVKK decision URL format. URL must start with https://www.kvkk.gov.tr/"
)
result = await kvkk_client_instance.get_decision_document(decision_url, page_number or 1)
logger.info(f"KVKK document retrieved successfully. Page {result.current_page}/{result.total_pages}, Content length: {len(result.markdown_chunk) if result.markdown_chunk else 0}")
return result
except Exception as e:
logger.exception(f"Error retrieving KVKK document: {e}")
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number or 1,
total_pages=0,
is_paginated=False,
error_message=f"Error retrieving KVKK document: {str(e)}"
)
# --- ChatGPT Deep Research Compatible Tools ---
def get_preview_text(markdown_content: str, skip_chars: int = 100, preview_chars: int = 200) -> str:
+8 -15
View File
@@ -6,8 +6,7 @@ from bs4 import BeautifulSoup
from typing import Dict, Any, List, Optional, Tuple
import logging
import html
import tempfile
import os
import io
from urllib.parse import urlencode, urljoin
from markitdown import MarkItDown
@@ -532,21 +531,18 @@ class SayistayApiClient:
raise
def _convert_html_to_markdown(self, html_content: str) -> Optional[str]:
"""Convert HTML content to Markdown using MarkItDown."""
"""Convert HTML content to Markdown using MarkItDown with BytesIO to avoid filename length issues."""
if not html_content:
return None
temp_file_path = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
# Write HTML to temp file
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp:
tmp.write(html_content)
temp_file_path = tmp.name
# Convert
result = md_converter.convert(temp_file_path)
result = md_converter.convert(html_stream)
markdown_content = result.text_content
logger.info("Successfully converted HTML to Markdown")
@@ -555,9 +551,6 @@ class SayistayApiClient:
except Exception as e:
logger.error(f"Error converting HTML to Markdown: {e}")
return f"Error converting HTML content: {str(e)}"
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
async def get_document_as_markdown(self, decision_id: str, decision_type: str) -> SayistayDocumentMarkdown:
"""
+7 -11
View File
@@ -7,8 +7,7 @@ from typing import Dict, Any, List, Optional, Union, Tuple
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from urllib.parse import urljoin, urlencode # urlencode for aiohttp form data
@@ -196,21 +195,18 @@ class UyusmazlikApiClient:
html_input_for_markdown = processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_input_for_markdown.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
conversion_result = md_converter.convert(temp_file_path)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
logger.info("UyusmazlikApiClient: HTML to Markdown conversion successful.")
except Exception as e:
logger.error(f"UyusmazlikApiClient: Error during MarkItDown HTML to Markdown conversion: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(self, document_url: str) -> UyusmazlikDocumentMarkdown:
+7 -13
View File
@@ -6,8 +6,7 @@ from typing import Dict, Any, List, Optional
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from .models import (
@@ -108,25 +107,20 @@ class YargitayOfficialApiClient:
html_to_convert = processed_html
markdown_output = None
temp_file_path = None
try:
md_converter = MarkItDown() # Plugins disabled as per basic usage
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_to_convert.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Write the HTML to a temporary file for MarkItDown to process
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_html_file:
tmp_html_file.write(html_to_convert)
temp_file_path = tmp_html_file.name
conversion_result = md_converter.convert(temp_file_path)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_output = conversion_result.text_content
logger.info("Successfully converted HTML to Markdown.")
except Exception as e:
logger.error(f"Error during MarkItDown HTML to Markdown conversion: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path) # Clean up the temporary file
return markdown_output