39 Commits
Author SHA1 Message Date
saidsurucu d590702272 Bump version to 0.1.3
- Add auto-install Playwright browsers functionality
- Fix KIK tool page_number parameter descriptions
- Add 'accepts int' notes to all page_number parameters
- Improve PyPI deployment experience
2025-07-12 11:41:35 +03:00
saidsurucu 1ebe8847fb fix kik 2025-07-12 11:40:11 +03:00
saidsurucu cb318faeba add kvkk module, several bug fix 2025-07-11 23:50:56 +03:00
saidsurucu 83f54a86a8 Update mcp_server_main.py 2025-07-11 21:22:13 +03:00
saidsurucu f5f0f99678 add redis 2025-07-11 21:19:16 +03:00
saidsurucu b0d7151ba1 OAuth callback JWT token fix 2025-07-09 22:59:30 +03:00
saidsurucu de9e337163 fix auth 2025-07-09 22:37:26 +03:00
saidsurucu a7ebb8b27e fix auth 2025-07-09 21:36:05 +03:00
saidsurucu 01e58ab5ed Update mcp_auth_http_simple.py 2025-07-09 20:47:26 +03:00
saidsurucu 7081bbbd91 fix auth 2025-07-09 20:34:36 +03:00
saidsurucu ca89c9480d fix auth 2025-07-09 20:16:10 +03:00
saidsurucu b751e94847 Update mcp_server_main.py 2025-07-09 19:58:07 +03:00
saidsurucu 3ccb52e719 fix auth 2025-07-09 19:55:09 +03:00
saidsurucu e8b92e347c Update asgi_app.py 2025-07-09 19:44:31 +03:00
saidsurucu e65b42abc8 Update asgi_app.py 2025-07-09 19:26:24 +03:00
saidsurucu 71096ae67d Update asgi_app.py 2025-07-09 19:25:46 +03:00
saidsurucu 9fb23dadae Update asgi_app.py 2025-07-09 19:25:21 +03:00
saidsurucu d77781709a Update asgi_app.py 2025-07-09 19:15:19 +03:00
saidsurucu c059ec30a7 Update asgi_app.py 2025-07-09 19:09:37 +03:00
saidsurucu 6842ad207d Update asgi_app.py 2025-07-09 18:52:52 +03:00
saidsurucu bbc56d9409 Update asgi_app.py 2025-07-09 18:48:44 +03:00
saidsurucu 66b5e2631e Update asgi_app.py 2025-07-09 17:55:45 +03:00
saidsurucu a17853b918 Update asgi_app.py 2025-07-09 17:44:34 +03:00
saidsurucu 0a302bb13b Update asgi_app.py 2025-07-09 17:35:12 +03:00
saidsurucu 28e9a47464 Update asgi_app.py 2025-07-09 17:27:30 +03:00
saidsurucu 6c1efece31 Update mcp_server_main.py 2025-07-09 17:15:06 +03:00
saidsurucu eb9441a6f3 Update mcp_server_main.py 2025-07-09 16:27:14 +03:00
saidsurucu d006dc8a55 Update asgi_app.py 2025-07-09 15:44:09 +03:00
saidsurucu 55cee6933d upgrade fastmcp 2025-07-09 15:38:04 +03:00
saidsurucu 2f375d74e5 Update asgi_app.py 2025-07-09 14:54:39 +03:00
saidsurucu 82856b25e9 Update asgi_app.py 2025-07-09 14:24:31 +03:00
saidsurucu 8464aedceb Update asgi_app.py 2025-07-09 13:46:04 +03:00
saidsurucu 64c3a2c138 Update asgi_app.py 2025-07-08 23:52:08 +03:00
saidsurucu 318bedd4c5 add jwt support 2025-07-08 23:18:44 +03:00
saidsurucu 91895f6c1a Update asgi_app.py 2025-07-08 22:46:58 +03:00
saidsurucu b86c18c842 update auth logic 2025-07-08 22:34:02 +03:00
saidsurucu 5509380cef Rename fastapi_app.py to example_fastapi_app.py 2025-07-03 17:54:54 +03:00
saidsurucu 1c818756e4 Update README.md 2025-07-03 12:53:38 +03:00
saidsurucu 34ac65dc02 Update README.md 2025-07-03 12:52:40 +03:00
23 changed files with 3228 additions and 352 deletions
+3
View File
@@ -56,6 +56,9 @@ LOG_LEVEL=info
# Base URL for the application (used for OAuth callbacks and API URLs)
BASE_URL=http://localhost:8000
# JWT Secret for MCP token generation
JWT_SECRET_KEY=your_jwt_secret_key_here
# =============================================================================
# MCP SERVER SETTINGS
# =============================================================================
+1
View File
@@ -190,3 +190,4 @@ GEMINI.md
fly.toml
scripts/deploy-flyio.sh
docs/DEPLOYMENT_FLYIO.md
setup_jwt_template.py
+1 -4
View File
@@ -48,7 +48,7 @@ Bu bölüm, Yargı MCP aracını 5ire gibi Claude Desktop dışındaki MCP istem
* **Name:** `Yargı MCP`
* **Command:**
```
uvx --from git+https://github.com/saidsurucu/yargi-mcp yargi-mcp
uvx yargi-mcp
```
* **Save** butonuna basarak kaydedin.
![5ire ayarları](./5ire-settings.png)
@@ -70,7 +70,6 @@ Bu bölüm, Yargı MCP aracını 5ire gibi Claude Desktop dışındaki MCP istem
"Yargı MCP": {
"command": "uvx",
"args": [
"--from", "git+https://github.com/saidsurucu/yargi-mcp",
"yargi-mcp"
]
}
@@ -101,8 +100,6 @@ Yargı MCP'yi Gemini CLI ile kullanmak için:
"yargi_mcp": {
"command": "uvx",
"args": [
"--from",
"git+https://github.com/saidsurucu/yargi-mcp",
"yargi-mcp"
]
}
+11 -12
View File
@@ -7,8 +7,7 @@ from typing import Dict, Any, List, Optional, Tuple
import logging
import html
import re
import tempfile
import os
import io
from urllib.parse import urlencode, urljoin, quote
from markitdown import MarkItDown
import math # For math.ceil for pagination
@@ -230,23 +229,23 @@ class AnayasaBireyselBasvuruApiClient:
html_input_for_markdown = processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
# Ensure the content is wrapped in basic HTML structure if it's not already
if not html_input_for_markdown.strip().lower().startswith(("<html", "<!doctype")):
tmp_file.write(f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>")
html_content = f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>"
else:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
html_content = html_input_for_markdown
conversion_result = md_converter.convert(temp_file_path)
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
except Exception as e:
logger.error(f"AnayasaBireyselBasvuruApiClient: MarkItDown conversion error: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(
+19 -22
View File
@@ -7,8 +7,7 @@ from typing import Dict, Any, List, Optional, Tuple
import logging
import html
import re
import tempfile
import os
import io
from urllib.parse import urlencode, urljoin, quote
from markitdown import MarkItDown
import math # For math.ceil for pagination
@@ -82,6 +81,13 @@ class AnayasaMahkemesiApiClient:
if params.has_dissenting_opinion and params.has_dissenting_opinion.value and params.has_dissenting_opinion.value != "ALL": query_params.append(("KarsiOy", params.has_dissenting_opinion.value))
if params.has_different_reasoning and params.has_different_reasoning.value and params.has_different_reasoning.value != "ALL": query_params.append(("FarkliGerekce", params.has_different_reasoning.value))
# Add pagination and sorting parameters as query params instead of URL path
if params.results_per_page and params.results_per_page != 10:
query_params.append(("SatirSayisi", str(params.results_per_page)))
if params.sort_by_criteria and params.sort_by_criteria != "KararTarihi":
query_params.append(("Siralama", params.sort_by_criteria))
if params.page_to_fetch and params.page_to_fetch > 1:
query_params.append(("page", str(params.page_to_fetch)))
return query_params
@@ -90,16 +96,8 @@ class AnayasaMahkemesiApiClient:
self,
params: AnayasaNormDenetimiSearchRequest
) -> AnayasaSearchResult:
path_segments = []
if params.results_per_page and params.results_per_page != 10: # Default is 10
path_segments.append(f"SatirSayisi/{params.results_per_page}")
if params.sort_by_criteria and params.sort_by_criteria != "KararTarihi": # Default is KararTarihi
# Ensure correct quoting for criteria that might have Turkish chars or spaces
path_segments.append(f"Siralama/{quote(params.sort_by_criteria)}")
path_segments.append(self.SEARCH_PATH_SEGMENT)
request_path = "/" + "/".join(path_segments)
# Use simple /Ara endpoint - the complex path structure seems to cause 404s
request_path = f"/{self.SEARCH_PATH_SEGMENT}"
final_query_params = self._build_search_query_params_for_aym(params)
logger.info(f"AnayasaMahkemesiApiClient: Performing Norm Denetimi search. Path: {request_path}, Params: {final_query_params}")
@@ -222,24 +220,23 @@ class AnayasaMahkemesiApiClient:
html_input_for_markdown = str(body_tag) if body_tag else processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
# Ensure the content is wrapped in basic HTML structure if it's not already
if not html_input_for_markdown.strip().lower().startswith(("<html", "<!doctype")):
tmp_file.write(f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>")
html_content = f"<html><head><meta charset=\"UTF-8\"></head><body>{html_input_for_markdown}</body></html>"
else:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
html_content = html_input_for_markdown
conversion_result = md_converter.convert(temp_file_path)
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
except Exception as e:
logger.error(f"AnayasaMahkemesiApiClient: MarkItDown conversion error: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(
+379 -34
View File
@@ -10,8 +10,11 @@ Usage:
"""
import os
from fastapi import FastAPI, Request, HTTPException
from fastapi.responses import JSONResponse
import time
import logging
from datetime import datetime, timedelta
from fastapi import FastAPI, Request, HTTPException, Query
from fastapi.responses import JSONResponse, HTMLResponse
from fastapi.exception_handlers import http_exception_handler
from starlette.middleware import Middleware
from starlette.middleware.cors import CORSMiddleware
@@ -23,13 +26,16 @@ from mcp_server_main import app as mcp_server
# Import Stripe webhook router
from stripe_webhook import router as stripe_router
# Import MCP Auth HTTP adapter
from mcp_auth_http_adapter import router as mcp_auth_router
# Import simplified MCP Auth HTTP adapter
from mcp_auth_http_simple import router as mcp_auth_router
# OAuth configuration from environment variables
CLERK_ISSUER = os.getenv("CLERK_ISSUER", "https://accounts.yargimcp.com")
BASE_URL = os.getenv("BASE_URL", "https://yargimcp.com")
# Setup logging
logger = logging.getLogger(__name__)
# Configure CORS middleware
cors_origins = os.getenv("ALLOWED_ORIGINS", "*").split(",")
custom_middleware = [
@@ -42,25 +48,46 @@ custom_middleware = [
),
]
# Create MCP Starlette sub-application first
# Create MCP Starlette sub-application (without auth wrapper)
mcp_app = mcp_server.http_app(
path="/",
middleware=custom_middleware
)
# Create FastAPI wrapper application with MCP app's lifespan
# Configure JSON encoder for proper Turkish character support
import json
from fastapi.responses import JSONResponse
class UTF8JSONResponse(JSONResponse):
def __init__(self, content=None, status_code=200, headers=None, **kwargs):
if headers is None:
headers = {}
headers["Content-Type"] = "application/json; charset=utf-8"
super().__init__(content, status_code, headers, **kwargs)
def render(self, content) -> bytes:
return json.dumps(
content,
ensure_ascii=False,
allow_nan=False,
indent=None,
separators=(",", ":"),
).encode("utf-8")
# Create FastAPI wrapper application with MCP lifespan
app = FastAPI(
title="Yargı MCP Server",
description="MCP server for Turkish legal databases with OAuth authentication",
version="0.1.0",
middleware=custom_middleware,
lifespan=mcp_app.lifespan # Critical: Get lifespan from mcp_app, not mcp_server
lifespan=mcp_app.lifespan, # MCP app lifespan
default_response_class=UTF8JSONResponse # Use UTF-8 JSON encoder
)
# Add Stripe webhook router to FastAPI
app.include_router(stripe_router, prefix="/api")
# Add MCP Auth HTTP adapter to FastAPI (replaces old OAuth router)
# Add MCP Auth HTTP adapter to FastAPI (handles OAuth endpoints)
app.include_router(mcp_auth_router)
# Custom 401 exception handler for MCP spec compliance
@@ -80,23 +107,110 @@ async def custom_401_handler(request: Request, exc: HTTPException):
return response
# Mount MCP app as sub-application
app.mount("/mcp", mcp_app)
# Mount MCP app as sub-application at /mcp-server to avoid path conflicts
app.mount("/mcp-server", mcp_app)
# Add POST handler for /mcp to forward to mounted app
@app.post("/mcp")
async def mcp_post_handler(request: Request):
"""Forward POST /mcp requests to mounted MCP app"""
# Forward to the mounted app by calling it directly
# Add custom route to handle /mcp requests and forward to mounted app
@app.api_route("/mcp", methods=["POST", "DELETE", "OPTIONS"])
@app.api_route("/mcp/", methods=["POST", "DELETE", "OPTIONS"])
async def mcp_protocol_handler(request: Request):
"""Handle MCP protocol requests by forwarding to mounted app"""
# Handle DELETE requests for session termination
if request.method == "DELETE":
logger.info("DELETE request received for session termination")
# For session termination, we just return 200 OK
# The actual session cleanup is handled by the underlying MCP transport
from starlette.responses import Response
return Response(
status_code=200,
content="Session terminated successfully"
)
# REQUIRED: Validate Bearer JWT tokens for all MCP requests
auth_header = request.headers.get("Authorization")
if not auth_header or not auth_header.startswith("Bearer "):
logger.error("Missing or invalid Authorization header")
raise HTTPException(
status_code=401,
detail="Missing or invalid Authorization header. Bearer token required."
)
token = auth_header.split(" ")[1]
try:
# Check if this is a mock token for development/testing
if token.startswith("mock_clerk_jwt_"):
logger.info(f"Using mock JWT token for development: {token[:30]}...")
# For mock tokens, we'll allow access with a mock user
request.state.user_id = "mock_user_dev"
request.state.session_id = "mock_session_dev"
request.state.token_scopes = ["read", "search"]
logger.info("Mock JWT token accepted for development")
elif token.startswith("eyJ"):
# This looks like a real JWT token (starts with eyJ which is base64 encoded '{"')
logger.info(f"Processing real JWT token: {token[:30]}...")
# Validate real Clerk JWT token
from clerk_backend_api import Clerk, models
import jwt
# Decode JWT token and extract user info
try:
decoded_token = jwt.decode(token, options={"verify_signature": False})
user_id = decoded_token.get("user_id") or decoded_token.get("sub")
user_email = decoded_token.get("email")
token_scopes = decoded_token.get("scopes", ["read", "search"])
session_id = decoded_token.get("sid", "jwt_session")
logger.info(f"JWT token claims - user_id: {user_id}, email: {user_email}, scopes: {token_scopes}")
if user_id and user_email:
# JWT token is signed by Clerk and contains valid user info
request.state.user_id = user_id
request.state.user_email = user_email
request.state.session_id = session_id
request.state.token_scopes = token_scopes
logger.info(f"Real JWT token accepted for user: {user_id}")
else:
logger.error(f"Missing required fields in JWT token - user_id: {bool(user_id)}, email: {bool(user_email)}")
raise HTTPException(
status_code=401,
detail="Invalid token - missing user_id or email in claims"
)
except Exception as e:
logger.error(f"JWT token decoding failed: {e}")
raise HTTPException(
status_code=401,
detail="Invalid JWT token format"
)
else:
# Invalid token format - doesn't start with expected patterns
logger.error(f"Invalid token format: {token[:30]}...")
raise HTTPException(
status_code=401,
detail="Invalid token format - must be a valid JWT token"
)
except HTTPException:
# Re-raise HTTPException as-is
raise
except Exception as e:
logger.error(f"Bearer token validation failed: {str(e)}")
raise HTTPException(
status_code=401,
detail=f"Token validation failed: {str(e)}"
)
# Forward the request to the mounted MCP app
async def receive():
return await request.receive()
# Create a new scope for the mounted app
# Create new scope for the mounted app
scope = request.scope.copy()
scope["path"] = "/" # Root path for the mounted app
scope["path"] = "/" # Root path for mounted app
scope["path_info"] = "/"
# Capture response
# Capture the response
response_parts = {"status": 200, "headers": [], "body": b""}
async def send(message):
@@ -124,6 +238,9 @@ async def mcp_post_handler(request: Request):
)
# SSE transport deprecated - removed
# FastAPI health check endpoint
@app.get("/health")
async def health_check():
@@ -153,6 +270,9 @@ async def root():
"oauth_google": "/auth/google/login",
"user_info": "/auth/user"
},
"transports": {
"http": "/mcp"
},
"supported_databases": [
"Yargıtay (Court of Cassation)",
"Danıştay (Council of State)",
@@ -174,13 +294,14 @@ async def root():
})
# OAuth 2.0 Authorization Server Metadata proxy (for MCP clients that can't reach Clerk directly)
@app.get("/.well-known/oauth-authorization-server")
# MCP Auth Toolkit expects this to be under /mcp/.well-known/oauth-authorization-server
@app.get("/mcp/.well-known/oauth-authorization-server")
async def oauth_authorization_server():
"""OAuth 2.0 Authorization Server Metadata proxy to Clerk"""
"""OAuth 2.0 Authorization Server Metadata proxy to Clerk - MCP Auth Toolkit standard location"""
return JSONResponse({
"issuer": CLERK_ISSUER,
"authorization_endpoint": f"{BASE_URL}/auth/login",
"token_endpoint": f"{BASE_URL}/auth/callback",
"issuer": BASE_URL,
"authorization_endpoint": "https://yargimcp.com/mcp-callback",
"token_endpoint": f"{BASE_URL}/token",
"jwks_uri": f"{CLERK_ISSUER}/.well-known/jwks.json",
"response_types_supported": ["code"],
"grant_types_supported": ["authorization_code", "refresh_token"],
@@ -191,7 +312,65 @@ async def oauth_authorization_server():
"claims_supported": ["sub", "iss", "aud", "exp", "iat", "email", "name"],
"code_challenge_methods_supported": ["S256"],
"service_documentation": f"{BASE_URL}/mcp",
"registration_endpoint": f"{BASE_URL}/auth/register",
"registration_endpoint": f"{BASE_URL}/register",
"resource_documentation": f"{BASE_URL}/mcp"
})
# Claude AI MCP specific endpoint format
@app.get("/.well-known/oauth-authorization-server/mcp")
async def oauth_authorization_server_mcp_suffix():
"""OAuth 2.0 Authorization Server Metadata - Claude AI MCP specific format"""
return JSONResponse({
"issuer": BASE_URL,
"authorization_endpoint": "https://yargimcp.com/mcp-callback",
"token_endpoint": f"{BASE_URL}/token",
"jwks_uri": f"{CLERK_ISSUER}/.well-known/jwks.json",
"response_types_supported": ["code"],
"grant_types_supported": ["authorization_code", "refresh_token"],
"token_endpoint_auth_methods_supported": ["client_secret_basic", "none"],
"scopes_supported": ["read", "search", "openid", "profile", "email"],
"subject_types_supported": ["public"],
"id_token_signing_alg_values_supported": ["RS256"],
"claims_supported": ["sub", "iss", "aud", "exp", "iat", "email", "name"],
"code_challenge_methods_supported": ["S256"],
"service_documentation": f"{BASE_URL}/mcp",
"registration_endpoint": f"{BASE_URL}/register",
"resource_documentation": f"{BASE_URL}/mcp"
})
@app.get("/.well-known/oauth-protected-resource/mcp")
async def oauth_protected_resource_mcp_suffix():
"""OAuth 2.0 Protected Resource Metadata - Claude AI MCP specific format"""
return JSONResponse({
"resource": BASE_URL,
"authorization_servers": [
BASE_URL
],
"scopes_supported": ["read", "search"],
"bearer_methods_supported": ["header"],
"resource_documentation": f"{BASE_URL}/mcp",
"resource_policy_uri": f"{BASE_URL}/privacy"
})
# Keep root level for compatibility with some MCP clients
@app.get("/.well-known/oauth-authorization-server")
async def oauth_authorization_server_root():
"""OAuth 2.0 Authorization Server Metadata proxy to Clerk - root level for compatibility"""
return JSONResponse({
"issuer": BASE_URL,
"authorization_endpoint": "https://yargimcp.com/mcp-callback",
"token_endpoint": f"{BASE_URL}/token",
"jwks_uri": f"{CLERK_ISSUER}/.well-known/jwks.json",
"response_types_supported": ["code"],
"grant_types_supported": ["authorization_code", "refresh_token"],
"token_endpoint_auth_methods_supported": ["client_secret_basic", "none"],
"scopes_supported": ["read", "search", "openid", "profile", "email"],
"subject_types_supported": ["public"],
"id_token_signing_alg_values_supported": ["RS256"],
"claims_supported": ["sub", "iss", "aud", "exp", "iat", "email", "name"],
"code_challenge_methods_supported": ["S256"],
"service_documentation": f"{BASE_URL}/mcp",
"registration_endpoint": f"{BASE_URL}/register",
"resource_documentation": f"{BASE_URL}/mcp"
})
@@ -205,12 +384,12 @@ async def mcp_info():
"version": "0.1.0",
"description": "MCP server for Turkish legal databases",
"protocol": "mcp/1.0",
"transport": "http",
"transport": ["http"],
"authentication_required": True,
"authentication": {
"type": "oauth2",
"authorization_url": f"{BASE_URL}/auth/login",
"token_url": f"{BASE_URL}/auth/callback",
"authorization_url": "https://yargimcp.com/sign-in?redirect_url=https://api.yargimcp.com/auth/mcp-callback",
"token_url": f"{BASE_URL}/auth/mcp-token",
"scopes": ["read", "search"],
"provider": "clerk"
},
@@ -231,7 +410,7 @@ async def mcp_info():
"note": "This is an MCP server. Use POST to /mcp/ with proper MCP protocol headers.",
"headers_required": [
"Content-Type: application/json",
"Accept: application/json, text/event-stream",
"Accept: application/json",
"Authorization: Bearer <token>",
"X-Session-ID: <session-id>"
]
@@ -322,12 +501,178 @@ async def status():
"auth_status": "enabled" if os.getenv("ENABLE_AUTH", "false").lower() == "true" else "disabled"
})
# Alternative: SSE transport (for compatibility)
sse_app = mcp_server.http_app(
path="/sse",
transport="sse",
middleware=custom_middleware
# Note: JWT token validation is now handled entirely by Clerk
# All authentication flows use Clerk JWT tokens directly
async def validate_clerk_session(request: Request, clerk_token: str = None) -> str:
"""Validate Clerk session from cookies or JWT token and return user_id"""
logger.info(f"Validating Clerk session - token provided: {bool(clerk_token)}")
try:
# Try to import Clerk SDK
from clerk_backend_api import Clerk
clerk = Clerk(bearer_auth=os.getenv("CLERK_SECRET_KEY"))
# Try JWT token first (from URL parameter)
if clerk_token:
logger.info("Validating Clerk JWT token from URL parameter")
try:
# Extract session_id from JWT token and verify with Clerk
import jwt
decoded_token = jwt.decode(clerk_token, options={"verify_signature": False})
session_id = decoded_token.get("sid") # Use standard JWT 'sid' claim
if session_id:
# Verify with Clerk using session_id
session = clerk.sessions.verify(session_id=session_id, token=clerk_token)
user_id = session.user_id if session else None
if user_id:
logger.info(f"JWT token validation successful - user_id: {user_id}")
return user_id
else:
logger.error("JWT token validation failed - no user_id in session")
else:
logger.error("No session_id found in JWT token")
except Exception as e:
logger.error(f"JWT token validation failed: {str(e)}")
# Fall through to cookie validation
# Fallback to cookie validation
logger.info("Attempting cookie-based session validation")
clerk_session = request.cookies.get("__session")
if not clerk_session:
logger.error("No Clerk session cookie found")
raise HTTPException(status_code=401, detail="No Clerk session found")
# Validate session with Clerk
session = clerk.sessions.verify_session(clerk_session)
logger.info(f"Cookie session validation successful - user_id: {session.user_id}")
return session.user_id
except ImportError:
# Fallback for development without Clerk SDK
logger.warning("Clerk SDK not available - using development fallback")
return "dev_user_123"
except Exception as e:
logger.error(f"Session validation failed: {str(e)}")
raise HTTPException(status_code=401, detail=f"Session validation failed: {str(e)}")
# MCP OAuth Callback Endpoint
@app.get("/auth/mcp-callback")
async def mcp_oauth_callback(request: Request, clerk_token: str = Query(None)):
"""Handle OAuth callback for MCP token generation"""
logger.info(f"MCP OAuth callback - clerk_token provided: {bool(clerk_token)}")
try:
# Validate Clerk session with JWT token support
user_id = await validate_clerk_session(request, clerk_token)
logger.info(f"User authenticated successfully - user_id: {user_id}")
# Use the Clerk JWT token directly (no need to generate custom token)
logger.info("User authenticated successfully via Clerk")
# Return success response
return HTMLResponse(f"""
<html>
<head>
<title>MCP Connection Successful</title>
<style>
body {{ font-family: Arial, sans-serif; text-align: center; padding: 50px; }}
.success {{ color: #28a745; }}
.token {{ background: #f8f9fa; padding: 15px; border-radius: 5px; margin: 20px 0; word-break: break-all; }}
</style>
</head>
<body>
<h1 class="success">✅ MCP Connection Successful!</h1>
<p>Your Yargı MCP integration is now active.</p>
<div class="token">
<strong>Authentication:</strong><br>
<code>Use your Clerk JWT token directly with Bearer authentication</code>
</div>
<p>You can now close this window and return to your MCP client.</p>
<script>
// Try to close the popup if opened as such
if (window.opener) {{
window.opener.postMessage({{
type: 'MCP_AUTH_SUCCESS',
token: 'use_clerk_jwt_token'
}}, '*');
setTimeout(() => window.close(), 3000);
}}
</script>
</body>
</html>
""")
except HTTPException as e:
logger.error(f"MCP OAuth callback failed: {e.detail}")
return HTMLResponse(f"""
<html>
<head>
<title>MCP Connection Failed</title>
<style>
body {{ font-family: Arial, sans-serif; text-align: center; padding: 50px; }}
.error {{ color: #dc3545; }}
.debug {{ background: #f8f9fa; padding: 10px; margin: 20px 0; border-radius: 5px; font-family: monospace; }}
</style>
</head>
<body>
<h1 class="error">❌ MCP Connection Failed</h1>
<p>{e.detail}</p>
<div class="debug">
<strong>Debug Info:</strong><br>
Clerk Token: {'✅ Provided' if clerk_token else '❌ Missing'}<br>
Error: {e.detail}<br>
Status: {e.status_code}
</div>
<p>Please try again or contact support.</p>
<a href="https://yargimcp.com/sign-in">Return to Sign In</a>
</body>
</html>
""", status_code=e.status_code)
except Exception as e:
logger.error(f"Unexpected error in MCP OAuth callback: {str(e)}")
return HTMLResponse(f"""
<html>
<head>
<title>MCP Connection Error</title>
<style>
body {{ font-family: Arial, sans-serif; text-align: center; padding: 50px; }}
.error {{ color: #dc3545; }}
</style>
</head>
<body>
<h1 class="error">❌ Unexpected Error</h1>
<p>An unexpected error occurred during authentication.</p>
<p>Error: {str(e)}</p>
<a href="https://yargimcp.com/sign-in">Return to Sign In</a>
</body>
</html>
""", status_code=500)
# OAuth2 Token Endpoint - Now uses Clerk JWT tokens directly
@app.post("/auth/mcp-token")
async def mcp_token_endpoint(request: Request):
"""OAuth2 token endpoint for MCP clients - returns Clerk JWT token info"""
try:
# Validate Clerk session
user_id = await validate_clerk_session(request)
return JSONResponse({
"message": "Use your Clerk JWT token directly with Bearer authentication",
"token_type": "Bearer",
"scope": "yargi.read",
"user_id": user_id,
"instructions": "Include 'Authorization: Bearer YOUR_CLERK_JWT_TOKEN' in your requests"
})
except HTTPException as e:
return JSONResponse(
status_code=e.status_code,
content={"error": "invalid_request", "error_description": e.detail}
)
# Note: Only HTTP transport supported - SSE transport deprecated
# Export for uvicorn
__all__ = ["app", "sse_app"]
__all__ = ["app"]
+12 -27
View File
@@ -5,8 +5,7 @@ import base64
from typing import Optional
import logging
from markitdown import MarkItDown
import tempfile
import os
import io
from .models import (
BedestenSearchRequest, BedestenSearchResponse,
@@ -125,17 +124,14 @@ class BedestenApiClient:
if not html_content:
return None
temp_file_path = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
# Write HTML to temp file
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp:
tmp.write(html_content)
temp_file_path = tmp.name
# Convert
result = md_converter.convert(temp_file_path)
result = md_converter.convert(html_stream)
markdown_content = result.text_content
logger.info("Successfully converted HTML to Markdown")
@@ -144,27 +140,19 @@ class BedestenApiClient:
except Exception as e:
logger.error(f"Error converting HTML to Markdown: {e}")
return f"Error converting HTML content: {str(e)}"
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
def _convert_pdf_to_markdown(self, pdf_bytes: bytes) -> Optional[str]:
"""Convert PDF to Markdown using MarkItDown"""
if not pdf_bytes:
return None
temp_file_path = None
try:
# MarkItDown supports PDF with markitdown[pdf]
# Create BytesIO stream from PDF bytes
pdf_stream = io.BytesIO(pdf_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
# Write PDF to temp file
with tempfile.NamedTemporaryFile(mode="wb", delete=False, suffix=".pdf") as tmp:
tmp.write(pdf_bytes)
temp_file_path = tmp.name
# Convert
result = md_converter.convert(temp_file_path)
result = md_converter.convert(pdf_stream)
markdown_content = result.text_content
logger.info("Successfully converted PDF to Markdown")
@@ -173,9 +161,6 @@ class BedestenApiClient:
except Exception as e:
logger.error(f"Error converting PDF to Markdown: {e}")
return f"Error converting PDF content: {str(e)}. The document may be corrupted or in an unsupported format."
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
async def close_client_session(self):
"""Close HTTP client session"""
+10 -14
View File
@@ -6,8 +6,7 @@ from typing import Dict, Any, List, Optional
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from .models import (
@@ -124,31 +123,28 @@ class DanistayApiClient:
html_input_for_markdown = processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown() # Basic conversion
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_input_for_markdown.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
tmp_file.write(html_input_for_markdown) # Write the full HTML string
temp_file_path = tmp_file.name
conversion_result = md_converter.convert(temp_file_path)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
logger.info("DanistayApiClient: HTML to Markdown conversion successful.")
except Exception as e:
logger.error(f"DanistayApiClient: Error during MarkItDown HTML to Markdown conversion: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(self, id: str) -> DanistayDocumentMarkdown:
"""
Retrieves a specific Danıştay decision by ID and returns its content as Markdown.
The /getDokuman endpoint for Danıştay returns direct HTML.
The /getDokuman endpoint for Danıştay requires arananKelime parameter.
"""
document_api_url = f"{self.DOCUMENT_ENDPOINT}?id={id}"
# Add required arananKelime parameter - using empty string as minimum requirement
document_api_url = f"{self.DOCUMENT_ENDPOINT}?id={id}&arananKelime="
source_url = f"{self.BASE_URL}{document_api_url}"
logger.info(f"DanistayApiClient: Fetching Danistay document for Markdown (ID: {id}) from {source_url}")
+1 -1
View File
@@ -97,7 +97,7 @@ class DanistayApiResponseInnerData(BaseModel):
class DanistayApiResponse(BaseModel):
"""Model for the complete search response from the Danistay API."""
data: DanistayApiResponseInnerData
data: Optional[DanistayApiResponseInnerData] = Field(None, description="Response data, can be null when no results found")
metadata: Optional[Dict[str, Any]] = Field(None, description="Optional metadata (Meta Veri) from API.")
class DanistayDocumentMarkdown(BaseModel):
+7 -12
View File
@@ -6,8 +6,7 @@ from typing import Dict, Any, List, Optional
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from .models import (
@@ -114,22 +113,18 @@ class EmsalApiClient:
html_input_for_markdown = content
markdown_text = None
temp_file_path = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_input_for_markdown.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
conversion_result = md_converter.convert(temp_file_path)
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
logger.info("EmsalApiClient: HTML to Markdown conversion successful.")
except Exception as e:
logger.error(f"EmsalApiClient: Error during MarkItDown HTML to Markdown conversion for Emsal: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
+792 -47
View File
@@ -18,7 +18,8 @@ import html as html_parser
from markitdown import MarkItDown
import os
import math
import tempfile
import io
import random
from .models import (
KikSearchRequest,
@@ -69,14 +70,79 @@ class KikApiClient:
self.playwright_instance = await async_playwright().start()
if not self.browser or not self.browser.is_connected():
if self.browser: await self.browser.close()
self.browser = await self.playwright_instance.chromium.launch(headless=True)
# Ultra stealth browser configuration
self.browser = await self.playwright_instance.chromium.launch(
headless=True,
args=[
# Disable automation indicators
'--no-first-run',
'--no-default-browser-check',
'--disable-dev-shm-usage',
'--disable-extensions',
'--disable-gpu',
'--disable-default-apps',
'--disable-translate',
'--disable-blink-features=AutomationControlled',
'--disable-ipc-flooding-protection',
'--disable-renderer-backgrounding',
'--disable-backgrounding-occluded-windows',
'--disable-client-side-phishing-detection',
'--disable-sync',
'--disable-features=TranslateUI,BlinkGenPropertyTrees',
'--disable-component-extensions-with-background-pages',
'--no-sandbox', # Sometimes needed for headless
'--disable-web-security',
'--disable-features=VizDisplayCompositor',
# Language and locale
'--lang=tr-TR',
'--accept-lang=tr-TR,tr;q=0.9,en;q=0.8',
# Performance optimizations
'--memory-pressure-off',
'--max_old_space_size=4096',
]
)
browser_recreated = True
if not self.context or browser_recreated:
if self.context: await self.context.close()
if not self.browser: raise PlaywrightError("Browser not initialized.")
# Ultra realistic context configuration
self.context = await self.browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36",
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
viewport={'width': 1920, 'height': 1080},
screen={'width': 1920, 'height': 1080},
device_scale_factor=1.0,
is_mobile=False,
has_touch=False,
# Localization
locale='tr-TR',
timezone_id='Europe/Istanbul',
# Realistic browser features
java_script_enabled=True,
accept_downloads=True,
ignore_https_errors=True,
# Color scheme and media
color_scheme='light',
reduced_motion='no-preference',
forced_colors='none',
# Additional headers for realism
extra_http_headers={
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'tr-TR,tr;q=0.9,en;q=0.8',
'Cache-Control': 'max-age=0',
'DNT': '1',
'Upgrade-Insecure-Requests': '1',
'Sec-Ch-Ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Windows"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
},
# Permissions to appear realistic
permissions=['geolocation'],
geolocation={'latitude': 41.0082, 'longitude': 28.9784}, # Istanbul
)
context_recreated = True
if not self.page or self.page.is_closed() or force_new_page or context_recreated or browser_recreated:
@@ -86,6 +152,9 @@ class KikApiClient:
if not self.page: raise PlaywrightError("Failed to create new page.")
self.page.set_default_navigation_timeout(self.request_timeout)
self.page.set_default_timeout(self.request_timeout)
# CRITICAL: Anti-detection JavaScript injection
await self._inject_stealth_scripts()
if not self.page or self.page.is_closed():
raise PlaywrightError("Playwright page initialization failed.")
logger.debug("_ensure_playwright_ready completed.")
@@ -99,29 +168,623 @@ class KikApiClient:
if self.playwright_instance: await self.playwright_instance.stop(); self.playwright_instance = None
logger.info("KikApiClient (Playwright): Resources closed.")
async def _inject_stealth_scripts(self):
"""
Inject comprehensive stealth JavaScript to evade bot detection.
Overrides navigator properties and other fingerprinting vectors.
"""
if not self.page:
logger.warning("Cannot inject stealth scripts: page is None")
return
logger.debug("Injecting comprehensive stealth scripts...")
stealth_script = '''
// Override navigator.webdriver
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
configurable: true
});
// Override navigator properties to appear more human
Object.defineProperty(navigator, 'languages', {
get: () => ['tr-TR', 'tr', 'en-US', 'en'],
configurable: true
});
Object.defineProperty(navigator, 'platform', {
get: () => 'Win32',
configurable: true
});
Object.defineProperty(navigator, 'vendor', {
get: () => 'Google Inc.',
configurable: true
});
Object.defineProperty(navigator, 'deviceMemory', {
get: () => 8,
configurable: true
});
Object.defineProperty(navigator, 'hardwareConcurrency', {
get: () => 8,
configurable: true
});
Object.defineProperty(navigator, 'maxTouchPoints', {
get: () => 0,
configurable: true
});
// Override plugins to appear realistic
Object.defineProperty(navigator, 'plugins', {
get: () => {
return [
{
0: {type: "application/x-google-chrome-pdf", suffixes: "pdf", description: "Portable Document Format", enabledPlugin: Plugin},
description: "Portable Document Format",
filename: "internal-pdf-viewer",
length: 1,
name: "Chrome PDF Plugin"
},
{
0: {type: "application/pdf", suffixes: "pdf", description: "", enabledPlugin: Plugin},
description: "",
filename: "mhjfbmdgcfjbbpaeojofohoefgiehjai",
length: 1,
name: "Chrome PDF Viewer"
}
];
},
configurable: true
});
// Override permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
// Override WebGL rendering context
const getParameter = WebGLRenderingContext.prototype.getParameter;
WebGLRenderingContext.prototype.getParameter = function(parameter) {
if (parameter === 37445) { // UNMASKED_VENDOR_WEBGL
return 'Intel Inc.';
}
if (parameter === 37446) { // UNMASKED_RENDERER_WEBGL
return 'Intel(R) Iris(R) Plus Graphics 640';
}
return getParameter(parameter);
};
// Override canvas fingerprinting
const toBlob = HTMLCanvasElement.prototype.toBlob;
const toDataURL = HTMLCanvasElement.prototype.toDataURL;
const getImageData = CanvasRenderingContext2D.prototype.getImageData;
const noisify = (canvas, context) => {
const imageData = context.getImageData(0, 0, canvas.width, canvas.height);
for (let i = 0; i < imageData.data.length; i += 4) {
imageData.data[i] += Math.floor(Math.random() * 10) - 5;
imageData.data[i + 1] += Math.floor(Math.random() * 10) - 5;
imageData.data[i + 2] += Math.floor(Math.random() * 10) - 5;
}
context.putImageData(imageData, 0, 0);
};
Object.defineProperty(HTMLCanvasElement.prototype, 'toBlob', {
value: function(callback, type, encoderOptions) {
noisify(this, this.getContext('2d'));
return toBlob.apply(this, arguments);
}
});
Object.defineProperty(HTMLCanvasElement.prototype, 'toDataURL', {
value: function(type, encoderOptions) {
noisify(this, this.getContext('2d'));
return toDataURL.apply(this, arguments);
}
});
// Override AudioContext for audio fingerprinting
const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
const originalAnalyser = audioCtx.createAnalyser;
audioCtx.createAnalyser = function() {
const analyser = originalAnalyser.apply(this, arguments);
const getFloatFrequencyData = analyser.getFloatFrequencyData;
analyser.getFloatFrequencyData = function(array) {
getFloatFrequencyData.apply(this, arguments);
for (let i = 0; i < array.length; i++) {
array[i] += Math.random() * 0.0001;
}
};
return analyser;
};
// Override screen properties
Object.defineProperty(window.screen, 'colorDepth', {
get: () => 24,
configurable: true
});
Object.defineProperty(window.screen, 'pixelDepth', {
get: () => 24,
configurable: true
});
// Override timezone
Date.prototype.getTimezoneOffset = function() {
return -180; // UTC+3 (Istanbul)
};
// Override document.cookie to prevent tracking
const originalCookieDescriptor = Object.getOwnPropertyDescriptor(Document.prototype, 'cookie') ||
Object.getOwnPropertyDescriptor(HTMLDocument.prototype, 'cookie');
if (originalCookieDescriptor && originalCookieDescriptor.configurable) {
Object.defineProperty(document, 'cookie', {
get: function() {
return originalCookieDescriptor.get.call(this);
},
set: function(val) {
console.log('Cookie set blocked:', val);
return originalCookieDescriptor.set.call(this, val);
},
configurable: true
});
}
// Remove automation traces
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_JSON;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Object;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Proxy;
// Add realistic performance timing
if (window.performance && window.performance.timing) {
const timing = window.performance.timing;
const now = Date.now();
Object.defineProperty(timing, 'navigationStart', { value: now - Math.floor(Math.random() * 1000) + 1000, configurable: false });
Object.defineProperty(timing, 'loadEventEnd', { value: now - Math.floor(Math.random() * 100) + 100, configurable: false });
}
console.log('✓ Stealth scripts injected successfully');
'''
try:
await self.page.add_init_script(stealth_script)
logger.debug("✅ Stealth scripts injected successfully")
except Exception as e:
logger.warning(f"⚠️ Failed to inject stealth scripts: {e}")
async def _simulate_human_behavior(self, fast_mode: bool = True):
"""
Simulate realistic human behavior patterns to avoid detection.
Includes mouse movements, typing patterns, and natural delays.
Args:
fast_mode: If True, use minimal timing for speed optimization
"""
if not self.page:
logger.warning("Cannot simulate human behavior: page is None")
return
logger.debug("🤖 Simulating human behavior patterns...")
try:
if fast_mode:
# ULTRA-FAST MODE: Minimal human behavior
viewport_size = self.page.viewport_size
if viewport_size and random.random() < 0.7: # 70% chance to do movement
width, height = viewport_size['width'], viewport_size['height']
# Single quick mouse movement
x = random.randint(200, width - 200)
y = random.randint(200, height - 200)
await self.page.mouse.move(x, y)
# Brief scroll (50% chance)
if random.random() < 0.5:
await self.page.mouse.wheel(0, random.randint(50, 100))
# Ultra-minimal delay
await asyncio.sleep(random.uniform(0.05, 0.15)) # Reduced from 0.1-0.3
else:
# FULL MODE: Original comprehensive behavior
viewport_size = self.page.viewport_size
if viewport_size:
width, height = viewport_size['width'], viewport_size['height']
# Generate 3-5 random mouse movements
movements = random.randint(3, 5)
logger.debug(f" 🖱️ Performing {movements} random mouse movements")
for i in range(movements):
x = random.randint(100, width - 100)
y = random.randint(100, height - 100)
# Move mouse with realistic speed (not instant)
await self.page.mouse.move(x, y)
await asyncio.sleep(random.uniform(0.1, 0.3))
# 2. Scroll simulation
logger.debug(" 📜 Simulating scroll behavior")
scroll_amount = random.randint(100, 300)
await self.page.mouse.wheel(0, scroll_amount)
await asyncio.sleep(random.uniform(0.2, 0.5))
# Scroll back up
await self.page.mouse.wheel(0, -scroll_amount)
await asyncio.sleep(random.uniform(0.2, 0.4))
# 3. Random page interaction delays
await asyncio.sleep(random.uniform(0.5, 1.5))
logger.debug("✅ Human behavior simulation completed")
except Exception as e:
logger.warning(f"⚠️ Human behavior simulation failed: {e}")
async def _human_type(self, selector: str, text: str, clear_first: bool = True, fast_mode: bool = True):
"""
Type text with human-like patterns and delays.
Args:
selector: CSS selector for the input element
text: Text to type
clear_first: Whether to clear the field first
fast_mode: If True, use minimal delays for speed optimization
"""
if not self.page:
logger.warning("Cannot perform human typing: page is None")
return
try:
if fast_mode:
# FAST MODE: Direct fill for speed
await self.page.fill(selector, text)
await asyncio.sleep(random.uniform(0.02, 0.05)) # Reduced from 0.05-0.1
else:
# FULL MODE: Character-by-character human typing
# Focus on the element first
await self.page.focus(selector)
await asyncio.sleep(random.uniform(0.1, 0.3))
# Clear field if requested
if clear_first:
await self.page.keyboard.press('Control+a')
await asyncio.sleep(random.uniform(0.05, 0.15))
await self.page.keyboard.press('Delete')
await asyncio.sleep(random.uniform(0.05, 0.15))
# Type each character with human-like delays
for char in text:
await self.page.keyboard.type(char)
# Human typing speed: 50-150ms between characters
delay = random.uniform(0.05, 0.15)
# Occasional longer pauses (thinking)
if random.random() < 0.1: # 10% chance
delay += random.uniform(0.2, 0.8)
await asyncio.sleep(delay)
# Brief pause after typing
await asyncio.sleep(random.uniform(0.2, 0.6))
logger.debug(f"✅ Human-typed '{text}' into {selector}")
except Exception as e:
logger.warning(f"⚠️ Human typing failed: {e}")
async def _human_click(self, selector: str, wait_before: bool = True, wait_after: bool = True, fast_mode: bool = True):
"""
Perform a human-like click with realistic delays and mouse movement.
Args:
selector: CSS selector or element to click
wait_before: Whether to wait before clicking
wait_after: Whether to wait after clicking
fast_mode: If True, use minimal delays for speed optimization
"""
if not self.page:
logger.warning("Cannot perform human click: page is None")
return
try:
if fast_mode:
# FAST MODE: Direct click with minimal delay
if wait_before:
await asyncio.sleep(random.uniform(0.02, 0.08)) # Reduced from 0.05-0.15
await self.page.click(selector)
if wait_after:
await asyncio.sleep(random.uniform(0.02, 0.08)) # Reduced from 0.05-0.15
else:
# FULL MODE: Realistic mouse movement and timing
# Wait before clicking (thinking time)
if wait_before:
await asyncio.sleep(random.uniform(0.3, 0.8))
# Get element bounds for realistic mouse movement
element = await self.page.query_selector(selector)
if element:
box = await element.bounding_box()
if box:
# Move to element with slight randomness
center_x = box['x'] + box['width'] / 2
center_y = box['y'] + box['height'] / 2
# Add small random offset
offset_x = random.uniform(-10, 10)
offset_y = random.uniform(-5, 5)
await self.page.mouse.move(center_x + offset_x, center_y + offset_y)
await asyncio.sleep(random.uniform(0.1, 0.3))
# Perform click
await self.page.mouse.click(center_x + offset_x, center_y + offset_y)
logger.debug(f"✅ Human-clicked {selector}")
else:
# Fallback to regular click
await self.page.click(selector)
logger.debug(f"✅ Fallback-clicked {selector}")
else:
logger.warning(f"⚠️ Element not found for human click: {selector}")
return
# Wait after clicking (processing time)
if wait_after:
await asyncio.sleep(random.uniform(0.2, 0.6))
logger.debug(f"✅ Human-clicked {selector}")
except Exception as e:
logger.warning(f"⚠️ Human click failed: {e}")
async def _simulate_page_exploration(self, fast_mode: bool = True):
"""
Simulate natural page exploration before performing the main task.
This helps establish a more human-like session.
Args:
fast_mode: If True, use minimal exploration for speed optimization
"""
if not self.page:
return
logger.debug("🕵️ Simulating page exploration...")
try:
if fast_mode:
# ULTRA-FAST MODE: Minimal exploration
await asyncio.sleep(random.uniform(0.05, 0.1)) # Reduced from 0.1-0.3
# Single mouse movement (optional)
try:
elements = await self.page.query_selector_all("input, button")
if elements and random.random() < 0.5: # 50% chance to skip
element = random.choice(elements)
box = await element.bounding_box()
if box:
center_x = box['x'] + box['width'] / 2
center_y = box['y'] + box['height'] / 2
await self.page.mouse.move(center_x, center_y)
except:
pass
await asyncio.sleep(random.uniform(0.02, 0.05)) # Reduced from 0.05-0.15
else:
# FULL MODE: Comprehensive exploration
# 1. Brief pause to "read" the page
await asyncio.sleep(random.uniform(1.0, 2.5))
# 2. Move mouse to various UI elements (like a human would explore)
explore_selectors = [
"h1", "h2", ".navbar", "#header", ".logo",
"input", "button", "a", ".form-group"
]
explored = 0
for selector in explore_selectors:
elements = await self.page.query_selector_all(selector)
if elements and explored < 3: # Explore max 3 elements
element = random.choice(elements)
box = await element.bounding_box()
if box:
center_x = box['x'] + box['width'] / 2
center_y = box['y'] + box['height'] / 2
await self.page.mouse.move(center_x, center_y)
await asyncio.sleep(random.uniform(0.3, 0.8))
explored += 1
# 3. Small scroll to simulate reading
await self.page.mouse.wheel(0, random.randint(50, 150))
await asyncio.sleep(random.uniform(0.5, 1.2))
logger.debug("✅ Page exploration completed")
except Exception as e:
logger.debug(f"⚠️ Page exploration failed: {e}")
def _parse_decision_entries_from_soup(self, soup: BeautifulSoup, search_karar_tipi: KikKararTipi) -> List[KikDecisionEntry]:
entries: List[KikDecisionEntry] = []
table = soup.find("table", {"id": self.RESULTS_TABLE_ID})
if not table: return entries
logger.debug(f"Looking for table with ID: {self.RESULTS_TABLE_ID}")
if not table:
logger.warning(f"Table with ID '{self.RESULTS_TABLE_ID}' not found in HTML")
# Log available tables for debugging
all_tables = soup.find_all("table")
logger.debug(f"Found {len(all_tables)} tables in HTML")
for idx, tbl in enumerate(all_tables):
table_id = tbl.get('id', 'no-id')
table_class = tbl.get('class', 'no-class')
rows = tbl.find_all('tr')
logger.debug(f"Table {idx}: id='{table_id}', class='{table_class}', rows={len(rows)}")
# If this looks like a results table, try to use it
if (table_id and ('grd' in table_id.lower() or 'kurul' in table_id.lower() or 'sonuc' in table_id.lower())) or \
(isinstance(table_class, list) and any('grid' in cls.lower() or 'result' in cls.lower() for cls in table_class)) or \
len(rows) > 3: # Table with multiple rows might be results
logger.info(f"Trying to parse table {idx} as potential results table: id='{table_id}'")
table = tbl
break
if not table:
logger.error("No suitable results table found")
return entries
rows = table.find_all("tr")
logger.info(f"Found {len(rows)} rows in results table")
for row_idx, row in enumerate(rows):
if row_idx < 2: continue
# Skip first row (search bar with colspan=7) and second row (header with 6 cells)
if row_idx < 2:
logger.debug(f"Skipping header row {row_idx}")
continue
cells = row.find_all("td")
if len(cells) == 6:
logger.debug(f"Row {row_idx}: Found {len(cells)} cells")
# Log cell contents for debugging
if cells and row_idx < 5: # Log first few data rows
for cell_idx, cell in enumerate(cells):
cell_text = cell.get_text(strip=True)[:50] # First 50 chars
logger.debug(f" Cell {cell_idx}: '{cell_text}...'")
# Be more flexible with cell count - try 6 cells first, then adapt
if len(cells) >= 5: # At least 5 cells for minimum required data
try:
preview_button_tag = cells[0].find("a", id=re.compile(r"btnOnizle$"))
# Try to find preview button in first cell or any cell with a link
preview_button_tag = None
event_target = ""
if preview_button_tag and preview_button_tag.has_attr('href'):
match = re.search(r"__doPostBack\('([^']*)','([^']*)'\)", preview_button_tag['href'])
if match: event_target = match.group(1)
karar_no_span = cells[1].find("span", id=re.compile(r"lblKno$"))
karar_tarihi_span = cells[2].find("span", id=re.compile(r"lblKtar$"))
idare_span = cells[3].find("span", id=re.compile(r"lblIdare$"))
basvuru_sahibi_span = cells[4].find("span", id=re.compile(r"lblSikayetci$"))
ihale_span = cells[5].find("span", id=re.compile(r"lblIhale$"))
if not (event_target and karar_no_span and karar_tarihi_span): continue
# Look for preview button in first few cells
for cell_idx in range(min(3, len(cells))):
cell = cells[cell_idx]
# Try multiple patterns for preview button (based on actual HTML structure)
preview_candidates = [
cell.find("a", id="btnOnizle"), # Exact match
cell.find("a", id=re.compile(r"btnOnizle$")),
cell.find("a", id=re.compile(r"btn.*Onizle")),
cell.find("a", id=re.compile(r".*Onizle.*")),
cell.find("a", href=re.compile(r"__doPostBack"))
]
for candidate in preview_candidates:
if candidate and candidate.has_attr('href'):
match = re.search(r"__doPostBack\('([^']*)','([^']*)'\)", candidate['href'])
if match:
event_target = match.group(1)
preview_button_tag = candidate
logger.debug(f"Row {row_idx}: Found event_target '{event_target}' in cell {cell_idx}")
break
if preview_button_tag:
break
if not preview_button_tag:
logger.debug(f"Row {row_idx}: No preview button found in any cell")
# Log what links we found
for cell_idx, cell in enumerate(cells[:3]):
links_in_cell = cell.find_all("a")
logger.debug(f" Cell {cell_idx}: {len(links_in_cell)} links")
for link in links_in_cell[:2]:
logger.debug(f" Link id='{link.get('id')}', href='{link.get('href', '')[:50]}...'")
# Try to find decision data spans with more flexible patterns
karar_no_span = None
karar_tarihi_span = None
idare_span = None
basvuru_sahibi_span = None
ihale_span = None
# Try different span patterns for karar no (usually in cell 1)
for cell_idx in range(min(4, len(cells))):
if not karar_no_span:
cell = cells[cell_idx]
candidates = [
cell.find("span", id="lblKno"), # Exact match based on actual HTML
cell.find("span", id=re.compile(r"lblKno$")),
cell.find("span", id=re.compile(r".*Kno.*")),
cell.find("span", id=re.compile(r".*KararNo.*")),
cell.find("span", id=re.compile(r".*No.*"))
]
for candidate in candidates:
if candidate and candidate.get_text(strip=True):
karar_no_span = candidate
logger.debug(f"Row {row_idx}: Found karar_no in cell {cell_idx}")
break
# Try different patterns for karar tarihi (usually in cell 2)
for cell_idx in range(min(4, len(cells))):
if not karar_tarihi_span:
cell = cells[cell_idx]
candidates = [
cell.find("span", id="lblKtar"), # Exact match based on actual HTML
cell.find("span", id=re.compile(r"lblKtar$")),
cell.find("span", id=re.compile(r".*Ktar.*")),
cell.find("span", id=re.compile(r".*Tarih.*")),
cell.find("span", id=re.compile(r".*Date.*"))
]
for candidate in candidates:
if candidate and candidate.get_text(strip=True):
# Check if it looks like a date
text = candidate.get_text(strip=True)
if re.match(r'\d{1,2}[./]\d{1,2}[./]\d{4}', text):
karar_tarihi_span = candidate
logger.debug(f"Row {row_idx}: Found karar_tarihi in cell {cell_idx}")
break
# Find other spans in remaining cells (if we have 6 cells) - using exact IDs
if len(cells) >= 6:
idare_span = cells[3].find("span", id="lblIdare") or cells[3].find("span")
basvuru_sahibi_span = cells[4].find("span", id="lblSikayetci") or cells[4].find("span")
ihale_span = cells[5].find("span", id="lblIhale") or cells[5].find("span")
elif len(cells) == 5:
# Adjust for 5-cell layout
idare_span = cells[2].find("span") if cells[2] != cells[1] else None
basvuru_sahibi_span = cells[3].find("span") if len(cells) > 3 else None
ihale_span = cells[4].find("span") if len(cells) > 4 else None
# Log what we found
logger.debug(f"Row {row_idx}: karar_no_span={karar_no_span is not None}, "
f"karar_tarihi_span={karar_tarihi_span is not None}, "
f"event_target={bool(event_target)}")
# For KIK, we need at least karar_no and karar_tarihi, event_target is helpful but not critical
if not (karar_no_span and karar_tarihi_span):
logger.debug(f"Row {row_idx}: Missing required fields (karar_no or karar_tarihi), skipping")
# Log what spans we found in cells
for i, cell in enumerate(cells):
spans = cell.find_all("span")
if spans:
span_info = []
for s in spans:
span_id = s.get('id', 'no-id')
span_text = s.get_text(strip=True)[:20]
span_info.append(f"{span_id}:'{span_text}...'")
logger.debug(f" Cell {i} spans: {span_info}")
continue
# If we don't have event_target, we can still create an entry but mark it specially
if not event_target:
logger.warning(f"Row {row_idx}: No event_target found, document retrieval won't work")
event_target = f"missing_target_row_{row_idx}" # Placeholder
# Karar tipini arama parametresinden alıyoruz, çünkü HTML'de direkt olarak bulunmuyor.
try:
entry = KikDecisionEntry(
preview_event_target=event_target,
kararNo=karar_no_span.get_text(strip=True),
@@ -132,8 +795,17 @@ class KikApiClient:
ihaleKonusu=ihale_span.get_text(strip=True) if ihale_span else None,
)
entries.append(entry)
logger.info(f"Row {row_idx}: Successfully parsed decision: {entry.karar_no_str}")
except Exception as e:
logger.error(f"Error parsing a KIK decision entry row: {e}", exc_info=True)
logger.error(f"Row {row_idx}: Error creating KikDecisionEntry: {e}")
continue
except Exception as e:
logger.error(f"Error parsing row {row_idx}: {e}", exc_info=True)
else:
logger.debug(f"Row {row_idx}: Expected at least 5 cells but found {len(cells)}, skipping")
logger.info(f"Parsed {len(entries)} decision entries from {len(rows)} rows")
return entries
def _parse_total_records_from_soup(self, soup: BeautifulSoup) -> int:
@@ -163,6 +835,10 @@ class KikApiClient:
try:
if page.url != search_url:
await page.goto(search_url, wait_until="networkidle", timeout=self.request_timeout)
# Simulate natural page exploration after navigation (FAST MODE)
await self._simulate_page_exploration(fast_mode=True)
search_button_selector = f"a[id='{self.FIELD_LOCATORS['search_button_id']}']"
await page.wait_for_selector(search_button_selector, state="visible", timeout=self.request_timeout)
@@ -170,12 +846,18 @@ class KikApiClient:
radio_locator_selector = f"{self.FIELD_LOCATORS['karar_tipi_radio_group']}[value='{current_karar_tipi_value}']"
if not await page.locator(radio_locator_selector).is_checked():
js_target_radio = f"ctl00$ContentPlaceHolder1${current_karar_tipi_value}"
logger.info(f"Selecting radio button: {js_target_radio}")
async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout):
await page.evaluate(f"javascript:__doPostBack('{js_target_radio}','')")
await page.wait_for_timeout(1000)
# Ultra-fast wait for page to stabilize after radio button change
await page.wait_for_timeout(300) # Reduced from 1000ms
logger.info("Radio button selection completed")
async def fill_if_value(selector_key: str, value: Optional[str]):
if value is not None: await page.fill(self.FIELD_LOCATORS[selector_key], value)
# Helper function for human-like form filling (FAST MODE)
async def human_fill_if_value(selector_key: str, value: Optional[str]):
if value is not None:
selector = self.FIELD_LOCATORS[selector_key]
await self._human_type(selector, value, fast_mode=True)
# Karar No'yu KİK sitesine göndermeden önce '_' -> '/' dönüşümü yap
karar_no_for_kik_form = None
@@ -183,43 +865,105 @@ class KikApiClient:
karar_no_for_kik_form = search_params.karar_no.replace('_', '/')
logger.info(f"Using karar_no '{karar_no_for_kik_form}' (transformed from '{search_params.karar_no}') for KIK form.")
await fill_if_value('karar_metni', search_params.karar_metni)
await fill_if_value('karar_no', karar_no_for_kik_form) # Dönüştürülmüş halini kullan
# ... (diğer fill_if_value çağrıları aynı) ...
await fill_if_value('karar_tarihi_baslangic', search_params.karar_tarihi_baslangic)
await fill_if_value('karar_tarihi_bitis', search_params.karar_tarihi_bitis)
await fill_if_value('resmi_gazete_sayisi', search_params.resmi_gazete_sayisi)
await fill_if_value('resmi_gazete_tarihi', search_params.resmi_gazete_tarihi)
await fill_if_value('basvuru_konusu_ihale', search_params.basvuru_konusu_ihale)
await fill_if_value('basvuru_sahibi', search_params.basvuru_sahibi)
await fill_if_value('ihaleyi_yapan_idare', search_params.ihaleyi_yapan_idare)
# Fill form fields with FAST human-like behavior
logger.info("Filling form fields with fast mode...")
# Start with FAST mouse behavior simulation
await self._simulate_human_behavior(fast_mode=True)
await human_fill_if_value('karar_metni', search_params.karar_metni)
await human_fill_if_value('karar_no', karar_no_for_kik_form) # Dönüştürülmüş halini kullan
await human_fill_if_value('karar_tarihi_baslangic', search_params.karar_tarihi_baslangic)
await human_fill_if_value('karar_tarihi_bitis', search_params.karar_tarihi_bitis)
await human_fill_if_value('resmi_gazete_sayisi', search_params.resmi_gazete_sayisi)
await human_fill_if_value('resmi_gazete_tarihi', search_params.resmi_gazete_tarihi)
await human_fill_if_value('basvuru_konusu_ihale', search_params.basvuru_konusu_ihale)
await human_fill_if_value('basvuru_sahibi', search_params.basvuru_sahibi)
await human_fill_if_value('ihaleyi_yapan_idare', search_params.ihaleyi_yapan_idare)
if search_params.yil:
await page.select_option(self.FIELD_LOCATORS['yil'], value=search_params.yil)
await page.wait_for_timeout(50) # Reduced from 100ms
logger.info("Form filling completed, preparing for search...")
# Additional FAST human behavior before search
await self._simulate_human_behavior(fast_mode=True)
action_is_search_button_click = (search_params.page == 1)
event_target_for_submit: str
if action_is_search_button_click:
event_target_for_submit = self.FIELD_LOCATORS['search_button_id']
else: # Pagination
page_link_ctl_number = search_params.page + 2
event_target_for_submit = f"ctl00$ContentPlaceHolder1$grdKurulKararSorguSonuc$ctl14$ctl{page_link_ctl_number:02d}"
try:
async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout):
if action_is_search_button_click:
await page.locator(search_button_selector).click()
else:
event_target_for_submit = self.FIELD_LOCATORS['search_button_id']
# Use human-like clicking for search button
search_button_selector = f"a[id='{event_target_for_submit}']"
logger.info(f"Performing human-like search button click...")
try:
# FAST Human-like click on search button
await self._human_click(search_button_selector, wait_before=True, wait_after=False, fast_mode=True)
# Wait for navigation
await page.wait_for_load_state("networkidle", timeout=self.request_timeout)
logger.info("Search navigation completed successfully")
except Exception as e:
logger.warning(f"Human click failed, falling back to JavaScript: {e}")
# Fallback to original method
async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout):
await page.evaluate(f"javascript:__doPostBack('{event_target_for_submit}','')")
logger.info("Search navigation completed via fallback")
else:
# Pagination - use original method for consistency
page_link_ctl_number = search_params.page + 2
event_target_for_submit = f"ctl00$ContentPlaceHolder1$grdKurulKararSorguSonuc$ctl14$ctl{page_link_ctl_number:02d}"
logger.info(f"Executing pagination with event target: {event_target_for_submit}")
async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout):
await page.evaluate(f"javascript:__doPostBack('{event_target_for_submit}','')")
logger.info("Pagination navigation completed successfully")
except PlaywrightTimeoutError:
await page.wait_for_timeout(2000)
logger.warning("Search navigation timed out, but continuing...")
await page.wait_for_timeout(5000) # Longer wait if navigation fails
# Ultra-fast wait time for results to load
logger.info("Waiting for search results to load...")
await page.wait_for_timeout(500) # Reduced from 1000ms
results_table_dom_selector = f"table#{self.RESULTS_TABLE_ID}"
try:
await page.wait_for_selector(results_table_dom_selector, timeout=30000, state="attached")
await page.wait_for_timeout(2000)
# First wait for any tables to appear (more general check)
logger.info("Waiting for any tables to appear...")
await page.wait_for_function("""
() => document.querySelectorAll('table').length > 0
""", timeout=4000) # Reduced from 8000ms
logger.info("At least one table appeared")
# Then wait for our specific table
await page.wait_for_selector(results_table_dom_selector, timeout=4000, state="attached") # Reduced from 8000ms
logger.debug("Results table attached to DOM")
# Wait for table to have some content (more than just headers)
await page.wait_for_function(f"""
() => {{
const table = document.querySelector('{results_table_dom_selector}');
return table && table.querySelectorAll('tr').length > 2;
}}
""", timeout=4000) # Reduced from 20000ms
logger.debug("Results table populated with data")
# Ultra-fast additional wait for any remaining JavaScript
await page.wait_for_timeout(500) # Reduced from 3000ms
except PlaywrightTimeoutError:
logger.warning(f"Timeout waiting for results table '{results_table_dom_selector}'.")
# Try one more wait for content placeholder
try:
await page.wait_for_selector("#ctl00_ContentPlaceHolder1", timeout=10000)
logger.info("ContentPlaceHolder1 found, checking for tables...")
await page.wait_for_timeout(5000)
except PlaywrightTimeoutError:
logger.warning("ContentPlaceHolder1 also not found - content may not have loaded")
html_content = await page.content()
soup = BeautifulSoup(html_content, "html.parser")
@@ -251,16 +995,17 @@ class KikApiClient:
# ... (öncekiyle aynı) ...
if not html_fragment: return None
cleaned_html = self._clean_html_for_markdown(html_fragment)
markdown_output = None; temp_file_path = None
markdown_output = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = cleaned_html.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown(enable_plugins=True, remove_alt_whitespace=True, keep_underline=True)
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_html_file:
tmp_html_file.write(cleaned_html); temp_file_path = tmp_html_file.name
markdown_output = md_converter.convert(temp_file_path).text_content
markdown_output = md_converter.convert(html_stream).text_content
if markdown_output: markdown_output = re.sub(r'\n{3,}', '\n\n', markdown_output).strip()
except Exception as e: logger.error(f"MarkItDown conversion error: {e}", exc_info=True)
finally:
if temp_file_path and os.path.exists(temp_file_path): os.remove(temp_file_path)
return markdown_output
+1
View File
@@ -0,0 +1 @@
# kvkk_mcp_module/__init__.py
+372
View File
@@ -0,0 +1,372 @@
# kvkk_mcp_module/client.py
import httpx
from bs4 import BeautifulSoup
from typing import List, Optional, Dict, Any
import logging
import os
import re
import io
import math
from urllib.parse import urljoin, urlparse, parse_qs
from markitdown import MarkItDown
from pydantic import HttpUrl
from .models import (
KvkkSearchRequest,
KvkkDecisionSummary,
KvkkSearchResult,
KvkkDocumentMarkdown
)
logger = logging.getLogger(__name__)
if not logger.hasHandlers():
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
class KvkkApiClient:
"""
API client for searching and retrieving KVKK (Personal Data Protection Authority) decisions
using Brave Search API for discovery and direct HTTP requests for content retrieval.
"""
BRAVE_API_URL = "https://api.search.brave.com/res/v1/web/search"
KVKK_BASE_URL = "https://www.kvkk.gov.tr"
DOCUMENT_MARKDOWN_CHUNK_SIZE = 5000 # Character limit per page
def __init__(self, request_timeout: float = 60.0):
"""Initialize the KVKK API client."""
self.brave_api_token = os.getenv("BRAVE_API_TOKEN")
if not self.brave_api_token:
# Fallback to provided free token
self.brave_api_token = "BSAuaRKB-dvSDSQxIN0ft1p2k6N82Kq"
logger.info("Using fallback Brave API token (limited free token)")
else:
logger.info("Using Brave API token from environment variable")
self.http_client = httpx.AsyncClient(
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
},
timeout=request_timeout,
verify=True,
follow_redirects=True
)
def _construct_search_query(self, keywords: str) -> str:
"""Construct the search query for Brave API."""
base_query = 'site:kvkk.gov.tr "karar özeti"'
if keywords.strip():
return f"{base_query} {keywords.strip()}"
return base_query
def _extract_decision_id_from_url(self, url: str) -> Optional[str]:
"""Extract decision ID from KVKK decision URL."""
try:
# Example URL: https://www.kvkk.gov.tr/Icerik/7288/2021-1303
parsed_url = urlparse(url)
path_parts = parsed_url.path.strip('/').split('/')
if len(path_parts) >= 3 and path_parts[0] == 'Icerik':
# Extract the decision ID from the path
decision_id = '/'.join(path_parts[1:]) # e.g., "7288/2021-1303"
return decision_id
except Exception as e:
logger.debug(f"Could not extract decision ID from URL {url}: {e}")
return None
def _extract_decision_metadata_from_title(self, title: str) -> Dict[str, Optional[str]]:
"""Extract decision metadata from title string."""
metadata = {
"decision_date": None,
"decision_number": None
}
if not title:
return metadata
# Extract decision date (DD/MM/YYYY format)
date_match = re.search(r'(\d{1,2}/\d{1,2}/\d{4})', title)
if date_match:
metadata["decision_date"] = date_match.group(1)
# Extract decision number (YYYY/XXXX format)
number_match = re.search(r'(\d{4}/\d+)', title)
if number_match:
metadata["decision_number"] = number_match.group(1)
return metadata
async def search_decisions(self, params: KvkkSearchRequest) -> KvkkSearchResult:
"""Search for KVKK decisions using Brave API."""
search_query = self._construct_search_query(params.keywords)
logger.info(f"KvkkApiClient: Searching with query: {search_query}")
try:
# Calculate offset for pagination
offset = (params.page - 1) * params.pageSize
response = await self.http_client.get(
self.BRAVE_API_URL,
headers={
"Accept": "application/json",
"Accept-Encoding": "gzip",
"x-subscription-token": self.brave_api_token
},
params={
"q": search_query,
"country": "TR",
"search_lang": "tr",
"ui_lang": "tr-TR",
"offset": offset,
"count": params.pageSize
}
)
response.raise_for_status()
data = response.json()
# Extract search results
decisions = []
web_results = data.get("web", {}).get("results", [])
for result in web_results:
title = result.get("title", "")
url = result.get("url", "")
description = result.get("description", "")
# Extract metadata from title
metadata = self._extract_decision_metadata_from_title(title)
# Extract decision ID from URL
decision_id = self._extract_decision_id_from_url(url)
decision = KvkkDecisionSummary(
title=title,
url=HttpUrl(url) if url else None,
description=description,
decision_id=decision_id,
publication_date=metadata.get("decision_date"),
decision_number=metadata.get("decision_number")
)
decisions.append(decision)
# Get total results if available
total_results = None
query_info = data.get("query", {})
if "total_results" in query_info:
total_results = query_info["total_results"]
return KvkkSearchResult(
decisions=decisions,
total_results=total_results,
page=params.page,
pageSize=params.pageSize,
query=search_query
)
except httpx.RequestError as e:
logger.error(f"KvkkApiClient: HTTP request error during search: {e}")
return KvkkSearchResult(
decisions=[],
total_results=0,
page=params.page,
pageSize=params.pageSize,
query=search_query
)
except Exception as e:
logger.error(f"KvkkApiClient: Unexpected error during search: {e}")
return KvkkSearchResult(
decisions=[],
total_results=0,
page=params.page,
pageSize=params.pageSize,
query=search_query
)
def _extract_decision_content_from_html(self, html: str, url: str) -> Dict[str, Any]:
"""Extract decision content from KVKK decision page HTML."""
try:
soup = BeautifulSoup(html, 'html.parser')
# Extract title
title = None
title_element = soup.find('h3', class_='blog-post-title')
if title_element:
title = title_element.get_text(strip=True)
elif soup.title:
title = soup.title.get_text(strip=True)
# Extract decision content from the main content div
content_div = soup.find('div', class_='blog-post-inner')
if not content_div:
# Fallback to other possible content containers
content_div = soup.find('div', style='text-align:justify;')
if not content_div:
logger.warning(f"Could not find decision content div in {url}")
return {
"title": title,
"decision_date": None,
"decision_number": None,
"subject_summary": None,
"html_content": None
}
# Extract decision metadata from table
decision_date = None
decision_number = None
subject_summary = None
table = content_div.find('table')
if table:
rows = table.find_all('tr')
for row in rows:
cells = row.find_all('td')
if len(cells) >= 3:
field_name = cells[0].get_text(strip=True)
field_value = cells[2].get_text(strip=True)
if 'Karar Tarihi' in field_name:
decision_date = field_value
elif 'Karar No' in field_name:
decision_number = field_value
elif 'Konu Özeti' in field_name:
subject_summary = field_value
return {
"title": title,
"decision_date": decision_date,
"decision_number": decision_number,
"subject_summary": subject_summary,
"html_content": str(content_div)
}
except Exception as e:
logger.error(f"Error extracting content from HTML for {url}: {e}")
return {
"title": None,
"decision_date": None,
"decision_number": None,
"subject_summary": None,
"html_content": None
}
def _convert_html_to_markdown(self, html_content: str) -> Optional[str]:
"""Convert HTML content to Markdown using MarkItDown with BytesIO to avoid filename length issues."""
if not html_content:
return None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown(enable_plugins=False)
result = md_converter.convert(html_stream)
return result.text_content
except Exception as e:
logger.error(f"Error converting HTML to Markdown: {e}")
return None
async def get_decision_document(self, decision_url: str, page_number: int = 1) -> KvkkDocumentMarkdown:
"""Retrieve and convert a KVKK decision document to paginated Markdown."""
logger.info(f"KvkkApiClient: Getting decision document from: {decision_url}, page: {page_number}")
try:
# Fetch the decision page
response = await self.http_client.get(decision_url)
response.raise_for_status()
# Extract content from HTML
extracted_data = self._extract_decision_content_from_html(response.text, decision_url)
# Convert HTML content to Markdown
full_markdown_content = None
if extracted_data["html_content"]:
full_markdown_content = self._convert_html_to_markdown(extracted_data["html_content"])
if not full_markdown_content:
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=extracted_data["title"],
decision_date=extracted_data["decision_date"],
decision_number=extracted_data["decision_number"],
subject_summary=extracted_data["subject_summary"],
markdown_chunk=None,
current_page=page_number,
total_pages=0,
is_paginated=False,
error_message="Could not convert document content to Markdown"
)
# Calculate pagination
content_length = len(full_markdown_content)
total_pages = math.ceil(content_length / self.DOCUMENT_MARKDOWN_CHUNK_SIZE)
if total_pages == 0:
total_pages = 1
# Clamp page number to valid range
current_page_clamped = max(1, min(page_number, total_pages))
# Extract the requested chunk
start_index = (current_page_clamped - 1) * self.DOCUMENT_MARKDOWN_CHUNK_SIZE
end_index = start_index + self.DOCUMENT_MARKDOWN_CHUNK_SIZE
markdown_chunk = full_markdown_content[start_index:end_index]
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=extracted_data["title"],
decision_date=extracted_data["decision_date"],
decision_number=extracted_data["decision_number"],
subject_summary=extracted_data["subject_summary"],
markdown_chunk=markdown_chunk,
current_page=current_page_clamped,
total_pages=total_pages,
is_paginated=(total_pages > 1),
error_message=None
)
except httpx.HTTPStatusError as e:
error_msg = f"HTTP error {e.response.status_code} when fetching decision document"
logger.error(f"KvkkApiClient: {error_msg}")
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number,
total_pages=0,
is_paginated=False,
error_message=error_msg
)
except Exception as e:
error_msg = f"Unexpected error when fetching decision document: {str(e)}"
logger.error(f"KvkkApiClient: {error_msg}")
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number,
total_pages=0,
is_paginated=False,
error_message=error_msg
)
async def close_client_session(self):
"""Close the HTTP client session."""
if hasattr(self, 'http_client') and self.http_client and not self.http_client.is_closed:
await self.http_client.aclose()
logger.info("KvkkApiClient: HTTP client session closed.")
+49
View File
@@ -0,0 +1,49 @@
# kvkk_mcp_module/models.py
from pydantic import BaseModel, Field, HttpUrl
from typing import List, Optional, Any
class KvkkSearchRequest(BaseModel):
"""Model for KVKK (Personal Data Protection Authority) search request via Brave API."""
keywords: str = Field(..., description="""
Keywords to search for in KVKK decisions.
The search will automatically include 'site:kvkk.gov.tr "karar özeti"' to target KVKK decision summaries.
Examples: "açık rıza", "veri güvenliği", "kişisel veri işleme"
""")
page: int = Field(1, ge=1, le=50, description="Page number for search results (1-50).")
pageSize: int = Field(10, ge=1, le=20, description="Number of results per page (1-20).")
class KvkkDecisionSummary(BaseModel):
"""Model for a single KVKK decision summary from Brave search results."""
title: Optional[str] = Field(None, description="Decision title from search results.")
url: Optional[HttpUrl] = Field(None, description="URL to the KVKK decision page.")
description: Optional[str] = Field(None, description="Brief description or snippet from search results.")
decision_id: Optional[str] = Field(None, description="Extracted decision ID from URL (e.g., Icerik/7288/2021-1303).")
publication_date: Optional[str] = Field(None, description="Publication date if extractable from title or description.")
decision_number: Optional[str] = Field(None, description="Decision number if extractable from title or description.")
class KvkkSearchResult(BaseModel):
"""Model for the overall search result for KVKK decisions."""
decisions: List[KvkkDecisionSummary] = Field(default_factory=list, description="List of KVKK decisions found.")
total_results: Optional[int] = Field(None, description="Total number of results available (if provided by Brave API).")
page: int = Field(1, description="Current page number of results.")
pageSize: int = Field(10, description="Number of results per page.")
query: Optional[str] = Field(None, description="The actual search query sent to Brave API.")
class KvkkDocumentMarkdown(BaseModel):
"""Model for KVKK decision document content converted to paginated Markdown."""
source_url: HttpUrl = Field(description="URL of the original KVKK decision page.")
title: Optional[str] = Field(None, description="Title of the KVKK decision.")
decision_date: Optional[str] = Field(None, description="Decision date (Karar Tarihi).")
decision_number: Optional[str] = Field(None, description="Decision number (Karar No).")
subject_summary: Optional[str] = Field(None, description="Subject summary (Konu Özeti).")
markdown_chunk: Optional[str] = Field(None, description="A 5,000 character chunk of the Markdown content.")
current_page: int = Field(description="The current page number of the markdown chunk (1-indexed).")
total_pages: int = Field(description="Total number of pages for the full markdown content.")
is_paginated: bool = Field(description="True if the full markdown content is split into multiple pages.")
error_message: Optional[str] = Field(None, description="Error message if document retrieval or conversion failed.")
class Config:
json_encoders = {
HttpUrl: str
}
+107 -62
View File
@@ -135,19 +135,17 @@ async def authorize_endpoint(
@router.get("/auth/callback")
async def oauth_callback(
request: Request,
state: Optional[str] = Query(None)
state: Optional[str] = Query(None),
clerk_token: Optional[str] = Query(None)
):
"""Handle OAuth callback from Clerk - simplified for custom domains"""
"""Handle OAuth callback from Clerk - supports both JWT token and cookie auth"""
logger.info(f"OAuth callback received - state: {state}")
logger.info(f"Query params: {dict(request.query_params)}")
logger.info(f"Cookies: {dict(request.cookies)}")
logger.info(f"Clerk JWT token provided: {bool(clerk_token)}")
# For Clerk custom domains, we'll assume authentication succeeded
# if Clerk redirected the user to our callback URL
# For custom domains, we'll skip complex session verification
# and rely on the fact that Clerk only redirects here after successful auth
# Support both JWT token (for cross-domain) and cookie auth (for subdomain)
try:
if not state:
@@ -189,17 +187,79 @@ async def oauth_callback(
content={"error": "invalid_request", "error_description": "OAuth session expired or not found"}
)
# Check if we have a JWT token (for cross-domain auth)
user_authenticated = False
auth_method = "none"
if clerk_token:
logger.info("Attempting JWT token validation")
try:
# Validate JWT token with Clerk
from clerk_backend_api import Clerk
clerk = Clerk(bearer_auth=os.getenv("CLERK_SECRET_KEY"))
# Extract session_id from JWT token and verify with Clerk
import jwt
decoded_token = jwt.decode(clerk_token, options={"verify_signature": False})
session_id = decoded_token.get("sid") or decoded_token.get("session_id")
if session_id:
# Verify with Clerk using session_id
session = clerk.sessions.verify(session_id=session_id, token=clerk_token)
user_id = session.user_id if session else None
else:
user_id = None
if user_id:
logger.info(f"JWT token validation successful - user_id: {user_id}")
user_authenticated = True
auth_method = "jwt_token"
# Store user info in session for token exchange
oauth_session["user_id"] = user_id
oauth_session["auth_method"] = "jwt_token"
else:
logger.error("JWT token validation failed - no user_id in claims")
except Exception as e:
logger.error(f"JWT token validation failed: {str(e)}")
# Fall through to cookie validation
# If no JWT token or validation failed, check cookies
if not user_authenticated:
logger.info("Checking for Clerk session cookies")
# Check for Clerk session cookies (for subdomain auth)
clerk_session_cookie = request.cookies.get("__session")
if clerk_session_cookie:
logger.info("Found Clerk session cookie, assuming authenticated")
user_authenticated = True
auth_method = "cookie"
oauth_session["auth_method"] = "cookie"
else:
logger.info("No Clerk session cookie found")
# For custom domains, we'll also trust that Clerk redirected here
if not user_authenticated:
logger.info("Trusting Clerk redirect for custom domain flow")
user_authenticated = True
auth_method = "trusted_redirect"
oauth_session["auth_method"] = "trusted_redirect"
logger.info(f"User authenticated: {user_authenticated}, method: {auth_method}")
# Generate simple authorization code for custom domain flow
auth_code = f"clerk_custom_{session_id}_{int(time.time())}"
# Store the code mapping for token exchange
code_data = {
"session_id": session_id,
"clerk_authenticated": True,
"clerk_authenticated": user_authenticated,
"auth_method": auth_method,
"custom_domain_flow": True,
"created_at": time.time(),
"expires_at": (datetime.utcnow() + timedelta(minutes=5)).timestamp(),
}
if "user_id" in oauth_session:
code_data["user_id"] = oauth_session["user_id"]
oauth_provider.storage.set_session(f"code_{auth_code}", code_data)
# Build redirect URL back to Claude
@@ -264,70 +324,55 @@ async def token_endpoint(request: Request):
)
try:
# Import here to avoid circular imports
from mcp_server_main import app as mcp_app
from mcp_auth_factory import get_oauth_provider
# OAuth token exchange - validate code and return Clerk JWT
# This supports proper OAuth flow while using Clerk JWT tokens
# Get OAuth provider
oauth_provider = get_oauth_provider(mcp_app)
if not oauth_provider:
raise HTTPException(status_code=500, detail="OAuth provider not configured")
# Extract session info from code
code_session = None
if code.startswith("clerk_"):
# Get the code mapping
code_session = oauth_provider.storage.get_session(f"code_{code}")
if code_session:
session_id = code_session.get("session_id")
else:
logger.error(f"Code mapping not found for: {code}")
if not code or not redirect_uri:
logger.error("Missing required parameters: code or redirect_uri")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Invalid authorization code"}
)
else:
session_id = code
session = oauth_provider.storage.get_session(session_id)
if not session:
logger.error(f"Session {session_id} not found for token exchange")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Invalid authorization code"}
content={"error": "invalid_request", "error_description": "Missing code or redirect_uri"}
)
# Validate PKCE if present
if "pkce_challenge" in session and code_verifier:
# Validate PKCE challenge
if not oauth_provider.validate_pkce(code_verifier, session["pkce_challenge"]):
logger.error("PKCE challenge validation failed")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Invalid code verifier"}
)
logger.info("PKCE validation successful")
else:
logger.info("No PKCE validation required")
# Validate OAuth code with Clerk
if CLERK_AVAILABLE:
try:
clerk = Clerk(bearer_auth=os.getenv("CLERK_SECRET_KEY"))
# Create JWT token
access_token = oauth_provider._create_mcp_token(
session["scopes"],
session.get("clerk_token", ""),
session_id
)
# Clean up sessions
oauth_provider.storage.delete_session(session_id)
if code_session:
oauth_provider.storage.delete_session(f"code_{code}")
# In a real implementation, you'd validate the code with Clerk
# For now, we'll assume the code is valid if it looks like a Clerk code
if len(code) > 10: # Basic validation
# Create a mock session with the code
# In practice, this would be validated with Clerk's OAuth flow
# Return Clerk JWT token format
# This should be the actual Clerk JWT token from the OAuth flow
return JSONResponse({
"access_token": access_token,
"access_token": f"mock_clerk_jwt_{code}",
"token_type": "Bearer",
"expires_in": 3600,
"scope": " ".join(session["scopes"])
"scope": "yargi.read yargi.search"
})
else:
logger.error(f"Invalid code format: {code}")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Invalid authorization code"}
)
except Exception as e:
logger.error(f"Clerk validation failed: {e}")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Authorization code validation failed"}
)
else:
logger.warning("Clerk SDK not available, using mock response")
return JSONResponse({
"access_token": "mock_jwt_token_for_development",
"token_type": "Bearer",
"expires_in": 3600,
"scope": "yargi.read yargi.search"
})
except Exception as e:
+522
View File
@@ -0,0 +1,522 @@
"""
Simplified MCP OAuth HTTP adapter - only Clerk JWT based authentication
Uses Redis for authorization code storage to support multi-machine deployment
"""
import os
import logging
from typing import Optional
from urllib.parse import urlencode, quote
from fastapi import APIRouter, Request, Query, HTTPException
from fastapi.responses import RedirectResponse, JSONResponse
# Import Redis session store
from redis_session_store import get_redis_store
# Try to import Clerk SDK
try:
from clerk_backend_api import Clerk
CLERK_AVAILABLE = True
except ImportError:
CLERK_AVAILABLE = False
Clerk = None
logger = logging.getLogger(__name__)
router = APIRouter()
# OAuth configuration
BASE_URL = os.getenv("BASE_URL", "https://api.yargimcp.com")
CLERK_DOMAIN = os.getenv("CLERK_DOMAIN", "accounts.yargimcp.com")
# Initialize Redis store
redis_store = None
def get_redis_session_store():
"""Get Redis store instance with lazy initialization."""
global redis_store
if redis_store is None:
try:
import concurrent.futures
import functools
# Use thread pool with timeout to prevent hanging
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(get_redis_store)
try:
# 5 second timeout for Redis initialization
redis_store = future.result(timeout=5.0)
if redis_store:
logger.info("Redis session store initialized for OAuth handler")
else:
logger.warning("Redis store initialization returned None")
except concurrent.futures.TimeoutError:
logger.error("Redis initialization timed out after 5 seconds")
redis_store = None
future.cancel() # Try to cancel the hanging operation
except Exception as e:
logger.error(f"Failed to initialize Redis store: {e}")
redis_store = None
if redis_store is None:
# Fall back to in-memory storage with warning
logger.warning("Falling back to in-memory storage - multi-machine deployment will not work")
return redis_store
@router.get("/.well-known/oauth-authorization-server")
async def get_oauth_metadata():
"""OAuth 2.0 Authorization Server Metadata (RFC 8414)"""
return JSONResponse({
"issuer": BASE_URL,
"authorization_endpoint": "https://yargimcp.com/mcp-callback",
"token_endpoint": f"{BASE_URL}/token",
"registration_endpoint": f"{BASE_URL}/register",
"response_types_supported": ["code"],
"grant_types_supported": ["authorization_code"],
"code_challenge_methods_supported": ["S256"],
"token_endpoint_auth_methods_supported": ["none"],
"scopes_supported": ["read", "search", "openid", "profile", "email"],
"service_documentation": f"{BASE_URL}/mcp/"
})
@router.get("/auth/login")
async def oauth_authorize(
request: Request,
client_id: str = Query(...),
redirect_uri: str = Query(...),
response_type: str = Query("code"),
scope: Optional[str] = Query("read search"),
state: Optional[str] = Query(None),
code_challenge: Optional[str] = Query(None),
code_challenge_method: Optional[str] = Query(None)
):
"""OAuth 2.1 Authorization Endpoint - redirects to Clerk"""
logger.info(f"OAuth authorize request - client_id: {client_id}")
logger.info(f"Redirect URI: {redirect_uri}")
logger.info(f"State: {state}")
logger.info(f"PKCE Challenge: {bool(code_challenge)}")
try:
# Build callback URL with all necessary parameters
callback_url = f"{BASE_URL}/auth/callback"
callback_params = {
"client_id": client_id,
"redirect_uri": redirect_uri,
"state": state or "",
"scope": scope or "read search"
}
# Add PKCE parameters if present
if code_challenge:
callback_params["code_challenge"] = code_challenge
callback_params["code_challenge_method"] = code_challenge_method or "S256"
# Encode callback URL as redirect_url for Clerk
callback_with_params = f"{callback_url}?{urlencode(callback_params)}"
# Build Clerk sign-in URL - use yargimcp.com frontend for JWT token generation
clerk_params = {
"redirect_url": callback_with_params
}
# Use frontend sign-in page that handles JWT token generation
clerk_signin_url = f"https://yargimcp.com/sign-in?{urlencode(clerk_params)}"
logger.info(f"Redirecting to Clerk: {clerk_signin_url}")
return RedirectResponse(url=clerk_signin_url)
except Exception as e:
logger.exception(f"Authorization failed: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/auth/callback")
async def oauth_callback(
request: Request,
client_id: str = Query(...),
redirect_uri: str = Query(...),
state: Optional[str] = Query(None),
scope: Optional[str] = Query("read search"),
code_challenge: Optional[str] = Query(None),
code_challenge_method: Optional[str] = Query(None),
clerk_token: Optional[str] = Query(None)
):
"""OAuth callback from Clerk - generates authorization code"""
logger.info(f"OAuth callback - client_id: {client_id}")
logger.info(f"Clerk token provided: {bool(clerk_token)}")
try:
# Validate user with Clerk and generate real JWT token
user_authenticated = False
user_id = None
session_id = None
real_jwt_token = None
if clerk_token and CLERK_AVAILABLE:
try:
# Extract user info from JWT token (no Clerk session verification needed)
import jwt
decoded_token = jwt.decode(clerk_token, options={"verify_signature": False})
user_id = decoded_token.get("user_id") or decoded_token.get("sub")
user_email = decoded_token.get("email")
token_scopes = decoded_token.get("scopes", ["read", "search"])
logger.info(f"JWT token claims - user_id: {user_id}, email: {user_email}, scopes: {token_scopes}")
if user_id and user_email:
# JWT token is already signed by Clerk and contains valid user info
user_authenticated = True
logger.info(f"User authenticated via JWT token - user_id: {user_id}")
# Use the JWT token directly as the real token (it's already from Clerk template)
real_jwt_token = clerk_token
logger.info("Using Clerk JWT token directly (already real token)")
else:
logger.error(f"Missing required fields in JWT token - user_id: {bool(user_id)}, email: {bool(user_email)}")
except Exception as e:
logger.error(f"JWT validation failed: {e}")
# Fallback to cookie validation
if not user_authenticated:
clerk_session = request.cookies.get("__session")
if clerk_session:
user_authenticated = True
logger.info("User authenticated via cookie")
# Try to get session from cookie and generate JWT
if CLERK_AVAILABLE:
try:
clerk = Clerk(bearer_auth=os.getenv("CLERK_SECRET_KEY"))
# Note: sessions.verify_session is deprecated, but we'll try
# In practice, you'd need to extract session_id from cookie
logger.info("Cookie authentication - JWT generation not implemented yet")
except Exception as e:
logger.warning(f"Failed to generate JWT from cookie: {e}")
# Only generate authorization code if we have a real JWT token
if user_authenticated and real_jwt_token:
# Generate authorization code
auth_code = f"clerk_auth_{os.urandom(16).hex()}"
# Prepare code data
import time
code_data = {
"user_id": user_id,
"session_id": session_id,
"real_jwt_token": real_jwt_token,
"user_authenticated": user_authenticated,
"client_id": client_id,
"redirect_uri": redirect_uri,
"scope": scope or "read search"
}
# Try to store in Redis, fall back to in-memory if Redis unavailable
store = get_redis_session_store()
if store:
# Store in Redis with automatic expiration
success = store.set_oauth_code(auth_code, code_data)
if success:
logger.info(f"Stored authorization code {auth_code[:10]}... in Redis with real JWT token")
else:
logger.error(f"Failed to store authorization code in Redis, falling back to in-memory")
# Fall back to in-memory storage
if not hasattr(oauth_callback, '_code_storage'):
oauth_callback._code_storage = {}
oauth_callback._code_storage[auth_code] = code_data
else:
# Fall back to in-memory storage
logger.warning("Redis not available, using in-memory storage")
if not hasattr(oauth_callback, '_code_storage'):
oauth_callback._code_storage = {}
oauth_callback._code_storage[auth_code] = code_data
logger.info(f"Stored authorization code in memory (fallback)")
# Redirect back to client with authorization code
redirect_params = {
"code": auth_code,
"state": state or ""
}
final_redirect_url = f"{redirect_uri}?{urlencode(redirect_params)}"
logger.info(f"Redirecting back to client: {final_redirect_url}")
return RedirectResponse(url=final_redirect_url)
else:
# No JWT token yet - redirect back to sign-in page to wait for authentication
logger.info("No JWT token provided - redirecting back to sign-in to complete authentication")
# Keep the same redirect URL so the flow continues
sign_in_params = {
"redirect_url": f"{request.url._url}" # Current callback URL with all params
}
sign_in_url = f"https://yargimcp.com/sign-in?{urlencode(sign_in_params)}"
logger.info(f"Redirecting back to sign-in: {sign_in_url}")
return RedirectResponse(url=sign_in_url)
except Exception as e:
logger.exception(f"Callback processing failed: {e}")
return JSONResponse(
status_code=500,
content={"error": "server_error", "error_description": str(e)}
)
@router.post("/auth/register")
async def register_client(request: Request):
"""Dynamic Client Registration (RFC 7591)"""
data = await request.json()
logger.info(f"Client registration request: {data}")
# Simple dynamic registration - accept any client
client_id = f"mcp-client-{os.urandom(8).hex()}"
return JSONResponse({
"client_id": client_id,
"client_secret": None, # Public client
"redirect_uris": data.get("redirect_uris", []),
"grant_types": ["authorization_code"],
"response_types": ["code"],
"client_name": data.get("client_name", "MCP Client"),
"token_endpoint_auth_method": "none"
})
@router.post("/auth/callback")
async def oauth_callback_post(request: Request):
"""OAuth callback POST endpoint for token exchange"""
# Parse form data (standard OAuth token exchange format)
form_data = await request.form()
grant_type = form_data.get("grant_type")
code = form_data.get("code")
redirect_uri = form_data.get("redirect_uri")
client_id = form_data.get("client_id")
code_verifier = form_data.get("code_verifier")
logger.info(f"OAuth callback POST - grant_type: {grant_type}")
logger.info(f"Code: {code[:20] if code else 'None'}...")
logger.info(f"Client ID: {client_id}")
logger.info(f"PKCE verifier: {bool(code_verifier)}")
if grant_type != "authorization_code":
return JSONResponse(
status_code=400,
content={"error": "unsupported_grant_type"}
)
if not code or not redirect_uri:
return JSONResponse(
status_code=400,
content={"error": "invalid_request", "error_description": "Missing code or redirect_uri"}
)
try:
# Validate authorization code
if not code.startswith("clerk_auth_"):
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Invalid authorization code"}
)
# Retrieve stored JWT token using authorization code from Redis or in-memory fallback
stored_code_data = None
# Try to get from Redis first, then fall back to in-memory
store = get_redis_session_store()
if store:
stored_code_data = store.get_oauth_code(code, delete_after_use=True)
if stored_code_data:
logger.info(f"Retrieved authorization code {code[:10]}... from Redis")
else:
logger.warning(f"Authorization code {code[:10]}... not found in Redis")
# Fall back to in-memory storage if Redis unavailable or code not found
if not stored_code_data and hasattr(oauth_callback, '_code_storage'):
stored_code_data = oauth_callback._code_storage.get(code)
if stored_code_data:
# Clean up in-memory storage
oauth_callback._code_storage.pop(code, None)
logger.info(f"Retrieved authorization code {code[:10]}... from in-memory storage")
if not stored_code_data:
logger.error(f"No stored data found for authorization code: {code}")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Authorization code not found or expired"}
)
# Note: Redis TTL handles expiration automatically, but check for manual expiration for in-memory fallback
import time
expires_at = stored_code_data.get("expires_at", 0)
if expires_at and time.time() > expires_at:
logger.error(f"Authorization code expired: {code}")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Authorization code expired"}
)
# Get the real JWT token
real_jwt_token = stored_code_data.get("real_jwt_token")
if real_jwt_token:
logger.info("Returning real Clerk JWT token")
# Note: Code already deleted from Redis, clean up in-memory fallback if used
if hasattr(oauth_callback, '_code_storage'):
oauth_callback._code_storage.pop(code, None)
return JSONResponse({
"access_token": real_jwt_token,
"token_type": "Bearer",
"expires_in": 3600,
"scope": "read search"
})
else:
logger.warning("No real JWT token found, generating mock token")
# Fallback to mock token for testing
mock_token = f"mock_clerk_jwt_{code}"
return JSONResponse({
"access_token": mock_token,
"token_type": "Bearer",
"expires_in": 3600,
"scope": "read search"
})
except Exception as e:
logger.exception(f"OAuth callback POST failed: {e}")
return JSONResponse(
status_code=500,
content={"error": "server_error", "error_description": str(e)}
)
@router.post("/register")
async def register_client(request: Request):
"""Dynamic Client Registration (RFC 7591)"""
data = await request.json()
logger.info(f"Client registration request: {data}")
# Simple dynamic registration - accept any client
client_id = f"mcp-client-{os.urandom(8).hex()}"
return JSONResponse({
"client_id": client_id,
"client_secret": None, # Public client
"redirect_uris": data.get("redirect_uris", []),
"grant_types": ["authorization_code"],
"response_types": ["code"],
"client_name": data.get("client_name", "MCP Client"),
"token_endpoint_auth_method": "none"
})
@router.post("/token")
async def token_endpoint(request: Request):
"""OAuth 2.1 Token Endpoint - exchanges code for Clerk JWT"""
# Parse form data
form_data = await request.form()
grant_type = form_data.get("grant_type")
code = form_data.get("code")
redirect_uri = form_data.get("redirect_uri")
client_id = form_data.get("client_id")
code_verifier = form_data.get("code_verifier")
logger.info(f"Token exchange - grant_type: {grant_type}")
logger.info(f"Code: {code[:20] if code else 'None'}...")
if grant_type != "authorization_code":
return JSONResponse(
status_code=400,
content={"error": "unsupported_grant_type"}
)
if not code or not redirect_uri:
return JSONResponse(
status_code=400,
content={"error": "invalid_request", "error_description": "Missing code or redirect_uri"}
)
try:
# Validate authorization code
if not code.startswith("clerk_auth_"):
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Invalid authorization code"}
)
# Retrieve stored JWT token using authorization code from Redis or in-memory fallback
stored_code_data = None
# Try to get from Redis first, then fall back to in-memory
store = get_redis_session_store()
if store:
stored_code_data = store.get_oauth_code(code, delete_after_use=True)
if stored_code_data:
logger.info(f"Retrieved authorization code {code[:10]}... from Redis (/token endpoint)")
else:
logger.warning(f"Authorization code {code[:10]}... not found in Redis (/token endpoint)")
# Fall back to in-memory storage if Redis unavailable or code not found
if not stored_code_data and hasattr(oauth_callback, '_code_storage'):
stored_code_data = oauth_callback._code_storage.get(code)
if stored_code_data:
# Clean up in-memory storage
oauth_callback._code_storage.pop(code, None)
logger.info(f"Retrieved authorization code {code[:10]}... from in-memory storage (/token endpoint)")
if not stored_code_data:
logger.error(f"No stored data found for authorization code: {code}")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Authorization code not found or expired"}
)
# Note: Redis TTL handles expiration automatically, but check for manual expiration for in-memory fallback
import time
expires_at = stored_code_data.get("expires_at", 0)
if expires_at and time.time() > expires_at:
logger.error(f"Authorization code expired: {code}")
return JSONResponse(
status_code=400,
content={"error": "invalid_grant", "error_description": "Authorization code expired"}
)
# Get the real JWT token
real_jwt_token = stored_code_data.get("real_jwt_token")
if real_jwt_token:
logger.info("Returning real Clerk JWT token from /token endpoint")
# Note: Code already deleted from Redis, clean up in-memory fallback if used
if hasattr(oauth_callback, '_code_storage'):
oauth_callback._code_storage.pop(code, None)
return JSONResponse({
"access_token": real_jwt_token,
"token_type": "Bearer",
"expires_in": 3600,
"scope": "read search"
})
else:
logger.warning("No real JWT token found in /token endpoint, generating mock token")
# Fallback to mock token for testing
mock_token = f"mock_clerk_jwt_{code}"
return JSONResponse({
"access_token": mock_token,
"token_type": "Bearer",
"expires_in": 3600,
"scope": "read search"
})
except Exception as e:
logger.exception(f"Token exchange failed: {e}")
return JSONResponse(
status_code=500,
content={"error": "server_error", "error_description": str(e)}
)
+428 -52
View File
@@ -4,7 +4,7 @@ import atexit
import logging
import os
from pydantic import HttpUrl, Field
from typing import Optional, Dict, List, Literal
from typing import Optional, Dict, List, Literal, Any, Union
import urllib.parse
# --- Logging Configuration Start ---
@@ -31,7 +31,12 @@ root_logger.addHandler(console_handler)
logger = logging.getLogger(__name__)
# --- Logging Configuration End ---
from mcp_auth_factory import create_app
# Create FastMCP app directly without authentication wrapper
from fastmcp import FastMCP
def create_app():
"""Create basic FastMCP app without authentication wrapper"""
return FastMCP("Yargı MCP Server")
# --- Module Imports ---
from yargitay_mcp_module.client import YargitayOfficialApiClient
@@ -96,6 +101,14 @@ from sayistay_mcp_module.models import (
)
from sayistay_mcp_module.enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
# KVKK Module Imports
from kvkk_mcp_module.client import KvkkApiClient
from kvkk_mcp_module.models import (
KvkkSearchRequest,
KvkkSearchResult,
KvkkDocumentMarkdown
)
app = create_app()
@@ -110,6 +123,7 @@ kik_client_instance = KikApiClient()
rekabet_client_instance = RekabetKurumuApiClient()
bedesten_client_instance = BedestenApiClient()
sayistay_client_instance = SayistayApiClient()
kvkk_client_instance = KvkkApiClient()
KARAR_TURU_ADI_TO_GUID_ENUM_MAP = {
@@ -538,7 +552,7 @@ async def search_emsal_detailed_decisions(
end_date: Optional[str] = Field(None, description="End date for decision (DD.MM.YYYY)."),
sort_criteria: str = Field("1", description="Sorting criteria (e.g., 1: Esas No)."),
sort_direction: str = Field("desc", description="Sorting direction ('asc' or 'desc')."),
page_number: int = Field(1, ge=1, description="Page number."),
page_number: int = Field(1, ge=1, description="Page number (accepts int)."),
page_size: int = Field(10, ge=1, le=100, description="Results per page.")
) -> CompactEmsalSearchResult:
"""
@@ -604,7 +618,7 @@ async def search_emsal_detailed_decisions(
if api_response.data:
return CompactEmsalSearchResult(
decisions=api_response.data.data,
total_records=api_response.data.totalRecords if api_response.data.totalRecords is not None else 0,
total_records=api_response.data.recordsTotal if api_response.data.recordsTotal is not None else 0,
requested_page=search_query.page_number,
page_size=search_query.page_size
)
@@ -970,7 +984,7 @@ async def search_anayasa_norm_denetimi_decisions(
)
async def get_anayasa_norm_denetimi_document_markdown(
document_url: str = Field(..., description="The URL path (e.g., /ND/YYYY/NN) or full https URL of the AYM Norm Denetimi decision from normkararlarbilgibankasi.anayasa.gov.tr."),
page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters).")
page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).")
) -> AnayasaDocumentMarkdown:
"""
Retrieves the full text of a Constitutional Court norm control decision in paginated Markdown format.
@@ -1090,7 +1104,7 @@ async def search_anayasa_bireysel_basvuru_report(
)
async def get_anayasa_bireysel_basvuru_document_markdown(
document_url_path: str = Field(..., description="The URL path (e.g., /BB/YYYY/NNNN) of the AYM Bireysel Başvuru decision from kararlarbilgibankasi.anayasa.gov.tr."),
page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters).")
page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).")
) -> AnayasaBireyselBasvuruDocumentMarkdown:
"""
Retrieves the full text of a Constitutional Court individual application decision in paginated Markdown format.
@@ -1128,7 +1142,14 @@ async def get_anayasa_bireysel_basvuru_document_markdown(
logger.info(f"Tool 'get_anayasa_bireysel_basvuru_document_markdown' called for URL path: {document_url_path}, Page: {page_number}")
if not document_url_path or not document_url_path.strip() or not document_url_path.startswith("/BB/"):
raise ValueError("Document URL path (e.g., /BB/YYYY/NNNN) is required for Anayasa Bireysel Başvuru document retrieval.")
current_page_to_fetch = page_number if page_number is not None and page_number >= 1 else 1
# Handle both int and string page_number inputs
try:
current_page_to_fetch = int(page_number) if page_number is not None else 1
if current_page_to_fetch < 1:
current_page_to_fetch = 1
except (ValueError, TypeError):
current_page_to_fetch = 1
try:
return await anayasa_bireysel_client_instance.get_decision_document_as_markdown(document_url_path, page_number=current_page_to_fetch)
except Exception as e:
@@ -1242,7 +1263,7 @@ async def search_kik_decisions(
)
async def get_kik_document_markdown(
karar_id: str = Field(..., description="The Base64 encoded KIK decision identifier."),
page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed). Default is 1.")
page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1.")
) -> KikDocumentMarkdown:
"""
Retrieves the full text of a KIK (Public Procurement Authority) decision in paginated Markdown format.
@@ -1417,7 +1438,7 @@ async def search_rekabet_kurumu_decisions(
)
async def get_rekabet_kurumu_document(
karar_id: str = Field(..., description="GUID (kararId) of the Rekabet Kurumu decision. This ID is obtained from search results."),
page_number: Optional[int] = Field(1, ge=1, description="Requested page number for the Markdown content converted from PDF (1-indexed). Default is 1.")
page_number: Optional[int] = Field(1, ge=1, description="Requested page number for the Markdown content converted from PDF (1-indexed, accepts int). Default is 1.")
) -> RekabetDocument:
"""
Retrieves the full text of a Turkish Competition Authority decision in paginated Markdown format.
@@ -2437,7 +2458,8 @@ def perform_cleanup():
globals().get('kik_client_instance'),
globals().get('rekabet_client_instance'),
globals().get('bedesten_client_instance'),
globals().get('sayistay_client_instance')
globals().get('sayistay_client_instance'),
globals().get('kvkk_client_instance')
]
async def close_all_clients_async():
tasks = []
@@ -2466,10 +2488,273 @@ def perform_cleanup():
atexit.register(perform_cleanup)
# --- ChatGPT Deep Research Compatible Tools ---
# --- MCP Tools for KVKK ---
@app.tool(
description="Search KVKK (Personal Data Protection Authority) decisions using Brave Search API with advanced filtering and Turkish language support. KVKK is Turkey's data protection authority enforcing personal data protection laws equivalent to GDPR",
annotations={
"readOnlyHint": True,
"openWorldHint": True,
"idempotentHint": True
}
)
async def search_kvkk_decisions(
keywords: str = Field(..., description="""
Keywords to search for in KVKK decisions. The search automatically targets KVKK decision summaries.
Search Tips:
• Use Turkish legal terms: "açık rıza" (explicit consent), "veri güvenliği" (data security)
• Combine relevant terms: "kişisel veri işleme" (personal data processing)
• Use specific concepts: "GDPR", "veri ihlali" (data breach), "aydınlatma yükümlülüğü"
Examples:
"açık rıza" - Explicit consent decisions
"veri güvenliği" - Data security cases
"kişisel veri işleme" - Personal data processing
"GDPR uyum" - GDPR compliance
"veri ihlali bildirimi" - Data breach notifications
"""),
page: int = Field(1, ge=1, le=50, description="Page number for results (1-50)."),
pageSize: int = Field(10, ge=1, le=20, description="Number of results per page (1-20).")
) -> KvkkSearchResult:
"""
Searches KVKK (Personal Data Protection Authority) decisions using Brave Search API.
KVKK is Turkey's data protection authority, equivalent to European Data Protection Authorities.
It enforces the Turkish Personal Data Protection Law (KVKK - Kişisel Verilerin Korunması Kanunu)
which is Turkey's GDPR-equivalent legislation.
Key Features:
• Brave Search API integration for comprehensive coverage
• Turkish language search with automatic site targeting
• Decision summaries with metadata extraction
• Pagination support for large result sets
• URL-based decision identification
Search Coverage:
• Administrative fines and penalties
• Data processing compliance decisions
• Data breach notification requirements
• Consent and transparency obligations
• International data transfer decisions
• Data subject rights enforcement
Use Cases:
• Research Turkish data protection precedents
• Analyze KVKK enforcement patterns
• Find specific data protection decisions
• Study compliance requirements and penalties
• Compare with GDPR implementation
Returns structured data with decision titles, URLs, descriptions, and extractable metadata
including decision dates and numbers where available.
"""
logger.info(f"KVKK search tool called with keywords: {keywords}")
search_request = KvkkSearchRequest(
keywords=keywords,
page=page,
pageSize=pageSize
)
try:
result = await kvkk_client_instance.search_decisions(search_request)
logger.info(f"KVKK search completed. Found {len(result.decisions)} decisions on page {page}")
return result
except Exception as e:
logger.exception(f"Error in KVKK search: {e}")
# Return empty result on error
return KvkkSearchResult(
decisions=[],
total_results=0,
page=page,
pageSize=pageSize,
query=keywords
)
@app.tool(
description="ChatGPT Deep Research search for Turkish legal databases via Bedesten API - returns numeric document IDs and supports advanced search operators",
description="Retrieve the full text content of a KVKK decision document converted to Markdown format with metadata extraction and proper legal document formatting",
annotations={
"readOnlyHint": True,
"openWorldHint": False,
"idempotentHint": True
}
)
async def get_kvkk_document_markdown(
decision_url: str = Field(..., description="""
URL of the KVKK decision document to retrieve.
Expected URL format:
• Full KVKK decision page URL (e.g., https://www.kvkk.gov.tr/Icerik/7288/2021-1303)
• URL must point to a valid KVKK decision page
• URLs are typically obtained from search_kvkk_decisions results
Examples:
• https://www.kvkk.gov.tr/Icerik/7288/2021-1303
• https://www.kvkk.gov.tr/Icerik/8043/2023-1356
Note: The URL should be a complete KVKK decision page URL, not just a decision ID.
"""),
page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).")
) -> KvkkDocumentMarkdown:
"""
Retrieves the full text of a KVKK decision document in paginated Markdown format.
This tool fetches complete KVKK decision content from the official KVKK website
and converts it to clean, readable Markdown format. Content is paginated into
5,000-character chunks for easier processing.
Input Requirements:
• decision_url: Complete KVKK decision page URL from search_kvkk_decisions results
• page_number: Page number for pagination (1-indexed, default: 1)
Output Format:
• Clean Markdown text with proper KVKK decision formatting
• Pagination information (current_page, total_pages, is_paginated)
• Decision metadata (title, date, number, subject summary)
Content Processing:
• Fetches HTML content from KVKK decision pages
• Extracts decision metadata (date, number, subject summary)
• Converts legal document content to properly formatted Markdown
• Preserves document structure and important formatting
• Removes navigation elements and website artifacts
Use Cases:
• Reading full KVKK decision texts with proper formatting
• Legal analysis of personal data protection decisions
• Content analysis and case summarization
• Citation extraction and legal reference building
Returns structured document with paginated Markdown content and extracted metadata.
"""
logger.info(f"KVKK document retrieval tool called for URL: {decision_url}")
# Handle page_number type conversion (Union[int, str] -> int)
if isinstance(page_number, str):
try:
page_number = int(page_number)
except ValueError:
logger.warning(f"Invalid page_number string '{page_number}', defaulting to 1")
page_number = 1
if not decision_url or not decision_url.strip():
return KvkkDocumentMarkdown(
source_url=HttpUrl("https://www.kvkk.gov.tr"),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number or 1,
total_pages=0,
is_paginated=False,
error_message="Decision URL is required and cannot be empty."
)
try:
# Validate URL format
if not decision_url.startswith("https://www.kvkk.gov.tr/"):
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number or 1,
total_pages=0,
is_paginated=False,
error_message="Invalid KVKK decision URL format. URL must start with https://www.kvkk.gov.tr/"
)
result = await kvkk_client_instance.get_decision_document(decision_url, page_number or 1)
logger.info(f"KVKK document retrieved successfully. Page {result.current_page}/{result.total_pages}, Content length: {len(result.markdown_chunk) if result.markdown_chunk else 0}")
return result
except Exception as e:
logger.exception(f"Error retrieving KVKK document: {e}")
return KvkkDocumentMarkdown(
source_url=HttpUrl(decision_url),
title=None,
decision_date=None,
decision_number=None,
subject_summary=None,
markdown_chunk=None,
current_page=page_number or 1,
total_pages=0,
is_paginated=False,
error_message=f"Error retrieving KVKK document: {str(e)}"
)
# --- ChatGPT Deep Research Compatible Tools ---
def get_preview_text(markdown_content: str, skip_chars: int = 100, preview_chars: int = 200) -> str:
"""
Extract a preview of document text by skipping headers and showing meaningful content.
Args:
markdown_content: Full document content in markdown format
skip_chars: Number of characters to skip from the beginning (default: 100)
preview_chars: Number of characters to show in preview (default: 200)
Returns:
Preview text suitable for ChatGPT Deep Research
"""
if not markdown_content:
return ""
# Remove common markdown artifacts and clean up
cleaned_content = markdown_content.strip()
# Skip the first N characters (usually headers, metadata)
if len(cleaned_content) > skip_chars:
content_start = cleaned_content[skip_chars:]
else:
content_start = cleaned_content
# Get the next N characters for preview
if len(content_start) > preview_chars:
preview = content_start[:preview_chars]
else:
preview = content_start
# Clean up the preview - remove incomplete sentences at the end
preview = preview.strip()
# If preview ends mid-sentence, try to end at last complete sentence
if preview and not preview.endswith('.'):
last_period = preview.rfind('.')
if last_period > 50: # Only if there's a reasonable sentence
preview = preview[:last_period + 1]
# Add ellipsis if content was truncated
if len(content_start) > preview_chars:
preview += "..."
return preview.strip()
@app.tool(
description="""
Search Turkish legal databases for court decisions and legal precedents.
This tool searches across all major Turkish courts and returns document IDs for ChatGPT Deep Research.
SEARCH LANGUAGE: Queries must be in Turkish - English terms will not work.
SEARCH STRATEGY:
• Use specific legal terms: "mülkiyet hakkı" (property rights), "sözleşme ihlali" (contract breach)
• Try exact phrases in quotes: "\"idari işlem\"" for precise administrative law terms
• Combine multiple concepts: "+\"mülkiyet hakkı\" +\"anayasa\"" for constitutional property rights
• Search by legal areas: "\"ticaret hukuku\"", "\"medeni hukuk\"", "\"ceza hukuku\""
COURT COVERAGE:
• Yargıtay: Supreme Court (civil/criminal final appeals)
• Danıştay: Council of State (administrative law)
• Yerel Hukuk: Local Civil Courts (first instance)
• İstinaf Hukuk: Civil Appeals Courts (intermediate appeals)
• KYB: Extraordinary appeals (rare prosecutorial challenges)
Returns document IDs that can be fetched with the fetch tool for full text analysis.
""",
annotations={
"readOnlyHint": True,
"openWorldHint": True,
@@ -2502,7 +2787,7 @@ async def search(
• Yerel Hukuk (Local Civil Courts) - First instance civil decisions
• İstinaf Hukuk (Civil Appeals Courts) - Appellate court decisions
• Kanun Yararına Bozma (KYB) - Extraordinary appeal decisions""")
) -> List[Dict[str, str]]:
) -> Dict[str, List[Dict[str, str]]]:
"""
Bedesten API search tool for ChatGPT Deep Research compatibility.
@@ -2513,7 +2798,7 @@ async def search(
For regular legal research, use specific court tools like search_yargitay_bedesten.
Returns:
Array of search result objects with numeric id, title, text snippet, and mevzuat.adalet.gov.tr url fields
Object with "results" field containing a list of documents with id, title, text preview, and url
as required by ChatGPT Deep Research specification.
"""
logger.info(f"ChatGPT Deep Research search tool called with query: {query}")
@@ -2545,37 +2830,45 @@ async def search(
# Add results from this court type (limit to top 5 per court)
for decision in search_results.data.emsalKararList[:5]:
# Embed all metadata into title for ChatGPT Deep Research compatibility
title_parts = [
court_name,
decision.birimAdi or 'Bilinmeyen Daire',
f"Esas: {decision.esasNo or 'N/A'}",
f"Karar: {decision.kararNo or 'N/A'}",
f"Tarih: {decision.kararTarihiStr or 'N/A'}"
]
# Add finalization status if available
if decision.kesinlesmeDurumu:
title_parts.append(f"Durum: {decision.kesinlesmeDurumu}")
# Fetch first 1000 characters of document content
document_preview = ""
# For ChatGPT Deep Research, fetch document content for preview
try:
# Fetch document content for preview
doc = await bedesten_client_instance.get_document_as_markdown(decision.documentId)
if doc.markdown_content:
# Get first 1000 characters of the document content
document_preview = doc.markdown_content[:1000]
# If truncated, add ellipsis
if len(doc.markdown_content) > 1000:
document_preview += "..."
except Exception as e:
logger.warning(f"Failed to fetch document preview for {decision.documentId}: {e}")
document_preview = f"{court_name} decision on '{query}' - Date: {decision.kararTarihi} - Court: {decision.birimAdi or 'Unknown'}"
# Generate preview text (skip first 100 chars, show next 200)
preview_text = get_preview_text(doc.markdown_content, skip_chars=100, preview_chars=200)
# Build title from metadata
title_parts = []
if decision.birimAdi:
title_parts.append(decision.birimAdi)
if decision.esasNo:
title_parts.append(f"Esas: {decision.esasNo}")
if decision.kararNo:
title_parts.append(f"Karar: {decision.kararNo}")
if decision.kararTarihiStr:
title_parts.append(f"Tarih: {decision.kararTarihiStr}")
if title_parts:
title = " - ".join(title_parts)
else:
title = f"{court_name} - Document {decision.documentId}"
# Add to results in OpenAI format
results.append({
"id": decision.documentId,
"title": " - ".join(title_parts),
"text": document_preview,
"title": title,
"text": preview_text,
"url": f"https://mevzuat.adalet.gov.tr/ictihat/{decision.documentId}"
})
except Exception as e:
logger.warning(f"Could not fetch preview for document {decision.documentId}: {e}")
# Add minimal result without preview
results.append({
"id": decision.documentId,
"title": f"{court_name} - Document {decision.documentId}",
"text": "Document preview not available",
"url": f"https://mevzuat.adalet.gov.tr/ictihat/{decision.documentId}"
})
@@ -2600,17 +2893,37 @@ async def search(
"""
logger.info(f"ChatGPT Deep Research search completed. Found {len(results)} results via Bedesten API.")
return results
return {"results": results}
except Exception as e:
logger.exception("Error in ChatGPT Deep Research search tool")
# Return partial results if any were found
if results:
return results
return {"results": results}
raise
@app.tool(
description="ChatGPT Deep Research fetch for Turkish legal documents via Bedesten API - accepts numeric document IDs and retrieves complete text in Markdown format",
description="""
Retrieve full text of Turkish legal documents using document IDs from search results.
This tool fetches complete court decisions in clean Markdown format for analysis.
INPUT: Numeric document ID from search tool results (e.g., "730113500", "1149020800")
OUTPUT: Complete legal document with:
• Full decision text in readable Markdown format
• Court metadata (chamber, case numbers, dates)
• Legal reasoning and conclusions
• Citations and legal references
DOCUMENT TYPES:
• Supreme Court opinions with detailed legal analysis
• Administrative court decisions on government actions
• Civil court rulings on private disputes
• Criminal court decisions and sentencing rationale
• Extraordinary appeal reviews by prosecutors
Use this tool after searching to get the complete text of relevant legal decisions for analysis, citation, and research.
""",
annotations={
"readOnlyHint": True,
"openWorldHint": False, # Retrieves specific documents, not exploring
@@ -2627,7 +2940,7 @@ async def fetch(
• Numeric document ID only (e.g., "730113500", "71370900")
• IDs are obtained from the search tool results
• Works for all Turkish court types via unified Bedesten API""")
) -> Dict[str, str]:
) -> Dict[str, Any]:
"""
Bedesten API fetch tool for ChatGPT Deep Research compatibility.
@@ -2653,9 +2966,44 @@ async def fetch(
# Use the numeric ID directly with Bedesten API
doc = await bedesten_client_instance.get_document_as_markdown(id)
# Try to get additional metadata by searching for this specific document
title = f"Turkish Legal Document {id}"
try:
# Quick search to get metadata for better title
search_results = await bedesten_client_instance.search_documents(
BedestenSearchRequest(
data=BedestenSearchData(
phrase=id, # Search by document ID
pageSize=1,
pageNumber=1
)
)
)
if search_results.data.emsalKararList:
decision = search_results.data.emsalKararList[0]
if decision.documentId == id:
# Build a proper title from metadata
title_parts = []
if decision.birimAdi:
title_parts.append(decision.birimAdi)
if decision.esasNo:
title_parts.append(f"Esas: {decision.esasNo}")
if decision.kararNo:
title_parts.append(f"Karar: {decision.kararNo}")
if decision.kararTarihiStr:
title_parts.append(f"Tarih: {decision.kararTarihiStr}")
if title_parts:
title = " - ".join(title_parts)
else:
title = f"Turkish Legal Decision {id}"
except Exception as e:
logger.warning(f"Could not fetch metadata for document {id}: {e}")
return {
"id": id,
"title": f"Turkish Legal Database - Document {id}",
"title": title,
"text": doc.markdown_content,
"url": f"https://mevzuat.adalet.gov.tr/ictihat/{id}",
"metadata": {
@@ -2664,8 +3012,7 @@ async def fetch(
"source_url": doc.source_url,
"mime_type": doc.mime_type,
"api_source": "Bedesten Unified API",
"chatgpt_deep_research": True,
"note": "For detailed metadata (birimAdi, esasNo, kararNo, etc.), use the search tool results"
"chatgpt_deep_research": True
}
}
@@ -2709,14 +3056,43 @@ async def fetch(
logger.exception(f"Error fetching ChatGPT Deep Research document {id}")
raise
def ensure_playwright_browsers():
"""Ensure Playwright browsers are installed for KIK tool functionality."""
try:
import subprocess
import os
# Check if chromium is already installed
chromium_path = os.path.expanduser("~/Library/Caches/ms-playwright/chromium-1179")
if os.path.exists(chromium_path):
logger.info("Playwright Chromium browser already installed.")
return
logger.info("Installing Playwright Chromium browser for KIK tool...")
result = subprocess.run(
["python", "-m", "playwright", "install", "chromium"],
capture_output=True,
text=True,
timeout=300 # 5 minutes timeout
)
if result.returncode == 0:
logger.info("Playwright Chromium browser installed successfully.")
else:
logger.warning(f"Failed to install Playwright browser: {result.stderr}")
logger.warning("KIK tool may not work properly without Playwright browsers.")
except Exception as e:
logger.warning(f"Could not auto-install Playwright browsers: {e}")
logger.warning("KIK tool may not work properly. Manual installation: 'playwright install chromium'")
def main():
from mcp_auth_factory import enable_tool_authentication
# Enable authentication on all tools now that they're defined
enable_tool_authentication(app)
logger.info(f"Starting {app.name} server via main() function...")
logger.info(f"Logs will be written to: {LOG_FILE_PATH}")
# Ensure Playwright browsers are installed
ensure_playwright_browsers()
try:
app.run()
except KeyboardInterrupt:
+3 -2
View File
@@ -1,6 +1,6 @@
[project]
name = "yargi-mcp"
version = "0.1.2"
version = "0.1.3"
description = "MCP Server For Turkish Legal Databases"
readme = "README.md"
requires-python = ">=3.11"
@@ -26,7 +26,7 @@ dependencies = [
"pydantic>=2.11.4",
"aiohttp>=3.11.18",
"playwright>=1.52.0",
"fastmcp>=2.9.2",
"fastmcp>=2.10.3",
"pypdf>=5.5.0",
"fastapi>=0.115.14",
"PyJWT>=2.8.0",
@@ -48,6 +48,7 @@ production = [
saas = [
"clerk-backend-api>=3.0.0",
"stripe>=9.1.0",
"upstash-redis>=1.1.0",
]
[project.scripts]
+464
View File
@@ -0,0 +1,464 @@
"""
Redis Session Store for OAuth Authorization Codes and User Sessions
This module provides Redis-based storage for OAuth authorization codes and user sessions,
enabling multi-machine deployment support by replacing in-memory storage.
Uses Upstash Redis via REST API for serverless-friendly operation.
"""
import os
import json
import time
import logging
from typing import Optional, Dict, Any, Union
from datetime import datetime, timedelta
logger = logging.getLogger(__name__)
try:
from upstash_redis import Redis
UPSTASH_AVAILABLE = True
except ImportError:
UPSTASH_AVAILABLE = False
Redis = None
# Use standard Python exceptions for Redis connection errors
import socket
from requests.exceptions import ConnectionError as RequestsConnectionError, Timeout as RequestsTimeout
class RedisSessionStore:
"""
Redis-based session store for OAuth flows and user sessions.
Uses Upstash Redis REST API for connection-free operation suitable for
multi-instance deployments on platforms like Fly.io.
"""
def __init__(self):
"""Initialize Redis connection using environment variables."""
if not UPSTASH_AVAILABLE:
raise ImportError("upstash-redis package is required. Install with: pip install upstash-redis")
# Initialize Upstash Redis client from environment with optimized connection settings
try:
# Get Upstash Redis configuration
redis_url = os.getenv("UPSTASH_REDIS_REST_URL")
redis_token = os.getenv("UPSTASH_REDIS_REST_TOKEN")
if not redis_url or not redis_token:
raise ValueError("UPSTASH_REDIS_REST_URL and UPSTASH_REDIS_REST_TOKEN must be set")
logger.info(f"Connecting to Upstash Redis at {redis_url[:30]}...")
# Initialize with explicit configuration for better SSL handling
self.redis = Redis(
url=redis_url,
token=redis_token
)
logger.info("Upstash Redis client created")
# Skip connection test during initialization to prevent server hang
# Connection will be tested during first actual operation
logger.info("Redis client initialized - connection will be tested on first use")
except Exception as e:
logger.error(f"Failed to initialize Upstash Redis: {e}")
raise
# TTL values (in seconds)
self.oauth_code_ttl = int(os.getenv("OAUTH_CODE_TTL", "600")) # 10 minutes
self.session_ttl = int(os.getenv("SESSION_TTL", "3600")) # 1 hour
def _serialize_data(self, data: Dict[str, Any]) -> Dict[str, str]:
"""Convert data to Redis-compatible string format."""
serialized = {}
for key, value in data.items():
if isinstance(value, (dict, list)):
serialized[key] = json.dumps(value)
elif isinstance(value, (int, float)):
serialized[key] = str(value)
elif isinstance(value, bool):
serialized[key] = "true" if value else "false"
else:
serialized[key] = str(value)
return serialized
def _deserialize_data(self, data: Dict[str, str]) -> Dict[str, Any]:
"""Convert Redis string data back to original types."""
if not data:
return {}
deserialized = {}
for key, value in data.items():
if not isinstance(value, str):
deserialized[key] = value
continue
# Try to deserialize JSON
if value.startswith(('[', '{')):
try:
deserialized[key] = json.loads(value)
continue
except json.JSONDecodeError:
pass
# Try to convert numbers
if value.isdigit():
deserialized[key] = int(value)
continue
if value.replace('.', '').isdigit():
try:
deserialized[key] = float(value)
continue
except ValueError:
pass
# Handle booleans
if value in ("true", "false"):
deserialized[key] = value == "true"
continue
# Keep as string
deserialized[key] = value
return deserialized
# OAuth Authorization Code Methods
def set_oauth_code(self, code: str, data: Dict[str, Any]) -> bool:
"""
Store OAuth authorization code with automatic expiration.
Args:
code: Authorization code string
data: Code data including user_id, client_id, etc.
Returns:
True if stored successfully, False otherwise
"""
try:
key = f"oauth:code:{code}"
# Add timestamp for debugging
data_with_timestamp = data.copy()
data_with_timestamp.update({
"created_at": time.time(),
"expires_at": time.time() + self.oauth_code_ttl
})
# Serialize and store - Upstash Redis doesn't support mapping parameter
serialized_data = self._serialize_data(data_with_timestamp)
# Use individual hset calls for each field with retry logic
max_retries = 3
for attempt in range(max_retries):
try:
# Clear any existing data first
self.redis.delete(key)
# Set all fields in a pipeline-like manner
for field, value in serialized_data.items():
self.redis.hset(key, field, value)
# Set expiration
self.redis.expire(key, self.oauth_code_ttl)
logger.info(f"Stored OAuth code {code[:10]}... with TTL {self.oauth_code_ttl}s (attempt {attempt + 1})")
return True
except (RequestsConnectionError, RequestsTimeout, OSError, socket.error) as e:
logger.warning(f"Redis connection error on attempt {attempt + 1}: {e}")
if attempt == max_retries - 1:
raise # Re-raise on final attempt
time.sleep(0.5 * (attempt + 1)) # Exponential backoff
except Exception as e:
logger.error(f"Failed to store OAuth code {code[:10]}... after {max_retries} attempts: {e}")
return False
def get_oauth_code(self, code: str, delete_after_use: bool = True) -> Optional[Dict[str, Any]]:
"""
Retrieve OAuth authorization code data.
Args:
code: Authorization code string
delete_after_use: If True, delete the code after retrieval (one-time use)
Returns:
Code data dictionary or None if not found/expired
"""
max_retries = 3
for attempt in range(max_retries):
try:
key = f"oauth:code:{code}"
# Get all hash fields with retry
data = self.redis.hgetall(key)
if not data:
logger.warning(f"OAuth code {code[:10]}... not found or expired (attempt {attempt + 1})")
return None
# Deserialize data
deserialized_data = self._deserialize_data(data)
# Check manual expiration (in case Redis TTL failed)
expires_at = deserialized_data.get("expires_at", 0)
if expires_at and time.time() > expires_at:
logger.warning(f"OAuth code {code[:10]}... manually expired")
try:
self.redis.delete(key)
except Exception as del_error:
logger.warning(f"Failed to delete expired code: {del_error}")
return None
# Delete after use for security (one-time use)
if delete_after_use:
try:
self.redis.delete(key)
logger.info(f"Retrieved and deleted OAuth code {code[:10]}... (attempt {attempt + 1})")
except Exception as del_error:
logger.warning(f"Failed to delete code after use: {del_error}")
# Continue anyway since we got the data
else:
logger.info(f"Retrieved OAuth code {code[:10]}... (not deleted, attempt {attempt + 1})")
return deserialized_data
except (RequestsConnectionError, RequestsTimeout, OSError, socket.error) as e:
logger.warning(f"Redis connection error on retrieval attempt {attempt + 1}: {e}")
if attempt == max_retries - 1:
logger.error(f"Failed to retrieve OAuth code {code[:10]}... after {max_retries} attempts: {e}")
return None
time.sleep(0.5 * (attempt + 1)) # Exponential backoff
except Exception as e:
logger.error(f"Failed to retrieve OAuth code {code[:10]}... on attempt {attempt + 1}: {e}")
if attempt == max_retries - 1:
return None
time.sleep(0.5 * (attempt + 1))
return None
# User Session Methods
def set_session(self, session_id: str, user_data: Dict[str, Any]) -> bool:
"""
Store user session data with sliding expiration.
Args:
session_id: Unique session identifier
user_data: User session data (user_id, email, scopes, etc.)
Returns:
True if stored successfully, False otherwise
"""
try:
key = f"session:{session_id}"
# Add session metadata
session_data = user_data.copy()
session_data.update({
"session_id": session_id,
"created_at": time.time(),
"last_accessed": time.time()
})
# Serialize and store - Upstash Redis doesn't support mapping parameter
serialized_data = self._serialize_data(session_data)
# Use individual hset calls for each field (Upstash compatibility)
for field, value in serialized_data.items():
self.redis.hset(key, field, value)
self.redis.expire(key, self.session_ttl)
logger.info(f"Stored session {session_id[:10]}... with TTL {self.session_ttl}s")
return True
except Exception as e:
logger.error(f"Failed to store session {session_id[:10]}...: {e}")
return False
def get_session(self, session_id: str, refresh_ttl: bool = True) -> Optional[Dict[str, Any]]:
"""
Retrieve user session data.
Args:
session_id: Session identifier
refresh_ttl: If True, extend session TTL on access
Returns:
Session data dictionary or None if not found/expired
"""
try:
key = f"session:{session_id}"
# Get session data
data = self.redis.hgetall(key)
if not data:
logger.warning(f"Session {session_id[:10]}... not found or expired")
return None
# Deserialize data
session_data = self._deserialize_data(data)
# Update last accessed time and refresh TTL
if refresh_ttl:
session_data["last_accessed"] = time.time()
self.redis.hset(key, "last_accessed", str(time.time()))
self.redis.expire(key, self.session_ttl)
logger.debug(f"Refreshed session {session_id[:10]}... TTL")
return session_data
except Exception as e:
logger.error(f"Failed to retrieve session {session_id[:10]}...: {e}")
return None
def delete_session(self, session_id: str) -> bool:
"""
Delete user session (logout).
Args:
session_id: Session identifier
Returns:
True if deleted successfully, False otherwise
"""
try:
key = f"session:{session_id}"
result = self.redis.delete(key)
if result:
logger.info(f"Deleted session {session_id[:10]}...")
return True
else:
logger.warning(f"Session {session_id[:10]}... not found for deletion")
return False
except Exception as e:
logger.error(f"Failed to delete session {session_id[:10]}...: {e}")
return False
# Health Check Methods
def health_check(self) -> Dict[str, Any]:
"""
Perform Redis health check.
Returns:
Health status dictionary
"""
try:
# Test basic operations
test_key = f"health:check:{int(time.time())}"
test_value = {"timestamp": time.time(), "test": True}
# Test set - Use individual hset calls for Upstash compatibility
serialized_test = self._serialize_data(test_value)
for field, value in serialized_test.items():
self.redis.hset(test_key, field, value)
# Test get
retrieved = self.redis.hgetall(test_key)
# Test delete
self.redis.delete(test_key)
return {
"status": "healthy",
"redis_connected": True,
"operations_working": bool(retrieved),
"timestamp": datetime.utcnow().isoformat()
}
except Exception as e:
logger.error(f"Redis health check failed: {e}")
return {
"status": "unhealthy",
"redis_connected": False,
"error": str(e),
"timestamp": datetime.utcnow().isoformat()
}
def get_stats(self) -> Dict[str, Any]:
"""
Get Redis usage statistics.
Returns:
Statistics dictionary
"""
try:
# Get basic info (not all Upstash plans support INFO command)
stats = {
"oauth_codes_pattern": "oauth:code:*",
"sessions_pattern": "session:*",
"timestamp": datetime.utcnow().isoformat()
}
try:
# Try to get counts (may fail on some Upstash plans)
oauth_keys = self.redis.keys("oauth:code:*")
session_keys = self.redis.keys("session:*")
stats.update({
"active_oauth_codes": len(oauth_keys) if oauth_keys else 0,
"active_sessions": len(session_keys) if session_keys else 0
})
except Exception as e:
logger.warning(f"Could not get detailed stats: {e}")
stats["warning"] = "Detailed stats not available on this Redis plan"
return stats
except Exception as e:
logger.error(f"Failed to get Redis stats: {e}")
return {"error": str(e), "timestamp": datetime.utcnow().isoformat()}
# Global instance for easy importing
redis_store = None
def get_redis_store() -> Optional[RedisSessionStore]:
"""
Get global Redis store instance (singleton pattern).
Returns:
RedisSessionStore instance or None if initialization fails
"""
global redis_store
if redis_store is None:
try:
logger.info("Initializing Redis store...")
redis_store = RedisSessionStore()
logger.info("Redis store initialized successfully")
except Exception as e:
logger.error(f"Failed to initialize Redis store: {e}")
redis_store = None
return redis_store
def init_redis_store() -> RedisSessionStore:
"""
Initialize Redis store and perform health check.
Returns:
RedisSessionStore instance
Raises:
Exception if Redis is not available or unhealthy
"""
store = get_redis_store()
# Perform health check
health = store.health_check()
if health["status"] != "healthy":
raise Exception(f"Redis health check failed: {health}")
logger.info("Redis session store initialized and healthy")
return store
+8 -15
View File
@@ -6,8 +6,7 @@ from bs4 import BeautifulSoup
from typing import Dict, Any, List, Optional, Tuple
import logging
import html
import tempfile
import os
import io
from urllib.parse import urlencode, urljoin
from markitdown import MarkItDown
@@ -532,21 +531,18 @@ class SayistayApiClient:
raise
def _convert_html_to_markdown(self, html_content: str) -> Optional[str]:
"""Convert HTML content to Markdown using MarkItDown."""
"""Convert HTML content to Markdown using MarkItDown with BytesIO to avoid filename length issues."""
if not html_content:
return None
temp_file_path = None
try:
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_content.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
# Write HTML to temp file
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp:
tmp.write(html_content)
temp_file_path = tmp.name
# Convert
result = md_converter.convert(temp_file_path)
result = md_converter.convert(html_stream)
markdown_content = result.text_content
logger.info("Successfully converted HTML to Markdown")
@@ -555,9 +551,6 @@ class SayistayApiClient:
except Exception as e:
logger.error(f"Error converting HTML to Markdown: {e}")
return f"Error converting HTML content: {str(e)}"
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
async def get_document_as_markdown(self, decision_id: str, decision_type: str) -> SayistayDocumentMarkdown:
"""
+7 -11
View File
@@ -7,8 +7,7 @@ from typing import Dict, Any, List, Optional, Union, Tuple
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from urllib.parse import urljoin, urlencode # urlencode for aiohttp form data
@@ -196,21 +195,18 @@ class UyusmazlikApiClient:
html_input_for_markdown = processed_html
markdown_text = None
temp_file_path = None
try:
md_converter = MarkItDown()
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_file:
tmp_file.write(html_input_for_markdown)
temp_file_path = tmp_file.name
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_input_for_markdown.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
conversion_result = md_converter.convert(temp_file_path)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_text = conversion_result.text_content
logger.info("UyusmazlikApiClient: HTML to Markdown conversion successful.")
except Exception as e:
logger.error(f"UyusmazlikApiClient: Error during MarkItDown HTML to Markdown conversion: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path)
return markdown_text
async def get_decision_document_as_markdown(self, document_url: str) -> UyusmazlikDocumentMarkdown:
+7 -13
View File
@@ -6,8 +6,7 @@ from typing import Dict, Any, List, Optional
import logging
import html
import re
import tempfile
import os
import io
from markitdown import MarkItDown
from .models import (
@@ -108,25 +107,20 @@ class YargitayOfficialApiClient:
html_to_convert = processed_html
markdown_output = None
temp_file_path = None
try:
md_converter = MarkItDown() # Plugins disabled as per basic usage
# Convert HTML string to bytes and create BytesIO stream
html_bytes = html_to_convert.encode('utf-8')
html_stream = io.BytesIO(html_bytes)
# Write the HTML to a temporary file for MarkItDown to process
with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".html", encoding="utf-8") as tmp_html_file:
tmp_html_file.write(html_to_convert)
temp_file_path = tmp_html_file.name
conversion_result = md_converter.convert(temp_file_path)
# Pass BytesIO stream to MarkItDown to avoid temp file creation
md_converter = MarkItDown()
conversion_result = md_converter.convert(html_stream)
markdown_output = conversion_result.text_content
logger.info("Successfully converted HTML to Markdown.")
except Exception as e:
logger.error(f"Error during MarkItDown HTML to Markdown conversion: {e}")
finally:
if temp_file_path and os.path.exists(temp_file_path):
os.remove(temp_file_path) # Clean up the temporary file
return markdown_output