Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4d57a3939f | ||
|
|
50c6963eee | ||
|
|
def7e7d65e | ||
|
|
82a0d13d25 | ||
|
|
18b552ca2f | ||
|
|
1e96b1888e | ||
|
|
815786a09d | ||
|
|
260adb3ac9 | ||
|
|
7164205425 | ||
|
|
3961a23d3a | ||
|
|
25723f070f | ||
|
|
6376037ccf | ||
|
|
d1728ce114 | ||
|
|
69b5da5cef | ||
|
|
91564bf0a1 |
@@ -0,0 +1 @@
|
|||||||
|
/cache
|
||||||
@@ -0,0 +1,84 @@
|
|||||||
|
# list of languages for which language servers are started; choose from:
|
||||||
|
# al bash clojure cpp csharp csharp_omnisharp
|
||||||
|
# dart elixir elm erlang fortran go
|
||||||
|
# haskell java julia kotlin lua markdown
|
||||||
|
# nix perl php python python_jedi r
|
||||||
|
# rego ruby ruby_solargraph rust scala swift
|
||||||
|
# terraform typescript typescript_vts yaml zig
|
||||||
|
# Note:
|
||||||
|
# - For C, use cpp
|
||||||
|
# - For JavaScript, use typescript
|
||||||
|
# Special requirements:
|
||||||
|
# - csharp: Requires the presence of a .sln file in the project folder.
|
||||||
|
# When using multiple languages, the first language server that supports a given file will be used for that file.
|
||||||
|
# The first language is the default language and the respective language server will be used as a fallback.
|
||||||
|
# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
|
||||||
|
languages:
|
||||||
|
- python
|
||||||
|
|
||||||
|
# the encoding used by text files in the project
|
||||||
|
# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
|
||||||
|
encoding: "utf-8"
|
||||||
|
|
||||||
|
# whether to use the project's gitignore file to ignore files
|
||||||
|
# Added on 2025-04-07
|
||||||
|
ignore_all_files_in_gitignore: true
|
||||||
|
|
||||||
|
# list of additional paths to ignore
|
||||||
|
# same syntax as gitignore, so you can use * and **
|
||||||
|
# Was previously called `ignored_dirs`, please update your config if you are using that.
|
||||||
|
# Added (renamed) on 2025-04-07
|
||||||
|
ignored_paths: []
|
||||||
|
|
||||||
|
# whether the project is in read-only mode
|
||||||
|
# If set to true, all editing tools will be disabled and attempts to use them will result in an error
|
||||||
|
# Added on 2025-04-18
|
||||||
|
read_only: false
|
||||||
|
|
||||||
|
# list of tool names to exclude. We recommend not excluding any tools, see the readme for more details.
|
||||||
|
# Below is the complete list of tools for convenience.
|
||||||
|
# To make sure you have the latest list of tools, and to view their descriptions,
|
||||||
|
# execute `uv run scripts/print_tool_overview.py`.
|
||||||
|
#
|
||||||
|
# * `activate_project`: Activates a project by name.
|
||||||
|
# * `check_onboarding_performed`: Checks whether project onboarding was already performed.
|
||||||
|
# * `create_text_file`: Creates/overwrites a file in the project directory.
|
||||||
|
# * `delete_lines`: Deletes a range of lines within a file.
|
||||||
|
# * `delete_memory`: Deletes a memory from Serena's project-specific memory store.
|
||||||
|
# * `execute_shell_command`: Executes a shell command.
|
||||||
|
# * `find_referencing_code_snippets`: Finds code snippets in which the symbol at the given location is referenced.
|
||||||
|
# * `find_referencing_symbols`: Finds symbols that reference the symbol at the given location (optionally filtered by type).
|
||||||
|
# * `find_symbol`: Performs a global (or local) search for symbols with/containing a given name/substring (optionally filtered by type).
|
||||||
|
# * `get_current_config`: Prints the current configuration of the agent, including the active and available projects, tools, contexts, and modes.
|
||||||
|
# * `get_symbols_overview`: Gets an overview of the top-level symbols defined in a given file.
|
||||||
|
# * `initial_instructions`: Gets the initial instructions for the current project.
|
||||||
|
# Should only be used in settings where the system prompt cannot be set,
|
||||||
|
# e.g. in clients you have no control over, like Claude Desktop.
|
||||||
|
# * `insert_after_symbol`: Inserts content after the end of the definition of a given symbol.
|
||||||
|
# * `insert_at_line`: Inserts content at a given line in a file.
|
||||||
|
# * `insert_before_symbol`: Inserts content before the beginning of the definition of a given symbol.
|
||||||
|
# * `list_dir`: Lists files and directories in the given directory (optionally with recursion).
|
||||||
|
# * `list_memories`: Lists memories in Serena's project-specific memory store.
|
||||||
|
# * `onboarding`: Performs onboarding (identifying the project structure and essential tasks, e.g. for testing or building).
|
||||||
|
# * `prepare_for_new_conversation`: Provides instructions for preparing for a new conversation (in order to continue with the necessary context).
|
||||||
|
# * `read_file`: Reads a file within the project directory.
|
||||||
|
# * `read_memory`: Reads the memory with the given name from Serena's project-specific memory store.
|
||||||
|
# * `remove_project`: Removes a project from the Serena configuration.
|
||||||
|
# * `replace_lines`: Replaces a range of lines within a file with new content.
|
||||||
|
# * `replace_symbol_body`: Replaces the full definition of a symbol.
|
||||||
|
# * `restart_language_server`: Restarts the language server, may be necessary when edits not through Serena happen.
|
||||||
|
# * `search_for_pattern`: Performs a search for a pattern in the project.
|
||||||
|
# * `summarize_changes`: Provides instructions for summarizing the changes made to the codebase.
|
||||||
|
# * `switch_modes`: Activates modes by providing a list of their names
|
||||||
|
# * `think_about_collected_information`: Thinking tool for pondering the completeness of collected information.
|
||||||
|
# * `think_about_task_adherence`: Thinking tool for determining whether the agent is still on track with the current task.
|
||||||
|
# * `think_about_whether_you_are_done`: Thinking tool for determining whether the task is truly completed.
|
||||||
|
# * `write_memory`: Writes a named memory (for future reference) to Serena's project-specific memory store.
|
||||||
|
excluded_tools: []
|
||||||
|
|
||||||
|
# initial prompt for the project. It will always be given to the LLM upon activating the project
|
||||||
|
# (contrary to the memories, which are loaded on demand).
|
||||||
|
initial_prompt: ""
|
||||||
|
|
||||||
|
project_name: "yargi-mcp"
|
||||||
|
included_optional_tools: []
|
||||||
+2
-1
@@ -9,10 +9,11 @@ COPY pyproject.toml poetry.lock* requirements*.txt* ./
|
|||||||
|
|
||||||
# Fast, deterministic install with `uv`
|
# Fast, deterministic install with `uv`
|
||||||
RUN pip install --no-cache-dir uv && \
|
RUN pip install --no-cache-dir uv && \
|
||||||
|
uv pip install --system --no-cache-dir . && \
|
||||||
uv pip install --system --no-cache-dir .[asgi,saas]
|
uv pip install --system --no-cache-dir .[asgi,saas]
|
||||||
|
|
||||||
# Cache buster - force rebuild
|
# Cache buster - force rebuild
|
||||||
ARG CACHE_BUST=202507221015
|
ARG CACHE_BUST=202510061202
|
||||||
RUN echo "Cache bust: $CACHE_BUST"
|
RUN echo "Cache bust: $CACHE_BUST"
|
||||||
|
|
||||||
# Copy application source
|
# Copy application source
|
||||||
|
|||||||
@@ -4,6 +4,28 @@
|
|||||||
|
|
||||||
Bu proje, çeşitli Türk hukuk kaynaklarına (Yargıtay, Danıştay, Emsal Kararlar, Uyuşmazlık Mahkemesi, Anayasa Mahkemesi - Norm Denetimi ile Bireysel Başvuru Kararları, Kamu İhale Kurulu Kararları, Rekabet Kurumu Kararları, Sayıştay Kararları, KVKK Kararları ve BDDK Kararları) erişimi kolaylaştıran bir [FastMCP](https://gofastmcp.com/) sunucusu oluşturur. Bu sayede, bu kaynaklardan veri arama ve belge getirme işlemleri, Model Context Protocol (MCP) destekleyen LLM (Büyük Dil Modeli) uygulamaları (örneğin Claude Desktop veya [5ire](https://5ire.app)) ve diğer istemciler tarafından araç (tool) olarak kullanılabilir hale gelir.
|
Bu proje, çeşitli Türk hukuk kaynaklarına (Yargıtay, Danıştay, Emsal Kararlar, Uyuşmazlık Mahkemesi, Anayasa Mahkemesi - Norm Denetimi ile Bireysel Başvuru Kararları, Kamu İhale Kurulu Kararları, Rekabet Kurumu Kararları, Sayıştay Kararları, KVKK Kararları ve BDDK Kararları) erişimi kolaylaştıran bir [FastMCP](https://gofastmcp.com/) sunucusu oluşturur. Bu sayede, bu kaynaklardan veri arama ve belge getirme işlemleri, Model Context Protocol (MCP) destekleyen LLM (Büyük Dil Modeli) uygulamaları (örneğin Claude Desktop veya [5ire](https://5ire.app)) ve diğer istemciler tarafından araç (tool) olarak kullanılabilir hale gelir.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🚀 5 Dakikada Başla (Remote MCP)
|
||||||
|
|
||||||
|
### ✅ Kurulum Gerektirmez! Hemen Kullan!
|
||||||
|
|
||||||
|
🔗 **Remote MCP Adresi:** `https://yargimcp.fastmcp.app/mcp`
|
||||||
|
|
||||||
|
### Claude Desktop ile Kullanım
|
||||||
|
|
||||||
|
1. **Claude Desktop'ı açın**
|
||||||
|
2. **Settings → Connectors → Add Custom Connector**
|
||||||
|
3. **Bilgileri girin:**
|
||||||
|
- **Name:** `Yargı MCP`
|
||||||
|
- **URL:** `https://yargimcp.fastmcp.app/mcp`
|
||||||
|
4. **Add** butonuna tıklayın
|
||||||
|
5. **Hemen kullanmaya başlayın!** 🎉
|
||||||
|
|
||||||
|
> 💡 **İpucu:** Remote MCP sayesinde Python, uv veya herhangi bir kurulum yapmadan doğrudan Claude Desktop üzerinden Türk hukuk kaynaklarına erişebilirsiniz!
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||

|

|
||||||
|
|
||||||
🎯 **Temel Özellikler**
|
🎯 **Temel Özellikler**
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ and trying to reverse engineer the hash generation logic.
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import hmac
|
||||||
import base64
|
import base64
|
||||||
from fastmcp import Client
|
from fastmcp import Client
|
||||||
from mcp_server_main import app
|
from mcp_server_main import app
|
||||||
@@ -151,7 +152,7 @@ async def test_hash_generation_comprehensive():
|
|||||||
|
|
||||||
# Test with first decision
|
# Test with first decision
|
||||||
sample_decision = decisions[0]
|
sample_decision = decisions[0]
|
||||||
print("\n📋 Sample decision for hash analysis:")
|
print(f"\n📋 Sample decision for hash analysis:")
|
||||||
for key, value in sample_decision.items():
|
for key, value in sample_decision.items():
|
||||||
print(f" {key}: {value}")
|
print(f" {key}: {value}")
|
||||||
|
|
||||||
@@ -161,20 +162,20 @@ async def test_hash_generation_comprehensive():
|
|||||||
all_hashes = {}
|
all_hashes = {}
|
||||||
|
|
||||||
# Test different hash generation methods
|
# Test different hash generation methods
|
||||||
print("\n🔨 Testing webpack-style hashing...")
|
print(f"\n🔨 Testing webpack-style hashing...")
|
||||||
webpack_hashes = test_webpack_style_hashing(sample_decision)
|
webpack_hashes = test_webpack_style_hashing(sample_decision)
|
||||||
all_hashes.update(webpack_hashes)
|
all_hashes.update(webpack_hashes)
|
||||||
|
|
||||||
print("🔨 Testing Angular routing hashes...")
|
print(f"🔨 Testing Angular routing hashes...")
|
||||||
angular_hashes = test_angular_routing_hashes(sample_decision)
|
angular_hashes = test_angular_routing_hashes(sample_decision)
|
||||||
all_hashes.update(angular_hashes)
|
all_hashes.update(angular_hashes)
|
||||||
|
|
||||||
print("🔨 Testing base64 encoding variants...")
|
print(f"🔨 Testing base64 encoding variants...")
|
||||||
b64_hashes = test_base64_encoding_variants(sample_decision)
|
b64_hashes = test_base64_encoding_variants(sample_decision)
|
||||||
all_hashes.update(b64_hashes)
|
all_hashes.update(b64_hashes)
|
||||||
|
|
||||||
# Check for matches
|
# Check for matches
|
||||||
print("\n🎯 Checking for hash matches...")
|
print(f"\n🎯 Checking for hash matches...")
|
||||||
matches_found = []
|
matches_found = []
|
||||||
partial_matches = []
|
partial_matches = []
|
||||||
|
|
||||||
@@ -190,13 +191,13 @@ async def test_hash_generation_comprehensive():
|
|||||||
print(f" 🔍 Partial match (last 8): {hash_name} -> ...{hash_value[-16:]}")
|
print(f" 🔍 Partial match (last 8): {hash_name} -> ...{hash_value[-16:]}")
|
||||||
|
|
||||||
if not matches_found and not partial_matches:
|
if not matches_found and not partial_matches:
|
||||||
print(" ❌ No matches found")
|
print(f" ❌ No matches found")
|
||||||
print("\n📝 Sample generated hashes (first 10):")
|
print(f"\n📝 Sample generated hashes (first 10):")
|
||||||
for i, (hash_name, hash_value) in enumerate(list(all_hashes.items())[:10]):
|
for i, (hash_name, hash_value) in enumerate(list(all_hashes.items())[:10]):
|
||||||
print(f" {hash_name}: {hash_value}")
|
print(f" {hash_name}: {hash_value}")
|
||||||
|
|
||||||
# Try combinations with other decisions
|
# Try combinations with other decisions
|
||||||
print("\n🔄 Testing hash combinations with multiple decisions...")
|
print(f"\n🔄 Testing hash combinations with multiple decisions...")
|
||||||
if len(decisions) > 1:
|
if len(decisions) > 1:
|
||||||
for i, decision in enumerate(decisions[1:3]): # Test 2 more
|
for i, decision in enumerate(decisions[1:3]): # Test 2 more
|
||||||
print(f"\n Testing decision {i+2}: {decision.get('kararNo')}")
|
print(f"\n Testing decision {i+2}: {decision.get('kararNo')}")
|
||||||
@@ -208,7 +209,7 @@ async def test_hash_generation_comprehensive():
|
|||||||
matches_found.append((f"decision_{i+2}_{hash_name}", hash_value))
|
matches_found.append((f"decision_{i+2}_{hash_name}", hash_value))
|
||||||
|
|
||||||
# Try composite hashes (combining multiple fields)
|
# Try composite hashes (combining multiple fields)
|
||||||
print("\n🔗 Testing composite hash generation...")
|
print(f"\n🔗 Testing composite hash generation...")
|
||||||
composite_tests = [
|
composite_tests = [
|
||||||
f"{sample_decision.get('gundemMaddesiId')}_{sample_decision.get('kararNo')}",
|
f"{sample_decision.get('gundemMaddesiId')}_{sample_decision.get('kararNo')}",
|
||||||
f"{sample_decision.get('kararNo')}_{sample_decision.get('kararTarihi')}",
|
f"{sample_decision.get('kararNo')}_{sample_decision.get('kararTarihi')}",
|
||||||
@@ -223,7 +224,7 @@ async def test_hash_generation_comprehensive():
|
|||||||
print(f" 🎉 COMPOSITE MATCH FOUND: test_{i} -> {composite_str[:50]}...")
|
print(f" 🎉 COMPOSITE MATCH FOUND: test_{i} -> {composite_str[:50]}...")
|
||||||
matches_found.append((f"composite_{i}", composite_hash))
|
matches_found.append((f"composite_{i}", composite_hash))
|
||||||
|
|
||||||
print("\n🎯 Hash analysis completed!")
|
print(f"\n🎯 Hash analysis completed!")
|
||||||
print(f" Total matches found: {len(matches_found)}")
|
print(f" Total matches found: {len(matches_found)}")
|
||||||
print(f" Partial matches: {len(partial_matches)}")
|
print(f" Partial matches: {len(partial_matches)}")
|
||||||
|
|
||||||
|
|||||||
@@ -2,13 +2,13 @@
|
|||||||
# This client is for Bireysel Başvuru: https://kararlarbilgibankasi.anayasa.gov.tr
|
# This client is for Bireysel Başvuru: https://kararlarbilgibankasi.anayasa.gov.tr
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup, Tag
|
||||||
from typing import List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Tuple
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urlencode, urljoin, quote
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
import math # For math.ceil for pagination
|
import math # For math.ceil for pagination
|
||||||
|
|
||||||
|
|||||||
@@ -3,12 +3,12 @@
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Tuple
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urlencode, urljoin, quote
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
import math # For math.ceil for pagination
|
import math # For math.ceil for pagination
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# Unified client for both Norm Denetimi and Bireysel Başvuru
|
# Unified client for both Norm Denetimi and Bireysel Başvuru
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Optional
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
|
|||||||
+37
-61
@@ -10,22 +10,38 @@ Usage:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
import logging
|
import logging
|
||||||
import json
|
import json
|
||||||
|
from datetime import datetime, timedelta
|
||||||
from fastapi import FastAPI, Request, HTTPException, Query
|
from fastapi import FastAPI, Request, HTTPException, Query
|
||||||
from fastapi.responses import JSONResponse, HTMLResponse, Response
|
from fastapi.responses import JSONResponse, HTMLResponse, Response
|
||||||
from fastapi.exception_handlers import http_exception_handler
|
from fastapi.exception_handlers import http_exception_handler
|
||||||
from starlette.middleware import Middleware
|
from starlette.middleware import Middleware
|
||||||
from starlette.middleware.cors import CORSMiddleware
|
from starlette.middleware.cors import CORSMiddleware
|
||||||
|
from starlette.middleware.base import BaseHTTPMiddleware
|
||||||
|
|
||||||
# Import the proper create_app function that includes all middleware
|
# Import the proper create_app function that includes all middleware
|
||||||
from mcp_server_main import create_app
|
from mcp_server_main import create_app
|
||||||
|
|
||||||
# Import Stripe webhook router
|
# Conditional auth-related imports (only if auth enabled)
|
||||||
from stripe_webhook import router as stripe_router
|
_auth_check = os.getenv("ENABLE_AUTH", "false").lower() == "true"
|
||||||
|
|
||||||
# Import simplified MCP Auth HTTP adapter
|
if _auth_check:
|
||||||
from mcp_auth_http_simple import router as mcp_auth_router
|
# Import MCP Auth HTTP adapter (OAuth endpoints)
|
||||||
|
try:
|
||||||
|
from mcp_auth_http_simple import router as mcp_auth_router
|
||||||
|
except ImportError:
|
||||||
|
mcp_auth_router = None
|
||||||
|
|
||||||
|
# Import Stripe webhook router
|
||||||
|
try:
|
||||||
|
from stripe_webhook import router as stripe_router
|
||||||
|
except ImportError:
|
||||||
|
stripe_router = None
|
||||||
|
else:
|
||||||
|
mcp_auth_router = None
|
||||||
|
stripe_router = None
|
||||||
|
|
||||||
# OAuth configuration from environment variables
|
# OAuth configuration from environment variables
|
||||||
CLERK_ISSUER = os.getenv("CLERK_ISSUER", "https://clerk.yargimcp.com")
|
CLERK_ISSUER = os.getenv("CLERK_ISSUER", "https://clerk.yargimcp.com")
|
||||||
@@ -59,15 +75,13 @@ if CLERK_SECRET_KEY and CLERK_ISSUER:
|
|||||||
# Production: Use Clerk JWKS endpoint for token validation
|
# Production: Use Clerk JWKS endpoint for token validation
|
||||||
bearer_auth = BearerAuthProvider(
|
bearer_auth = BearerAuthProvider(
|
||||||
jwks_uri=f"{CLERK_ISSUER}/.well-known/jwks.json",
|
jwks_uri=f"{CLERK_ISSUER}/.well-known/jwks.json",
|
||||||
issuer=None, # Disable issuer validation - Clerk uses different issuer format
|
issuer=None,
|
||||||
algorithm="RS256",
|
algorithm="RS256",
|
||||||
audience=None, # Disable audience validation - Clerk uses different audience format
|
audience=None,
|
||||||
required_scopes=[] # Disable scope validation - Clerk JWT has ['read', 'search']
|
required_scopes=[]
|
||||||
)
|
)
|
||||||
logger.info(f"Bearer auth configured with Clerk JWKS: {CLERK_ISSUER}/.well-known/jwks.json")
|
|
||||||
else:
|
else:
|
||||||
# Development: Generate RSA key pair for testing
|
# Development: Generate RSA key pair for testing
|
||||||
logger.warning("No Clerk credentials found - using development RSA key pair")
|
|
||||||
dev_key_pair = RSAKeyPair.generate()
|
dev_key_pair = RSAKeyPair.generate()
|
||||||
bearer_auth = BearerAuthProvider(
|
bearer_auth = BearerAuthProvider(
|
||||||
public_key=dev_key_pair.public_key,
|
public_key=dev_key_pair.public_key,
|
||||||
@@ -76,30 +90,12 @@ else:
|
|||||||
required_scopes=["yargi.read"]
|
required_scopes=["yargi.read"]
|
||||||
)
|
)
|
||||||
|
|
||||||
# Generate a test token for development
|
|
||||||
dev_token = dev_key_pair.create_token(
|
|
||||||
subject="dev-user",
|
|
||||||
issuer="https://dev.yargimcp.com",
|
|
||||||
audience="dev-mcp-server",
|
|
||||||
scopes=["yargi.read", "yargi.search"],
|
|
||||||
expires_in_seconds=3600 * 24 # 24 hours for development
|
|
||||||
)
|
|
||||||
logger.info(f"Development Bearer token: {dev_token}")
|
|
||||||
|
|
||||||
# Create MCP app with Bearer authentication
|
# Create MCP app with Bearer authentication
|
||||||
mcp_server = create_app(auth=bearer_auth if auth_enabled else None)
|
mcp_server = create_app(auth=bearer_auth if auth_enabled else None)
|
||||||
|
|
||||||
# Create MCP Starlette sub-application with root path - mount will add /mcp prefix
|
# Create MCP Starlette sub-application with root path - mount will add /mcp prefix
|
||||||
mcp_app = mcp_server.http_app(path="/")
|
mcp_app = mcp_server.http_app(path="/")
|
||||||
logger.info(f"MCP Starlette app created - type: {type(mcp_app)}, has routes: {hasattr(mcp_app, 'routes')}")
|
|
||||||
|
|
||||||
# Debug FastMCP routes
|
|
||||||
if hasattr(mcp_app, 'routes'):
|
|
||||||
logger.info(f"MCP app route count: {len(mcp_app.routes)}")
|
|
||||||
for i, route in enumerate(mcp_app.routes):
|
|
||||||
logger.info(f"Route {i}: {route.path if hasattr(route, 'path') else 'unknown'} - {type(route)}")
|
|
||||||
else:
|
|
||||||
logger.warning("MCP app has no routes attribute")
|
|
||||||
|
|
||||||
# Configure JSON encoder for proper Turkish character support
|
# Configure JSON encoder for proper Turkish character support
|
||||||
class UTF8JSONResponse(JSONResponse):
|
class UTF8JSONResponse(JSONResponse):
|
||||||
@@ -138,11 +134,12 @@ app = FastAPI(
|
|||||||
redirect_slashes=False # Disable to prevent 307 redirects on /mcp endpoint
|
redirect_slashes=False # Disable to prevent 307 redirects on /mcp endpoint
|
||||||
)
|
)
|
||||||
|
|
||||||
# Add Stripe webhook router to FastAPI
|
# Add auth-related routers to FastAPI (only if available)
|
||||||
app.include_router(stripe_router, prefix="/api/stripe")
|
if stripe_router:
|
||||||
|
app.include_router(stripe_router, prefix="/api/stripe")
|
||||||
|
|
||||||
# Add MCP Auth HTTP adapter to FastAPI (handles OAuth endpoints)
|
if mcp_auth_router:
|
||||||
app.include_router(mcp_auth_router)
|
app.include_router(mcp_auth_router)
|
||||||
|
|
||||||
# Custom 401 exception handler for MCP spec compliance
|
# Custom 401 exception handler for MCP spec compliance
|
||||||
@app.exception_handler(401)
|
@app.exception_handler(401)
|
||||||
@@ -254,7 +251,6 @@ async def clerk_cors_proxy(request: Request, path: str):
|
|||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Clerk proxy error: {e}")
|
|
||||||
return JSONResponse(
|
return JSONResponse(
|
||||||
{"error": "proxy_error", "message": str(e)},
|
{"error": "proxy_error", "message": str(e)},
|
||||||
status_code=500,
|
status_code=500,
|
||||||
@@ -448,7 +444,6 @@ async def status():
|
|||||||
# Simplified OAuth session validation for callback endpoints only
|
# Simplified OAuth session validation for callback endpoints only
|
||||||
async def validate_clerk_session_for_oauth(request: Request, clerk_token: str = None) -> str:
|
async def validate_clerk_session_for_oauth(request: Request, clerk_token: str = None) -> str:
|
||||||
"""Validate Clerk session for OAuth callback endpoints only (not for MCP endpoints)"""
|
"""Validate Clerk session for OAuth callback endpoints only (not for MCP endpoints)"""
|
||||||
logger.info(f"OAuth callback session validation - token provided: {bool(clerk_token)}")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Use Clerk SDK if available
|
# Use Clerk SDK if available
|
||||||
@@ -458,58 +453,43 @@ async def validate_clerk_session_for_oauth(request: Request, clerk_token: str =
|
|||||||
|
|
||||||
# Try JWT token first (from URL parameter)
|
# Try JWT token first (from URL parameter)
|
||||||
if clerk_token:
|
if clerk_token:
|
||||||
logger.info("Validating Clerk JWT token for OAuth callback")
|
|
||||||
try:
|
try:
|
||||||
# Trust OAuth flow redirect - FastMCP handles full JWT validation for MCP endpoints
|
|
||||||
logger.info("OAuth JWT token accepted for callback")
|
|
||||||
return "oauth_user_from_token"
|
return "oauth_user_from_token"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"OAuth JWT token validation failed: {str(e)}")
|
pass
|
||||||
# Fall through to cookie validation
|
|
||||||
|
|
||||||
# Fallback to cookie validation
|
# Fallback to cookie validation
|
||||||
logger.info("Attempting cookie-based session validation for OAuth")
|
|
||||||
clerk_session = request.cookies.get("__session")
|
clerk_session = request.cookies.get("__session")
|
||||||
if not clerk_session:
|
if not clerk_session:
|
||||||
logger.error("No Clerk session cookie found")
|
|
||||||
raise HTTPException(status_code=401, detail="No Clerk session found")
|
raise HTTPException(status_code=401, detail="No Clerk session found")
|
||||||
|
|
||||||
# Validate session with Clerk
|
# Validate session with Clerk
|
||||||
session = clerk.sessions.verify_session(clerk_session)
|
session = clerk.sessions.verify_session(clerk_session)
|
||||||
logger.info(f"OAuth cookie session validation successful - user_id: {session.user_id}")
|
|
||||||
return session.user_id
|
return session.user_id
|
||||||
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
# Fallback for development without Clerk SDK
|
|
||||||
logger.warning("Clerk SDK not available - using development fallback for OAuth")
|
|
||||||
return "dev_user_123"
|
return "dev_user_123"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"OAuth session validation failed: {str(e)}")
|
|
||||||
raise HTTPException(status_code=401, detail=f"OAuth session validation failed: {str(e)}")
|
raise HTTPException(status_code=401, detail=f"OAuth session validation failed: {str(e)}")
|
||||||
|
|
||||||
# MCP OAuth Callback Endpoint
|
# MCP OAuth Callback Endpoint
|
||||||
@app.get("/auth/mcp-callback")
|
@app.get("/auth/mcp-callback")
|
||||||
async def mcp_oauth_callback(request: Request, clerk_token: str = Query(None)):
|
async def mcp_oauth_callback(request: Request, clerk_token: str = Query(None)):
|
||||||
"""Handle OAuth callback for MCP token generation"""
|
"""Handle OAuth callback for MCP token generation"""
|
||||||
logger.info(f"MCP OAuth callback - clerk_token provided: {bool(clerk_token)}")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Validate Clerk session with JWT token support
|
# Validate Clerk session with JWT token support
|
||||||
user_id = await validate_clerk_session_for_oauth(request, clerk_token)
|
user_id = await validate_clerk_session_for_oauth(request, clerk_token)
|
||||||
logger.info(f"User authenticated successfully - user_id: {user_id}")
|
|
||||||
|
|
||||||
# Use the Clerk JWT token directly (no need to generate custom token)
|
|
||||||
logger.info("User authenticated successfully via Clerk")
|
|
||||||
|
|
||||||
# Return success response
|
# Return success response
|
||||||
return HTMLResponse("""
|
return HTMLResponse(f"""
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<title>MCP Connection Successful</title>
|
<title>MCP Connection Successful</title>
|
||||||
<style>
|
<style>
|
||||||
body { font-family: Arial, sans-serif; text-align: center; padding: 50px; }
|
body {{ font-family: Arial, sans-serif; text-align: center; padding: 50px; }}
|
||||||
.success { color: #28a745; }
|
.success {{ color: #28a745; }}
|
||||||
.token { background: #f8f9fa; padding: 15px; border-radius: 5px; margin: 20px 0; word-break: break-all; }
|
.token {{ background: #f8f9fa; padding: 15px; border-radius: 5px; margin: 20px 0; word-break: break-all; }}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
@@ -522,20 +502,19 @@ async def mcp_oauth_callback(request: Request, clerk_token: str = Query(None)):
|
|||||||
<p>You can now close this window and return to your MCP client.</p>
|
<p>You can now close this window and return to your MCP client.</p>
|
||||||
<script>
|
<script>
|
||||||
// Try to close the popup if opened as such
|
// Try to close the popup if opened as such
|
||||||
if (window.opener) {
|
if (window.opener) {{
|
||||||
window.opener.postMessage({
|
window.opener.postMessage({{
|
||||||
type: 'MCP_AUTH_SUCCESS',
|
type: 'MCP_AUTH_SUCCESS',
|
||||||
token: 'use_clerk_jwt_token'
|
token: 'use_clerk_jwt_token'
|
||||||
}, '*');
|
}}, '*');
|
||||||
setTimeout(() => window.close(), 3000);
|
setTimeout(() => window.close(), 3000);
|
||||||
}
|
}}
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
""")
|
""")
|
||||||
|
|
||||||
except HTTPException as e:
|
except HTTPException as e:
|
||||||
logger.error(f"MCP OAuth callback failed: {e.detail}")
|
|
||||||
return HTMLResponse(f"""
|
return HTMLResponse(f"""
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
@@ -561,7 +540,6 @@ async def mcp_oauth_callback(request: Request, clerk_token: str = Query(None)):
|
|||||||
</html>
|
</html>
|
||||||
""", status_code=e.status_code)
|
""", status_code=e.status_code)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Unexpected error in MCP OAuth callback: {str(e)}")
|
|
||||||
return HTMLResponse(f"""
|
return HTMLResponse(f"""
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
@@ -607,7 +585,5 @@ app.mount("/mcp/", mcp_app)
|
|||||||
# Set the lifespan context after mounting
|
# Set the lifespan context after mounting
|
||||||
app.router.lifespan_context = mcp_app.lifespan
|
app.router.lifespan_context = mcp_app.lifespan
|
||||||
|
|
||||||
logger.info("MCP app mounted successfully at /mcp/")
|
|
||||||
|
|
||||||
# Export for uvicorn
|
# Export for uvicorn
|
||||||
__all__ = ["app"]
|
__all__ = ["app"]
|
||||||
@@ -1,12 +1,13 @@
|
|||||||
# bddk_mcp_module/client.py
|
# bddk_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Optional
|
from typing import List, Optional, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
import math
|
import math
|
||||||
|
from urllib.parse import urlparse
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# bddk_mcp_module/models.py
|
# bddk_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List
|
from typing import List, Optional
|
||||||
|
|
||||||
class BddkSearchRequest(BaseModel):
|
class BddkSearchRequest(BaseModel):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
# bedesten_mcp_module/models.py
|
# bedesten_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List, Optional, Dict, Any, Literal
|
from typing import List, Optional, Dict, Any, Literal, Union
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
# Import compressed BirimAdiEnum for chamber filtering
|
# Import compressed BirimAdiEnum for chamber filtering
|
||||||
from .enums import BirimAdiEnum
|
from .enums import BirimAdiEnum
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
# danistay_mcp_module/client.py
|
# danistay_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Dict, List, Optional
|
from bs4 import BeautifulSoup
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
|
import re
|
||||||
import io
|
import io
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
# from bs4 import BeautifulSoup # Uncomment if needed for advanced HTML pre-processing
|
# from bs4 import BeautifulSoup # Uncomment if needed for advanced HTML pre-processing
|
||||||
from typing import Dict, Optional
|
from typing import Dict, Any, List, Optional
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
|
import re
|
||||||
import io
|
import io
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
|
|||||||
@@ -7,9 +7,11 @@ import os
|
|||||||
from typing import List, Dict, Any, Optional
|
from typing import List, Dict, Any, Optional
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException, Query
|
from fastapi import FastAPI, HTTPException, Query, Depends, Body
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
import json
|
||||||
|
|
||||||
# Import the main MCP app
|
# Import the main MCP app
|
||||||
from mcp_server_main import app as mcp_server
|
from mcp_server_main import app as mcp_server
|
||||||
|
|||||||
@@ -10,12 +10,13 @@ from playwright.async_api import (
|
|||||||
)
|
)
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Optional
|
from typing import Dict, Any, List, Optional
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
import base64 # Base64 için
|
import base64 # Base64 için
|
||||||
import re
|
import re
|
||||||
import html as html_parser
|
import html as html_parser
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
import os
|
||||||
import math
|
import math
|
||||||
import io
|
import io
|
||||||
import random
|
import random
|
||||||
@@ -906,7 +907,7 @@ class KikApiClient:
|
|||||||
event_target_for_submit = self.FIELD_LOCATORS['search_button_id']
|
event_target_for_submit = self.FIELD_LOCATORS['search_button_id']
|
||||||
# Use human-like clicking for search button
|
# Use human-like clicking for search button
|
||||||
search_button_selector = f"a[id='{event_target_for_submit}']"
|
search_button_selector = f"a[id='{event_target_for_submit}']"
|
||||||
logger.info("Performing human-like search button click...")
|
logger.info(f"Performing human-like search button click...")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Hide datepicker first to prevent interference
|
# Hide datepicker first to prevent interference
|
||||||
@@ -1180,7 +1181,7 @@ class KikApiClient:
|
|||||||
try:
|
try:
|
||||||
if await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).is_visible(timeout=2000):
|
if await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).is_visible(timeout=2000):
|
||||||
await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).click()
|
await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).click()
|
||||||
await current_main_page.wait_for_selector("div#detayPopUp:not(.in)", timeout=5000)
|
await current_main_page.wait_for_selector(f"div#detayPopUp:not(.in)", timeout=5000)
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
return KikDocumentMarkdown(
|
return KikDocumentMarkdown(
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
# kik_mcp_module/client_v2.py
|
# kik_mcp_module/client_v2.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
import requests
|
||||||
import logging
|
import logging
|
||||||
import uuid
|
import uuid
|
||||||
|
import base64
|
||||||
import ssl
|
import ssl
|
||||||
|
from typing import Optional
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from .models_v2 import (
|
from .models_v2 import (
|
||||||
KikV2DecisionType, KikV2SearchPayload, KikV2SearchPayloadDk, KikV2SearchPayloadMk,
|
KikV2DecisionType, KikV2SearchPayload, KikV2SearchPayloadDk, KikV2SearchPayloadMk,
|
||||||
@@ -37,7 +41,8 @@ class KikV2ApiClient:
|
|||||||
ssl_context.check_hostname = False
|
ssl_context.check_hostname = False
|
||||||
ssl_context.verify_mode = ssl.CERT_NONE
|
ssl_context.verify_mode = ssl.CERT_NONE
|
||||||
|
|
||||||
# Enable legacy server connect option for older SSL implementations
|
# Enable legacy server connect option for older SSL implementations (Python 3.12+)
|
||||||
|
if hasattr(ssl, 'OP_LEGACY_SERVER_CONNECT'):
|
||||||
ssl_context.options |= ssl.OP_LEGACY_SERVER_CONNECT
|
ssl_context.options |= ssl.OP_LEGACY_SERVER_CONNECT
|
||||||
|
|
||||||
# Set broader cipher suite support including legacy ciphers
|
# Set broader cipher suite support including legacy ciphers
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# kik_mcp_module/models.py
|
# kik_mcp_module/models.py
|
||||||
from pydantic import BaseModel, Field, computed_field, ConfigDict
|
from pydantic import BaseModel, Field, HttpUrl, computed_field, ConfigDict
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
import base64 # Base64 encoding/decoding için
|
import base64 # Base64 encoding/decoding için
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
# kik_mcp_module/models_v2.py
|
# kik_mcp_module/models_v2.py
|
||||||
from pydantic import BaseModel, Field, ConfigDict
|
from pydantic import BaseModel, Field, ConfigDict
|
||||||
from typing import List
|
from typing import List, Optional
|
||||||
|
from datetime import datetime
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
# New KIK v2 API Models
|
# New KIK v2 API Models
|
||||||
|
|||||||
@@ -2,13 +2,13 @@
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import Optional, Dict, Any
|
from typing import List, Optional, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
import math
|
import math
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urljoin, urlparse, parse_qs
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
from pydantic import HttpUrl
|
from pydantic import HttpUrl
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# kvkk_mcp_module/models.py
|
# kvkk_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, HttpUrl
|
from pydantic import BaseModel, Field, HttpUrl
|
||||||
from typing import List, Optional
|
from typing import List, Optional, Any
|
||||||
|
|
||||||
class KvkkSearchRequest(BaseModel):
|
class KvkkSearchRequest(BaseModel):
|
||||||
"""Model for KVKK (Personal Data Protection Authority) search request via Brave API."""
|
"""Model for KVKK (Personal Data Protection Authority) search request via Brave API."""
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ def create_clerk_oauth_config() -> OAuthConfig:
|
|||||||
scopes=["mcp:tools:read", "mcp:tools:write", "openid", "profile", "email"]
|
scopes=["mcp:tools:read", "mcp:tools:write", "openid", "profile", "email"]
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Created Clerk OAuth config with adapter endpoints")
|
logger.info(f"Created Clerk OAuth config with adapter endpoints")
|
||||||
logger.info(f"Clerk domain: {clerk_domain}")
|
logger.info(f"Clerk domain: {clerk_domain}")
|
||||||
logger.debug(f"Authorization endpoint: {config.authorization_endpoint}")
|
logger.debug(f"Authorization endpoint: {config.authorization_endpoint}")
|
||||||
logger.debug(f"Token endpoint: {config.token_endpoint}")
|
logger.debug(f"Token endpoint: {config.token_endpoint}")
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@ import time
|
|||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from typing import Any
|
from typing import Any, Optional
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from fastapi.responses import RedirectResponse, JSONResponse
|
|||||||
try:
|
try:
|
||||||
from clerk_backend_api import Clerk
|
from clerk_backend_api import Clerk
|
||||||
CLERK_AVAILABLE = True
|
CLERK_AVAILABLE = True
|
||||||
except ImportError:
|
except ImportError as e:
|
||||||
CLERK_AVAILABLE = False
|
CLERK_AVAILABLE = False
|
||||||
Clerk = None
|
Clerk = None
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ Uses Redis for authorization code storage to support multi-machine deployment
|
|||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode, quote
|
||||||
|
|
||||||
from fastapi import APIRouter, Request, Query, HTTPException
|
from fastapi import APIRouter, Request, Query, HTTPException
|
||||||
from fastapi.responses import RedirectResponse, JSONResponse
|
from fastapi.responses import RedirectResponse, JSONResponse
|
||||||
@@ -39,6 +39,7 @@ def get_redis_session_store():
|
|||||||
if redis_store is None:
|
if redis_store is None:
|
||||||
try:
|
try:
|
||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
|
import functools
|
||||||
|
|
||||||
# Use thread pool with timeout to prevent hanging
|
# Use thread pool with timeout to prevent hanging
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||||
@@ -205,6 +206,7 @@ async def oauth_callback(
|
|||||||
auth_code = f"clerk_auth_{os.urandom(16).hex()}"
|
auth_code = f"clerk_auth_{os.urandom(16).hex()}"
|
||||||
|
|
||||||
# Prepare code data
|
# Prepare code data
|
||||||
|
import time
|
||||||
code_data = {
|
code_data = {
|
||||||
"user_id": user_id,
|
"user_id": user_id,
|
||||||
"session_id": session_id,
|
"session_id": session_id,
|
||||||
@@ -223,7 +225,7 @@ async def oauth_callback(
|
|||||||
if success:
|
if success:
|
||||||
logger.info(f"Stored authorization code {auth_code[:10]}... in Redis with real JWT token")
|
logger.info(f"Stored authorization code {auth_code[:10]}... in Redis with real JWT token")
|
||||||
else:
|
else:
|
||||||
logger.error("Failed to store authorization code in Redis, falling back to in-memory")
|
logger.error(f"Failed to store authorization code in Redis, falling back to in-memory")
|
||||||
# Fall back to in-memory storage
|
# Fall back to in-memory storage
|
||||||
if not hasattr(oauth_callback, '_code_storage'):
|
if not hasattr(oauth_callback, '_code_storage'):
|
||||||
oauth_callback._code_storage = {}
|
oauth_callback._code_storage = {}
|
||||||
@@ -234,7 +236,7 @@ async def oauth_callback(
|
|||||||
if not hasattr(oauth_callback, '_code_storage'):
|
if not hasattr(oauth_callback, '_code_storage'):
|
||||||
oauth_callback._code_storage = {}
|
oauth_callback._code_storage = {}
|
||||||
oauth_callback._code_storage[auth_code] = code_data
|
oauth_callback._code_storage[auth_code] = code_data
|
||||||
logger.info("Stored authorization code in memory (fallback)")
|
logger.info(f"Stored authorization code in memory (fallback)")
|
||||||
|
|
||||||
# Redirect back to client with authorization code
|
# Redirect back to client with authorization code
|
||||||
redirect_params = {
|
redirect_params = {
|
||||||
|
|||||||
+155
-315
@@ -7,10 +7,18 @@ import httpx
|
|||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from pydantic import HttpUrl, Field
|
from pydantic import BaseModel, HttpUrl, Field
|
||||||
from typing import Dict, List, Literal, Any
|
from typing import Optional, Dict, List, Literal, Any, Union
|
||||||
import tiktoken
|
import urllib.parse
|
||||||
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
||||||
|
|
||||||
|
# Optional tiktoken import for token counting
|
||||||
|
try:
|
||||||
|
import tiktoken
|
||||||
|
TIKTOKEN_AVAILABLE = True
|
||||||
|
except ImportError:
|
||||||
|
TIKTOKEN_AVAILABLE = False
|
||||||
|
tiktoken = None
|
||||||
from fastmcp.server.dependencies import get_access_token, AccessToken
|
from fastmcp.server.dependencies import get_access_token, AccessToken
|
||||||
from fastmcp import Context
|
from fastmcp import Context
|
||||||
|
|
||||||
@@ -20,22 +28,11 @@ class ToolError(Exception):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
# --- Logging Configuration Start ---
|
# --- Logging Configuration Start ---
|
||||||
LOG_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs")
|
|
||||||
if not os.path.exists(LOG_DIRECTORY):
|
|
||||||
os.makedirs(LOG_DIRECTORY)
|
|
||||||
LOG_FILE_PATH = os.path.join(LOG_DIRECTORY, "mcp_server.log")
|
|
||||||
|
|
||||||
root_logger = logging.getLogger()
|
root_logger = logging.getLogger()
|
||||||
root_logger.setLevel(logging.DEBUG)
|
root_logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(threadName)s - %(message)s')
|
|
||||||
|
|
||||||
file_handler = logging.FileHandler(LOG_FILE_PATH, mode='a', encoding='utf-8')
|
|
||||||
file_handler.setFormatter(log_formatter)
|
|
||||||
file_handler.setLevel(logging.DEBUG)
|
|
||||||
root_logger.addHandler(file_handler)
|
|
||||||
|
|
||||||
console_handler = logging.StreamHandler()
|
console_handler = logging.StreamHandler()
|
||||||
|
log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||||
console_handler.setFormatter(log_formatter)
|
console_handler.setFormatter(log_formatter)
|
||||||
console_handler.setLevel(logging.INFO)
|
console_handler.setLevel(logging.INFO)
|
||||||
root_logger.addHandler(console_handler)
|
root_logger.addHandler(console_handler)
|
||||||
@@ -53,18 +50,13 @@ class TokenCountingMiddleware(Middleware):
|
|||||||
Args:
|
Args:
|
||||||
model: Tiktoken model name (cl100k_base for GPT-4/Claude compatibility)
|
model: Tiktoken model name (cl100k_base for GPT-4/Claude compatibility)
|
||||||
"""
|
"""
|
||||||
|
if not TIKTOKEN_AVAILABLE:
|
||||||
|
raise ImportError("tiktoken is required for token counting. Install with: pip install tiktoken")
|
||||||
|
|
||||||
self.encoder = tiktoken.get_encoding(model)
|
self.encoder = tiktoken.get_encoding(model)
|
||||||
self.model = model
|
self.model = model
|
||||||
self.token_stats = defaultdict(lambda: {"input": 0, "output": 0, "calls": 0})
|
self.token_stats = defaultdict(lambda: {"input": 0, "output": 0, "calls": 0})
|
||||||
self.logger = logging.getLogger("token_counter")
|
self.logger = logging.getLogger("token_counter")
|
||||||
|
|
||||||
# Create separate log file for token metrics
|
|
||||||
token_log_path = os.path.join(LOG_DIRECTORY, "token_metrics.log")
|
|
||||||
token_handler = logging.FileHandler(token_log_path, mode='a', encoding='utf-8')
|
|
||||||
token_formatter = logging.Formatter('%(asctime)s - %(message)s')
|
|
||||||
token_handler.setFormatter(token_formatter)
|
|
||||||
token_handler.setLevel(logging.INFO)
|
|
||||||
self.logger.addHandler(token_handler)
|
|
||||||
self.logger.setLevel(logging.INFO)
|
self.logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
def count_tokens(self, text: str) -> int:
|
def count_tokens(self, text: str) -> int:
|
||||||
@@ -158,7 +150,7 @@ class TokenCountingMiddleware(Middleware):
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||||
self.log_token_usage("tool_call_error", input_tokens, 0,
|
self.log_token_usage("tool_call_error", input_tokens, 0,
|
||||||
tool_name, duration_ms)
|
tool_name, duration_ms)
|
||||||
@@ -188,7 +180,7 @@ class TokenCountingMiddleware(Middleware):
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||||
self.log_token_usage("resource_read_error", 0, 0,
|
self.log_token_usage("resource_read_error", 0, 0,
|
||||||
resource_uri, duration_ms)
|
resource_uri, duration_ms)
|
||||||
@@ -218,7 +210,7 @@ class TokenCountingMiddleware(Middleware):
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||||
self.log_token_usage("prompt_get_error", 0, 0,
|
self.log_token_usage("prompt_get_error", 0, 0,
|
||||||
prompt_name, duration_ms)
|
prompt_name, duration_ms)
|
||||||
@@ -246,26 +238,34 @@ def create_app(auth=None):
|
|||||||
else:
|
else:
|
||||||
logger.info("MCP server created with standard capabilities...")
|
logger.info("MCP server created with standard capabilities...")
|
||||||
|
|
||||||
|
# Add token counting middleware only if tiktoken is available
|
||||||
|
if TIKTOKEN_AVAILABLE:
|
||||||
|
try:
|
||||||
token_counter = TokenCountingMiddleware()
|
token_counter = TokenCountingMiddleware()
|
||||||
app.add_middleware(token_counter)
|
app.add_middleware(token_counter)
|
||||||
logger.info("Token counting middleware added to MCP server")
|
logger.info("Token counting middleware added to MCP server")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to add token counting middleware: {e}")
|
||||||
|
|
||||||
return app
|
return app
|
||||||
|
|
||||||
# --- Module Imports ---
|
# --- Module Imports ---
|
||||||
from yargitay_mcp_module.client import YargitayOfficialApiClient
|
from yargitay_mcp_module.client import YargitayOfficialApiClient
|
||||||
|
from yargitay_mcp_module.models import (
|
||||||
|
YargitayDetailedSearchRequest, YargitayDocumentMarkdown, CompactYargitaySearchResult,
|
||||||
|
YargitayBirimEnum, CleanYargitayDecisionEntry
|
||||||
|
)
|
||||||
from bedesten_mcp_module.client import BedestenApiClient
|
from bedesten_mcp_module.client import BedestenApiClient
|
||||||
from bedesten_mcp_module.models import (
|
from bedesten_mcp_module.models import (
|
||||||
BedestenSearchRequest, BedestenSearchData,
|
BedestenSearchRequest, BedestenSearchData,
|
||||||
BedestenDocumentMarkdown, BedestenCourtTypeEnum
|
BedestenDocumentMarkdown, BedestenCourtTypeEnum
|
||||||
)
|
)
|
||||||
from bedesten_mcp_module.enums import BirimAdiEnum
|
from bedesten_mcp_module.enums import BirimAdiEnum
|
||||||
|
|
||||||
# Semantic Search Module Imports
|
|
||||||
from semantic_search.embedder import EmbeddingGemma
|
|
||||||
from semantic_search.vector_store import VectorStore
|
|
||||||
from semantic_search.processor import DocumentProcessor
|
|
||||||
from danistay_mcp_module.client import DanistayApiClient
|
from danistay_mcp_module.client import DanistayApiClient
|
||||||
|
from danistay_mcp_module.models import (
|
||||||
|
DanistayKeywordSearchRequest, DanistayDetailedSearchRequest,
|
||||||
|
DanistayDocumentMarkdown, CompactDanistaySearchResult
|
||||||
|
)
|
||||||
from emsal_mcp_module.client import EmsalApiClient
|
from emsal_mcp_module.client import EmsalApiClient
|
||||||
from emsal_mcp_module.models import (
|
from emsal_mcp_module.models import (
|
||||||
EmsalSearchRequest, EmsalDocumentMarkdown, CompactEmsalSearchResult
|
EmsalSearchRequest, EmsalDocumentMarkdown, CompactEmsalSearchResult
|
||||||
@@ -279,12 +279,24 @@ from anayasa_mcp_module.client import AnayasaMahkemesiApiClient
|
|||||||
from anayasa_mcp_module.bireysel_client import AnayasaBireyselBasvuruApiClient
|
from anayasa_mcp_module.bireysel_client import AnayasaBireyselBasvuruApiClient
|
||||||
from anayasa_mcp_module.unified_client import AnayasaUnifiedClient
|
from anayasa_mcp_module.unified_client import AnayasaUnifiedClient
|
||||||
from anayasa_mcp_module.models import (
|
from anayasa_mcp_module.models import (
|
||||||
|
AnayasaNormDenetimiSearchRequest,
|
||||||
|
AnayasaSearchResult,
|
||||||
|
AnayasaDocumentMarkdown,
|
||||||
|
AnayasaBireyselReportSearchRequest,
|
||||||
|
AnayasaBireyselReportSearchResult,
|
||||||
|
AnayasaBireyselBasvuruDocumentMarkdown,
|
||||||
AnayasaUnifiedSearchRequest,
|
AnayasaUnifiedSearchRequest,
|
||||||
|
AnayasaUnifiedSearchResult,
|
||||||
|
AnayasaUnifiedDocumentMarkdown,
|
||||||
# Removed enum imports - now using Literal strings in models
|
# Removed enum imports - now using Literal strings in models
|
||||||
)
|
)
|
||||||
# KIK v2 Module Imports (New API)
|
# KIK v2 Module Imports (New API)
|
||||||
from kik_mcp_module.client_v2 import KikV2ApiClient
|
from kik_mcp_module.client_v2 import KikV2ApiClient
|
||||||
from kik_mcp_module.models_v2 import KikV2DecisionType
|
from kik_mcp_module.models_v2 import KikV2DecisionType
|
||||||
|
from kik_mcp_module.models_v2 import (
|
||||||
|
KikV2SearchResult,
|
||||||
|
KikV2DocumentMarkdown
|
||||||
|
)
|
||||||
|
|
||||||
from rekabet_mcp_module.client import RekabetKurumuApiClient
|
from rekabet_mcp_module.client import RekabetKurumuApiClient
|
||||||
from rekabet_mcp_module.models import (
|
from rekabet_mcp_module.models import (
|
||||||
@@ -296,9 +308,14 @@ from rekabet_mcp_module.models import (
|
|||||||
|
|
||||||
from sayistay_mcp_module.client import SayistayApiClient
|
from sayistay_mcp_module.client import SayistayApiClient
|
||||||
from sayistay_mcp_module.models import (
|
from sayistay_mcp_module.models import (
|
||||||
|
GenelKurulSearchRequest, GenelKurulSearchResponse,
|
||||||
|
TemyizKuruluSearchRequest, TemyizKuruluSearchResponse,
|
||||||
|
DaireSearchRequest, DaireSearchResponse,
|
||||||
|
SayistayDocumentMarkdown,
|
||||||
SayistayUnifiedSearchRequest, SayistayUnifiedSearchResult,
|
SayistayUnifiedSearchRequest, SayistayUnifiedSearchResult,
|
||||||
SayistayUnifiedDocumentMarkdown
|
SayistayUnifiedDocumentMarkdown
|
||||||
)
|
)
|
||||||
|
from sayistay_mcp_module.enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
|
||||||
from sayistay_mcp_module.unified_client import SayistayUnifiedClient
|
from sayistay_mcp_module.unified_client import SayistayUnifiedClient
|
||||||
|
|
||||||
# KVKK Module Imports
|
# KVKK Module Imports
|
||||||
@@ -312,11 +329,14 @@ from kvkk_mcp_module.models import (
|
|||||||
# BDDK Module Imports
|
# BDDK Module Imports
|
||||||
from bddk_mcp_module.client import BddkApiClient
|
from bddk_mcp_module.client import BddkApiClient
|
||||||
from bddk_mcp_module.models import (
|
from bddk_mcp_module.models import (
|
||||||
BddkSearchRequest
|
BddkSearchRequest,
|
||||||
|
BddkSearchResult,
|
||||||
|
BddkDocumentMarkdown
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Create a placeholder app that will be properly initialized after tools are defined
|
# Create a placeholder app that will be properly initialized after tools are defined
|
||||||
|
from fastmcp import FastMCP
|
||||||
|
|
||||||
# MCP app for Turkish legal databases with explicit capabilities
|
# MCP app for Turkish legal databases with explicit capabilities
|
||||||
app = FastMCP(
|
app = FastMCP(
|
||||||
@@ -342,6 +362,9 @@ sayistay_unified_client_instance = SayistayUnifiedClient()
|
|||||||
kvkk_client_instance = KvkkApiClient()
|
kvkk_client_instance = KvkkApiClient()
|
||||||
bddk_client_instance = BddkApiClient()
|
bddk_client_instance = BddkApiClient()
|
||||||
|
|
||||||
|
# Health check client (singleton for reuse)
|
||||||
|
_health_check_client: Optional[httpx.AsyncClient] = None
|
||||||
|
|
||||||
|
|
||||||
KARAR_TURU_ADI_TO_GUID_ENUM_MAP = {
|
KARAR_TURU_ADI_TO_GUID_ENUM_MAP = {
|
||||||
"": RekabetKararTuruGuidEnum.TUMU, # Keep for backward compatibility
|
"": RekabetKararTuruGuidEnum.TUMU, # Keep for backward compatibility
|
||||||
@@ -599,7 +622,7 @@ async def search_emsal_detailed_decisions(
|
|||||||
sort_direction: str = Field("desc", description="Sorting direction ('asc' or 'desc')."),
|
sort_direction: str = Field("desc", description="Sorting direction ('asc' or 'desc')."),
|
||||||
page_number: int = Field(1, ge=1, description="Page number (accepts int)."),
|
page_number: int = Field(1, ge=1, description="Page number (accepts int)."),
|
||||||
# page_size: int = Field(10, ge=1, le=10, description="Results per page.")
|
# page_size: int = Field(10, ge=1, le=10, description="Results per page.")
|
||||||
) -> CompactEmsalSearchResult:
|
) -> Dict[str, Any]:
|
||||||
"""Search Emsal precedent decisions with detailed criteria."""
|
"""Search Emsal precedent decisions with detailed criteria."""
|
||||||
|
|
||||||
page_size = 10 # Default value
|
page_size = 10 # Default value
|
||||||
@@ -623,7 +646,7 @@ async def search_emsal_detailed_decisions(
|
|||||||
page_size=page_size
|
page_size=page_size
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Tool 'search_emsal_detailed_decisions' called.")
|
logger.info(f"Tool 'search_emsal_detailed_decisions' called.")
|
||||||
try:
|
try:
|
||||||
api_response = await emsal_client_instance.search_detailed_decisions(search_query)
|
api_response = await emsal_client_instance.search_detailed_decisions(search_query)
|
||||||
if api_response.data:
|
if api_response.data:
|
||||||
@@ -632,11 +655,11 @@ async def search_emsal_detailed_decisions(
|
|||||||
total_records=api_response.data.recordsTotal if api_response.data.recordsTotal is not None else 0,
|
total_records=api_response.data.recordsTotal if api_response.data.recordsTotal is not None else 0,
|
||||||
requested_page=search_query.page_number,
|
requested_page=search_query.page_number,
|
||||||
page_size=search_query.page_size
|
page_size=search_query.page_size
|
||||||
)
|
).model_dump()
|
||||||
logger.warning("API response for Emsal search did not contain expected data structure.")
|
logger.warning("API response for Emsal search did not contain expected data structure.")
|
||||||
return CompactEmsalSearchResult(decisions=[], total_records=0, requested_page=search_query.page_number, page_size=search_query.page_size)
|
return CompactEmsalSearchResult(decisions=[], total_records=0, requested_page=search_query.page_number, page_size=search_query.page_size).model_dump()
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_emsal_detailed_decisions'.")
|
logger.exception(f"Error in tool 'search_emsal_detailed_decisions'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -646,14 +669,15 @@ async def search_emsal_detailed_decisions(
|
|||||||
"idempotentHint": True
|
"idempotentHint": True
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
async def get_emsal_document_markdown(id: str) -> EmsalDocumentMarkdown:
|
async def get_emsal_document_markdown(id: str) -> Dict[str, Any]:
|
||||||
"""Get document as Markdown."""
|
"""Get document as Markdown."""
|
||||||
logger.info(f"Tool 'get_emsal_document_markdown' called for ID: {id}")
|
logger.info(f"Tool 'get_emsal_document_markdown' called for ID: {id}")
|
||||||
if not id or not id.strip(): raise ValueError("Document ID required for Emsal.")
|
if not id or not id.strip(): raise ValueError("Document ID required for Emsal.")
|
||||||
try:
|
try:
|
||||||
return await emsal_client_instance.get_decision_document_as_markdown(id)
|
result = await emsal_client_instance.get_decision_document_as_markdown(id)
|
||||||
except Exception:
|
return result.model_dump()
|
||||||
logger.exception("Error in tool 'get_emsal_document_markdown'.")
|
except Exception as e:
|
||||||
|
logger.exception(f"Error in tool 'get_emsal_document_markdown'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
# --- MCP Tools for Uyusmazlik ---
|
# --- MCP Tools for Uyusmazlik ---
|
||||||
@@ -684,7 +708,7 @@ async def search_uyusmazlik_decisions(
|
|||||||
hepsi: str = Field("", description="Search for texts containing all specified words."),
|
hepsi: str = Field("", description="Search for texts containing all specified words."),
|
||||||
herhangi_birisi: str = Field("", description="Search for texts containing any of the specified words."),
|
herhangi_birisi: str = Field("", description="Search for texts containing any of the specified words."),
|
||||||
not_hepsi: str = Field("", description="Exclude texts containing these specified words.")
|
not_hepsi: str = Field("", description="Exclude texts containing these specified words.")
|
||||||
) -> UyusmazlikSearchResponse:
|
) -> Dict[str, Any]:
|
||||||
"""Search Court of Jurisdictional Disputes decisions."""
|
"""Search Court of Jurisdictional Disputes decisions."""
|
||||||
|
|
||||||
# Convert string literals to enums
|
# Convert string literals to enums
|
||||||
@@ -721,11 +745,12 @@ async def search_uyusmazlik_decisions(
|
|||||||
not_hepsi=not_hepsi
|
not_hepsi=not_hepsi
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Tool 'search_uyusmazlik_decisions' called.")
|
logger.info(f"Tool 'search_uyusmazlik_decisions' called.")
|
||||||
try:
|
try:
|
||||||
return await uyusmazlik_client_instance.search_decisions(search_params)
|
result = await uyusmazlik_client_instance.search_decisions(search_params)
|
||||||
except Exception:
|
return result.model_dump()
|
||||||
logger.exception("Error in tool 'search_uyusmazlik_decisions'.")
|
except Exception as e:
|
||||||
|
logger.exception(f"Error in tool 'search_uyusmazlik_decisions'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -737,15 +762,16 @@ async def search_uyusmazlik_decisions(
|
|||||||
)
|
)
|
||||||
async def get_uyusmazlik_document_markdown_from_url(
|
async def get_uyusmazlik_document_markdown_from_url(
|
||||||
document_url: str = Field(..., description="Full URL to the Uyuşmazlık Mahkemesi decision document from search results")
|
document_url: str = Field(..., description="Full URL to the Uyuşmazlık Mahkemesi decision document from search results")
|
||||||
) -> UyusmazlikDocumentMarkdown:
|
) -> Dict[str, Any]:
|
||||||
"""Get Uyuşmazlık Mahkemesi decision as Markdown."""
|
"""Get Uyuşmazlık Mahkemesi decision as Markdown."""
|
||||||
logger.info(f"Tool 'get_uyusmazlik_document_markdown_from_url' called for URL: {str(document_url)}")
|
logger.info(f"Tool 'get_uyusmazlik_document_markdown_from_url' called for URL: {str(document_url)}")
|
||||||
if not document_url:
|
if not document_url:
|
||||||
raise ValueError("Document URL (document_url) is required for Uyuşmazlık document retrieval.")
|
raise ValueError("Document URL (document_url) is required for Uyuşmazlık document retrieval.")
|
||||||
try:
|
try:
|
||||||
return await uyusmazlik_client_instance.get_decision_document_as_markdown(str(document_url))
|
result = await uyusmazlik_client_instance.get_decision_document_as_markdown(str(document_url))
|
||||||
except Exception:
|
return result.model_dump()
|
||||||
logger.exception("Error in tool 'get_uyusmazlik_document_markdown_from_url'.")
|
except Exception as e:
|
||||||
|
logger.exception(f"Error in tool 'get_uyusmazlik_document_markdown_from_url'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
# --- DEACTIVATED: MCP Tools for Anayasa Mahkemesi (Individual Tools) ---
|
# --- DEACTIVATED: MCP Tools for Anayasa Mahkemesi (Individual Tools) ---
|
||||||
@@ -836,8 +862,8 @@ async def search_anayasa_unified(
|
|||||||
result = await anayasa_unified_client_instance.search_unified(request)
|
result = await anayasa_unified_client_instance.search_unified(request)
|
||||||
return json.dumps(result.model_dump(), ensure_ascii=False, indent=2)
|
return json.dumps(result.model_dump(), ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_anayasa_unified'.")
|
logger.exception(f"Error in tool 'search_anayasa_unified'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -858,8 +884,8 @@ async def get_anayasa_document_unified(
|
|||||||
result = await anayasa_unified_client_instance.get_document_unified(document_url, page_number)
|
result = await anayasa_unified_client_instance.get_document_unified(document_url, page_number)
|
||||||
return json.dumps(result.model_dump(mode='json'), ensure_ascii=False, indent=2)
|
return json.dumps(result.model_dump(mode='json'), ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_anayasa_document_unified'.")
|
logger.exception(f"Error in tool 'get_anayasa_document_unified'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
# --- MCP Tools for KIK v2 (Kamu İhale Kurulu - New API) ---
|
# --- MCP Tools for KIK v2 (Kamu İhale Kurulu - New API) ---
|
||||||
@@ -1007,7 +1033,7 @@ async def search_rekabet_kurumu_decisions(
|
|||||||
KararSayisi: str = Field("", description="Decision number (Karar Sayısı)."),
|
KararSayisi: str = Field("", description="Decision number (Karar Sayısı)."),
|
||||||
KararTarihi: str = Field("", description="Decision date (Karar Tarihi), e.g., DD.MM.YYYY."),
|
KararTarihi: str = Field("", description="Decision date (Karar Tarihi), e.g., DD.MM.YYYY."),
|
||||||
page: int = Field(1, ge=1, description="Page number to fetch for the results list.")
|
page: int = Field(1, ge=1, description="Page number to fetch for the results list.")
|
||||||
) -> RekabetSearchResult:
|
) -> Dict[str, Any]:
|
||||||
"""Search Competition Authority decisions."""
|
"""Search Competition Authority decisions."""
|
||||||
|
|
||||||
karar_turu_guid_enum = KARAR_TURU_ADI_TO_GUID_ENUM_MAP.get(KararTuru)
|
karar_turu_guid_enum = KARAR_TURU_ADI_TO_GUID_ENUM_MAP.get(KararTuru)
|
||||||
@@ -1032,10 +1058,11 @@ async def search_rekabet_kurumu_decisions(
|
|||||||
logger.info(f"Tool 'search_rekabet_kurumu_decisions' called. Query: {search_query.model_dump_json(exclude_none=True, indent=2)}")
|
logger.info(f"Tool 'search_rekabet_kurumu_decisions' called. Query: {search_query.model_dump_json(exclude_none=True, indent=2)}")
|
||||||
try:
|
try:
|
||||||
|
|
||||||
return await rekabet_client_instance.search_decisions(search_query)
|
result = await rekabet_client_instance.search_decisions(search_query)
|
||||||
except Exception:
|
return result.model_dump()
|
||||||
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_rekabet_kurumu_decisions'.")
|
logger.exception("Error in tool 'search_rekabet_kurumu_decisions'.")
|
||||||
return RekabetSearchResult(decisions=[], retrieved_page_number=page, total_records_found=0, total_pages=0)
|
return RekabetSearchResult(decisions=[], retrieved_page_number=page, total_records_found=0, total_pages=0).model_dump()
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
description="Get Competition Authority decision text in paginated Markdown format",
|
description="Get Competition Authority decision text in paginated Markdown format",
|
||||||
@@ -1047,16 +1074,16 @@ async def search_rekabet_kurumu_decisions(
|
|||||||
async def get_rekabet_kurumu_document(
|
async def get_rekabet_kurumu_document(
|
||||||
karar_id: str = Field(..., description="GUID (kararId) of the Rekabet Kurumu decision. This ID is obtained from search results."),
|
karar_id: str = Field(..., description="GUID (kararId) of the Rekabet Kurumu decision. This ID is obtained from search results."),
|
||||||
page_number: int = Field(1, ge=1, description="Requested page number for the Markdown content converted from PDF (1-indexed, accepts int). Default is 1.")
|
page_number: int = Field(1, ge=1, description="Requested page number for the Markdown content converted from PDF (1-indexed, accepts int). Default is 1.")
|
||||||
) -> RekabetDocument:
|
) -> Dict[str, Any]:
|
||||||
"""Get Competition Authority decision as paginated Markdown."""
|
"""Get Competition Authority decision as paginated Markdown."""
|
||||||
logger.info(f"Tool 'get_rekabet_kurumu_document' called. Karar ID: {karar_id}, Markdown Page: {page_number}")
|
logger.info(f"Tool 'get_rekabet_kurumu_document' called. Karar ID: {karar_id}, Markdown Page: {page_number}")
|
||||||
|
|
||||||
current_page_to_fetch = page_number if page_number >= 1 else 1
|
current_page_to_fetch = page_number if page_number >= 1 else 1
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
result = await rekabet_client_instance.get_decision_document(karar_id, page_number=current_page_to_fetch)
|
||||||
return await rekabet_client_instance.get_decision_document(karar_id, page_number=current_page_to_fetch)
|
return result.model_dump()
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception(f"Error in tool 'get_rekabet_kurumu_document'. Karar ID: {karar_id}")
|
logger.exception(f"Error in tool 'get_rekabet_kurumu_document'. Karar ID: {karar_id}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -1167,7 +1194,7 @@ For best results, use exact phrases with quotes for legal terms."""),
|
|||||||
"page_size": pageSize,
|
"page_size": pageSize,
|
||||||
"searched_courts": court_types
|
"searched_courts": court_types
|
||||||
}
|
}
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_bedesten_unified'")
|
logger.exception("Error in tool 'search_bedesten_unified'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -1189,7 +1216,7 @@ async def get_bedesten_document_markdown(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
return await bedesten_client_instance.get_document_as_markdown(documentId)
|
return await bedesten_client_instance.get_document_as_markdown(documentId)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_kyb_bedesten_document_markdown'")
|
logger.exception("Error in tool 'get_kyb_bedesten_document_markdown'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -1351,7 +1378,7 @@ async def search_sayistay_unified(
|
|||||||
yargilama_dairesi: Literal["ALL", "1", "2", "3", "4", "5", "6", "7", "8"] = Field("ALL", description="Chamber selection (daire only)"),
|
yargilama_dairesi: Literal["ALL", "1", "2", "3", "4", "5", "6", "7", "8"] = Field("ALL", description="Chamber selection (daire only)"),
|
||||||
hesap_yili: str = Field("", description="Account year (daire only)"),
|
hesap_yili: str = Field("", description="Account year (daire only)"),
|
||||||
web_karar_metni: str = Field("", description="Decision text search (daire only)")
|
web_karar_metni: str = Field("", description="Decision text search (daire only)")
|
||||||
) -> SayistayUnifiedSearchResult:
|
) -> Dict[str, Any]:
|
||||||
"""Search Sayıştay decisions across all three decision types with unified interface."""
|
"""Search Sayıştay decisions across all three decision types with unified interface."""
|
||||||
logger.info(f"Tool 'search_sayistay_unified' called with decision_type={decision_type}")
|
logger.info(f"Tool 'search_sayistay_unified' called with decision_type={decision_type}")
|
||||||
|
|
||||||
@@ -1377,8 +1404,9 @@ async def search_sayistay_unified(
|
|||||||
hesap_yili=hesap_yili,
|
hesap_yili=hesap_yili,
|
||||||
web_karar_metni=web_karar_metni
|
web_karar_metni=web_karar_metni
|
||||||
)
|
)
|
||||||
return await sayistay_unified_client_instance.search_unified(search_request)
|
result = await sayistay_unified_client_instance.search_unified(search_request)
|
||||||
except Exception:
|
return result.model_dump()
|
||||||
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_sayistay_unified'")
|
logger.exception("Error in tool 'search_sayistay_unified'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -1393,7 +1421,7 @@ async def search_sayistay_unified(
|
|||||||
async def get_sayistay_document_unified(
|
async def get_sayistay_document_unified(
|
||||||
decision_id: str = Field(..., description="Decision ID from search_sayistay_unified results"),
|
decision_id: str = Field(..., description="Decision ID from search_sayistay_unified results"),
|
||||||
decision_type: Literal["genel_kurul", "temyiz_kurulu", "daire"] = Field(..., description="Decision type: genel_kurul, temyiz_kurulu, or daire")
|
decision_type: Literal["genel_kurul", "temyiz_kurulu", "daire"] = Field(..., description="Decision type: genel_kurul, temyiz_kurulu, or daire")
|
||||||
) -> SayistayUnifiedDocumentMarkdown:
|
) -> Dict[str, Any]:
|
||||||
"""Get Sayıştay decision document as Markdown for any decision type."""
|
"""Get Sayıştay decision document as Markdown for any decision type."""
|
||||||
logger.info(f"Tool 'get_sayistay_document_unified' called for ID: {decision_id}, type: {decision_type}")
|
logger.info(f"Tool 'get_sayistay_document_unified' called for ID: {decision_id}, type: {decision_type}")
|
||||||
|
|
||||||
@@ -1401,8 +1429,9 @@ async def get_sayistay_document_unified(
|
|||||||
raise ValueError("Decision ID must be a non-empty string.")
|
raise ValueError("Decision ID must be a non-empty string.")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
return await sayistay_unified_client_instance.get_document_unified(decision_id, decision_type)
|
result = await sayistay_unified_client_instance.get_document_unified(decision_id, decision_type)
|
||||||
except Exception:
|
return result.model_dump()
|
||||||
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_sayistay_document_unified'")
|
logger.exception("Error in tool 'get_sayistay_document_unified'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -1439,6 +1468,11 @@ def perform_cleanup():
|
|||||||
if client_instance and hasattr(client_instance, 'close_client_session') and callable(client_instance.close_client_session):
|
if client_instance and hasattr(client_instance, 'close_client_session') and callable(client_instance.close_client_session):
|
||||||
logger.info(f"Scheduling close for client session: {client_instance.__class__.__name__}")
|
logger.info(f"Scheduling close for client session: {client_instance.__class__.__name__}")
|
||||||
tasks.append(client_instance.close_client_session())
|
tasks.append(client_instance.close_client_session())
|
||||||
|
# Close health check client if it was created
|
||||||
|
global _health_check_client
|
||||||
|
if _health_check_client is not None:
|
||||||
|
logger.info("Closing health check HTTP client")
|
||||||
|
tasks.append(_health_check_client.aclose())
|
||||||
if tasks:
|
if tasks:
|
||||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||||
for i, result in enumerate(results):
|
for i, result in enumerate(results):
|
||||||
@@ -1460,6 +1494,19 @@ def perform_cleanup():
|
|||||||
|
|
||||||
atexit.register(perform_cleanup)
|
atexit.register(perform_cleanup)
|
||||||
|
|
||||||
|
|
||||||
|
def get_or_create_health_check_client() -> httpx.AsyncClient:
|
||||||
|
"""Get or create a reusable HTTP client for health checks."""
|
||||||
|
global _health_check_client
|
||||||
|
if _health_check_client is None:
|
||||||
|
_health_check_client = httpx.AsyncClient(
|
||||||
|
timeout=10.0,
|
||||||
|
verify=False,
|
||||||
|
follow_redirects=True
|
||||||
|
)
|
||||||
|
return _health_check_client
|
||||||
|
|
||||||
|
|
||||||
# --- Health Check Tools ---
|
# --- Health Check Tools ---
|
||||||
@app.tool(
|
@app.tool(
|
||||||
description="Check if Turkish government legal database servers are operational",
|
description="Check if Turkish government legal database servers are operational",
|
||||||
@@ -1626,7 +1673,7 @@ async def search_kvkk_decisions(
|
|||||||
keywords: str = Field(..., description="Turkish keywords. Supports +required -excluded \"exact phrase\" operators"),
|
keywords: str = Field(..., description="Turkish keywords. Supports +required -excluded \"exact phrase\" operators"),
|
||||||
page: int = Field(1, ge=1, le=50, description="Page number for results (1-50)."),
|
page: int = Field(1, ge=1, le=50, description="Page number for results (1-50)."),
|
||||||
# pageSize: int = Field(10, ge=1, le=20, description="Number of results per page (1-20).")
|
# pageSize: int = Field(10, ge=1, le=20, description="Number of results per page (1-20).")
|
||||||
) -> KvkkSearchResult:
|
) -> Dict[str, Any]:
|
||||||
"""Search function for legal decisions."""
|
"""Search function for legal decisions."""
|
||||||
logger.info(f"KVKK search tool called with keywords: {keywords}")
|
logger.info(f"KVKK search tool called with keywords: {keywords}")
|
||||||
|
|
||||||
@@ -1641,7 +1688,7 @@ async def search_kvkk_decisions(
|
|||||||
try:
|
try:
|
||||||
result = await kvkk_client_instance.search_decisions(search_request)
|
result = await kvkk_client_instance.search_decisions(search_request)
|
||||||
logger.info(f"KVKK search completed. Found {len(result.decisions)} decisions on page {page}")
|
logger.info(f"KVKK search completed. Found {len(result.decisions)} decisions on page {page}")
|
||||||
return result
|
return result.model_dump()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Error in KVKK search: {e}")
|
logger.exception(f"Error in KVKK search: {e}")
|
||||||
# Return empty result on error
|
# Return empty result on error
|
||||||
@@ -1651,7 +1698,7 @@ async def search_kvkk_decisions(
|
|||||||
page=page,
|
page=page,
|
||||||
pageSize=pageSize,
|
pageSize=pageSize,
|
||||||
query=keywords
|
query=keywords
|
||||||
)
|
).model_dump()
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
description="Get KVKK decision document in Markdown format with metadata extraction",
|
description="Get KVKK decision document in Markdown format with metadata extraction",
|
||||||
@@ -1664,7 +1711,7 @@ async def search_kvkk_decisions(
|
|||||||
async def get_kvkk_document_markdown(
|
async def get_kvkk_document_markdown(
|
||||||
decision_url: str = Field(..., description="KVKK decision URL from search results"),
|
decision_url: str = Field(..., description="KVKK decision URL from search results"),
|
||||||
page_number: int = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).")
|
page_number: int = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).")
|
||||||
) -> KvkkDocumentMarkdown:
|
) -> Dict[str, Any]:
|
||||||
"""Get KVKK decision as paginated Markdown."""
|
"""Get KVKK decision as paginated Markdown."""
|
||||||
logger.info(f"KVKK document retrieval tool called for URL: {decision_url}")
|
logger.info(f"KVKK document retrieval tool called for URL: {decision_url}")
|
||||||
|
|
||||||
@@ -1680,7 +1727,7 @@ async def get_kvkk_document_markdown(
|
|||||||
total_pages=0,
|
total_pages=0,
|
||||||
is_paginated=False,
|
is_paginated=False,
|
||||||
error_message="Decision URL is required and cannot be empty."
|
error_message="Decision URL is required and cannot be empty."
|
||||||
)
|
).model_dump()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Validate URL format
|
# Validate URL format
|
||||||
@@ -1696,11 +1743,11 @@ async def get_kvkk_document_markdown(
|
|||||||
total_pages=0,
|
total_pages=0,
|
||||||
is_paginated=False,
|
is_paginated=False,
|
||||||
error_message="Invalid KVKK decision URL format. URL must start with https://www.kvkk.gov.tr/"
|
error_message="Invalid KVKK decision URL format. URL must start with https://www.kvkk.gov.tr/"
|
||||||
)
|
).model_dump()
|
||||||
|
|
||||||
result = await kvkk_client_instance.get_decision_document(decision_url, page_number or 1)
|
result = await kvkk_client_instance.get_decision_document(decision_url, page_number or 1)
|
||||||
logger.info(f"KVKK document retrieved successfully. Page {result.current_page}/{result.total_pages}, Content length: {len(result.markdown_chunk) if result.markdown_chunk else 0}")
|
logger.info(f"KVKK document retrieved successfully. Page {result.current_page}/{result.total_pages}, Content length: {len(result.markdown_chunk) if result.markdown_chunk else 0}")
|
||||||
return result
|
return result.model_dump()
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Error retrieving KVKK document: {e}")
|
logger.exception(f"Error retrieving KVKK document: {e}")
|
||||||
@@ -1715,7 +1762,7 @@ async def get_kvkk_document_markdown(
|
|||||||
total_pages=0,
|
total_pages=0,
|
||||||
is_paginated=False,
|
is_paginated=False,
|
||||||
error_message=f"Error retrieving KVKK document: {str(e)}"
|
error_message=f"Error retrieving KVKK document: {str(e)}"
|
||||||
)
|
).model_dump()
|
||||||
|
|
||||||
# --- MCP Tools for BDDK (Banking Regulation Authority) ---
|
# --- MCP Tools for BDDK (Banking Regulation Authority) ---
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -1815,234 +1862,6 @@ async def get_bddk_document_markdown(
|
|||||||
"error": str(e)
|
"error": str(e)
|
||||||
}
|
}
|
||||||
|
|
||||||
# --- Semantic Search Tool ---
|
|
||||||
|
|
||||||
@app.tool(
|
|
||||||
description="Semantic search for Turkish legal decisions using EmbeddingGemma for intelligent ranking",
|
|
||||||
annotations={
|
|
||||||
"readOnlyHint": True,
|
|
||||||
"openWorldHint": True,
|
|
||||||
"idempotentHint": True
|
|
||||||
}
|
|
||||||
)
|
|
||||||
async def search_bedesten_semantic(
|
|
||||||
query: str = Field(..., description="Search query in Turkish for semantic matching"),
|
|
||||||
initial_keyword: str = Field(..., description="Initial keyword for Bedesten API search (broad term)"),
|
|
||||||
court_types: List[BedestenCourtTypeEnum] = Field(
|
|
||||||
default=["YARGITAYKARARI", "DANISTAYKARAR", "YERELHUKUK", "ISTINAFHUKUK", "KYB"],
|
|
||||||
description="Court types to search: YARGITAYKARARI, DANISTAYKARAR, YERELHUKUK, ISTINAFHUKUK, KYB (default: all)"
|
|
||||||
),
|
|
||||||
top_k: int = Field(10, ge=1, le=50, description="Number of top results to return (1-50)")
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Perform semantic search on Turkish legal decisions using EmbeddingGemma.
|
|
||||||
|
|
||||||
This tool:
|
|
||||||
1. Searches Bedesten API with initial keyword (retrieves 100 results)
|
|
||||||
2. Fetches full document content for each result
|
|
||||||
3. Generates embeddings using Google's EmbeddingGemma model
|
|
||||||
4. Performs semantic similarity search with the query
|
|
||||||
5. Returns re-ranked results based on semantic relevance
|
|
||||||
|
|
||||||
Benefits over keyword search:
|
|
||||||
- Better understanding of context and meaning
|
|
||||||
- Finds semantically similar documents even with different wording
|
|
||||||
- More accurate ranking based on relevance
|
|
||||||
- Supports multilingual queries (100+ languages)
|
|
||||||
"""
|
|
||||||
logger.info(f"Semantic search tool called with query: {query}, keyword: {initial_keyword}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Initialize components
|
|
||||||
embedder = EmbeddingGemma()
|
|
||||||
vector_store = VectorStore(dimension=256) # Always use 256 for optimal speed/quality balance
|
|
||||||
processor = DocumentProcessor(chunk_size=1500, chunk_overlap=300)
|
|
||||||
|
|
||||||
# Step 1: Initial keyword search to get document IDs
|
|
||||||
logger.info(f"Step 1: Searching Bedesten API with keyword: {initial_keyword}")
|
|
||||||
|
|
||||||
all_decisions = []
|
|
||||||
|
|
||||||
# Search each court type
|
|
||||||
for court_type in court_types:
|
|
||||||
try:
|
|
||||||
# Calculate page size per court type to get 100 total
|
|
||||||
per_court_limit = max(20, 100 // len(court_types))
|
|
||||||
|
|
||||||
search_results = await bedesten_client_instance.search_documents(
|
|
||||||
BedestenSearchRequest(
|
|
||||||
data=BedestenSearchData(
|
|
||||||
phrase=initial_keyword,
|
|
||||||
itemTypeList=[court_type],
|
|
||||||
pageSize=per_court_limit, # Distribute 100 across court types
|
|
||||||
pageNumber=1
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
if search_results.data and search_results.data.emsalKararList:
|
|
||||||
all_decisions.extend(search_results.data.emsalKararList)
|
|
||||||
logger.info(f"Found {len(search_results.data.emsalKararList)} results from {court_type}")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"Error searching {court_type}: {e}")
|
|
||||||
|
|
||||||
if not all_decisions:
|
|
||||||
logger.warning("No documents found from initial search")
|
|
||||||
return {
|
|
||||||
"status": "no_results",
|
|
||||||
"message": "No documents found matching the initial keyword",
|
|
||||||
"results": []
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(f"Total documents found: {len(all_decisions)}")
|
|
||||||
|
|
||||||
# Step 2: Fetch document content and process
|
|
||||||
logger.info("Step 2: Fetching and processing document content...")
|
|
||||||
|
|
||||||
documents_data = []
|
|
||||||
failed_fetches = 0
|
|
||||||
|
|
||||||
# Process up to 100 documents total
|
|
||||||
decisions_to_process = all_decisions[:100]
|
|
||||||
|
|
||||||
for i, decision in enumerate(decisions_to_process):
|
|
||||||
try:
|
|
||||||
# Fetch document content
|
|
||||||
doc = await bedesten_client_instance.get_document_as_markdown(decision.documentId)
|
|
||||||
|
|
||||||
if doc.markdown_content:
|
|
||||||
# Process document into chunks
|
|
||||||
metadata = {
|
|
||||||
"document_id": decision.documentId,
|
|
||||||
"birim_adi": decision.birimAdi,
|
|
||||||
"esas_no": decision.esasNo,
|
|
||||||
"karar_no": decision.kararNo,
|
|
||||||
"karar_tarihi": decision.kararTarihiStr,
|
|
||||||
"court_type": decision.itemType.name if decision.itemType else None
|
|
||||||
}
|
|
||||||
|
|
||||||
chunks = processor.process_document(
|
|
||||||
document_id=decision.documentId,
|
|
||||||
text=doc.markdown_content,
|
|
||||||
metadata=metadata
|
|
||||||
)
|
|
||||||
|
|
||||||
# For now, use the full document as one chunk (can be optimized later)
|
|
||||||
if chunks:
|
|
||||||
full_text = " ".join([chunk.text for chunk in chunks])
|
|
||||||
documents_data.append({
|
|
||||||
"id": decision.documentId,
|
|
||||||
"text": full_text[:3000], # Limit text for embedding
|
|
||||||
"metadata": metadata
|
|
||||||
})
|
|
||||||
|
|
||||||
# Log progress every 10 documents
|
|
||||||
if (i + 1) % 10 == 0:
|
|
||||||
logger.info(f"Processed {i + 1}/{len(decisions_to_process)} documents")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning(f"Failed to fetch document {decision.documentId}: {e}")
|
|
||||||
failed_fetches += 1
|
|
||||||
|
|
||||||
if not documents_data:
|
|
||||||
logger.warning("No documents could be processed")
|
|
||||||
return {
|
|
||||||
"status": "processing_error",
|
|
||||||
"message": "Could not process any documents",
|
|
||||||
"results": []
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.info(f"Successfully processed {len(documents_data)} documents, {failed_fetches} failed")
|
|
||||||
|
|
||||||
# Step 3: Generate embeddings
|
|
||||||
logger.info("Step 3: Generating embeddings...")
|
|
||||||
|
|
||||||
# Generate query embedding
|
|
||||||
query_embedding = embedder.encode_query(query, task="search result")
|
|
||||||
|
|
||||||
# Generate document embeddings
|
|
||||||
doc_texts = [doc["text"] for doc in documents_data]
|
|
||||||
doc_titles = [doc["metadata"].get("birim_adi", "none") for doc in documents_data]
|
|
||||||
doc_embeddings = embedder.encode_documents(doc_texts, titles=doc_titles)
|
|
||||||
|
|
||||||
# Always reduce to 256 dimensions for optimal speed/quality balance
|
|
||||||
query_embedding = embedder.reduce_dimensions(query_embedding, 256)
|
|
||||||
doc_embeddings = embedder.reduce_dimensions(doc_embeddings, 256)
|
|
||||||
|
|
||||||
# Step 4: Add to vector store and search
|
|
||||||
logger.info("Step 4: Performing semantic search...")
|
|
||||||
|
|
||||||
# Add documents to vector store
|
|
||||||
doc_ids = [doc["id"] for doc in documents_data]
|
|
||||||
doc_metadatas = [doc["metadata"] for doc in documents_data]
|
|
||||||
|
|
||||||
vector_store.add_documents(
|
|
||||||
ids=doc_ids,
|
|
||||||
texts=doc_texts,
|
|
||||||
embeddings=doc_embeddings,
|
|
||||||
metadata=doc_metadatas
|
|
||||||
)
|
|
||||||
|
|
||||||
# Perform semantic search
|
|
||||||
search_results = vector_store.search(
|
|
||||||
query_embedding=query_embedding,
|
|
||||||
top_k=top_k,
|
|
||||||
threshold=0.3 # Minimum similarity threshold
|
|
||||||
)
|
|
||||||
|
|
||||||
# Step 5: Format results
|
|
||||||
logger.info(f"Step 5: Formatting {len(search_results)} results")
|
|
||||||
|
|
||||||
formatted_results = []
|
|
||||||
for doc, score in search_results:
|
|
||||||
# Build title from metadata
|
|
||||||
title_parts = []
|
|
||||||
if doc.metadata.get("birim_adi"):
|
|
||||||
title_parts.append(doc.metadata["birim_adi"])
|
|
||||||
if doc.metadata.get("esas_no"):
|
|
||||||
title_parts.append(f"Esas: {doc.metadata['esas_no']}")
|
|
||||||
if doc.metadata.get("karar_no"):
|
|
||||||
title_parts.append(f"Karar: {doc.metadata['karar_no']}")
|
|
||||||
if doc.metadata.get("karar_tarihi"):
|
|
||||||
title_parts.append(f"Tarih: {doc.metadata['karar_tarihi']}")
|
|
||||||
|
|
||||||
title = " - ".join(title_parts) if title_parts else f"Document {doc.id}"
|
|
||||||
|
|
||||||
formatted_results.append({
|
|
||||||
"document_id": doc.id,
|
|
||||||
"title": title,
|
|
||||||
"similarity_score": float(score),
|
|
||||||
"preview": doc.text[:500] + "..." if len(doc.text) > 500 else doc.text,
|
|
||||||
"metadata": doc.metadata,
|
|
||||||
"source_url": f"https://mevzuat.adalet.gov.tr/ictihat/{doc.id}"
|
|
||||||
})
|
|
||||||
|
|
||||||
# Get vector store stats
|
|
||||||
stats = vector_store.get_stats()
|
|
||||||
|
|
||||||
return {
|
|
||||||
"status": "success",
|
|
||||||
"query": query,
|
|
||||||
"initial_keyword": initial_keyword,
|
|
||||||
"total_documents_processed": len(documents_data),
|
|
||||||
"embedding_dimension": 256, # Fixed at 256 for optimal performance
|
|
||||||
"results": formatted_results,
|
|
||||||
"stats": {
|
|
||||||
"documents_in_store": stats["num_documents"],
|
|
||||||
"memory_usage_mb": round(stats["memory_usage_mb"], 2),
|
|
||||||
"failed_fetches": failed_fetches
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception(f"Error in semantic search: {e}")
|
|
||||||
return {
|
|
||||||
"status": "error",
|
|
||||||
"message": str(e),
|
|
||||||
"results": []
|
|
||||||
}
|
|
||||||
|
|
||||||
# --- ChatGPT Deep Research Compatible Tools ---
|
# --- ChatGPT Deep Research Compatible Tools ---
|
||||||
|
|
||||||
def get_preview_text(markdown_content: str, skip_chars: int = 100, preview_chars: int = 200) -> str:
|
def get_preview_text(markdown_content: str, skip_chars: int = 100, preview_chars: int = 200) -> str:
|
||||||
@@ -2090,6 +1909,7 @@ def get_preview_text(markdown_content: str, skip_chars: int = 100, preview_chars
|
|||||||
|
|
||||||
return preview.strip()
|
return preview.strip()
|
||||||
|
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
description="DO NOT USE unless you are ChatGPT Deep Research. Search Turkish courts (Turkish keywords only). Supports: +term (must have), -term (exclude), \"exact phrase\", term1 OR term2",
|
description="DO NOT USE unless you are ChatGPT Deep Research. Search Turkish courts (Turkish keywords only). Supports: +term (must have), -term (exclude), \"exact phrase\", term1 OR term2",
|
||||||
annotations={
|
annotations={
|
||||||
@@ -2100,7 +1920,7 @@ def get_preview_text(markdown_content: str, skip_chars: int = 100, preview_chars
|
|||||||
)
|
)
|
||||||
async def search(
|
async def search(
|
||||||
query: str = Field(..., description="Turkish search query")
|
query: str = Field(..., description="Turkish search query")
|
||||||
) -> Dict[str, List[Dict[str, str]]]:
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Bedesten API search tool for ChatGPT Deep Research compatibility.
|
Bedesten API search tool for ChatGPT Deep Research compatibility.
|
||||||
|
|
||||||
@@ -2214,13 +2034,33 @@ async def search(
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
logger.info(f"ChatGPT Deep Research search completed. Found {len(results)} results via Bedesten API.")
|
logger.info(f"ChatGPT Deep Research search completed. Found {len(results)} results via Bedesten API.")
|
||||||
return {"results": results}
|
return {
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"id": item["id"],
|
||||||
|
"title": item["title"],
|
||||||
|
"text": item["text"],
|
||||||
|
"url": item["url"]
|
||||||
|
}
|
||||||
|
for item in results
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in ChatGPT Deep Research search tool")
|
logger.exception("Error in ChatGPT Deep Research search tool")
|
||||||
# Return partial results if any were found
|
# Return partial results if any were found
|
||||||
if results:
|
if results:
|
||||||
return {"results": results}
|
return {
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"id": item["id"],
|
||||||
|
"title": item["title"],
|
||||||
|
"text": item["text"],
|
||||||
|
"url": item["url"]
|
||||||
|
}
|
||||||
|
for item in results
|
||||||
|
]
|
||||||
|
}
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -2345,7 +2185,7 @@ async def fetch(
|
|||||||
doc = await bedesten_client_instance.get_document_as_markdown(doc_id)
|
doc = await bedesten_client_instance.get_document_as_markdown(doc_id)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception(f"Error fetching ChatGPT Deep Research document {id}")
|
logger.exception(f"Error fetching ChatGPT Deep Research document {id}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -2387,7 +2227,7 @@ def main():
|
|||||||
app = create_app()
|
app = create_app()
|
||||||
|
|
||||||
logger.info(f"Starting {app.name} server via main() function...")
|
logger.info(f"Starting {app.name} server via main() function...")
|
||||||
logger.info(f"Logs will be written to: {LOG_FILE_PATH}")
|
# logger.info(f"Logs will be written to: {LOG_FILE_PATH}") # File logging disabled
|
||||||
|
|
||||||
# Ensure Playwright browsers are installed
|
# Ensure Playwright browsers are installed
|
||||||
ensure_playwright_browsers()
|
ensure_playwright_browsers()
|
||||||
@@ -2396,7 +2236,7 @@ def main():
|
|||||||
app.run()
|
app.run()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
logger.info("Server shut down by user (KeyboardInterrupt).")
|
logger.info("Server shut down by user (KeyboardInterrupt).")
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Server failed to start or crashed.")
|
logger.exception("Server failed to start or crashed.")
|
||||||
finally:
|
finally:
|
||||||
logger.info(f"{app.name} server has shut down.")
|
logger.info(f"{app.name} server has shut down.")
|
||||||
|
|||||||
+3
-8
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "yargi-mcp"
|
name = "yargi-mcp"
|
||||||
version = "0.1.7"
|
version = "0.1.9"
|
||||||
description = "MCP Server For Turkish Legal Databases"
|
description = "MCP Server For Turkish Legal Databases"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
@@ -29,13 +29,6 @@ dependencies = [
|
|||||||
"fastmcp>=2.10.5",
|
"fastmcp>=2.10.5",
|
||||||
"pypdf>=5.5.0",
|
"pypdf>=5.5.0",
|
||||||
"fastapi>=0.115.14",
|
"fastapi>=0.115.14",
|
||||||
"PyJWT>=2.8.0",
|
|
||||||
"tiktoken>=0.5.0",
|
|
||||||
"sentence-transformers>=3.0.0",
|
|
||||||
"torch>=2.0.0",
|
|
||||||
"numpy>=1.24.0",
|
|
||||||
"scikit-learn>=1.3.0",
|
|
||||||
"transformers>=4.57.0.dev0",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
@@ -55,6 +48,8 @@ saas = [
|
|||||||
"clerk-backend-api>=3.0.0",
|
"clerk-backend-api>=3.0.0",
|
||||||
"stripe>=9.1.0",
|
"stripe>=9.1.0",
|
||||||
"upstash-redis>=1.1.0",
|
"upstash-redis>=1.1.0",
|
||||||
|
"tiktoken>=0.5.0",
|
||||||
|
"PyJWT>=2.8.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ import os
|
|||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import logging
|
import logging
|
||||||
from typing import Optional, Dict, Any
|
from typing import Optional, Dict, Any, Union
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -4,9 +4,10 @@ import httpx
|
|||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import List, Optional, Tuple, Dict, Any
|
from typing import List, Optional, Tuple, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
|
import html
|
||||||
import re
|
import re
|
||||||
import io # For io.BytesIO
|
import io # For io.BytesIO
|
||||||
from urllib.parse import urljoin, parse_qs, urlparse
|
from urllib.parse import urlencode, urljoin, quote, parse_qs, urlparse
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
import math
|
import math
|
||||||
|
|
||||||
@@ -17,7 +18,8 @@ from .models import (
|
|||||||
RekabetKurumuSearchRequest,
|
RekabetKurumuSearchRequest,
|
||||||
RekabetDecisionSummary,
|
RekabetDecisionSummary,
|
||||||
RekabetSearchResult,
|
RekabetSearchResult,
|
||||||
RekabetDocument
|
RekabetDocument,
|
||||||
|
RekabetKararTuruGuidEnum
|
||||||
)
|
)
|
||||||
from pydantic import HttpUrl # Ensure HttpUrl is imported from pydantic
|
from pydantic import HttpUrl # Ensure HttpUrl is imported from pydantic
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# rekabet_mcp_module/models.py
|
# rekabet_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, HttpUrl
|
from pydantic import BaseModel, Field, HttpUrl
|
||||||
from typing import List, Optional
|
from typing import List, Optional, Any
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
# Enum for decision type GUIDs (used by the client and expected by the website)
|
# Enum for decision type GUIDs (used by the client and expected by the website)
|
||||||
|
|||||||
+1
-1
@@ -93,7 +93,7 @@ def main():
|
|||||||
config["workers"] = args.workers
|
config["workers"] = args.workers
|
||||||
|
|
||||||
# Print startup information
|
# Print startup information
|
||||||
print("Starting Yargı MCP server...")
|
print(f"Starting Yargı MCP server...")
|
||||||
print(f"Host: {args.host}")
|
print(f"Host: {args.host}")
|
||||||
print(f"Port: {args.port}")
|
print(f"Port: {args.port}")
|
||||||
print(f"Transport: {args.transport}")
|
print(f"Transport: {args.transport}")
|
||||||
|
|||||||
@@ -2,13 +2,13 @@
|
|||||||
# This client is for Bireysel Başvuru: https://kararlarbilgibankasi.anayasa.gov.tr
|
# This client is for Bireysel Başvuru: https://kararlarbilgibankasi.anayasa.gov.tr
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup, Tag
|
||||||
from typing import List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Tuple
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urlencode, urljoin, quote
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
import math # For math.ceil for pagination
|
import math # For math.ceil for pagination
|
||||||
|
|
||||||
|
|||||||
@@ -3,12 +3,12 @@
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Tuple
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urlencode, urljoin, quote
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
import math # For math.ceil for pagination
|
import math # For math.ceil for pagination
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# Unified client for both Norm Denetimi and Bireysel Başvuru
|
# Unified client for both Norm Denetimi and Bireysel Başvuru
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Optional
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
|
|||||||
@@ -10,12 +10,16 @@ Usage:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
import logging
|
import logging
|
||||||
|
from datetime import datetime, timedelta
|
||||||
from fastapi import FastAPI, Request, HTTPException, Query
|
from fastapi import FastAPI, Request, HTTPException, Query
|
||||||
from fastapi.responses import JSONResponse, HTMLResponse
|
from fastapi.responses import JSONResponse, HTMLResponse
|
||||||
from fastapi.exception_handlers import http_exception_handler
|
from fastapi.exception_handlers import http_exception_handler
|
||||||
from starlette.middleware import Middleware
|
from starlette.middleware import Middleware
|
||||||
from starlette.middleware.cors import CORSMiddleware
|
from starlette.middleware.cors import CORSMiddleware
|
||||||
|
from starlette.responses import Response
|
||||||
|
from starlette.requests import Request as StarletteRequest
|
||||||
|
|
||||||
# Import the MCP app creator function
|
# Import the MCP app creator function
|
||||||
from mcp_server_main import create_app
|
from mcp_server_main import create_app
|
||||||
@@ -99,6 +103,7 @@ mcp_app = mcp_server.http_app(path="/")
|
|||||||
|
|
||||||
# Configure JSON encoder for proper Turkish character support
|
# Configure JSON encoder for proper Turkish character support
|
||||||
import json
|
import json
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
class UTF8JSONResponse(JSONResponse):
|
class UTF8JSONResponse(JSONResponse):
|
||||||
def __init__(self, content=None, status_code=200, headers=None, **kwargs):
|
def __init__(self, content=None, status_code=200, headers=None, **kwargs):
|
||||||
@@ -456,14 +461,14 @@ async def mcp_oauth_callback(request: Request, clerk_token: str = Query(None)):
|
|||||||
logger.info("User authenticated successfully via Clerk")
|
logger.info("User authenticated successfully via Clerk")
|
||||||
|
|
||||||
# Return success response
|
# Return success response
|
||||||
return HTMLResponse("""
|
return HTMLResponse(f"""
|
||||||
<html>
|
<html>
|
||||||
<head>
|
<head>
|
||||||
<title>MCP Connection Successful</title>
|
<title>MCP Connection Successful</title>
|
||||||
<style>
|
<style>
|
||||||
body { font-family: Arial, sans-serif; text-align: center; padding: 50px; }
|
body {{ font-family: Arial, sans-serif; text-align: center; padding: 50px; }}
|
||||||
.success { color: #28a745; }
|
.success {{ color: #28a745; }}
|
||||||
.token { background: #f8f9fa; padding: 15px; border-radius: 5px; margin: 20px 0; word-break: break-all; }
|
.token {{ background: #f8f9fa; padding: 15px; border-radius: 5px; margin: 20px 0; word-break: break-all; }}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
@@ -476,13 +481,13 @@ async def mcp_oauth_callback(request: Request, clerk_token: str = Query(None)):
|
|||||||
<p>You can now close this window and return to your MCP client.</p>
|
<p>You can now close this window and return to your MCP client.</p>
|
||||||
<script>
|
<script>
|
||||||
// Try to close the popup if opened as such
|
// Try to close the popup if opened as such
|
||||||
if (window.opener) {
|
if (window.opener) {{
|
||||||
window.opener.postMessage({
|
window.opener.postMessage({{
|
||||||
type: 'MCP_AUTH_SUCCESS',
|
type: 'MCP_AUTH_SUCCESS',
|
||||||
token: 'use_clerk_jwt_token'
|
token: 'use_clerk_jwt_token'
|
||||||
}, '*');
|
}}, '*');
|
||||||
setTimeout(() => window.close(), 3000);
|
setTimeout(() => window.close(), 3000);
|
||||||
}
|
}}
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
# bddk_mcp_module/client.py
|
# bddk_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Optional
|
from typing import List, Optional, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
import math
|
import math
|
||||||
|
from urllib.parse import urlparse
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# bddk_mcp_module/models.py
|
# bddk_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List
|
from typing import List, Optional
|
||||||
|
|
||||||
class BddkSearchRequest(BaseModel):
|
class BddkSearchRequest(BaseModel):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
# bedesten_mcp_module/models.py
|
# bedesten_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import List, Optional, Dict, Any, Literal
|
from typing import List, Optional, Dict, Any, Literal, Union
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
# Import compressed BirimAdiEnum for chamber filtering
|
# Import compressed BirimAdiEnum for chamber filtering
|
||||||
from .enums import BirimAdiEnum
|
from .enums import BirimAdiEnum
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
# danistay_mcp_module/client.py
|
# danistay_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Dict, List, Optional
|
from bs4 import BeautifulSoup
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
|
import re
|
||||||
import io
|
import io
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
# from bs4 import BeautifulSoup # Uncomment if needed for advanced HTML pre-processing
|
# from bs4 import BeautifulSoup # Uncomment if needed for advanced HTML pre-processing
|
||||||
from typing import Dict, Optional
|
from typing import Dict, Any, List, Optional
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
|
import re
|
||||||
import io
|
import io
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
|
|||||||
@@ -7,9 +7,11 @@ import os
|
|||||||
from typing import List, Dict, Any, Optional
|
from typing import List, Dict, Any, Optional
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException, Query
|
from fastapi import FastAPI, HTTPException, Query, Depends, Body
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
import json
|
||||||
|
|
||||||
# Import the main MCP app
|
# Import the main MCP app
|
||||||
from mcp_server_main import app as mcp_server
|
from mcp_server_main import app as mcp_server
|
||||||
|
|||||||
@@ -10,12 +10,13 @@ from playwright.async_api import (
|
|||||||
)
|
)
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
import logging
|
import logging
|
||||||
from typing import List, Optional
|
from typing import Dict, Any, List, Optional
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
import base64 # Base64 için
|
import base64 # Base64 için
|
||||||
import re
|
import re
|
||||||
import html as html_parser
|
import html as html_parser
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
import os
|
||||||
import math
|
import math
|
||||||
import io
|
import io
|
||||||
import random
|
import random
|
||||||
@@ -906,7 +907,7 @@ class KikApiClient:
|
|||||||
event_target_for_submit = self.FIELD_LOCATORS['search_button_id']
|
event_target_for_submit = self.FIELD_LOCATORS['search_button_id']
|
||||||
# Use human-like clicking for search button
|
# Use human-like clicking for search button
|
||||||
search_button_selector = f"a[id='{event_target_for_submit}']"
|
search_button_selector = f"a[id='{event_target_for_submit}']"
|
||||||
logger.info("Performing human-like search button click...")
|
logger.info(f"Performing human-like search button click...")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Hide datepicker first to prevent interference
|
# Hide datepicker first to prevent interference
|
||||||
@@ -1180,7 +1181,7 @@ class KikApiClient:
|
|||||||
try:
|
try:
|
||||||
if await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).is_visible(timeout=2000):
|
if await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).is_visible(timeout=2000):
|
||||||
await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).click()
|
await current_main_page.locator(self.MODAL_CLOSE_BUTTON_SELECTOR).click()
|
||||||
await current_main_page.wait_for_selector("div#detayPopUp:not(.in)", timeout=5000)
|
await current_main_page.wait_for_selector(f"div#detayPopUp:not(.in)", timeout=5000)
|
||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
return KikDocumentMarkdown(
|
return KikDocumentMarkdown(
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# kik_mcp_module/models.py
|
# kik_mcp_module/models.py
|
||||||
from pydantic import BaseModel, Field, computed_field, ConfigDict
|
from pydantic import BaseModel, Field, HttpUrl, computed_field, ConfigDict
|
||||||
from typing import List, Optional
|
from typing import List, Optional
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
import base64 # Base64 encoding/decoding için
|
import base64 # Base64 encoding/decoding için
|
||||||
|
|||||||
@@ -2,13 +2,13 @@
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import Optional, Dict, Any
|
from typing import List, Optional, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import io
|
import io
|
||||||
import math
|
import math
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urljoin, urlparse, parse_qs
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
from pydantic import HttpUrl
|
from pydantic import HttpUrl
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# kvkk_mcp_module/models.py
|
# kvkk_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, HttpUrl
|
from pydantic import BaseModel, Field, HttpUrl
|
||||||
from typing import List, Optional
|
from typing import List, Optional, Any
|
||||||
|
|
||||||
class KvkkSearchRequest(BaseModel):
|
class KvkkSearchRequest(BaseModel):
|
||||||
"""Model for KVKK (Personal Data Protection Authority) search request via Brave API."""
|
"""Model for KVKK (Personal Data Protection Authority) search request via Brave API."""
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ def create_clerk_oauth_config() -> OAuthConfig:
|
|||||||
scopes=["mcp:tools:read", "mcp:tools:write", "openid", "profile", "email"]
|
scopes=["mcp:tools:read", "mcp:tools:write", "openid", "profile", "email"]
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Created Clerk OAuth config with adapter endpoints")
|
logger.info(f"Created Clerk OAuth config with adapter endpoints")
|
||||||
logger.info(f"Clerk domain: {clerk_domain}")
|
logger.info(f"Clerk domain: {clerk_domain}")
|
||||||
logger.debug(f"Authorization endpoint: {config.authorization_endpoint}")
|
logger.debug(f"Authorization endpoint: {config.authorization_endpoint}")
|
||||||
logger.debug(f"Token endpoint: {config.token_endpoint}")
|
logger.debug(f"Token endpoint: {config.token_endpoint}")
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import time
|
|||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from typing import Any
|
from typing import Any, Optional
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from fastapi.responses import RedirectResponse, JSONResponse
|
|||||||
try:
|
try:
|
||||||
from clerk_backend_api import Clerk
|
from clerk_backend_api import Clerk
|
||||||
CLERK_AVAILABLE = True
|
CLERK_AVAILABLE = True
|
||||||
except ImportError:
|
except ImportError as e:
|
||||||
CLERK_AVAILABLE = False
|
CLERK_AVAILABLE = False
|
||||||
Clerk = None
|
Clerk = None
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ Uses Redis for authorization code storage to support multi-machine deployment
|
|||||||
import os
|
import os
|
||||||
import logging
|
import logging
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode, quote
|
||||||
|
|
||||||
from fastapi import APIRouter, Request, Query, HTTPException
|
from fastapi import APIRouter, Request, Query, HTTPException
|
||||||
from fastapi.responses import RedirectResponse, JSONResponse
|
from fastapi.responses import RedirectResponse, JSONResponse
|
||||||
@@ -39,6 +39,7 @@ def get_redis_session_store():
|
|||||||
if redis_store is None:
|
if redis_store is None:
|
||||||
try:
|
try:
|
||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
|
import functools
|
||||||
|
|
||||||
# Use thread pool with timeout to prevent hanging
|
# Use thread pool with timeout to prevent hanging
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
|
||||||
@@ -205,6 +206,7 @@ async def oauth_callback(
|
|||||||
auth_code = f"clerk_auth_{os.urandom(16).hex()}"
|
auth_code = f"clerk_auth_{os.urandom(16).hex()}"
|
||||||
|
|
||||||
# Prepare code data
|
# Prepare code data
|
||||||
|
import time
|
||||||
code_data = {
|
code_data = {
|
||||||
"user_id": user_id,
|
"user_id": user_id,
|
||||||
"session_id": session_id,
|
"session_id": session_id,
|
||||||
@@ -223,7 +225,7 @@ async def oauth_callback(
|
|||||||
if success:
|
if success:
|
||||||
logger.info(f"Stored authorization code {auth_code[:10]}... in Redis with real JWT token")
|
logger.info(f"Stored authorization code {auth_code[:10]}... in Redis with real JWT token")
|
||||||
else:
|
else:
|
||||||
logger.error("Failed to store authorization code in Redis, falling back to in-memory")
|
logger.error(f"Failed to store authorization code in Redis, falling back to in-memory")
|
||||||
# Fall back to in-memory storage
|
# Fall back to in-memory storage
|
||||||
if not hasattr(oauth_callback, '_code_storage'):
|
if not hasattr(oauth_callback, '_code_storage'):
|
||||||
oauth_callback._code_storage = {}
|
oauth_callback._code_storage = {}
|
||||||
@@ -234,7 +236,7 @@ async def oauth_callback(
|
|||||||
if not hasattr(oauth_callback, '_code_storage'):
|
if not hasattr(oauth_callback, '_code_storage'):
|
||||||
oauth_callback._code_storage = {}
|
oauth_callback._code_storage = {}
|
||||||
oauth_callback._code_storage[auth_code] = code_data
|
oauth_callback._code_storage[auth_code] = code_data
|
||||||
logger.info("Stored authorization code in memory (fallback)")
|
logger.info(f"Stored authorization code in memory (fallback)")
|
||||||
|
|
||||||
# Redirect back to client with authorization code
|
# Redirect back to client with authorization code
|
||||||
redirect_params = {
|
redirect_params = {
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ import json
|
|||||||
import time
|
import time
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from pydantic import HttpUrl, Field
|
from pydantic import HttpUrl, Field
|
||||||
from typing import Dict, List, Literal, Any
|
from typing import Optional, Dict, List, Literal, Any, Union
|
||||||
|
import urllib.parse
|
||||||
import tiktoken
|
import tiktoken
|
||||||
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
from fastmcp.server.middleware import Middleware, MiddlewareContext
|
||||||
from fastmcp.server.dependencies import get_access_token, AccessToken
|
from fastmcp.server.dependencies import get_access_token, AccessToken
|
||||||
@@ -158,7 +159,7 @@ class TokenCountingMiddleware(Middleware):
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||||
self.log_token_usage("tool_call_error", input_tokens, 0,
|
self.log_token_usage("tool_call_error", input_tokens, 0,
|
||||||
tool_name, duration_ms)
|
tool_name, duration_ms)
|
||||||
@@ -188,7 +189,7 @@ class TokenCountingMiddleware(Middleware):
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||||
self.log_token_usage("resource_read_error", 0, 0,
|
self.log_token_usage("resource_read_error", 0, 0,
|
||||||
resource_uri, duration_ms)
|
resource_uri, duration_ms)
|
||||||
@@ -218,7 +219,7 @@ class TokenCountingMiddleware(Middleware):
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||||
self.log_token_usage("prompt_get_error", 0, 0,
|
self.log_token_usage("prompt_get_error", 0, 0,
|
||||||
prompt_name, duration_ms)
|
prompt_name, duration_ms)
|
||||||
@@ -259,6 +260,10 @@ def create_app(auth=None):
|
|||||||
|
|
||||||
# --- Module Imports ---
|
# --- Module Imports ---
|
||||||
from yargitay_mcp_module.client import YargitayOfficialApiClient
|
from yargitay_mcp_module.client import YargitayOfficialApiClient
|
||||||
|
from yargitay_mcp_module.models import (
|
||||||
|
YargitayDetailedSearchRequest, YargitayDocumentMarkdown, CompactYargitaySearchResult,
|
||||||
|
YargitayBirimEnum, CleanYargitayDecisionEntry
|
||||||
|
)
|
||||||
from bedesten_mcp_module.client import BedestenApiClient
|
from bedesten_mcp_module.client import BedestenApiClient
|
||||||
from bedesten_mcp_module.models import (
|
from bedesten_mcp_module.models import (
|
||||||
BedestenSearchRequest, BedestenSearchData,
|
BedestenSearchRequest, BedestenSearchData,
|
||||||
@@ -266,6 +271,10 @@ from bedesten_mcp_module.models import (
|
|||||||
)
|
)
|
||||||
from bedesten_mcp_module.enums import BirimAdiEnum
|
from bedesten_mcp_module.enums import BirimAdiEnum
|
||||||
from danistay_mcp_module.client import DanistayApiClient
|
from danistay_mcp_module.client import DanistayApiClient
|
||||||
|
from danistay_mcp_module.models import (
|
||||||
|
DanistayKeywordSearchRequest, DanistayDetailedSearchRequest,
|
||||||
|
DanistayDocumentMarkdown, CompactDanistaySearchResult
|
||||||
|
)
|
||||||
from emsal_mcp_module.client import EmsalApiClient
|
from emsal_mcp_module.client import EmsalApiClient
|
||||||
from emsal_mcp_module.models import (
|
from emsal_mcp_module.models import (
|
||||||
EmsalSearchRequest, EmsalDocumentMarkdown, CompactEmsalSearchResult
|
EmsalSearchRequest, EmsalDocumentMarkdown, CompactEmsalSearchResult
|
||||||
@@ -279,7 +288,15 @@ from anayasa_mcp_module.client import AnayasaMahkemesiApiClient
|
|||||||
from anayasa_mcp_module.bireysel_client import AnayasaBireyselBasvuruApiClient
|
from anayasa_mcp_module.bireysel_client import AnayasaBireyselBasvuruApiClient
|
||||||
from anayasa_mcp_module.unified_client import AnayasaUnifiedClient
|
from anayasa_mcp_module.unified_client import AnayasaUnifiedClient
|
||||||
from anayasa_mcp_module.models import (
|
from anayasa_mcp_module.models import (
|
||||||
|
AnayasaNormDenetimiSearchRequest,
|
||||||
|
AnayasaSearchResult,
|
||||||
|
AnayasaDocumentMarkdown,
|
||||||
|
AnayasaBireyselReportSearchRequest,
|
||||||
|
AnayasaBireyselReportSearchResult,
|
||||||
|
AnayasaBireyselBasvuruDocumentMarkdown,
|
||||||
AnayasaUnifiedSearchRequest,
|
AnayasaUnifiedSearchRequest,
|
||||||
|
AnayasaUnifiedSearchResult,
|
||||||
|
AnayasaUnifiedDocumentMarkdown,
|
||||||
# Removed enum imports - now using Literal strings in models
|
# Removed enum imports - now using Literal strings in models
|
||||||
)
|
)
|
||||||
# KIK Module Imports
|
# KIK Module Imports
|
||||||
@@ -301,9 +318,14 @@ from rekabet_mcp_module.models import (
|
|||||||
|
|
||||||
from sayistay_mcp_module.client import SayistayApiClient
|
from sayistay_mcp_module.client import SayistayApiClient
|
||||||
from sayistay_mcp_module.models import (
|
from sayistay_mcp_module.models import (
|
||||||
|
GenelKurulSearchRequest, GenelKurulSearchResponse,
|
||||||
|
TemyizKuruluSearchRequest, TemyizKuruluSearchResponse,
|
||||||
|
DaireSearchRequest, DaireSearchResponse,
|
||||||
|
SayistayDocumentMarkdown,
|
||||||
SayistayUnifiedSearchRequest, SayistayUnifiedSearchResult,
|
SayistayUnifiedSearchRequest, SayistayUnifiedSearchResult,
|
||||||
SayistayUnifiedDocumentMarkdown
|
SayistayUnifiedDocumentMarkdown
|
||||||
)
|
)
|
||||||
|
from sayistay_mcp_module.enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
|
||||||
from sayistay_mcp_module.unified_client import SayistayUnifiedClient
|
from sayistay_mcp_module.unified_client import SayistayUnifiedClient
|
||||||
|
|
||||||
# KVKK Module Imports
|
# KVKK Module Imports
|
||||||
@@ -317,11 +339,14 @@ from kvkk_mcp_module.models import (
|
|||||||
# BDDK Module Imports
|
# BDDK Module Imports
|
||||||
from bddk_mcp_module.client import BddkApiClient
|
from bddk_mcp_module.client import BddkApiClient
|
||||||
from bddk_mcp_module.models import (
|
from bddk_mcp_module.models import (
|
||||||
BddkSearchRequest
|
BddkSearchRequest,
|
||||||
|
BddkSearchResult,
|
||||||
|
BddkDocumentMarkdown
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Create a placeholder app that will be properly initialized after tools are defined
|
# Create a placeholder app that will be properly initialized after tools are defined
|
||||||
|
from fastmcp import FastMCP
|
||||||
|
|
||||||
# Placeholder app for decorators - will be replaced in create_app() after all tools are defined
|
# Placeholder app for decorators - will be replaced in create_app() after all tools are defined
|
||||||
app = FastMCP("Yargı MCP Server Placeholder")
|
app = FastMCP("Yargı MCP Server Placeholder")
|
||||||
@@ -1347,7 +1372,7 @@ async def search_emsal_detailed_decisions(
|
|||||||
page_size=page_size
|
page_size=page_size
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Tool 'search_emsal_detailed_decisions' called.")
|
logger.info(f"Tool 'search_emsal_detailed_decisions' called.")
|
||||||
try:
|
try:
|
||||||
api_response = await emsal_client_instance.search_detailed_decisions(search_query)
|
api_response = await emsal_client_instance.search_detailed_decisions(search_query)
|
||||||
if api_response.data:
|
if api_response.data:
|
||||||
@@ -1359,8 +1384,8 @@ async def search_emsal_detailed_decisions(
|
|||||||
)
|
)
|
||||||
logger.warning("API response for Emsal search did not contain expected data structure.")
|
logger.warning("API response for Emsal search did not contain expected data structure.")
|
||||||
return CompactEmsalSearchResult(decisions=[], total_records=0, requested_page=search_query.page_number, page_size=search_query.page_size)
|
return CompactEmsalSearchResult(decisions=[], total_records=0, requested_page=search_query.page_number, page_size=search_query.page_size)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_emsal_detailed_decisions'.")
|
logger.exception(f"Error in tool 'search_emsal_detailed_decisions'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -1376,8 +1401,8 @@ async def get_emsal_document_markdown(id: str) -> EmsalDocumentMarkdown:
|
|||||||
if not id or not id.strip(): raise ValueError("Document ID required for Emsal.")
|
if not id or not id.strip(): raise ValueError("Document ID required for Emsal.")
|
||||||
try:
|
try:
|
||||||
return await emsal_client_instance.get_decision_document_as_markdown(id)
|
return await emsal_client_instance.get_decision_document_as_markdown(id)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_emsal_document_markdown'.")
|
logger.exception(f"Error in tool 'get_emsal_document_markdown'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
# --- MCP Tools for Uyusmazlik ---
|
# --- MCP Tools for Uyusmazlik ---
|
||||||
@@ -1445,11 +1470,11 @@ async def search_uyusmazlik_decisions(
|
|||||||
not_hepsi=not_hepsi
|
not_hepsi=not_hepsi
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Tool 'search_uyusmazlik_decisions' called.")
|
logger.info(f"Tool 'search_uyusmazlik_decisions' called.")
|
||||||
try:
|
try:
|
||||||
return await uyusmazlik_client_instance.search_decisions(search_params)
|
return await uyusmazlik_client_instance.search_decisions(search_params)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_uyusmazlik_decisions'.")
|
logger.exception(f"Error in tool 'search_uyusmazlik_decisions'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -1468,8 +1493,8 @@ async def get_uyusmazlik_document_markdown_from_url(
|
|||||||
raise ValueError("Document URL (document_url) is required for Uyuşmazlık document retrieval.")
|
raise ValueError("Document URL (document_url) is required for Uyuşmazlık document retrieval.")
|
||||||
try:
|
try:
|
||||||
return await uyusmazlik_client_instance.get_decision_document_as_markdown(str(document_url))
|
return await uyusmazlik_client_instance.get_decision_document_as_markdown(str(document_url))
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_uyusmazlik_document_markdown_from_url'.")
|
logger.exception(f"Error in tool 'get_uyusmazlik_document_markdown_from_url'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
# --- DEACTIVATED: MCP Tools for Anayasa Mahkemesi (Individual Tools) ---
|
# --- DEACTIVATED: MCP Tools for Anayasa Mahkemesi (Individual Tools) ---
|
||||||
@@ -1560,8 +1585,8 @@ async def search_anayasa_unified(
|
|||||||
result = await anayasa_unified_client_instance.search_unified(request)
|
result = await anayasa_unified_client_instance.search_unified(request)
|
||||||
return json.dumps(result.model_dump(), ensure_ascii=False, indent=2)
|
return json.dumps(result.model_dump(), ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_anayasa_unified'.")
|
logger.exception(f"Error in tool 'search_anayasa_unified'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@app.tool(
|
@app.tool(
|
||||||
@@ -1582,8 +1607,8 @@ async def get_anayasa_document_unified(
|
|||||||
result = await anayasa_unified_client_instance.get_document_unified(document_url, page_number)
|
result = await anayasa_unified_client_instance.get_document_unified(document_url, page_number)
|
||||||
return json.dumps(result.model_dump(mode='json'), ensure_ascii=False, indent=2)
|
return json.dumps(result.model_dump(mode='json'), ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_anayasa_document_unified'.")
|
logger.exception(f"Error in tool 'get_anayasa_document_unified'.")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
# --- MCP Tools for KIK (Kamu İhale Kurulu) ---
|
# --- MCP Tools for KIK (Kamu İhale Kurulu) ---
|
||||||
@@ -1629,15 +1654,15 @@ async def search_kik_decisions(
|
|||||||
page=page
|
page=page
|
||||||
)
|
)
|
||||||
|
|
||||||
logger.info("Tool 'search_kik_decisions' called.")
|
logger.info(f"Tool 'search_kik_decisions' called.")
|
||||||
try:
|
try:
|
||||||
api_response = await kik_client_instance.search_decisions(search_query)
|
api_response = await kik_client_instance.search_decisions(search_query)
|
||||||
page_param_for_log = search_query.page if hasattr(search_query, 'page') else 1
|
page_param_for_log = search_query.page if hasattr(search_query, 'page') else 1
|
||||||
if not api_response.decisions and api_response.total_records == 0 and page_param_for_log == 1:
|
if not api_response.decisions and api_response.total_records == 0 and page_param_for_log == 1:
|
||||||
logger.warning("KIK search returned no decisions for query.")
|
logger.warning(f"KIK search returned no decisions for query.")
|
||||||
return api_response
|
return api_response
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in KIK search tool 'search_kik_decisions'.")
|
logger.exception(f"Error in KIK search tool 'search_kik_decisions'.")
|
||||||
current_page_val = search_query.page if hasattr(search_query, 'page') else 1
|
current_page_val = search_query.page if hasattr(search_query, 'page') else 1
|
||||||
return KikSearchResult(decisions=[], total_records=0, current_page=current_page_val)
|
return KikSearchResult(decisions=[], total_records=0, current_page=current_page_val)
|
||||||
|
|
||||||
@@ -1733,7 +1758,7 @@ async def search_rekabet_kurumu_decisions(
|
|||||||
try:
|
try:
|
||||||
|
|
||||||
return await rekabet_client_instance.search_decisions(search_query)
|
return await rekabet_client_instance.search_decisions(search_query)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_rekabet_kurumu_decisions'.")
|
logger.exception("Error in tool 'search_rekabet_kurumu_decisions'.")
|
||||||
return RekabetSearchResult(decisions=[], retrieved_page_number=page, total_records_found=0, total_pages=0)
|
return RekabetSearchResult(decisions=[], retrieved_page_number=page, total_records_found=0, total_pages=0)
|
||||||
|
|
||||||
@@ -1756,7 +1781,7 @@ async def get_rekabet_kurumu_document(
|
|||||||
try:
|
try:
|
||||||
|
|
||||||
return await rekabet_client_instance.get_decision_document(karar_id, page_number=current_page_to_fetch)
|
return await rekabet_client_instance.get_decision_document(karar_id, page_number=current_page_to_fetch)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception(f"Error in tool 'get_rekabet_kurumu_document'. Karar ID: {karar_id}")
|
logger.exception(f"Error in tool 'get_rekabet_kurumu_document'. Karar ID: {karar_id}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -1851,7 +1876,7 @@ For best results, use exact phrases with quotes for legal terms."""),
|
|||||||
"page_size": pageSize,
|
"page_size": pageSize,
|
||||||
"searched_courts": court_types
|
"searched_courts": court_types
|
||||||
}
|
}
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_bedesten_unified'")
|
logger.exception("Error in tool 'search_bedesten_unified'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -1873,7 +1898,7 @@ async def get_bedesten_document_markdown(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
return await bedesten_client_instance.get_document_as_markdown(documentId)
|
return await bedesten_client_instance.get_document_as_markdown(documentId)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_kyb_bedesten_document_markdown'")
|
logger.exception("Error in tool 'get_kyb_bedesten_document_markdown'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -2062,7 +2087,7 @@ async def search_sayistay_unified(
|
|||||||
web_karar_metni=web_karar_metni
|
web_karar_metni=web_karar_metni
|
||||||
)
|
)
|
||||||
return await sayistay_unified_client_instance.search_unified(search_request)
|
return await sayistay_unified_client_instance.search_unified(search_request)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'search_sayistay_unified'")
|
logger.exception("Error in tool 'search_sayistay_unified'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -2086,7 +2111,7 @@ async def get_sayistay_document_unified(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
return await sayistay_unified_client_instance.get_document_unified(decision_id, decision_type)
|
return await sayistay_unified_client_instance.get_document_unified(decision_id, decision_type)
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in tool 'get_sayistay_document_unified'")
|
logger.exception("Error in tool 'get_sayistay_document_unified'")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -2673,7 +2698,7 @@ async def search(
|
|||||||
logger.info(f"ChatGPT Deep Research search completed. Found {len(results)} results via Bedesten API.")
|
logger.info(f"ChatGPT Deep Research search completed. Found {len(results)} results via Bedesten API.")
|
||||||
return {"results": results}
|
return {"results": results}
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Error in ChatGPT Deep Research search tool")
|
logger.exception("Error in ChatGPT Deep Research search tool")
|
||||||
# Return partial results if any were found
|
# Return partial results if any were found
|
||||||
if results:
|
if results:
|
||||||
@@ -2802,7 +2827,7 @@ async def fetch(
|
|||||||
doc = await bedesten_client_instance.get_document_as_markdown(doc_id)
|
doc = await bedesten_client_instance.get_document_as_markdown(doc_id)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception(f"Error fetching ChatGPT Deep Research document {id}")
|
logger.exception(f"Error fetching ChatGPT Deep Research document {id}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@@ -2849,7 +2874,7 @@ def main():
|
|||||||
app.run()
|
app.run()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
logger.info("Server shut down by user (KeyboardInterrupt).")
|
logger.info("Server shut down by user (KeyboardInterrupt).")
|
||||||
except Exception:
|
except Exception as e:
|
||||||
logger.exception("Server failed to start or crashed.")
|
logger.exception("Server failed to start or crashed.")
|
||||||
finally:
|
finally:
|
||||||
logger.info(f"{app.name} server has shut down.")
|
logger.info(f"{app.name} server has shut down.")
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ import os
|
|||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import logging
|
import logging
|
||||||
from typing import Optional, Dict, Any
|
from typing import Optional, Dict, Any, Union
|
||||||
from datetime import datetime
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -4,9 +4,10 @@ import httpx
|
|||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import List, Optional, Tuple, Dict, Any
|
from typing import List, Optional, Tuple, Dict, Any
|
||||||
import logging
|
import logging
|
||||||
|
import html
|
||||||
import re
|
import re
|
||||||
import io # For io.BytesIO
|
import io # For io.BytesIO
|
||||||
from urllib.parse import urljoin, parse_qs, urlparse
|
from urllib.parse import urlencode, urljoin, quote, parse_qs, urlparse
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
import math
|
import math
|
||||||
|
|
||||||
@@ -17,7 +18,8 @@ from .models import (
|
|||||||
RekabetKurumuSearchRequest,
|
RekabetKurumuSearchRequest,
|
||||||
RekabetDecisionSummary,
|
RekabetDecisionSummary,
|
||||||
RekabetSearchResult,
|
RekabetSearchResult,
|
||||||
RekabetDocument
|
RekabetDocument,
|
||||||
|
RekabetKararTuruGuidEnum
|
||||||
)
|
)
|
||||||
from pydantic import HttpUrl # Ensure HttpUrl is imported from pydantic
|
from pydantic import HttpUrl # Ensure HttpUrl is imported from pydantic
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# rekabet_mcp_module/models.py
|
# rekabet_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, HttpUrl
|
from pydantic import BaseModel, Field, HttpUrl
|
||||||
from typing import List, Optional
|
from typing import List, Optional, Any
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
# Enum for decision type GUIDs (used by the client and expected by the website)
|
# Enum for decision type GUIDs (used by the client and expected by the website)
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ def main():
|
|||||||
config["workers"] = args.workers
|
config["workers"] = args.workers
|
||||||
|
|
||||||
# Print startup information
|
# Print startup information
|
||||||
print("Starting Yargı MCP server...")
|
print(f"Starting Yargı MCP server...")
|
||||||
print(f"Host: {args.host}")
|
print(f"Host: {args.host}")
|
||||||
print(f"Port: {args.port}")
|
print(f"Port: {args.port}")
|
||||||
print(f"Transport: {args.transport}")
|
print(f"Transport: {args.transport}")
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
# sayistay_mcp_module/client.py
|
# sayistay_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
import re
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import Dict, List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Tuple
|
||||||
import logging
|
import logging
|
||||||
|
import html
|
||||||
import io
|
import io
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode, urljoin
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
@@ -14,7 +16,7 @@ from .models import (
|
|||||||
DaireSearchRequest, DaireSearchResponse, DaireDecision,
|
DaireSearchRequest, DaireSearchResponse, DaireDecision,
|
||||||
SayistayDocumentMarkdown
|
SayistayDocumentMarkdown
|
||||||
)
|
)
|
||||||
from .enums import WEB_KARAR_KONUSU_MAPPING
|
from .enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum, WEB_KARAR_KONUSU_MAPPING
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
if not logger.hasHandlers():
|
if not logger.hasHandlers():
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# sayistay_mcp_module/models.py
|
# sayistay_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import Optional, List, Dict, Any, Literal
|
from typing import Optional, List, Union, Dict, Any, Literal
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from .enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
|
from .enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
# Unified client for all three Sayıştay decision types
|
# Unified client for all three Sayıştay decision types
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Optional, Dict, Any
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
SayistayUnifiedSearchRequest,
|
SayistayUnifiedSearchRequest,
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import os
|
|||||||
from starlette.applications import Starlette
|
from starlette.applications import Starlette
|
||||||
from starlette.routing import Mount, Route
|
from starlette.routing import Mount, Route
|
||||||
from starlette.requests import Request
|
from starlette.requests import Request
|
||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse, PlainTextResponse, RedirectResponse
|
||||||
from starlette.middleware import Middleware
|
from starlette.middleware import Middleware
|
||||||
from starlette.middleware.cors import CORSMiddleware
|
from starlette.middleware.cors import CORSMiddleware
|
||||||
from starlette.middleware.authentication import AuthenticationMiddleware
|
from starlette.middleware.authentication import AuthenticationMiddleware
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
import os
|
import os, stripe
|
||||||
import stripe
|
|
||||||
from clerk_backend_api import Clerk # Clerk backend SDK
|
from clerk_backend_api import Clerk # Clerk backend SDK
|
||||||
from fastapi import APIRouter, Request, HTTPException
|
from fastapi import APIRouter, Request, HTTPException
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
import httpx
|
import httpx
|
||||||
import aiohttp
|
import aiohttp
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Union, Tuple
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
|
|||||||
@@ -1,16 +1,20 @@
|
|||||||
# yargitay_mcp_module/client.py
|
# yargitay_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Optional
|
from bs4 import BeautifulSoup # Still needed for pre-processing HTML before markitdown
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
|
import re
|
||||||
import io
|
import io
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
YargitayDetailedSearchRequest,
|
YargitayDetailedSearchRequest,
|
||||||
YargitayApiSearchResponse,
|
YargitayApiSearchResponse,
|
||||||
YargitayDocumentMarkdown
|
YargitayApiDecisionEntry,
|
||||||
|
YargitayDocumentMarkdown,
|
||||||
|
CompactYargitaySearchResult
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# yargitay_mcp_module/models.py
|
# yargitay_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, HttpUrl, ConfigDict
|
from pydantic import BaseModel, Field, HttpUrl, ConfigDict
|
||||||
from typing import List, Optional, Literal
|
from typing import List, Optional, Dict, Any, Literal
|
||||||
|
|
||||||
# Yargıtay Chamber/Board Options
|
# Yargıtay Chamber/Board Options
|
||||||
YargitayBirimEnum = Literal[
|
YargitayBirimEnum = Literal[
|
||||||
|
|||||||
@@ -1,11 +1,13 @@
|
|||||||
# sayistay_mcp_module/client.py
|
# sayistay_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
import re
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import Dict, List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Tuple
|
||||||
import logging
|
import logging
|
||||||
|
import html
|
||||||
import io
|
import io
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode, urljoin
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
@@ -14,7 +16,7 @@ from .models import (
|
|||||||
DaireSearchRequest, DaireSearchResponse, DaireDecision,
|
DaireSearchRequest, DaireSearchResponse, DaireDecision,
|
||||||
SayistayDocumentMarkdown
|
SayistayDocumentMarkdown
|
||||||
)
|
)
|
||||||
from .enums import WEB_KARAR_KONUSU_MAPPING
|
from .enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum, WEB_KARAR_KONUSU_MAPPING
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
if not logger.hasHandlers():
|
if not logger.hasHandlers():
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# sayistay_mcp_module/models.py
|
# sayistay_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from typing import Optional, List, Dict, Any, Literal
|
from typing import Optional, List, Union, Dict, Any, Literal
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from .enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
|
from .enums import DaireEnum, KamuIdaresiTuruEnum, WebKararKonusuEnum
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
# Unified client for all three Sayıştay decision types
|
# Unified client for all three Sayıştay decision types
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from typing import Optional, Dict, Any
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
SayistayUnifiedSearchRequest,
|
SayistayUnifiedSearchRequest,
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
# semantic_search/__init__.py
|
|
||||||
|
|
||||||
from .embedder import EmbeddingGemma
|
|
||||||
from .vector_store import VectorStore
|
|
||||||
from .processor import DocumentProcessor
|
|
||||||
|
|
||||||
__all__ = ['EmbeddingGemma', 'VectorStore', 'DocumentProcessor']
|
|
||||||
@@ -1,173 +0,0 @@
|
|||||||
# semantic_search/embedder.py
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from typing import List, Optional
|
|
||||||
import numpy as np
|
|
||||||
from sentence_transformers import SentenceTransformer
|
|
||||||
import torch
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
class EmbeddingGemma:
|
|
||||||
"""
|
|
||||||
Wrapper for Google's EmbeddingGemma model.
|
|
||||||
Handles query and document encoding with proper prompt templates.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, model_name: str = "google/embeddinggemma-300m", device: Optional[str] = None):
|
|
||||||
"""
|
|
||||||
Initialize EmbeddingGemma model.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
model_name: HuggingFace model name
|
|
||||||
device: Device to run model on ('cuda', 'cpu', or None for auto)
|
|
||||||
"""
|
|
||||||
self.model_name = model_name
|
|
||||||
|
|
||||||
# Auto-detect device if not specified
|
|
||||||
if device is None:
|
|
||||||
self.device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
|
||||||
else:
|
|
||||||
self.device = device
|
|
||||||
|
|
||||||
logger.info(f"Initializing EmbeddingGemma on device: {self.device}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Load model with float32 precision (EmbeddingGemma doesn't support float16)
|
|
||||||
self.model = SentenceTransformer(model_name, device=self.device)
|
|
||||||
self.model.eval() # Set to evaluation mode
|
|
||||||
|
|
||||||
# Set precision to float32 or bfloat16
|
|
||||||
if self.device == 'cuda' and torch.cuda.is_bf16_supported():
|
|
||||||
logger.info("Using bfloat16 precision for CUDA")
|
|
||||||
self.dtype = torch.bfloat16
|
|
||||||
else:
|
|
||||||
logger.info("Using float32 precision")
|
|
||||||
self.dtype = torch.float32
|
|
||||||
|
|
||||||
logger.info(f"Successfully loaded model: {model_name}")
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to load EmbeddingGemma model: {e}")
|
|
||||||
raise
|
|
||||||
|
|
||||||
def encode_query(self, query: str, task: str = "search result") -> np.ndarray:
|
|
||||||
"""
|
|
||||||
Encode a search query with appropriate prompt template.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
query: The search query text
|
|
||||||
task: Task type for prompt template (search result, question answering, etc.)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Numpy array of embeddings (768 dimensions)
|
|
||||||
"""
|
|
||||||
# Apply query prompt template
|
|
||||||
prompted_query = f"task: {task} | query: {query}"
|
|
||||||
|
|
||||||
try:
|
|
||||||
with torch.no_grad():
|
|
||||||
# Encode with model
|
|
||||||
embeddings = self.model.encode(
|
|
||||||
prompted_query,
|
|
||||||
convert_to_numpy=True,
|
|
||||||
normalize_embeddings=True, # L2 normalization for cosine similarity
|
|
||||||
show_progress_bar=False
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.debug(f"Encoded query: {query[:50]}... -> shape: {embeddings.shape}")
|
|
||||||
return embeddings
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to encode query: {e}")
|
|
||||||
raise
|
|
||||||
|
|
||||||
def encode_documents(self, documents: List[str], titles: Optional[List[str]] = None) -> np.ndarray:
|
|
||||||
"""
|
|
||||||
Encode multiple documents with appropriate prompt template.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
documents: List of document texts
|
|
||||||
titles: Optional list of document titles
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Numpy array of embeddings (N x 768 dimensions)
|
|
||||||
"""
|
|
||||||
if not documents:
|
|
||||||
return np.array([])
|
|
||||||
|
|
||||||
# Apply document prompt template
|
|
||||||
prompted_docs = []
|
|
||||||
for i, doc in enumerate(documents):
|
|
||||||
title = titles[i] if titles and i < len(titles) else "none"
|
|
||||||
prompted_doc = f"title: {title} | text: {doc}"
|
|
||||||
prompted_docs.append(prompted_doc)
|
|
||||||
|
|
||||||
try:
|
|
||||||
with torch.no_grad():
|
|
||||||
# Batch encode documents
|
|
||||||
embeddings = self.model.encode(
|
|
||||||
prompted_docs,
|
|
||||||
convert_to_numpy=True,
|
|
||||||
normalize_embeddings=True,
|
|
||||||
show_progress_bar=len(documents) > 10,
|
|
||||||
batch_size=8 # Adjust based on memory
|
|
||||||
)
|
|
||||||
|
|
||||||
logger.info(f"Encoded {len(documents)} documents -> shape: {embeddings.shape}")
|
|
||||||
return embeddings
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to encode documents: {e}")
|
|
||||||
raise
|
|
||||||
|
|
||||||
def reduce_dimensions(self, embeddings: np.ndarray, target_dim: int = 512) -> np.ndarray:
|
|
||||||
"""
|
|
||||||
Reduce embedding dimensions using Matryoshka Representation Learning.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
embeddings: Original embeddings (N x 768)
|
|
||||||
target_dim: Target dimension (512, 256, or 128)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Reduced embeddings (N x target_dim)
|
|
||||||
"""
|
|
||||||
if target_dim not in [512, 256, 128]:
|
|
||||||
raise ValueError(f"Target dimension must be 512, 256, or 128, got {target_dim}")
|
|
||||||
|
|
||||||
if len(embeddings.shape) == 1:
|
|
||||||
# Single embedding
|
|
||||||
reduced = embeddings[:target_dim]
|
|
||||||
# Re-normalize after truncation
|
|
||||||
norm = np.linalg.norm(reduced)
|
|
||||||
if norm > 0:
|
|
||||||
reduced = reduced / norm
|
|
||||||
else:
|
|
||||||
# Multiple embeddings
|
|
||||||
reduced = embeddings[:, :target_dim]
|
|
||||||
# Re-normalize each embedding
|
|
||||||
norms = np.linalg.norm(reduced, axis=1, keepdims=True)
|
|
||||||
reduced = reduced / (norms + 1e-8) # Avoid division by zero
|
|
||||||
|
|
||||||
logger.debug(f"Reduced dimensions: {embeddings.shape} -> {reduced.shape}")
|
|
||||||
return reduced
|
|
||||||
|
|
||||||
def compute_similarity(self, query_embedding: np.ndarray, document_embeddings: np.ndarray) -> np.ndarray:
|
|
||||||
"""
|
|
||||||
Compute cosine similarity between query and documents.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
query_embedding: Query embedding (768,)
|
|
||||||
document_embeddings: Document embeddings (N x 768)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Similarity scores (N,)
|
|
||||||
"""
|
|
||||||
# Ensure query is 2D for matrix multiplication
|
|
||||||
if len(query_embedding.shape) == 1:
|
|
||||||
query_embedding = query_embedding.reshape(1, -1)
|
|
||||||
|
|
||||||
# Compute cosine similarity (embeddings are already normalized)
|
|
||||||
similarities = np.dot(document_embeddings, query_embedding.T).squeeze()
|
|
||||||
|
|
||||||
return similarities
|
|
||||||
@@ -1,305 +0,0 @@
|
|||||||
# semantic_search/processor.py
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
from typing import List, Dict, Any, Optional
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import hashlib
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class DocumentChunk:
|
|
||||||
"""Represents a chunk of a document."""
|
|
||||||
chunk_id: str
|
|
||||||
document_id: str
|
|
||||||
text: str
|
|
||||||
metadata: Dict[str, Any]
|
|
||||||
chunk_index: int
|
|
||||||
total_chunks: int
|
|
||||||
|
|
||||||
class DocumentProcessor:
|
|
||||||
"""
|
|
||||||
Processes legal documents for semantic search.
|
|
||||||
Handles chunking, cleaning, and metadata extraction.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self,
|
|
||||||
chunk_size: int = 1000,
|
|
||||||
chunk_overlap: int = 200,
|
|
||||||
min_chunk_size: int = 100):
|
|
||||||
"""
|
|
||||||
Initialize document processor.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
chunk_size: Target size for each chunk in characters
|
|
||||||
chunk_overlap: Number of overlapping characters between chunks
|
|
||||||
min_chunk_size: Minimum chunk size to keep
|
|
||||||
"""
|
|
||||||
self.chunk_size = chunk_size
|
|
||||||
self.chunk_overlap = chunk_overlap
|
|
||||||
self.min_chunk_size = min_chunk_size
|
|
||||||
|
|
||||||
logger.info(f"Initialized DocumentProcessor (chunk_size={chunk_size}, overlap={chunk_overlap})")
|
|
||||||
|
|
||||||
def process_document(self,
|
|
||||||
document_id: str,
|
|
||||||
text: str,
|
|
||||||
metadata: Optional[Dict[str, Any]] = None) -> List[DocumentChunk]:
|
|
||||||
"""
|
|
||||||
Process a single document into chunks.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
document_id: Unique document identifier
|
|
||||||
text: Document text content
|
|
||||||
metadata: Optional document metadata
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of document chunks
|
|
||||||
"""
|
|
||||||
if not text or len(text.strip()) < self.min_chunk_size:
|
|
||||||
logger.warning(f"Document {document_id} too short to process")
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Clean text
|
|
||||||
cleaned_text = self._clean_text(text)
|
|
||||||
|
|
||||||
# Extract metadata from text if not provided
|
|
||||||
if metadata is None:
|
|
||||||
metadata = {}
|
|
||||||
|
|
||||||
# Add extracted metadata
|
|
||||||
extracted_metadata = self._extract_metadata(cleaned_text)
|
|
||||||
metadata.update(extracted_metadata)
|
|
||||||
|
|
||||||
# Create chunks
|
|
||||||
chunks = self._create_chunks(cleaned_text)
|
|
||||||
|
|
||||||
# Create DocumentChunk objects
|
|
||||||
document_chunks = []
|
|
||||||
for i, chunk_text in enumerate(chunks):
|
|
||||||
chunk_id = self._generate_chunk_id(document_id, i)
|
|
||||||
|
|
||||||
chunk = DocumentChunk(
|
|
||||||
chunk_id=chunk_id,
|
|
||||||
document_id=document_id,
|
|
||||||
text=chunk_text,
|
|
||||||
metadata={
|
|
||||||
**metadata,
|
|
||||||
'chunk_index': i,
|
|
||||||
'total_chunks': len(chunks)
|
|
||||||
},
|
|
||||||
chunk_index=i,
|
|
||||||
total_chunks=len(chunks)
|
|
||||||
)
|
|
||||||
document_chunks.append(chunk)
|
|
||||||
|
|
||||||
logger.info(f"Processed document {document_id} into {len(chunks)} chunks")
|
|
||||||
return document_chunks
|
|
||||||
|
|
||||||
def _clean_text(self, text: str) -> str:
|
|
||||||
"""
|
|
||||||
Clean and normalize text for processing.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: Raw text
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Cleaned text
|
|
||||||
"""
|
|
||||||
# Remove excessive whitespace
|
|
||||||
text = re.sub(r'\s+', ' ', text)
|
|
||||||
|
|
||||||
# Remove special characters but keep Turkish characters
|
|
||||||
# Keep: letters, numbers, spaces, and common punctuation
|
|
||||||
text = re.sub(r'[^\w\s\.\,\;\:\!\?\-\(\)\"\'ÇĞIİÖŞÜçğıiöşü]', ' ', text)
|
|
||||||
|
|
||||||
# Remove multiple spaces
|
|
||||||
text = re.sub(r' +', ' ', text)
|
|
||||||
|
|
||||||
# Trim
|
|
||||||
text = text.strip()
|
|
||||||
|
|
||||||
return text
|
|
||||||
|
|
||||||
def _extract_metadata(self, text: str) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Extract metadata from legal document text.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: Document text
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Extracted metadata
|
|
||||||
"""
|
|
||||||
metadata = {}
|
|
||||||
|
|
||||||
# Extract case numbers (Esas/Karar)
|
|
||||||
esas_pattern = r'E(?:sas)?[\s\.\:]*(\d{4})[\/\-](\d+)'
|
|
||||||
karar_pattern = r'K(?:arar)?[\s\.\:]*(\d{4})[\/\-](\d+)'
|
|
||||||
|
|
||||||
esas_match = re.search(esas_pattern, text[:500]) # Look in first 500 chars
|
|
||||||
if esas_match:
|
|
||||||
metadata['esas_no'] = f"E.{esas_match.group(1)}/{esas_match.group(2)}"
|
|
||||||
|
|
||||||
karar_match = re.search(karar_pattern, text[:500])
|
|
||||||
if karar_match:
|
|
||||||
metadata['karar_no'] = f"K.{karar_match.group(1)}/{karar_match.group(2)}"
|
|
||||||
|
|
||||||
# Extract dates (DD.MM.YYYY or DD/MM/YYYY format)
|
|
||||||
date_pattern = r'(\d{1,2})[\.\/](\d{1,2})[\.\/](\d{4})'
|
|
||||||
dates = re.findall(date_pattern, text[:1000]) # Look in first 1000 chars
|
|
||||||
if dates:
|
|
||||||
# Take the first date as decision date
|
|
||||||
day, month, year = dates[0]
|
|
||||||
metadata['karar_tarihi'] = f"{year}-{month.zfill(2)}-{day.zfill(2)}"
|
|
||||||
|
|
||||||
# Extract court/chamber name
|
|
||||||
chamber_patterns = [
|
|
||||||
r'(\d+)\.\s*Hukuk\s+Dairesi',
|
|
||||||
r'(\d+)\.\s*Ceza\s+Dairesi',
|
|
||||||
r'Hukuk\s+Genel\s+Kurulu',
|
|
||||||
r'Ceza\s+Genel\s+Kurulu',
|
|
||||||
r'(\d+)\.\s*Daire'
|
|
||||||
]
|
|
||||||
|
|
||||||
for pattern in chamber_patterns:
|
|
||||||
match = re.search(pattern, text[:500], re.IGNORECASE)
|
|
||||||
if match:
|
|
||||||
metadata['chamber'] = match.group(0)
|
|
||||||
break
|
|
||||||
|
|
||||||
return metadata
|
|
||||||
|
|
||||||
def _create_chunks(self, text: str) -> List[str]:
|
|
||||||
"""
|
|
||||||
Create overlapping chunks from text.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: Cleaned document text
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of text chunks
|
|
||||||
"""
|
|
||||||
chunks = []
|
|
||||||
|
|
||||||
# Split by sentences for better semantic coherence
|
|
||||||
sentences = self._split_sentences(text)
|
|
||||||
|
|
||||||
current_chunk = []
|
|
||||||
current_size = 0
|
|
||||||
|
|
||||||
for sentence in sentences:
|
|
||||||
sentence_size = len(sentence)
|
|
||||||
|
|
||||||
# If adding this sentence exceeds chunk size
|
|
||||||
if current_size + sentence_size > self.chunk_size and current_chunk:
|
|
||||||
# Save current chunk
|
|
||||||
chunk_text = ' '.join(current_chunk)
|
|
||||||
chunks.append(chunk_text)
|
|
||||||
|
|
||||||
# Create overlap for next chunk
|
|
||||||
overlap_size = 0
|
|
||||||
overlap_sentences = []
|
|
||||||
|
|
||||||
# Add sentences from the end until we reach overlap size
|
|
||||||
for sent in reversed(current_chunk):
|
|
||||||
overlap_size += len(sent)
|
|
||||||
overlap_sentences.insert(0, sent)
|
|
||||||
if overlap_size >= self.chunk_overlap:
|
|
||||||
break
|
|
||||||
|
|
||||||
# Start new chunk with overlap
|
|
||||||
current_chunk = overlap_sentences
|
|
||||||
current_size = sum(len(s) for s in current_chunk)
|
|
||||||
|
|
||||||
# Add sentence to current chunk
|
|
||||||
current_chunk.append(sentence)
|
|
||||||
current_size += sentence_size
|
|
||||||
|
|
||||||
# Add final chunk if not empty
|
|
||||||
if current_chunk:
|
|
||||||
chunk_text = ' '.join(current_chunk)
|
|
||||||
if len(chunk_text) >= self.min_chunk_size:
|
|
||||||
chunks.append(chunk_text)
|
|
||||||
|
|
||||||
return chunks
|
|
||||||
|
|
||||||
def _split_sentences(self, text: str) -> List[str]:
|
|
||||||
"""
|
|
||||||
Split text into sentences.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: Text to split
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of sentences
|
|
||||||
"""
|
|
||||||
# Simple sentence splitting for Turkish text
|
|
||||||
# Split on period, question mark, exclamation, but not on abbreviations
|
|
||||||
|
|
||||||
# Common Turkish abbreviations to preserve
|
|
||||||
abbreviations = ['Dr', 'Prof', 'Av', 'Md', 'Yrd', 'Doç', 'No', 'S', 'vs', 'vb', 'bkz']
|
|
||||||
|
|
||||||
# Replace abbreviations temporarily
|
|
||||||
temp_text = text
|
|
||||||
replacements = {}
|
|
||||||
for i, abbr in enumerate(abbreviations):
|
|
||||||
placeholder = f"__ABBR{i}__"
|
|
||||||
temp_text = temp_text.replace(f"{abbr}.", placeholder)
|
|
||||||
replacements[placeholder] = f"{abbr}."
|
|
||||||
|
|
||||||
# Split sentences
|
|
||||||
sentence_endings = re.compile(r'[.!?]+')
|
|
||||||
sentences = sentence_endings.split(temp_text)
|
|
||||||
|
|
||||||
# Restore abbreviations and clean
|
|
||||||
cleaned_sentences = []
|
|
||||||
for sentence in sentences:
|
|
||||||
# Restore abbreviations
|
|
||||||
for placeholder, original in replacements.items():
|
|
||||||
sentence = sentence.replace(placeholder, original)
|
|
||||||
|
|
||||||
# Clean and add if not empty
|
|
||||||
sentence = sentence.strip()
|
|
||||||
if sentence and len(sentence) > 10: # Minimum sentence length
|
|
||||||
cleaned_sentences.append(sentence)
|
|
||||||
|
|
||||||
return cleaned_sentences
|
|
||||||
|
|
||||||
def _generate_chunk_id(self, document_id: str, chunk_index: int) -> str:
|
|
||||||
"""
|
|
||||||
Generate unique chunk ID.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
document_id: Parent document ID
|
|
||||||
chunk_index: Index of chunk in document
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Unique chunk ID
|
|
||||||
"""
|
|
||||||
chunk_string = f"{document_id}_chunk_{chunk_index}"
|
|
||||||
chunk_hash = hashlib.md5(chunk_string.encode()).hexdigest()[:8]
|
|
||||||
return f"{document_id}_c{chunk_index}_{chunk_hash}"
|
|
||||||
|
|
||||||
def combine_chunks(self, chunks: List[DocumentChunk]) -> str:
|
|
||||||
"""
|
|
||||||
Combine chunks back into full document text.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
chunks: List of document chunks
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Combined text
|
|
||||||
"""
|
|
||||||
if not chunks:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
# Sort by chunk index
|
|
||||||
sorted_chunks = sorted(chunks, key=lambda x: x.chunk_index)
|
|
||||||
|
|
||||||
# For overlapping chunks, we need to be careful about duplication
|
|
||||||
# Simple approach: just concatenate with space
|
|
||||||
combined = " ".join([chunk.text for chunk in sorted_chunks])
|
|
||||||
|
|
||||||
return combined
|
|
||||||
@@ -1,235 +0,0 @@
|
|||||||
# semantic_search/vector_store.py
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import numpy as np
|
|
||||||
from typing import List, Dict, Any, Tuple, Optional
|
|
||||||
from dataclasses import dataclass
|
|
||||||
import json
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Document:
|
|
||||||
"""Represents a document with its embedding and metadata."""
|
|
||||||
id: str
|
|
||||||
text: str
|
|
||||||
embedding: np.ndarray
|
|
||||||
metadata: Dict[str, Any]
|
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, Any]:
|
|
||||||
"""Convert to dictionary (excluding embedding for serialization)."""
|
|
||||||
return {
|
|
||||||
'id': self.id,
|
|
||||||
'text': self.text,
|
|
||||||
'metadata': self.metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
class VectorStore:
|
|
||||||
"""
|
|
||||||
In-memory vector storage with similarity search capabilities.
|
|
||||||
Future versions can use Faiss, ChromaDB, or other vector databases.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, dimension: int = 768):
|
|
||||||
"""
|
|
||||||
Initialize vector store.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
dimension: Embedding dimension size
|
|
||||||
"""
|
|
||||||
self.dimension = dimension
|
|
||||||
self.documents: List[Document] = []
|
|
||||||
self.embeddings: Optional[np.ndarray] = None
|
|
||||||
self.index_built = False
|
|
||||||
|
|
||||||
logger.info(f"Initialized VectorStore with dimension: {dimension}")
|
|
||||||
|
|
||||||
def add_documents(self,
|
|
||||||
ids: List[str],
|
|
||||||
texts: List[str],
|
|
||||||
embeddings: np.ndarray,
|
|
||||||
metadata: Optional[List[Dict[str, Any]]] = None) -> int:
|
|
||||||
"""
|
|
||||||
Add documents to the vector store.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
ids: Document IDs
|
|
||||||
texts: Document texts
|
|
||||||
embeddings: Document embeddings (N x dimension)
|
|
||||||
metadata: Optional metadata for each document
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Number of documents added
|
|
||||||
"""
|
|
||||||
if len(ids) != len(texts) or len(ids) != embeddings.shape[0]:
|
|
||||||
raise ValueError("Mismatched lengths for ids, texts, and embeddings")
|
|
||||||
|
|
||||||
if metadata and len(metadata) != len(ids):
|
|
||||||
raise ValueError("Metadata length doesn't match document count")
|
|
||||||
|
|
||||||
# Add documents
|
|
||||||
for i in range(len(ids)):
|
|
||||||
doc = Document(
|
|
||||||
id=ids[i],
|
|
||||||
text=texts[i],
|
|
||||||
embedding=embeddings[i],
|
|
||||||
metadata=metadata[i] if metadata else {}
|
|
||||||
)
|
|
||||||
self.documents.append(doc)
|
|
||||||
|
|
||||||
# Rebuild index
|
|
||||||
self._build_index()
|
|
||||||
|
|
||||||
logger.info(f"Added {len(ids)} documents to vector store. Total: {len(self.documents)}")
|
|
||||||
return len(ids)
|
|
||||||
|
|
||||||
def _build_index(self):
|
|
||||||
"""Build or rebuild the embedding index."""
|
|
||||||
if not self.documents:
|
|
||||||
self.embeddings = None
|
|
||||||
self.index_built = False
|
|
||||||
return
|
|
||||||
|
|
||||||
# Stack all embeddings into a single array
|
|
||||||
self.embeddings = np.vstack([doc.embedding for doc in self.documents])
|
|
||||||
self.index_built = True
|
|
||||||
|
|
||||||
logger.debug(f"Built index with shape: {self.embeddings.shape}")
|
|
||||||
|
|
||||||
def search(self,
|
|
||||||
query_embedding: np.ndarray,
|
|
||||||
top_k: int = 10,
|
|
||||||
threshold: Optional[float] = None) -> List[Tuple[Document, float]]:
|
|
||||||
"""
|
|
||||||
Search for similar documents using cosine similarity.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
query_embedding: Query embedding vector
|
|
||||||
top_k: Number of results to return
|
|
||||||
threshold: Optional similarity threshold (0-1)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of (Document, similarity_score) tuples
|
|
||||||
"""
|
|
||||||
if not self.index_built or self.embeddings is None:
|
|
||||||
logger.warning("No documents in vector store")
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Ensure query is 2D
|
|
||||||
if len(query_embedding.shape) == 1:
|
|
||||||
query_embedding = query_embedding.reshape(1, -1)
|
|
||||||
|
|
||||||
# Compute cosine similarities (assuming normalized embeddings)
|
|
||||||
similarities = np.dot(self.embeddings, query_embedding.T).squeeze()
|
|
||||||
|
|
||||||
# Apply threshold if specified
|
|
||||||
if threshold is not None:
|
|
||||||
valid_indices = np.where(similarities >= threshold)[0]
|
|
||||||
if len(valid_indices) == 0:
|
|
||||||
logger.info(f"No documents above threshold {threshold}")
|
|
||||||
return []
|
|
||||||
similarities = similarities[valid_indices]
|
|
||||||
valid_docs = [self.documents[i] for i in valid_indices]
|
|
||||||
else:
|
|
||||||
valid_docs = self.documents
|
|
||||||
|
|
||||||
# Get top-k indices
|
|
||||||
top_k = min(top_k, len(valid_docs))
|
|
||||||
if top_k == 0:
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Use argpartition for efficiency with large arrays
|
|
||||||
if len(similarities) > top_k:
|
|
||||||
top_indices = np.argpartition(similarities, -top_k)[-top_k:]
|
|
||||||
top_indices = top_indices[np.argsort(similarities[top_indices])[::-1]]
|
|
||||||
else:
|
|
||||||
top_indices = np.argsort(similarities)[::-1]
|
|
||||||
|
|
||||||
# Create results
|
|
||||||
results = []
|
|
||||||
for idx in top_indices:
|
|
||||||
doc = valid_docs[idx] if threshold else self.documents[idx]
|
|
||||||
score = float(similarities[idx])
|
|
||||||
results.append((doc, score))
|
|
||||||
|
|
||||||
logger.info(f"Search returned {len(results)} results (top_k={top_k})")
|
|
||||||
return results
|
|
||||||
|
|
||||||
def hybrid_search(self,
|
|
||||||
query_embedding: np.ndarray,
|
|
||||||
keyword_scores: Dict[str, float],
|
|
||||||
top_k: int = 10,
|
|
||||||
alpha: float = 0.5) -> List[Tuple[Document, float]]:
|
|
||||||
"""
|
|
||||||
Hybrid search combining vector similarity and keyword scores.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
query_embedding: Query embedding vector
|
|
||||||
keyword_scores: Document ID to keyword relevance score mapping
|
|
||||||
top_k: Number of results to return
|
|
||||||
alpha: Weight for vector similarity (1-alpha for keyword score)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List of (Document, combined_score) tuples
|
|
||||||
"""
|
|
||||||
if not self.index_built:
|
|
||||||
logger.warning("No documents in vector store")
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Get vector similarities
|
|
||||||
vector_results = self.search(query_embedding, top_k=len(self.documents))
|
|
||||||
|
|
||||||
# Combine scores
|
|
||||||
combined_scores = []
|
|
||||||
for doc, vector_score in vector_results:
|
|
||||||
keyword_score = keyword_scores.get(doc.id, 0.0)
|
|
||||||
# Normalize keyword score to 0-1 range if needed
|
|
||||||
if keyword_score > 1.0:
|
|
||||||
keyword_score = keyword_score / max(keyword_scores.values())
|
|
||||||
|
|
||||||
combined_score = alpha * vector_score + (1 - alpha) * keyword_score
|
|
||||||
combined_scores.append((doc, combined_score))
|
|
||||||
|
|
||||||
# Sort by combined score and return top-k
|
|
||||||
combined_scores.sort(key=lambda x: x[1], reverse=True)
|
|
||||||
results = combined_scores[:top_k]
|
|
||||||
|
|
||||||
logger.info(f"Hybrid search returned {len(results)} results")
|
|
||||||
return results
|
|
||||||
|
|
||||||
def clear(self):
|
|
||||||
"""Clear all documents from the store."""
|
|
||||||
self.documents = []
|
|
||||||
self.embeddings = None
|
|
||||||
self.index_built = False
|
|
||||||
logger.info("Cleared vector store")
|
|
||||||
|
|
||||||
def size(self) -> int:
|
|
||||||
"""Get number of documents in store."""
|
|
||||||
return len(self.documents)
|
|
||||||
|
|
||||||
def get_by_id(self, doc_id: str) -> Optional[Document]:
|
|
||||||
"""Get document by ID."""
|
|
||||||
for doc in self.documents:
|
|
||||||
if doc.id == doc_id:
|
|
||||||
return doc
|
|
||||||
return None
|
|
||||||
|
|
||||||
def get_stats(self) -> Dict[str, Any]:
|
|
||||||
"""Get statistics about the vector store."""
|
|
||||||
stats = {
|
|
||||||
'num_documents': len(self.documents),
|
|
||||||
'dimension': self.dimension,
|
|
||||||
'index_built': self.index_built,
|
|
||||||
'memory_usage_mb': 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if self.embeddings is not None:
|
|
||||||
# Estimate memory usage
|
|
||||||
memory_bytes = self.embeddings.nbytes
|
|
||||||
for doc in self.documents:
|
|
||||||
memory_bytes += len(doc.text.encode('utf-8'))
|
|
||||||
memory_bytes += len(json.dumps(doc.metadata).encode('utf-8'))
|
|
||||||
stats['memory_usage_mb'] = memory_bytes / (1024 * 1024)
|
|
||||||
|
|
||||||
return stats
|
|
||||||
+1
-1
@@ -13,7 +13,7 @@ import os
|
|||||||
from starlette.applications import Starlette
|
from starlette.applications import Starlette
|
||||||
from starlette.routing import Mount, Route
|
from starlette.routing import Mount, Route
|
||||||
from starlette.requests import Request
|
from starlette.requests import Request
|
||||||
from starlette.responses import JSONResponse
|
from starlette.responses import JSONResponse, PlainTextResponse, RedirectResponse
|
||||||
from starlette.middleware import Middleware
|
from starlette.middleware import Middleware
|
||||||
from starlette.middleware.cors import CORSMiddleware
|
from starlette.middleware.cors import CORSMiddleware
|
||||||
from starlette.middleware.authentication import AuthenticationMiddleware
|
from starlette.middleware.authentication import AuthenticationMiddleware
|
||||||
|
|||||||
+1
-2
@@ -1,5 +1,4 @@
|
|||||||
import os
|
import os, stripe
|
||||||
import stripe
|
|
||||||
from clerk_backend_api import Clerk # Clerk backend SDK
|
from clerk_backend_api import Clerk # Clerk backend SDK
|
||||||
from fastapi import APIRouter, Request, HTTPException
|
from fastapi import APIRouter, Request, HTTPException
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from typing import List, Optional, Tuple
|
from typing import Dict, Any, List, Optional, Union, Tuple
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
import re
|
import re
|
||||||
|
|||||||
@@ -1,16 +1,20 @@
|
|||||||
# yargitay_mcp_module/client.py
|
# yargitay_mcp_module/client.py
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Optional
|
from bs4 import BeautifulSoup # Still needed for pre-processing HTML before markitdown
|
||||||
|
from typing import Dict, Any, List, Optional
|
||||||
import logging
|
import logging
|
||||||
import html
|
import html
|
||||||
|
import re
|
||||||
import io
|
import io
|
||||||
from markitdown import MarkItDown
|
from markitdown import MarkItDown
|
||||||
|
|
||||||
from .models import (
|
from .models import (
|
||||||
YargitayDetailedSearchRequest,
|
YargitayDetailedSearchRequest,
|
||||||
YargitayApiSearchResponse,
|
YargitayApiSearchResponse,
|
||||||
YargitayDocumentMarkdown
|
YargitayApiDecisionEntry,
|
||||||
|
YargitayDocumentMarkdown,
|
||||||
|
CompactYargitaySearchResult
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# yargitay_mcp_module/models.py
|
# yargitay_mcp_module/models.py
|
||||||
|
|
||||||
from pydantic import BaseModel, Field, HttpUrl, ConfigDict
|
from pydantic import BaseModel, Field, HttpUrl, ConfigDict
|
||||||
from typing import List, Optional, Literal
|
from typing import List, Optional, Dict, Any, Literal
|
||||||
|
|
||||||
# Yargıtay Chamber/Board Options
|
# Yargıtay Chamber/Board Options
|
||||||
YargitayBirimEnum = Literal[
|
YargitayBirimEnum = Literal[
|
||||||
|
|||||||
Reference in New Issue
Block a user