From 1ebe8847fbf338f46979fd3853de84856f62426c Mon Sep 17 00:00:00 2001 From: saidsurucu Date: Sat, 12 Jul 2025 11:40:11 +0300 Subject: [PATCH] fix kik --- kik_mcp_module/client.py | 846 ++++++++++++++++++++++++++++++++++++--- mcp_server_main.py | 46 ++- 2 files changed, 836 insertions(+), 56 deletions(-) diff --git a/kik_mcp_module/client.py b/kik_mcp_module/client.py index ed32c88..e6c8108 100644 --- a/kik_mcp_module/client.py +++ b/kik_mcp_module/client.py @@ -19,6 +19,7 @@ from markitdown import MarkItDown import os import math import io +import random from .models import ( KikSearchRequest, @@ -69,14 +70,79 @@ class KikApiClient: self.playwright_instance = await async_playwright().start() if not self.browser or not self.browser.is_connected(): if self.browser: await self.browser.close() - self.browser = await self.playwright_instance.chromium.launch(headless=True) + # Ultra stealth browser configuration + self.browser = await self.playwright_instance.chromium.launch( + headless=True, + args=[ + # Disable automation indicators + '--no-first-run', + '--no-default-browser-check', + '--disable-dev-shm-usage', + '--disable-extensions', + '--disable-gpu', + '--disable-default-apps', + '--disable-translate', + '--disable-blink-features=AutomationControlled', + '--disable-ipc-flooding-protection', + '--disable-renderer-backgrounding', + '--disable-backgrounding-occluded-windows', + '--disable-client-side-phishing-detection', + '--disable-sync', + '--disable-features=TranslateUI,BlinkGenPropertyTrees', + '--disable-component-extensions-with-background-pages', + '--no-sandbox', # Sometimes needed for headless + '--disable-web-security', + '--disable-features=VizDisplayCompositor', + # Language and locale + '--lang=tr-TR', + '--accept-lang=tr-TR,tr;q=0.9,en;q=0.8', + # Performance optimizations + '--memory-pressure-off', + '--max_old_space_size=4096', + ] + ) browser_recreated = True if not self.context or browser_recreated: if self.context: await self.context.close() if not self.browser: raise PlaywrightError("Browser not initialized.") + # Ultra realistic context configuration self.context = await self.browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36", + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + viewport={'width': 1920, 'height': 1080}, + screen={'width': 1920, 'height': 1080}, + device_scale_factor=1.0, + is_mobile=False, + has_touch=False, + # Localization + locale='tr-TR', + timezone_id='Europe/Istanbul', + # Realistic browser features java_script_enabled=True, + accept_downloads=True, + ignore_https_errors=True, + # Color scheme and media + color_scheme='light', + reduced_motion='no-preference', + forced_colors='none', + # Additional headers for realism + extra_http_headers={ + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Encoding': 'gzip, deflate, br', + 'Accept-Language': 'tr-TR,tr;q=0.9,en;q=0.8', + 'Cache-Control': 'max-age=0', + 'DNT': '1', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Ch-Ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"', + 'Sec-Ch-Ua-Mobile': '?0', + 'Sec-Ch-Ua-Platform': '"Windows"', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + }, + # Permissions to appear realistic + permissions=['geolocation'], + geolocation={'latitude': 41.0082, 'longitude': 28.9784}, # Istanbul ) context_recreated = True if not self.page or self.page.is_closed() or force_new_page or context_recreated or browser_recreated: @@ -86,6 +152,9 @@ class KikApiClient: if not self.page: raise PlaywrightError("Failed to create new page.") self.page.set_default_navigation_timeout(self.request_timeout) self.page.set_default_timeout(self.request_timeout) + + # CRITICAL: Anti-detection JavaScript injection + await self._inject_stealth_scripts() if not self.page or self.page.is_closed(): raise PlaywrightError("Playwright page initialization failed.") logger.debug("_ensure_playwright_ready completed.") @@ -99,41 +168,644 @@ class KikApiClient: if self.playwright_instance: await self.playwright_instance.stop(); self.playwright_instance = None logger.info("KikApiClient (Playwright): Resources closed.") + async def _inject_stealth_scripts(self): + """ + Inject comprehensive stealth JavaScript to evade bot detection. + Overrides navigator properties and other fingerprinting vectors. + """ + if not self.page: + logger.warning("Cannot inject stealth scripts: page is None") + return + + logger.debug("Injecting comprehensive stealth scripts...") + + stealth_script = ''' + // Override navigator.webdriver + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + configurable: true + }); + + // Override navigator properties to appear more human + Object.defineProperty(navigator, 'languages', { + get: () => ['tr-TR', 'tr', 'en-US', 'en'], + configurable: true + }); + + Object.defineProperty(navigator, 'platform', { + get: () => 'Win32', + configurable: true + }); + + Object.defineProperty(navigator, 'vendor', { + get: () => 'Google Inc.', + configurable: true + }); + + Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8, + configurable: true + }); + + Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8, + configurable: true + }); + + Object.defineProperty(navigator, 'maxTouchPoints', { + get: () => 0, + configurable: true + }); + + // Override plugins to appear realistic + Object.defineProperty(navigator, 'plugins', { + get: () => { + return [ + { + 0: {type: "application/x-google-chrome-pdf", suffixes: "pdf", description: "Portable Document Format", enabledPlugin: Plugin}, + description: "Portable Document Format", + filename: "internal-pdf-viewer", + length: 1, + name: "Chrome PDF Plugin" + }, + { + 0: {type: "application/pdf", suffixes: "pdf", description: "", enabledPlugin: Plugin}, + description: "", + filename: "mhjfbmdgcfjbbpaeojofohoefgiehjai", + length: 1, + name: "Chrome PDF Viewer" + } + ]; + }, + configurable: true + }); + + // Override permissions + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => ( + parameters.name === 'notifications' ? + Promise.resolve({ state: Notification.permission }) : + originalQuery(parameters) + ); + + // Override WebGL rendering context + const getParameter = WebGLRenderingContext.prototype.getParameter; + WebGLRenderingContext.prototype.getParameter = function(parameter) { + if (parameter === 37445) { // UNMASKED_VENDOR_WEBGL + return 'Intel Inc.'; + } + if (parameter === 37446) { // UNMASKED_RENDERER_WEBGL + return 'Intel(R) Iris(R) Plus Graphics 640'; + } + return getParameter(parameter); + }; + + // Override canvas fingerprinting + const toBlob = HTMLCanvasElement.prototype.toBlob; + const toDataURL = HTMLCanvasElement.prototype.toDataURL; + const getImageData = CanvasRenderingContext2D.prototype.getImageData; + + const noisify = (canvas, context) => { + const imageData = context.getImageData(0, 0, canvas.width, canvas.height); + for (let i = 0; i < imageData.data.length; i += 4) { + imageData.data[i] += Math.floor(Math.random() * 10) - 5; + imageData.data[i + 1] += Math.floor(Math.random() * 10) - 5; + imageData.data[i + 2] += Math.floor(Math.random() * 10) - 5; + } + context.putImageData(imageData, 0, 0); + }; + + Object.defineProperty(HTMLCanvasElement.prototype, 'toBlob', { + value: function(callback, type, encoderOptions) { + noisify(this, this.getContext('2d')); + return toBlob.apply(this, arguments); + } + }); + + Object.defineProperty(HTMLCanvasElement.prototype, 'toDataURL', { + value: function(type, encoderOptions) { + noisify(this, this.getContext('2d')); + return toDataURL.apply(this, arguments); + } + }); + + // Override AudioContext for audio fingerprinting + const audioCtx = new (window.AudioContext || window.webkitAudioContext)(); + const originalAnalyser = audioCtx.createAnalyser; + audioCtx.createAnalyser = function() { + const analyser = originalAnalyser.apply(this, arguments); + const getFloatFrequencyData = analyser.getFloatFrequencyData; + analyser.getFloatFrequencyData = function(array) { + getFloatFrequencyData.apply(this, arguments); + for (let i = 0; i < array.length; i++) { + array[i] += Math.random() * 0.0001; + } + }; + return analyser; + }; + + // Override screen properties + Object.defineProperty(window.screen, 'colorDepth', { + get: () => 24, + configurable: true + }); + + Object.defineProperty(window.screen, 'pixelDepth', { + get: () => 24, + configurable: true + }); + + // Override timezone + Date.prototype.getTimezoneOffset = function() { + return -180; // UTC+3 (Istanbul) + }; + + // Override document.cookie to prevent tracking + const originalCookieDescriptor = Object.getOwnPropertyDescriptor(Document.prototype, 'cookie') || + Object.getOwnPropertyDescriptor(HTMLDocument.prototype, 'cookie'); + if (originalCookieDescriptor && originalCookieDescriptor.configurable) { + Object.defineProperty(document, 'cookie', { + get: function() { + return originalCookieDescriptor.get.call(this); + }, + set: function(val) { + console.log('Cookie set blocked:', val); + return originalCookieDescriptor.set.call(this, val); + }, + configurable: true + }); + } + + // Remove automation traces + delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array; + delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise; + delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol; + delete window.cdc_adoQpoasnfa76pfcZLmcfl_JSON; + delete window.cdc_adoQpoasnfa76pfcZLmcfl_Object; + delete window.cdc_adoQpoasnfa76pfcZLmcfl_Proxy; + + // Add realistic performance timing + if (window.performance && window.performance.timing) { + const timing = window.performance.timing; + const now = Date.now(); + Object.defineProperty(timing, 'navigationStart', { value: now - Math.floor(Math.random() * 1000) + 1000, configurable: false }); + Object.defineProperty(timing, 'loadEventEnd', { value: now - Math.floor(Math.random() * 100) + 100, configurable: false }); + } + + console.log('✓ Stealth scripts injected successfully'); + ''' + + try: + await self.page.add_init_script(stealth_script) + logger.debug("✅ Stealth scripts injected successfully") + except Exception as e: + logger.warning(f"⚠️ Failed to inject stealth scripts: {e}") + + async def _simulate_human_behavior(self, fast_mode: bool = True): + """ + Simulate realistic human behavior patterns to avoid detection. + Includes mouse movements, typing patterns, and natural delays. + + Args: + fast_mode: If True, use minimal timing for speed optimization + """ + if not self.page: + logger.warning("Cannot simulate human behavior: page is None") + return + + logger.debug("🤖 Simulating human behavior patterns...") + + try: + if fast_mode: + # ULTRA-FAST MODE: Minimal human behavior + viewport_size = self.page.viewport_size + if viewport_size and random.random() < 0.7: # 70% chance to do movement + width, height = viewport_size['width'], viewport_size['height'] + + # Single quick mouse movement + x = random.randint(200, width - 200) + y = random.randint(200, height - 200) + await self.page.mouse.move(x, y) + + # Brief scroll (50% chance) + if random.random() < 0.5: + await self.page.mouse.wheel(0, random.randint(50, 100)) + + # Ultra-minimal delay + await asyncio.sleep(random.uniform(0.05, 0.15)) # Reduced from 0.1-0.3 + + else: + # FULL MODE: Original comprehensive behavior + viewport_size = self.page.viewport_size + if viewport_size: + width, height = viewport_size['width'], viewport_size['height'] + + # Generate 3-5 random mouse movements + movements = random.randint(3, 5) + logger.debug(f" 🖱️ Performing {movements} random mouse movements") + + for i in range(movements): + x = random.randint(100, width - 100) + y = random.randint(100, height - 100) + + # Move mouse with realistic speed (not instant) + await self.page.mouse.move(x, y) + await asyncio.sleep(random.uniform(0.1, 0.3)) + + # 2. Scroll simulation + logger.debug(" 📜 Simulating scroll behavior") + scroll_amount = random.randint(100, 300) + await self.page.mouse.wheel(0, scroll_amount) + await asyncio.sleep(random.uniform(0.2, 0.5)) + + # Scroll back up + await self.page.mouse.wheel(0, -scroll_amount) + await asyncio.sleep(random.uniform(0.2, 0.4)) + + # 3. Random page interaction delays + await asyncio.sleep(random.uniform(0.5, 1.5)) + + logger.debug("✅ Human behavior simulation completed") + + except Exception as e: + logger.warning(f"⚠️ Human behavior simulation failed: {e}") + + async def _human_type(self, selector: str, text: str, clear_first: bool = True, fast_mode: bool = True): + """ + Type text with human-like patterns and delays. + + Args: + selector: CSS selector for the input element + text: Text to type + clear_first: Whether to clear the field first + fast_mode: If True, use minimal delays for speed optimization + """ + if not self.page: + logger.warning("Cannot perform human typing: page is None") + return + + try: + if fast_mode: + # FAST MODE: Direct fill for speed + await self.page.fill(selector, text) + await asyncio.sleep(random.uniform(0.02, 0.05)) # Reduced from 0.05-0.1 + else: + # FULL MODE: Character-by-character human typing + # Focus on the element first + await self.page.focus(selector) + await asyncio.sleep(random.uniform(0.1, 0.3)) + + # Clear field if requested + if clear_first: + await self.page.keyboard.press('Control+a') + await asyncio.sleep(random.uniform(0.05, 0.15)) + await self.page.keyboard.press('Delete') + await asyncio.sleep(random.uniform(0.05, 0.15)) + + # Type each character with human-like delays + for char in text: + await self.page.keyboard.type(char) + # Human typing speed: 50-150ms between characters + delay = random.uniform(0.05, 0.15) + + # Occasional longer pauses (thinking) + if random.random() < 0.1: # 10% chance + delay += random.uniform(0.2, 0.8) + + await asyncio.sleep(delay) + + # Brief pause after typing + await asyncio.sleep(random.uniform(0.2, 0.6)) + + logger.debug(f"✅ Human-typed '{text}' into {selector}") + + except Exception as e: + logger.warning(f"⚠️ Human typing failed: {e}") + + async def _human_click(self, selector: str, wait_before: bool = True, wait_after: bool = True, fast_mode: bool = True): + """ + Perform a human-like click with realistic delays and mouse movement. + + Args: + selector: CSS selector or element to click + wait_before: Whether to wait before clicking + wait_after: Whether to wait after clicking + fast_mode: If True, use minimal delays for speed optimization + """ + if not self.page: + logger.warning("Cannot perform human click: page is None") + return + + try: + if fast_mode: + # FAST MODE: Direct click with minimal delay + if wait_before: + await asyncio.sleep(random.uniform(0.02, 0.08)) # Reduced from 0.05-0.15 + + await self.page.click(selector) + + if wait_after: + await asyncio.sleep(random.uniform(0.02, 0.08)) # Reduced from 0.05-0.15 + + else: + # FULL MODE: Realistic mouse movement and timing + # Wait before clicking (thinking time) + if wait_before: + await asyncio.sleep(random.uniform(0.3, 0.8)) + + # Get element bounds for realistic mouse movement + element = await self.page.query_selector(selector) + if element: + box = await element.bounding_box() + if box: + # Move to element with slight randomness + center_x = box['x'] + box['width'] / 2 + center_y = box['y'] + box['height'] / 2 + + # Add small random offset + offset_x = random.uniform(-10, 10) + offset_y = random.uniform(-5, 5) + + await self.page.mouse.move(center_x + offset_x, center_y + offset_y) + await asyncio.sleep(random.uniform(0.1, 0.3)) + + # Perform click + await self.page.mouse.click(center_x + offset_x, center_y + offset_y) + + logger.debug(f"✅ Human-clicked {selector}") + else: + # Fallback to regular click + await self.page.click(selector) + logger.debug(f"✅ Fallback-clicked {selector}") + else: + logger.warning(f"⚠️ Element not found for human click: {selector}") + return + + # Wait after clicking (processing time) + if wait_after: + await asyncio.sleep(random.uniform(0.2, 0.6)) + + logger.debug(f"✅ Human-clicked {selector}") + + except Exception as e: + logger.warning(f"⚠️ Human click failed: {e}") + + async def _simulate_page_exploration(self, fast_mode: bool = True): + """ + Simulate natural page exploration before performing the main task. + This helps establish a more human-like session. + + Args: + fast_mode: If True, use minimal exploration for speed optimization + """ + if not self.page: + return + + logger.debug("🕵️ Simulating page exploration...") + + try: + if fast_mode: + # ULTRA-FAST MODE: Minimal exploration + await asyncio.sleep(random.uniform(0.05, 0.1)) # Reduced from 0.1-0.3 + + # Single mouse movement (optional) + try: + elements = await self.page.query_selector_all("input, button") + if elements and random.random() < 0.5: # 50% chance to skip + element = random.choice(elements) + box = await element.bounding_box() + if box: + center_x = box['x'] + box['width'] / 2 + center_y = box['y'] + box['height'] / 2 + await self.page.mouse.move(center_x, center_y) + except: + pass + + await asyncio.sleep(random.uniform(0.02, 0.05)) # Reduced from 0.05-0.15 + + else: + # FULL MODE: Comprehensive exploration + # 1. Brief pause to "read" the page + await asyncio.sleep(random.uniform(1.0, 2.5)) + + # 2. Move mouse to various UI elements (like a human would explore) + explore_selectors = [ + "h1", "h2", ".navbar", "#header", ".logo", + "input", "button", "a", ".form-group" + ] + + explored = 0 + for selector in explore_selectors: + elements = await self.page.query_selector_all(selector) + if elements and explored < 3: # Explore max 3 elements + element = random.choice(elements) + box = await element.bounding_box() + if box: + center_x = box['x'] + box['width'] / 2 + center_y = box['y'] + box['height'] / 2 + + await self.page.mouse.move(center_x, center_y) + await asyncio.sleep(random.uniform(0.3, 0.8)) + explored += 1 + + # 3. Small scroll to simulate reading + await self.page.mouse.wheel(0, random.randint(50, 150)) + await asyncio.sleep(random.uniform(0.5, 1.2)) + + logger.debug("✅ Page exploration completed") + + except Exception as e: + logger.debug(f"⚠️ Page exploration failed: {e}") + def _parse_decision_entries_from_soup(self, soup: BeautifulSoup, search_karar_tipi: KikKararTipi) -> List[KikDecisionEntry]: entries: List[KikDecisionEntry] = [] table = soup.find("table", {"id": self.RESULTS_TABLE_ID}) - if not table: return entries + + logger.debug(f"Looking for table with ID: {self.RESULTS_TABLE_ID}") + if not table: + logger.warning(f"Table with ID '{self.RESULTS_TABLE_ID}' not found in HTML") + # Log available tables for debugging + all_tables = soup.find_all("table") + logger.debug(f"Found {len(all_tables)} tables in HTML") + for idx, tbl in enumerate(all_tables): + table_id = tbl.get('id', 'no-id') + table_class = tbl.get('class', 'no-class') + rows = tbl.find_all('tr') + logger.debug(f"Table {idx}: id='{table_id}', class='{table_class}', rows={len(rows)}") + + # If this looks like a results table, try to use it + if (table_id and ('grd' in table_id.lower() or 'kurul' in table_id.lower() or 'sonuc' in table_id.lower())) or \ + (isinstance(table_class, list) and any('grid' in cls.lower() or 'result' in cls.lower() for cls in table_class)) or \ + len(rows) > 3: # Table with multiple rows might be results + logger.info(f"Trying to parse table {idx} as potential results table: id='{table_id}'") + table = tbl + break + + if not table: + logger.error("No suitable results table found") + return entries + rows = table.find_all("tr") + logger.info(f"Found {len(rows)} rows in results table") + for row_idx, row in enumerate(rows): - if row_idx < 2: continue + # Skip first row (search bar with colspan=7) and second row (header with 6 cells) + if row_idx < 2: + logger.debug(f"Skipping header row {row_idx}") + continue + cells = row.find_all("td") - if len(cells) == 6: + logger.debug(f"Row {row_idx}: Found {len(cells)} cells") + + # Log cell contents for debugging + if cells and row_idx < 5: # Log first few data rows + for cell_idx, cell in enumerate(cells): + cell_text = cell.get_text(strip=True)[:50] # First 50 chars + logger.debug(f" Cell {cell_idx}: '{cell_text}...'") + + # Be more flexible with cell count - try 6 cells first, then adapt + if len(cells) >= 5: # At least 5 cells for minimum required data try: - preview_button_tag = cells[0].find("a", id=re.compile(r"btnOnizle$")) + # Try to find preview button in first cell or any cell with a link + preview_button_tag = None event_target = "" - if preview_button_tag and preview_button_tag.has_attr('href'): - match = re.search(r"__doPostBack\('([^']*)','([^']*)'\)", preview_button_tag['href']) - if match: event_target = match.group(1) - karar_no_span = cells[1].find("span", id=re.compile(r"lblKno$")) - karar_tarihi_span = cells[2].find("span", id=re.compile(r"lblKtar$")) - idare_span = cells[3].find("span", id=re.compile(r"lblIdare$")) - basvuru_sahibi_span = cells[4].find("span", id=re.compile(r"lblSikayetci$")) - ihale_span = cells[5].find("span", id=re.compile(r"lblIhale$")) - if not (event_target and karar_no_span and karar_tarihi_span): continue + + # Look for preview button in first few cells + for cell_idx in range(min(3, len(cells))): + cell = cells[cell_idx] + # Try multiple patterns for preview button (based on actual HTML structure) + preview_candidates = [ + cell.find("a", id="btnOnizle"), # Exact match + cell.find("a", id=re.compile(r"btnOnizle$")), + cell.find("a", id=re.compile(r"btn.*Onizle")), + cell.find("a", id=re.compile(r".*Onizle.*")), + cell.find("a", href=re.compile(r"__doPostBack")) + ] + + for candidate in preview_candidates: + if candidate and candidate.has_attr('href'): + match = re.search(r"__doPostBack\('([^']*)','([^']*)'\)", candidate['href']) + if match: + event_target = match.group(1) + preview_button_tag = candidate + logger.debug(f"Row {row_idx}: Found event_target '{event_target}' in cell {cell_idx}") + break + + if preview_button_tag: + break + + if not preview_button_tag: + logger.debug(f"Row {row_idx}: No preview button found in any cell") + # Log what links we found + for cell_idx, cell in enumerate(cells[:3]): + links_in_cell = cell.find_all("a") + logger.debug(f" Cell {cell_idx}: {len(links_in_cell)} links") + for link in links_in_cell[:2]: + logger.debug(f" Link id='{link.get('id')}', href='{link.get('href', '')[:50]}...'") + + # Try to find decision data spans with more flexible patterns + karar_no_span = None + karar_tarihi_span = None + idare_span = None + basvuru_sahibi_span = None + ihale_span = None + + # Try different span patterns for karar no (usually in cell 1) + for cell_idx in range(min(4, len(cells))): + if not karar_no_span: + cell = cells[cell_idx] + candidates = [ + cell.find("span", id="lblKno"), # Exact match based on actual HTML + cell.find("span", id=re.compile(r"lblKno$")), + cell.find("span", id=re.compile(r".*Kno.*")), + cell.find("span", id=re.compile(r".*KararNo.*")), + cell.find("span", id=re.compile(r".*No.*")) + ] + for candidate in candidates: + if candidate and candidate.get_text(strip=True): + karar_no_span = candidate + logger.debug(f"Row {row_idx}: Found karar_no in cell {cell_idx}") + break + + # Try different patterns for karar tarihi (usually in cell 2) + for cell_idx in range(min(4, len(cells))): + if not karar_tarihi_span: + cell = cells[cell_idx] + candidates = [ + cell.find("span", id="lblKtar"), # Exact match based on actual HTML + cell.find("span", id=re.compile(r"lblKtar$")), + cell.find("span", id=re.compile(r".*Ktar.*")), + cell.find("span", id=re.compile(r".*Tarih.*")), + cell.find("span", id=re.compile(r".*Date.*")) + ] + for candidate in candidates: + if candidate and candidate.get_text(strip=True): + # Check if it looks like a date + text = candidate.get_text(strip=True) + if re.match(r'\d{1,2}[./]\d{1,2}[./]\d{4}', text): + karar_tarihi_span = candidate + logger.debug(f"Row {row_idx}: Found karar_tarihi in cell {cell_idx}") + break + + # Find other spans in remaining cells (if we have 6 cells) - using exact IDs + if len(cells) >= 6: + idare_span = cells[3].find("span", id="lblIdare") or cells[3].find("span") + basvuru_sahibi_span = cells[4].find("span", id="lblSikayetci") or cells[4].find("span") + ihale_span = cells[5].find("span", id="lblIhale") or cells[5].find("span") + elif len(cells) == 5: + # Adjust for 5-cell layout + idare_span = cells[2].find("span") if cells[2] != cells[1] else None + basvuru_sahibi_span = cells[3].find("span") if len(cells) > 3 else None + ihale_span = cells[4].find("span") if len(cells) > 4 else None + + # Log what we found + logger.debug(f"Row {row_idx}: karar_no_span={karar_no_span is not None}, " + f"karar_tarihi_span={karar_tarihi_span is not None}, " + f"event_target={bool(event_target)}") + + # For KIK, we need at least karar_no and karar_tarihi, event_target is helpful but not critical + if not (karar_no_span and karar_tarihi_span): + logger.debug(f"Row {row_idx}: Missing required fields (karar_no or karar_tarihi), skipping") + # Log what spans we found in cells + for i, cell in enumerate(cells): + spans = cell.find_all("span") + if spans: + span_info = [] + for s in spans: + span_id = s.get('id', 'no-id') + span_text = s.get_text(strip=True)[:20] + span_info.append(f"{span_id}:'{span_text}...'") + logger.debug(f" Cell {i} spans: {span_info}") + continue + + # If we don't have event_target, we can still create an entry but mark it specially + if not event_target: + logger.warning(f"Row {row_idx}: No event_target found, document retrieval won't work") + event_target = f"missing_target_row_{row_idx}" # Placeholder # Karar tipini arama parametresinden alıyoruz, çünkü HTML'de direkt olarak bulunmuyor. - entry = KikDecisionEntry( - preview_event_target=event_target, - kararNo=karar_no_span.get_text(strip=True), - karar_tipi=search_karar_tipi, # Arama yapılan karar tipini ekle - kararTarihi=karar_tarihi_span.get_text(strip=True), - idare=idare_span.get_text(strip=True) if idare_span else None, - basvuruSahibi=basvuru_sahibi_span.get_text(strip=True) if basvuru_sahibi_span else None, - ihaleKonusu=ihale_span.get_text(strip=True) if ihale_span else None, - ) - entries.append(entry) + try: + entry = KikDecisionEntry( + preview_event_target=event_target, + kararNo=karar_no_span.get_text(strip=True), + karar_tipi=search_karar_tipi, # Arama yapılan karar tipini ekle + kararTarihi=karar_tarihi_span.get_text(strip=True), + idare=idare_span.get_text(strip=True) if idare_span else None, + basvuruSahibi=basvuru_sahibi_span.get_text(strip=True) if basvuru_sahibi_span else None, + ihaleKonusu=ihale_span.get_text(strip=True) if ihale_span else None, + ) + entries.append(entry) + logger.info(f"Row {row_idx}: Successfully parsed decision: {entry.karar_no_str}") + except Exception as e: + logger.error(f"Row {row_idx}: Error creating KikDecisionEntry: {e}") + continue + except Exception as e: - logger.error(f"Error parsing a KIK decision entry row: {e}", exc_info=True) + logger.error(f"Error parsing row {row_idx}: {e}", exc_info=True) + else: + logger.debug(f"Row {row_idx}: Expected at least 5 cells but found {len(cells)}, skipping") + + logger.info(f"Parsed {len(entries)} decision entries from {len(rows)} rows") return entries def _parse_total_records_from_soup(self, soup: BeautifulSoup) -> int: @@ -163,6 +835,10 @@ class KikApiClient: try: if page.url != search_url: await page.goto(search_url, wait_until="networkidle", timeout=self.request_timeout) + + # Simulate natural page exploration after navigation (FAST MODE) + await self._simulate_page_exploration(fast_mode=True) + search_button_selector = f"a[id='{self.FIELD_LOCATORS['search_button_id']}']" await page.wait_for_selector(search_button_selector, state="visible", timeout=self.request_timeout) @@ -170,12 +846,18 @@ class KikApiClient: radio_locator_selector = f"{self.FIELD_LOCATORS['karar_tipi_radio_group']}[value='{current_karar_tipi_value}']" if not await page.locator(radio_locator_selector).is_checked(): js_target_radio = f"ctl00$ContentPlaceHolder1${current_karar_tipi_value}" + logger.info(f"Selecting radio button: {js_target_radio}") async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout): await page.evaluate(f"javascript:__doPostBack('{js_target_radio}','')") - await page.wait_for_timeout(1000) + # Ultra-fast wait for page to stabilize after radio button change + await page.wait_for_timeout(300) # Reduced from 1000ms + logger.info("Radio button selection completed") - async def fill_if_value(selector_key: str, value: Optional[str]): - if value is not None: await page.fill(self.FIELD_LOCATORS[selector_key], value) + # Helper function for human-like form filling (FAST MODE) + async def human_fill_if_value(selector_key: str, value: Optional[str]): + if value is not None: + selector = self.FIELD_LOCATORS[selector_key] + await self._human_type(selector, value, fast_mode=True) # Karar No'yu KİK sitesine göndermeden önce '_' -> '/' dönüşümü yap karar_no_for_kik_form = None @@ -183,41 +865,105 @@ class KikApiClient: karar_no_for_kik_form = search_params.karar_no.replace('_', '/') logger.info(f"Using karar_no '{karar_no_for_kik_form}' (transformed from '{search_params.karar_no}') for KIK form.") - await fill_if_value('karar_metni', search_params.karar_metni) - await fill_if_value('karar_no', karar_no_for_kik_form) # Dönüştürülmüş halini kullan - # ... (diğer fill_if_value çağrıları aynı) ... - await fill_if_value('karar_tarihi_baslangic', search_params.karar_tarihi_baslangic) - await fill_if_value('karar_tarihi_bitis', search_params.karar_tarihi_bitis) - await fill_if_value('resmi_gazete_sayisi', search_params.resmi_gazete_sayisi) - await fill_if_value('resmi_gazete_tarihi', search_params.resmi_gazete_tarihi) - await fill_if_value('basvuru_konusu_ihale', search_params.basvuru_konusu_ihale) - await fill_if_value('basvuru_sahibi', search_params.basvuru_sahibi) - await fill_if_value('ihaleyi_yapan_idare', search_params.ihaleyi_yapan_idare) + # Fill form fields with FAST human-like behavior + logger.info("Filling form fields with fast mode...") + + # Start with FAST mouse behavior simulation + await self._simulate_human_behavior(fast_mode=True) + + await human_fill_if_value('karar_metni', search_params.karar_metni) + await human_fill_if_value('karar_no', karar_no_for_kik_form) # Dönüştürülmüş halini kullan + await human_fill_if_value('karar_tarihi_baslangic', search_params.karar_tarihi_baslangic) + await human_fill_if_value('karar_tarihi_bitis', search_params.karar_tarihi_bitis) + await human_fill_if_value('resmi_gazete_sayisi', search_params.resmi_gazete_sayisi) + await human_fill_if_value('resmi_gazete_tarihi', search_params.resmi_gazete_tarihi) + await human_fill_if_value('basvuru_konusu_ihale', search_params.basvuru_konusu_ihale) + await human_fill_if_value('basvuru_sahibi', search_params.basvuru_sahibi) + await human_fill_if_value('ihaleyi_yapan_idare', search_params.ihaleyi_yapan_idare) if search_params.yil: await page.select_option(self.FIELD_LOCATORS['yil'], value=search_params.yil) + await page.wait_for_timeout(50) # Reduced from 100ms + + logger.info("Form filling completed, preparing for search...") + + # Additional FAST human behavior before search + await self._simulate_human_behavior(fast_mode=True) action_is_search_button_click = (search_params.page == 1) event_target_for_submit: str - if action_is_search_button_click: - event_target_for_submit = self.FIELD_LOCATORS['search_button_id'] - else: # Pagination - page_link_ctl_number = search_params.page + 2 - event_target_for_submit = f"ctl00$ContentPlaceHolder1$grdKurulKararSorguSonuc$ctl14$ctl{page_link_ctl_number:02d}" try: - async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout): - # Use __doPostBack for both search and pagination for consistency - await page.evaluate(f"javascript:__doPostBack('{event_target_for_submit}','')") + if action_is_search_button_click: + event_target_for_submit = self.FIELD_LOCATORS['search_button_id'] + # Use human-like clicking for search button + search_button_selector = f"a[id='{event_target_for_submit}']" + logger.info(f"Performing human-like search button click...") + + try: + # FAST Human-like click on search button + await self._human_click(search_button_selector, wait_before=True, wait_after=False, fast_mode=True) + + # Wait for navigation + await page.wait_for_load_state("networkidle", timeout=self.request_timeout) + logger.info("Search navigation completed successfully") + except Exception as e: + logger.warning(f"Human click failed, falling back to JavaScript: {e}") + # Fallback to original method + async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout): + await page.evaluate(f"javascript:__doPostBack('{event_target_for_submit}','')") + logger.info("Search navigation completed via fallback") + else: + # Pagination - use original method for consistency + page_link_ctl_number = search_params.page + 2 + event_target_for_submit = f"ctl00$ContentPlaceHolder1$grdKurulKararSorguSonuc$ctl14$ctl{page_link_ctl_number:02d}" + logger.info(f"Executing pagination with event target: {event_target_for_submit}") + + async with page.expect_navigation(wait_until="networkidle", timeout=self.request_timeout): + await page.evaluate(f"javascript:__doPostBack('{event_target_for_submit}','')") + logger.info("Pagination navigation completed successfully") except PlaywrightTimeoutError: - await page.wait_for_timeout(2000) + logger.warning("Search navigation timed out, but continuing...") + await page.wait_for_timeout(5000) # Longer wait if navigation fails + + # Ultra-fast wait time for results to load + logger.info("Waiting for search results to load...") + await page.wait_for_timeout(500) # Reduced from 1000ms results_table_dom_selector = f"table#{self.RESULTS_TABLE_ID}" try: - await page.wait_for_selector(results_table_dom_selector, timeout=30000, state="attached") - await page.wait_for_timeout(2000) + # First wait for any tables to appear (more general check) + logger.info("Waiting for any tables to appear...") + await page.wait_for_function(""" + () => document.querySelectorAll('table').length > 0 + """, timeout=4000) # Reduced from 8000ms + logger.info("At least one table appeared") + + # Then wait for our specific table + await page.wait_for_selector(results_table_dom_selector, timeout=4000, state="attached") # Reduced from 8000ms + logger.debug("Results table attached to DOM") + + # Wait for table to have some content (more than just headers) + await page.wait_for_function(f""" + () => {{ + const table = document.querySelector('{results_table_dom_selector}'); + return table && table.querySelectorAll('tr').length > 2; + }} + """, timeout=4000) # Reduced from 20000ms + logger.debug("Results table populated with data") + + # Ultra-fast additional wait for any remaining JavaScript + await page.wait_for_timeout(500) # Reduced from 3000ms + except PlaywrightTimeoutError: logger.warning(f"Timeout waiting for results table '{results_table_dom_selector}'.") + # Try one more wait for content placeholder + try: + await page.wait_for_selector("#ctl00_ContentPlaceHolder1", timeout=10000) + logger.info("ContentPlaceHolder1 found, checking for tables...") + await page.wait_for_timeout(5000) + except PlaywrightTimeoutError: + logger.warning("ContentPlaceHolder1 also not found - content may not have loaded") html_content = await page.content() soup = BeautifulSoup(html_content, "html.parser") diff --git a/mcp_server_main.py b/mcp_server_main.py index 09caea0..862a761 100644 --- a/mcp_server_main.py +++ b/mcp_server_main.py @@ -552,7 +552,7 @@ async def search_emsal_detailed_decisions( end_date: Optional[str] = Field(None, description="End date for decision (DD.MM.YYYY)."), sort_criteria: str = Field("1", description="Sorting criteria (e.g., 1: Esas No)."), sort_direction: str = Field("desc", description="Sorting direction ('asc' or 'desc')."), - page_number: int = Field(1, ge=1, description="Page number."), + page_number: int = Field(1, ge=1, description="Page number (accepts int)."), page_size: int = Field(10, ge=1, le=100, description="Results per page.") ) -> CompactEmsalSearchResult: """ @@ -984,7 +984,7 @@ async def search_anayasa_norm_denetimi_decisions( ) async def get_anayasa_norm_denetimi_document_markdown( document_url: str = Field(..., description="The URL path (e.g., /ND/YYYY/NN) or full https URL of the AYM Norm Denetimi decision from normkararlarbilgibankasi.anayasa.gov.tr."), - page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters).") + page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).") ) -> AnayasaDocumentMarkdown: """ Retrieves the full text of a Constitutional Court norm control decision in paginated Markdown format. @@ -1104,7 +1104,7 @@ async def search_anayasa_bireysel_basvuru_report( ) async def get_anayasa_bireysel_basvuru_document_markdown( document_url_path: str = Field(..., description="The URL path (e.g., /BB/YYYY/NNNN) of the AYM Bireysel Başvuru decision from kararlarbilgibankasi.anayasa.gov.tr."), - page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters). Accepts int.") + page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).") ) -> AnayasaBireyselBasvuruDocumentMarkdown: """ Retrieves the full text of a Constitutional Court individual application decision in paginated Markdown format. @@ -1263,7 +1263,7 @@ async def search_kik_decisions( ) async def get_kik_document_markdown( karar_id: str = Field(..., description="The Base64 encoded KIK decision identifier."), - page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed). Default is 1.") + page_number: Optional[int] = Field(1, ge=1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1.") ) -> KikDocumentMarkdown: """ Retrieves the full text of a KIK (Public Procurement Authority) decision in paginated Markdown format. @@ -1438,7 +1438,7 @@ async def search_rekabet_kurumu_decisions( ) async def get_rekabet_kurumu_document( karar_id: str = Field(..., description="GUID (kararId) of the Rekabet Kurumu decision. This ID is obtained from search results."), - page_number: Optional[int] = Field(1, ge=1, description="Requested page number for the Markdown content converted from PDF (1-indexed). Default is 1.") + page_number: Optional[int] = Field(1, ge=1, description="Requested page number for the Markdown content converted from PDF (1-indexed, accepts int). Default is 1.") ) -> RekabetDocument: """ Retrieves the full text of a Turkish Competition Authority decision in paginated Markdown format. @@ -2594,7 +2594,7 @@ async def get_kvkk_document_markdown( Note: The URL should be a complete KVKK decision page URL, not just a decision ID. """), - page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed). Default is 1 (first 5,000 characters). Accepts int.") + page_number: Union[int, str] = Field(1, description="Page number for paginated Markdown content (1-indexed, accepts int). Default is 1 (first 5,000 characters).") ) -> KvkkDocumentMarkdown: """ Retrieves the full text of a KVKK decision document in paginated Markdown format. @@ -3056,9 +3056,43 @@ async def fetch( logger.exception(f"Error fetching ChatGPT Deep Research document {id}") raise +def ensure_playwright_browsers(): + """Ensure Playwright browsers are installed for KIK tool functionality.""" + try: + import subprocess + import os + + # Check if chromium is already installed + chromium_path = os.path.expanduser("~/Library/Caches/ms-playwright/chromium-1179") + if os.path.exists(chromium_path): + logger.info("Playwright Chromium browser already installed.") + return + + logger.info("Installing Playwright Chromium browser for KIK tool...") + result = subprocess.run( + ["python", "-m", "playwright", "install", "chromium"], + capture_output=True, + text=True, + timeout=300 # 5 minutes timeout + ) + + if result.returncode == 0: + logger.info("Playwright Chromium browser installed successfully.") + else: + logger.warning(f"Failed to install Playwright browser: {result.stderr}") + logger.warning("KIK tool may not work properly without Playwright browsers.") + + except Exception as e: + logger.warning(f"Could not auto-install Playwright browsers: {e}") + logger.warning("KIK tool may not work properly. Manual installation: 'playwright install chromium'") + def main(): logger.info(f"Starting {app.name} server via main() function...") logger.info(f"Logs will be written to: {LOG_FILE_PATH}") + + # Ensure Playwright browsers are installed + ensure_playwright_browsers() + try: app.run() except KeyboardInterrupt: