first commit

src/services/scraper.ts

@@ -0,0 +1,100 @@
+import { chromium, Browser } from "playwright";
+
+let browser: Browser | null = null;
+
+async function getBrowser(): Promise<Browser> {
+  if (!browser || !browser.isConnected()) {
+    browser = await chromium.launch({
+      headless: true,
+      args: ["--no-sandbox", "--disable-setuid-sandbox"],
+    });
+  }
+  return browser;
+}
+
+// Navbar'da atlanacak anahtar kelimeler
+const SKIP_KEYWORDS = [
+  "kariyer", "/ik", "kvkk", "gizlilik", "cerez",
+  "blog", "haber", "saglikbulteni", "saglik-rehberi",
+  "rezervasyon", "randevu", "e-sonuc", "checkup",
+  "wp-admin", "wp-content", "feed", "sitemap",
+];
+
+const SKIP_SCHEMES = ["mailto:", "tel:", "javascript:", "#"];
+
+export interface NavLink {
+  label: string;
+  url: string;
+}
+
+export async function extractNavLinks(targetUrl: string): Promise<NavLink[]> {
+  const base = new URL(targetUrl);
+  const b = await getBrowser();
+  const page = await b.newPage();
+
+  try {
+    await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
+    await page.waitForTimeout(1200); // dropdown'ların render edilmesi için
+
+    // Navbar içindeki tüm <a> etiketlerini çek
+    const rawLinks = await page.evaluate(() => {
+      const selectors = [
+        "nav a", "header a",
+        "[class*='menu'] a", "[class*='navbar'] a",
+        "[class*='nav-'] a", "[id*='menu'] a", "[id*='nav'] a",
+      ];
+      const seen = new Set<string>();
+      const links: { href: string; label: string }[] = [];
+
+      for (const sel of selectors) {
+        document.querySelectorAll<HTMLAnchorElement>(sel).forEach((el) => {
+          const href = el.href || el.getAttribute("href") || "";
+          const label = el.textContent?.trim().replace(/\s+/g, " ") || "";
+          if (href && !seen.has(href)) {
+            seen.add(href);
+            links.push({ href, label });
+          }
+        });
+      }
+      return links;
+    });
+
+    // Filtrele ve temizle
+    const seen = new Set<string>();
+    const result: NavLink[] = [];
+
+    for (const { href, label } of rawLinks) {
+      if (!href) continue;
+      if (SKIP_SCHEMES.some((s) => href.startsWith(s))) continue;
+
+      let parsed: URL;
+      try { parsed = new URL(href, targetUrl); } catch { continue; }
+
+      // Sadece aynı domain
+      if (parsed.hostname !== base.hostname) continue;
+
+      const path = parsed.pathname.toLowerCase();
+
+      // Atlanacak sayfalar
+      if (SKIP_KEYWORDS.some((kw) => path.includes(kw))) continue;
+
+      // Fragment ve trailing slash temizle
+      parsed.hash = "";
+      const normalized = parsed.toString().replace(/\/$/, "");
+
+      if (seen.has(normalized)) continue;
+      seen.add(normalized);
+
+      result.push({ label: label || pathToLabel(parsed.pathname), url: normalized });
+    }
+
+    return result;
+  } finally {
+    await page.close();
+  }
+}
+
+function pathToLabel(pathname: string): string {
+  const last = pathname.split("/").filter(Boolean).pop() || "anasayfa";
+  return last.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+}