first commit

.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+.env

.gitignore

@@ -0,0 +1,9 @@
+node_modules
+dist
+.env
+.env.*
+.DS_Store
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+package-lock.json

Dockerfile

@@ -0,0 +1,13 @@
+FROM mcr.microsoft.com/playwright:v1.44.0-jammy
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm install
+RUN npx playwright install chromium
+
+COPY . .
+RUN npm run build
+
+EXPOSE 3001
+CMD ["node", "dist/index.js"]

README.md

@@ -0,0 +1,24 @@
+# Link Scraper
+
+A lightweight API built with Express and Playwright to scrape and extract links from web pages.
+
+## Installation
+
+```bash
+npm install
+```
+
+## Running the Application
+
+### Development Mode
+
+```bash
+npm run dev
+```
+
+### Production Build
+
+```bash
+npm run build
+npm start
+```

package.json

@@ -0,0 +1,21 @@
+{
+  "name": "link-scraper",
+  "version": "1.0.0",
+  "scripts": {
+    "dev": "ts-node-dev --respawn src/index.ts",
+    "build": "tsc",
+    "start": "node dist/index.js"
+  },
+  "dependencies": {
+    "express": "^4.18.2",
+    "playwright": "^1.44.0",
+    "cors": "^2.8.5"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/cors": "^2.8.17",
+    "@types/node": "^20.12.7",
+    "typescript": "^5.4.5",
+    "ts-node-dev": "^2.0.0"
+  }
+}

src/index.ts

@@ -0,0 +1,14 @@
+import express from "express";
+import cors from "cors";
+import { scrapeRoute } from "./routes/scrape";
+
+const app = express();
+const PORT = process.env.PORT || 3001;
+
+app.use(cors());
+app.use(express.json());
+
+app.get("/health", (_req, res) => res.json({ status: "ok" }));
+app.use("/scrape", scrapeRoute);
+
+app.listen(PORT, () => console.log(`Link scraper running on :${PORT}`));

src/routes/scrape.ts

@@ -0,0 +1,46 @@
+import { Router, Request, Response } from "express";
+import { extractNavLinks } from "../services/scraper";
+
+export const scrapeRoute = Router();
+
+/**
+ * POST /scrape
+ * Body: { "url": "https://firma.com" }
+ *
+ * Response:
+ * {
+ *   "source": "https://firma.com",
+ *   "count": 6,
+ *   "links": [
+ *     { "label": "Hakkımızda", "url": "https://firma.com/hakkimizda" },
+ *     ...
+ *   ]
+ * }
+ */
+scrapeRoute.post("/", async (req: Request, res: Response) => {
+  const { url } = req.body as { url?: string };
+
+  if (!url) {
+    return res.status(400).json({ error: "'url' alanı zorunlu" });
+  }
+
+  // Basit URL validasyonu
+  try { new URL(url); } catch {
+    return res.status(400).json({ error: "Geçersiz URL formatı" });
+  }
+
+  try {
+    const links = await extractNavLinks(url);
+
+    return res.json({
+      source: url,
+      count: links.length,
+      links,
+    });
+  } catch (err) {
+    return res.status(500).json({
+      error: "Link çekme başarısız",
+      detail: err instanceof Error ? err.message : String(err),
+    });
+  }
+});

src/services/scraper.ts

@@ -0,0 +1,100 @@
+import { chromium, Browser } from "playwright";
+
+let browser: Browser | null = null;
+
+async function getBrowser(): Promise<Browser> {
+  if (!browser || !browser.isConnected()) {
+    browser = await chromium.launch({
+      headless: true,
+      args: ["--no-sandbox", "--disable-setuid-sandbox"],
+    });
+  }
+  return browser;
+}
+
+// Navbar'da atlanacak anahtar kelimeler
+const SKIP_KEYWORDS = [
+  "kariyer", "/ik", "kvkk", "gizlilik", "cerez",
+  "blog", "haber", "saglikbulteni", "saglik-rehberi",
+  "rezervasyon", "randevu", "e-sonuc", "checkup",
+  "wp-admin", "wp-content", "feed", "sitemap",
+];
+
+const SKIP_SCHEMES = ["mailto:", "tel:", "javascript:", "#"];
+
+export interface NavLink {
+  label: string;
+  url: string;
+}
+
+export async function extractNavLinks(targetUrl: string): Promise<NavLink[]> {
+  const base = new URL(targetUrl);
+  const b = await getBrowser();
+  const page = await b.newPage();
+
+  try {
+    await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 30000 });
+    await page.waitForTimeout(1200); // dropdown'ların render edilmesi için
+
+    // Navbar içindeki tüm <a> etiketlerini çek
+    const rawLinks = await page.evaluate(() => {
+      const selectors = [
+        "nav a", "header a",
+        "[class*='menu'] a", "[class*='navbar'] a",
+        "[class*='nav-'] a", "[id*='menu'] a", "[id*='nav'] a",
+      ];
+      const seen = new Set<string>();
+      const links: { href: string; label: string }[] = [];
+
+      for (const sel of selectors) {
+        document.querySelectorAll<HTMLAnchorElement>(sel).forEach((el) => {
+          const href = el.href || el.getAttribute("href") || "";
+          const label = el.textContent?.trim().replace(/\s+/g, " ") || "";
+          if (href && !seen.has(href)) {
+            seen.add(href);
+            links.push({ href, label });
+          }
+        });
+      }
+      return links;
+    });
+
+    // Filtrele ve temizle
+    const seen = new Set<string>();
+    const result: NavLink[] = [];
+
+    for (const { href, label } of rawLinks) {
+      if (!href) continue;
+      if (SKIP_SCHEMES.some((s) => href.startsWith(s))) continue;
+
+      let parsed: URL;
+      try { parsed = new URL(href, targetUrl); } catch { continue; }
+
+      // Sadece aynı domain
+      if (parsed.hostname !== base.hostname) continue;
+
+      const path = parsed.pathname.toLowerCase();
+
+      // Atlanacak sayfalar
+      if (SKIP_KEYWORDS.some((kw) => path.includes(kw))) continue;
+
+      // Fragment ve trailing slash temizle
+      parsed.hash = "";
+      const normalized = parsed.toString().replace(/\/$/, "");
+
+      if (seen.has(normalized)) continue;
+      seen.add(normalized);
+
+      result.push({ label: label || pathToLabel(parsed.pathname), url: normalized });
+    }
+
+    return result;
+  } finally {
+    await page.close();
+  }
+}
+
+function pathToLabel(pathname: string): string {
+  const last = pathname.split("/").filter(Boolean).pop() || "anasayfa";
+  return last.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+}

tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true
+  }
+}