From 0773e7f6f3c7bd3f775e2aafc72c28f5a9b5be4a Mon Sep 17 00:00:00 2001 From: mstfyldz Date: Fri, 29 May 2026 23:23:50 +0300 Subject: [PATCH] first commit --- .dockerignore | 3 ++ .gitignore | 9 ++++ Dockerfile | 13 ++++++ README.md | 24 ++++++++++ package.json | 21 +++++++++ src/index.ts | 14 ++++++ src/routes/scrape.ts | 46 ++++++++++++++++++ src/services/scraper.ts | 100 ++++++++++++++++++++++++++++++++++++++++ tsconfig.json | 11 +++++ 9 files changed, 241 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 package.json create mode 100644 src/index.ts create mode 100644 src/routes/scrape.ts create mode 100644 src/services/scraper.ts create mode 100644 tsconfig.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9c97bbd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +node_modules +dist +.env diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..264fde9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules +dist +.env +.env.* +.DS_Store +npm-debug.log* +yarn-debug.log* +yarn-error.log* +package-lock.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6545eef --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM mcr.microsoft.com/playwright:v1.44.0-jammy + +WORKDIR /app + +COPY package*.json ./ +RUN npm install +RUN npx playwright install chromium + +COPY . . +RUN npm run build + +EXPOSE 3001 +CMD ["node", "dist/index.js"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..1f73b28 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# Link Scraper + +A lightweight API built with Express and Playwright to scrape and extract links from web pages. + +## Installation + +```bash +npm install +``` + +## Running the Application + +### Development Mode + +```bash +npm run dev +``` + +### Production Build + +```bash +npm run build +npm start +``` diff --git a/package.json b/package.json new file mode 100644 index 0000000..12f29d0 --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "name": "link-scraper", + "version": "1.0.0", + "scripts": { + "dev": "ts-node-dev --respawn src/index.ts", + "build": "tsc", + "start": "node dist/index.js" + }, + "dependencies": { + "express": "^4.18.2", + "playwright": "^1.44.0", + "cors": "^2.8.5" + }, + "devDependencies": { + "@types/express": "^4.17.21", + "@types/cors": "^2.8.17", + "@types/node": "^20.12.7", + "typescript": "^5.4.5", + "ts-node-dev": "^2.0.0" + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..a9587ad --- /dev/null +++ b/src/index.ts @@ -0,0 +1,14 @@ +import express from "express"; +import cors from "cors"; +import { scrapeRoute } from "./routes/scrape"; + +const app = express(); +const PORT = process.env.PORT || 3001; + +app.use(cors()); +app.use(express.json()); + +app.get("/health", (_req, res) => res.json({ status: "ok" })); +app.use("/scrape", scrapeRoute); + +app.listen(PORT, () => console.log(`Link scraper running on :${PORT}`)); diff --git a/src/routes/scrape.ts b/src/routes/scrape.ts new file mode 100644 index 0000000..366c903 --- /dev/null +++ b/src/routes/scrape.ts @@ -0,0 +1,46 @@ +import { Router, Request, Response } from "express"; +import { extractNavLinks } from "../services/scraper"; + +export const scrapeRoute = Router(); + +/** + * POST /scrape + * Body: { "url": "https://firma.com" } + * + * Response: + * { + * "source": "https://firma.com", + * "count": 6, + * "links": [ + * { "label": "Hakkımızda", "url": "https://firma.com/hakkimizda" }, + * ... + * ] + * } + */ +scrapeRoute.post("/", async (req: Request, res: Response) => { + const { url } = req.body as { url?: string }; + + if (!url) { + return res.status(400).json({ error: "'url' alanı zorunlu" }); + } + + // Basit URL validasyonu + try { new URL(url); } catch { + return res.status(400).json({ error: "Geçersiz URL formatı" }); + } + + try { + const links = await extractNavLinks(url); + + return res.json({ + source: url, + count: links.length, + links, + }); + } catch (err) { + return res.status(500).json({ + error: "Link çekme başarısız", + detail: err instanceof Error ? err.message : String(err), + }); + } +}); diff --git a/src/services/scraper.ts b/src/services/scraper.ts new file mode 100644 index 0000000..a9c722b --- /dev/null +++ b/src/services/scraper.ts @@ -0,0 +1,100 @@ +import { chromium, Browser } from "playwright"; + +let browser: Browser | null = null; + +async function getBrowser(): Promise { + if (!browser || !browser.isConnected()) { + browser = await chromium.launch({ + headless: true, + args: ["--no-sandbox", "--disable-setuid-sandbox"], + }); + } + return browser; +} + +// Navbar'da atlanacak anahtar kelimeler +const SKIP_KEYWORDS = [ + "kariyer", "/ik", "kvkk", "gizlilik", "cerez", + "blog", "haber", "saglikbulteni", "saglik-rehberi", + "rezervasyon", "randevu", "e-sonuc", "checkup", + "wp-admin", "wp-content", "feed", "sitemap", +]; + +const SKIP_SCHEMES = ["mailto:", "tel:", "javascript:", "#"]; + +export interface NavLink { + label: string; + url: string; +} + +export async function extractNavLinks(targetUrl: string): Promise { + const base = new URL(targetUrl); + const b = await getBrowser(); + const page = await b.newPage(); + + try { + await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 30000 }); + await page.waitForTimeout(1200); // dropdown'ların render edilmesi için + + // Navbar içindeki tüm etiketlerini çek + const rawLinks = await page.evaluate(() => { + const selectors = [ + "nav a", "header a", + "[class*='menu'] a", "[class*='navbar'] a", + "[class*='nav-'] a", "[id*='menu'] a", "[id*='nav'] a", + ]; + const seen = new Set(); + const links: { href: string; label: string }[] = []; + + for (const sel of selectors) { + document.querySelectorAll(sel).forEach((el) => { + const href = el.href || el.getAttribute("href") || ""; + const label = el.textContent?.trim().replace(/\s+/g, " ") || ""; + if (href && !seen.has(href)) { + seen.add(href); + links.push({ href, label }); + } + }); + } + return links; + }); + + // Filtrele ve temizle + const seen = new Set(); + const result: NavLink[] = []; + + for (const { href, label } of rawLinks) { + if (!href) continue; + if (SKIP_SCHEMES.some((s) => href.startsWith(s))) continue; + + let parsed: URL; + try { parsed = new URL(href, targetUrl); } catch { continue; } + + // Sadece aynı domain + if (parsed.hostname !== base.hostname) continue; + + const path = parsed.pathname.toLowerCase(); + + // Atlanacak sayfalar + if (SKIP_KEYWORDS.some((kw) => path.includes(kw))) continue; + + // Fragment ve trailing slash temizle + parsed.hash = ""; + const normalized = parsed.toString().replace(/\/$/, ""); + + if (seen.has(normalized)) continue; + seen.add(normalized); + + result.push({ label: label || pathToLabel(parsed.pathname), url: normalized }); + } + + return result; + } finally { + await page.close(); + } +} + +function pathToLabel(pathname: string): string { + const last = pathname.split("/").filter(Boolean).pop() || "anasayfa"; + return last.replace(/[-_]/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..b1755a3 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + } +}