feat: add sitemap.xml, robots.txt, full SEO metadata

2026-07-25 00:10:11 +03:00
parent 9511682906
commit d88cf955c7
3 changed files with 145 additions and 2 deletions
@@ -0,0 +1,32 @@
+import type { MetadataRoute } from 'next';
+
+const BASE_URL = 'https://ayrisai.xyz';
+const PROTECTED = ['/admin/', '/api/'];
+
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: [
+      // Genel arama botları — tüm public içerik açık
+      {
+        userAgent: '*',
+        allow: '/',
+        disallow: PROTECTED,
+      },
+      // AI arama / citation botları — izin ver
+      { userAgent: 'OAI-SearchBot', allow: '/', disallow: PROTECTED },
+      { userAgent: 'ChatGPT-User', allow: '/', disallow: PROTECTED },
+      { userAgent: 'PerplexityBot', allow: '/', disallow: PROTECTED },
+      { userAgent: 'Perplexity-User', allow: '/', disallow: PROTECTED },
+      { userAgent: 'Claude-SearchBot', allow: '/', disallow: PROTECTED },
+      { userAgent: 'Claude-User', allow: '/', disallow: PROTECTED },
+      // Training crawler'ları engelle
+      { userAgent: 'GPTBot', disallow: '/' },
+      { userAgent: 'ClaudeBot', disallow: '/' },
+      { userAgent: 'Google-Extended', disallow: '/' },
+      { userAgent: 'CCBot', disallow: '/' },
+      { userAgent: 'Bytespider', disallow: '/' },
+      { userAgent: 'Meta-ExternalAgent', disallow: '/' },
+    ],
+    sitemap: `${BASE_URL}/sitemap.xml`,
+  };
+}