33 lines
1.2 KiB
TypeScript
33 lines
1.2 KiB
TypeScript
import type { MetadataRoute } from 'next';
|
||
|
||
const BASE_URL = 'https://ayrisai.xyz';
|
||
const PROTECTED = ['/admin/', '/api/'];
|
||
|
||
export default function robots(): MetadataRoute.Robots {
|
||
return {
|
||
rules: [
|
||
// Genel arama botları — tüm public içerik açık
|
||
{
|
||
userAgent: '*',
|
||
allow: '/',
|
||
disallow: PROTECTED,
|
||
},
|
||
// AI arama / citation botları — izin ver
|
||
{ userAgent: 'OAI-SearchBot', allow: '/', disallow: PROTECTED },
|
||
{ userAgent: 'ChatGPT-User', allow: '/', disallow: PROTECTED },
|
||
{ userAgent: 'PerplexityBot', allow: '/', disallow: PROTECTED },
|
||
{ userAgent: 'Perplexity-User', allow: '/', disallow: PROTECTED },
|
||
{ userAgent: 'Claude-SearchBot', allow: '/', disallow: PROTECTED },
|
||
{ userAgent: 'Claude-User', allow: '/', disallow: PROTECTED },
|
||
// Training crawler'ları engelle
|
||
{ userAgent: 'GPTBot', disallow: '/' },
|
||
{ userAgent: 'ClaudeBot', disallow: '/' },
|
||
{ userAgent: 'Google-Extended', disallow: '/' },
|
||
{ userAgent: 'CCBot', disallow: '/' },
|
||
{ userAgent: 'Bytespider', disallow: '/' },
|
||
{ userAgent: 'Meta-ExternalAgent', disallow: '/' },
|
||
],
|
||
sitemap: `${BASE_URL}/sitemap.xml`,
|
||
};
|
||
}
|