24 lines
467 B
Plaintext
24 lines
467 B
Plaintext
# Hastebin robots.txt
|
|
# Updated: 2026-01-23
|
|
|
|
User-agent: *
|
|
|
|
# Allow static pages
|
|
Allow: /$
|
|
Allow: /about
|
|
|
|
# Allow MCP discovery for AI assistants and tools
|
|
Allow: /.well-known/
|
|
Allow: /.well-known/mcp.json
|
|
|
|
# Disallow individual pastes (ephemeral content)
|
|
Disallow: /raw/
|
|
Disallow: /documents/
|
|
|
|
# Disallow paste URLs (random keys)
|
|
# Pastes are identified by 10-character alphanumeric keys
|
|
Disallow: /*.*
|
|
|
|
# Sitemap location
|
|
Sitemap: https://haste.nixc.us/sitemap.xml
|