# Dickinson State University robots.txt
# Recommended search and AI visibility configuration
# Prepared: May 6, 2026
#
# Publish location:
# https://dickinsonstate.edu/robots.txt
#
# Purpose:
# Keep public recruitment, admissions, academic, catalog gateway, tuition,
# scholarship, housing, visit, news, and general public information crawlable
# by major search and AI search systems, while excluding Modern Campus/internal
# testing folders and utility paths that should not be crawled.
#
# Important:
# robots.txt is crawl guidance, not access control. Sensitive or private content
# should be protected through authentication, permissions, or server rules.

# ---------------------------------------------------------------------------
# Default rules for all crawlers
# ---------------------------------------------------------------------------
User-agent: *
Disallow: /_resources/
Disallow: /_showcase/
Disallow: /_qa/
Disallow: /_uat/
Disallow: /_training/
Disallow: /404.html
Allow: /

# ---------------------------------------------------------------------------
# Major search engines and AI search/answer crawlers
# These named crawler groups repeat the same DSU exclusions as the default group.
# This helps prevent crawler-specific groups from bypassing the default exclusions.
# ---------------------------------------------------------------------------
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
Disallow: /_resources/
Disallow: /_showcase/
Disallow: /_qa/
Disallow: /_uat/
Disallow: /_training/
Disallow: /404.html
Allow: /

# ---------------------------------------------------------------------------
# Optional OpenAI model training preference
# Keep this Disallow if DSU wants ChatGPT search visibility but does not want
# GPTBot access for OpenAI generative AI foundation model training.
# Change to Allow: / only if DSU intentionally wants to permit GPTBot.
# ---------------------------------------------------------------------------
User-agent: GPTBot
Disallow: /

# ---------------------------------------------------------------------------
# Sitemaps
# Google allows sitemap URLs in robots.txt to point to a different host.
# Keep the catalog sitemap line only if Modern Campus confirms it returns a
# valid public XML sitemap and 200 OK to search crawlers.
# ---------------------------------------------------------------------------
Sitemap: https://dickinsonstate.edu/sitemap.xml