# robots.txt for The Learning Network
# https://www.learning.net

User-agent: *
Allow: /

# Disallow admin, auth, and utility pages
Disallow: /admin/
Disallow: /auth
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /toolkit
Disallow: /complete-setup

# Disallow URL parameters that create duplicate content
# UTM tracking parameters
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?utm_content=
Disallow: /*?utm_term=
Disallow: /*?gclid=
Disallow: /*?fbclid=
Disallow: /*?ref=
Disallow: /*?source=

# Block crawling of URLs with any query parameters
# This prevents duplicate content from tracking/filter params
Disallow: /*?*

# Sitemap location
Sitemap: https://www.learning.net/sitemap.xml

# Specific bot configurations
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /auth
Disallow: /*?*

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /auth
Disallow: /*?*

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

# Crawl-delay for less aggressive crawling
User-agent: *
Crawl-delay: 1