User-agent: *
Allow: /

User-agent: Twitterbot
Allow: /

# Prevent search engines from indexing private or sensitive sections (optional)
Disallow: /admin/
Disallow: /login/
Disallow: /user-data/