# Allow all search engine crawlers
User-agent: *

# Disallow access to bin and js directories (usually unnecessary for crawling)
Disallow: /bin/
Disallow: /js/

# Disallow access to uploaded file directories which may contain user-uploaded files or sensitive information
Disallow: /UploadFile/

# Disallow access to backend or administrative directories (if applicable)
Disallow: /Areas/

# Disallow access to specific files
Disallow: /favicon.ico
Disallow: /web.config
Disallow: /PrecompiledApp.config

# Allow specific directories (e.g., public CSS and fonts)
Allow: /Content/
Allow: /fonts/
Allow: /webfonts/

# Specify the path to the website's Sitemap
Sitemap: https://www.example.com/sitemap.xml