# robots.txt - https://www.paragweld.com/
# Written by Powerhouse each time the sitemap updates. Disallow lines added
# here are kept on the next update; everything else is rewritten.
#
# Search engines and AI assistants may crawl the whole site except the paths
# below, which are form handlers, includes or pages with nothing to index.
# How the content may be USED once retrieved is stated in /ai.txt.

User-agent: *
Allow: /
Disallow: /cgi-bin/

# Assets stay crawlable: blocking CSS, JS or images stops search engines
# rendering the page correctly and hurts rankings.
Allow: /assets/


# ---------------------------------------------------------------------------
# AI assistants and answer engines
#
# Allowed so the business can be found and cited when someone asks an
# assistant about it. Being absent here does not protect the content: the
# answer just gets written without it, and usually gets the details wrong.
#
# Google-Extended is not a crawler. It decides whether pages Googlebot already
# fetched may be used by Gemini and AI Overviews. Blocking it removes the site
# from AI Overviews while changing nothing about Search.
#
# NOTE: a named group REPLACES the "User-agent: *" group rather than adding to
# it, so the same Disallow lines are repeated here on purpose.
# ---------------------------------------------------------------------------

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Bingbot
User-agent: CCBot
User-agent: Meta-ExternalAgent
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: Bytespider
Allow: /
Allow: /assets/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /cgi-bin/


Sitemap: https://www.paragweld.com/sitemap-index.xml