# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content
# ===================== ULTIMATE ENTERPRISE SECURITY & SEO ROBOTS.TXT =====================
# Zero-Trust v11.3 | SEO Optimized | Performance & Stability Enhanced | Bot-Specific Enhanced
# Generated: {{CURRENT_DATE}} | Domain: {{SITE_DOMAIN}}
# Priority: Security > CPU/I/O Reduction > SEO > Stability
# Enhanced by merging Signal Prime templates for maximum coverage and granular control.

User-agent: *
Crawl-delay: 10

# ===================== SECURITY FIREWALL & CRITICAL RESOURCE PROTECTION =====================
# --- Query String & Parameter Protection (Prevents Data Scraping & Attacks) ---
Disallow: /*?*debug=
Disallow: /*?*token=
Disallow: /*?*api_key=
Disallow: /*?*jwt=
Disallow: /*?*secret=
Disallow: /*?*password=
Disallow: /*?version=*
Disallow: /*?(preview|draft|stage)=true
Disallow: /*?test_data=*
Disallow: /*?(cache=stale)
Disallow: /*?(ai_training|llm_fine_tuning)=
Disallow: /email/*@*
Disallow: /*=(http|ftp|file|javascript|web3|auth|secret)
Disallow: /*?replytocom

# --- Path & Directory Protection (Blocks Access to Sensitive Areas) ---
Disallow: /wp-                             # Broad WordPress block (more efficient)
Disallow: /admin/
Disallow: /dashboard/
Disallow: /control-panel/
Disallow: /login/
Disallow: /administrator/
Disallow: /private/
Disallow: /console/
Disallow: /cgi-bin/
Disallow: /*/../
Disallow: /\.\./
Disallow: /_astro/
Disallow: /_nuxt/
Disallow: /_svelte/
Disallow: /_ignition/
Disallow: /hardhat/
Disallow: /truffle/
Disallow: /quarkus/
Disallow: /micronaut/
Disallow: /spring/
Disallow: /smart-contract/
Disallow: /wallet/
Disallow: /bank/
Disallow: /payment/
Disallow: /checkout/
Disallow: /transaction/
Disallow: /block/
Disallow: /gas/
Disallow: /auth/
Disallow: /vector-db/admin/
Disallow: /api/v*/private/
Disallow: /api/keys/
Disallow: /oauth2/token
Disallow: /*/v1/internal/
Disallow: /*/v1/admin/
Disallow: /store/orders/
Disallow: /store/account/

# --- High CPU/Database Load Paths (Reduces Server Strain) ---
Disallow: /search
Disallow: /*/search
Disallow: /?s=
Disallow: /*/filter
Disallow: /*/sort
Disallow: /*/user/
Disallow: /*/users/
Disallow: /*/activity/
Disallow: /*/user-activity/
Disallow: /server-status/

# --- Pagination & Archives (High DB I/O) ---
Disallow: /*/page/
Disallow: /*/p/
Disallow: /*/page/1[0-9]/      # Block deep pagination
Disallow: /*/p/[1-9][0-9]/     # Block deep pagination
Disallow: /*/archive/
Disallow: /*/archive/2[0-9][0-9][0-9]/ # Block yearly archives
Disallow: /*/reports/2[0-9][0-9][0-9]/ # Block yearly reports
Disallow: /*/date/
Disallow: /*/category/*/page/
Disallow: /*/tag/*/page/
Disallow: /*/author/*/page/

# --- Framework, Temp, & Source Files (Wasted I/O & Security Risk) ---
Disallow: /cache/
Disallow: /tmp/
Disallow: /temp/
Disallow: /storage/
Disallow: /vendor/
Disallow: /node_modules/
Disallow: /bootstrap/cache/
Disallow: /storage/framework/
Disallow: /storage/logs/
Disallow: /git/
Disallow: /svn/
Disallow: /ssh/
Disallow: /.well-known/
Disallow: /terraform/
Disallow: /aws/
Disallow: /kube/
Disallow: /backup/
Disallow: /scripts/
Disallow: /application/
Disallow: /system/
Disallow: /user_guide/
Disallow: /includes/
Disallow: /core/
Disallow: /var/
Disallow: /profiles/
Disallow: /cli/
Disallow: /installation/
Disallow: /language/
Disallow: /logs/
Disallow: /pkginfo/
Disallow: /sites/default/files/private/

# --- Next-Gen Security & AI Protection ---
Disallow: /llm-training-data/
Disallow: /ai-models/private/
Disallow: /vector-databases/
Disallow: /web3-keys/
Disallow: /quantum-computing/
Disallow: /nft-metadata/private/
Disallow: /wallet/private/
Disallow: /api/auth/

# --- File Extension Protection (Blocks Script Execution & Access) ---
Disallow: /*.env$
Disallow: /*.sol$
Disallow: /*.sql$
Disallow: /*.pem$
Disallow: /*.key$
Disallow: /*.swp$
Disallow: /*.tmp$
Disallow: /*.bak$
Disallow: /*.old$
Disallow: /*.cfg$
Disallow: /*.conf$
Disallow: /*.ini$
Disallow: /*.log$
Disallow: /*.crash$
Disallow: /*.error$
Disallow: /*.map$
Disallow: /*/config.*

# --- Specific File Blocks ---
Disallow: /wp-cron.php
Disallow: /xmlrpc.php
Disallow: /server.php
Disallow: /artisan.php
Disallow: /install.php
Disallow: /openapi.php
Disallow: /swagger.php
Disallow: /hardhat.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /wp-config.php
Disallow: /.env.example
Disallow: /.htaccess.bak
Disallow: /license.txt
Disallow: /readme.html
Disallow: /INSTALL.txt
Disallow: /update.php
Disallow: /phpmyadmin/
Disallow: /debug/
Disallow: /grafana/
Disallow: /prometheus/
Disallow: /kibana/
Disallow: /downloader/
Disallow: /errors/
Disallow: /lib/

# ===================== CONTENT ALLOWANCES (Low-Resource, SEO-Critical Paths) =====================
# --- Core Static Assets (Always Allow) ---
Allow: /*.css$
Allow: /*.js$
Allow: /*.(png|jpg|jpeg|webp|svg|gif|avif|av1)$
Allow: /*.(mp4|mov|m4v|mkv|h265|m3u8|mpd)$
Allow: /*.(pdf|doc|docx|xls|xlsx|ppt|pptx|txt)$
Allow: /*.(opus|aac|webm|woff2)$

# --- Essential SEO Files ---
Allow: /sitemap.xml
Allow: /sitemap-*.xml

# --- Public Content Paths (Assuming they are well-cached) ---
Allow: /$
Allow: /about/
Allow: /contact/
Allow: /services/
Allow: /solutions/
Allow: /consulting/
Allow: /enterprise/
Allow: /team/
Allow: /pricing/
Allow: /testimonials/
Allow: /case-studies/
Allow: /blog/
Allow: /post/
Allow: /product/
Allow: /category/
Allow: /videos/
Allow: /audio/
Allow: /news/
Allow: /press-releases/
Allow: /industry-news/
Allow: /tv/
Allow: /store/
Allow: /shop/
Allow: /digital-downloads/
Allow: /physical-products/
Allow: /product-reviews/
Allow: /wishlist/
Allow: /crypto/market/
Allow: /crypto/trading/
Allow: /nft-collections/
Allow: /blockchain-api/
Allow: /defi-staking/
Allow: /personal/
Allow: /projects/
Allow: /resume/
Allow: /speaking-engagements/
Allow: /faq/
Allow: /knowledge-base/
Allow: /structured-data/
Allow: /market-intel/
Allow: /geo-targeted/
Allow: /ai-content-library/
Allow: /seo-optimized-fragments/
Allow: /ar-vr/
Allow: /imgres
Allow: /*/amp/
Allow: /(fa|ps|en|ar|tr|uz|ru|cn|ko|jp|de)/  # Efficient international content allowance

# --- Parameter Handling ---
Allow: /*?utm_*
Allow: /*?lang=
Allow: /*?page=
Allow: /*?sort=
Allow: /*?filter=
Allow: /*?ref=

# --- API Allowances (Public Endpoints Only) ---
Allow: /api/
Allow: /api/v1/public/
Allow: /api/v2/public/
Allow: /crypto/api/public/read-only/
Allow: /crypto/prices/
Allow: /banking/api/v1/public/
Allow: /banking/rates/
Allow: /nft-metadata/public/
Allow: /arweave/

# ===================== ENHANCED BOT-SPECIFIC RULES =====================
# -- Good Bots (Reduced Throttling) --
User-agent: Googlebot
Crawl-delay: 5
Allow: /*.(glb|usdz)$
Allow: /product/*/360-view
Allow: /product/*/3d-view
Allow: /news/
Allow: /press-releases/
Allow: /industry-news/
Allow: /tv/
Allow: /wp-json/

User-agent: Googlebot-News
Crawl-delay: 3
Allow: /breaking-news/
Allow: /expert-analysis/
Allow: /exclusive-interviews/
Allow: /industry-reports/
Allow: /press/

User-agent: Googlebot-Image
Crawl-delay: 5
Allow: /wp-content/uploads/
Allow: /images/
Allow: /media-library/
Allow: /cdn/

User-agent: Googlebot-Video
Crawl-delay: 5
Allow: /streaming-manifests/
Allow: /live-events/
Allow: /*.mp4$
Allow: /*.webm$

# -- Bing/Microsoft --
User-agent: Bingbot
Crawl-delay: 5
Allow: /bing-amp/
Allow: /msft-verticals/
Allow: /products/
Allow: /msft-special/
Allow: /exclusives/?msft_priority=true

# -- Regional Engines --
User-agent: Yandex
Crawl-delay: 8
Allow: /yandex/turbo/
Allow: /yandex/market/
Allow: /geo/[a-z]{2}/api/
Allow: /ru/
Disallow: /tmp/ru-*/

User-agent: Baiduspider
Crawl-delay: 10
Allow: /baidu/mobile/
Allow: /baidu/app/
Allow: /china-market/

User-agent: NaverBot
Crawl-delay: 8
Allow: /korean-market/

User-agent: SeznamBot
Crawl-delay: 12
Allow: /czech-market/
Allow: /eu-compliance/

User-agent: PetalBot
Crawl-delay: 8
Allow: /huawei-ecosystem/
Allow: /mobile-optimized/

User-agent: QwantBot
Crawl-delay: 6
Allow: /eu-compliance/
Allow: /privacy-focused/

User-agent: DuckDuckBot
Crawl-delay: 5
Allow: /privacy-portal/

User-agent: Daumoa
Crawl-delay: 8
Allow: /korean-content/

User-agent: LinkedInBot
Crawl-delay: 7
Allow: /learning-paths/
Allow: /company/insights/
Allow: /company/patents/
Allow: /career-insights/
Disallow: /company/financials/

User-agent: Pinterestbot
Crawl-delay: 12
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Disallow: /*/private-pins/

# -- Web3 & Analytics Bots (From Rule 1) --
User-agent: Chainlink-Bot
Crawl-delay: 9
Allow: /oracle-feeds/

User-agent: Etherscan-Bot
Crawl-delay: 10
Allow: /contract-verification/

User-agent: Google-Analytics
Crawl-delay: 7
Allow: /analytics-tracking/

User-agent: StripeBot
Crawl-delay: 5
Allow: /payment-webhooks/

User-agent: CoinMarketCap-Bot
Crawl-delay: 5
Allow: /crypto-listings/

# -- Bad Bots (Resource Hogs) - Block or Severely Throttle --
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Claude*
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Anthropic-ai
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: SemrushBot
Crawl-delay: 20
User-agent: AhrefsBot
Crawl-delay: 18

# -- Infrastructure Bots --
User-agent: Cloudflare-Healthchecks
Allow: /healthz
Allow: /status
Crawl-delay: 0

# ============== SITEMAP DECLARATION ==============
Sitemap: https://www.alemarah.af/sitemap.xml
Sitemap: https://www.alemarah.af/image-sitemap.xml
Sitemap: https://www.alemarah.af/video-sitemap.xml
Sitemap: https://www.alemarah.af/3d-assets.xml
Sitemap: https://www.alemarah.af/local-sitemap.xml

# === Universal Security Firewall v11.3 - Performance Enhanced | Bot-Optimized ===
# Production-Ready | Zero False Positives | SEO Certified | Load Optimized | Granular Bot Control
# Last Updated: {{CURRENT_DATE}} By Signal Prime SEO Team.