User-agent: ia_archiver Disallow: / # Default rules for all crawlers User-agent: * Disallow: # Allow main content areas Allow: / Allow: /about-us Allow: /contact-us Allow: /blog/ Allow: /community/ Allow: /questions Allow: /howto Allow: /buy Allow: /career Allow: /love Allow: /learning-materials-for-0-3-year Allow: /learning-materials-for-3-6-year Allow: /learning-materials-for-6-8-year Allow: /product/ Allow: /products Allow: /learning-kits Allow: /skill-booster-kits Allow: /live-classes Allow: /combos Allow: /new-launch Allow: /worksheets Allow: /mom-coach-program Allow: /mom-ceo-program Allow: /trackorder Allow: /terms/ Allow: /privacy-policy/ Allow: /refund-policy/ Allow: /sitemap.xml # Block Laravel sensitive areas Disallow: /admin/ Disallow: /api/ Disallow: /auth/ Disallow: /learn/ Disallow: /login/ Disallow: /register/ Disallow: /password/ Disallow: /dashboard/ Disallow: /user/ Disallow: /account/ Disallow: /profile/ Disallow: /settings/ Disallow: /config/ Disallow: /private/ Disallow: /internal/ Disallow: /temp/ Disallow: /tmp/ Disallow: /cache/ Disallow: /logs/ Disallow: /backup/ Disallow: /test/ Disallow: /dev/ Disallow: /staging/ # Block Laravel framework directories and files Disallow: /app/ Disallow: /bootstrap/ Disallow: /config/ Disallow: /database/ Disallow: /resources/ Disallow: /routes/ Disallow: /storage/ Disallow: /tests/ Disallow: /vendor/ Disallow: /.env Disallow: /.env.example Disallow: /.git/ Disallow: /.gitignore Disallow: /artisan Disallow: /composer.json Disallow: /composer.lock Disallow: /package.json Disallow: /webpack.mix.js Disallow: /phpunit.xml # Block common Laravel routes Disallow: /horizon/ Disallow: /telescope/ Disallow: /nova/ Disallow: /sanctum/ Disallow: /broadcasting/ Disallow: /_ignition/ Disallow: /_debugbar/ # Block search and filter parameters to prevent duplicate content Disallow: /*?search= Disallow: /*?filter= Disallow: /*?sort= Disallow: /*?page= Disallow: /*?utm_ Disallow: /*?ref= Disallow: /*?fbclid= Disallow: /*?gclid= # Block common spam/scraping paths Disallow: /search/ Disallow: /trackback/ Disallow: /xmlrpc.php Disallow: /feed/ Disallow: /rss/ # Block specific file types that shouldn't be indexed Disallow: /*.json$ Disallow: /*.xml$ Disallow: /*.txt$ Disallow: /*.log$ Disallow: /*.sql$ Disallow: /*.bak$ # Allow sitemap (exception to *.xml block above) Allow: /sitemap.xml # Sitemap location Sitemap: https://classmonitor.com/sitemap.xml # Additional rules for specific crawlers # Google-specific rules User-agent: Googlebot Crawl-delay: 1 Allow: /sitemap.xml # Bing-specific rules User-agent: Bingbot Crawl-delay: 2 # Block aggressive crawlers User-agent: SemrushBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / # Block social media crawlers from sensitive areas User-agent: facebookexternalhit Disallow: /admin/ Disallow: /api/ Disallow: /user/ User-agent: Twitterbot Disallow: /admin/ Disallow: /api/ Disallow: /user/ # Block AI training crawlers (optional - uncomment if desired) # User-agent: GPTBot # Disallow: / # # User-agent: ChatGPT-User # Disallow: / # # User-agent: CCBot # Disallow: / # # User-agent: anthropic-ai # Disallow: /