# robots.txt file
# Generally speaking, don't disallow any image directories.
# update May 20 '18 -- Removed because Google needs access to /js to render pages! Don't disallow /js !!!
# update Mar 15 '20 -- Prior to this there was just 1 line ifateaps/app_data .. but we were getting a lot of garbage in site search ... added back many dirs
# update Jul 27 '20 -- removed full http://www.ifate.com* urls .. replaced with relative.
# update Jul 27 '20 -- Added user agent *
# update Oct 8 '20 --  Allow everything
# update Spring '23 -- disallow semrush, common crawl, majestic, ahrefs
# update April 24 -- Add delays to Proximic and grapeshot who are hammering the servers
# update August 29 2024 -- Added /oranum_click.php because these were getting crawled in GSC and it's just a click tracker
# update August 29 2024 -- Added /daily as this is all repeat content. The root horoscopes page is all that needs to be indexed.
# update August 29 2024 -- Added /numerology_reading.html as this is the 2nd page and can't be accessed directly without URL vars.
# update August 29 2024 -- Added /js/min/ifateprimary-min.js?  This prevents the parameter at the end from being counted as a new file
# update Aug 20 2025 -- Removed GPTbot... at the end of the day it's a source of traffic.
# update Aug 20 2025 -- Removed ifateprimary.min.js

# General rules

User-agent: *
Allow: /
Disallow: /oranum_click.php
Disallow: /daily
Disallow: /numerology_reading.html
Disallow: /i-ching-changes/iching-changes-headers/*.jpg
Disallow: /astrology-aspects/astrology-aspect-images/*.jpg



# Allow specific bots
User-agent: Mediapartners-Google
Allow: /

User-agent: Google-display-ads-bot
Allow: /



# Block specific bots
User-agent: SemrushBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /



# Set crawl delay for specific bots
User-agent: Proximic
Crawl-delay: 2

User-agent: grapeshot
Crawl-delay: 2