User-agent: * Crawl-delay: 5 # Then we start disallowing stuff # Directories Disallow: /min/ Disallow: /jobseekers_pictures/ Disallow: /stylesheets/ Disallow: /images/ # Disallow bots and crawlers User-agent: SEMrushBot Disallow: / User-agent: SemrushBot-SA Disallow: /