diff --git a/static/robots.txt b/static/robots.txt index 18f53c6..8aca9bf 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -8,11 +8,6 @@ User-Agent: Adsbot Disallow: / Allow: /ads.txt -# the UA should not be case-sensitive, but I gotta cover my bases. -User-Agent: AdsBot -Disallow: / -Allow: /ads.txt - # > This robot collects content from the Internet for the sole purpose of # helping educational institutions prevent plagiarism. [...] we compare student papers against the content we find on the Internet to see if we # can find similarities. (http://www.turnitin.com/robot/crawlerinfo.html) # --> fuck off. User-Agent: TurnitinBot