From c648324fe681c40e1b775f31a19c5666484ed41b Mon Sep 17 00:00:00 2001 From: Rohan Kumar Date: Sun, 24 Jul 2022 11:32:16 -0700 Subject: [PATCH] remove redundant robots.txt entry --- static/robots.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/static/robots.txt b/static/robots.txt index 18f53c6..8aca9bf 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -8,11 +8,6 @@ User-Agent: Adsbot Disallow: / Allow: /ads.txt -# the UA should not be case-sensitive, but I gotta cover my bases. -User-Agent: AdsBot -Disallow: / -Allow: /ads.txt - # > This robot collects content from the Internet for the sole purpose of # helping educational institutions prevent plagiarism. [...] we compare student papers against the content we find on the Internet to see if we # can find similarities. (http://www.turnitin.com/robot/crawlerinfo.html) # --> fuck off. User-Agent: TurnitinBot