From 4851ef772cc077c852468c8e8285a981f21ced6a Mon Sep 17 00:00:00 2001 From: Zachary Vance Date: Thu, 23 Apr 2020 02:24:09 -0700 Subject: [PATCH] Simplify robots.txt --- robots.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/robots.txt b/robots.txt index 62a5e12..06e4ae9 100644 --- a/robots.txt +++ b/robots.txt @@ -1,5 +1,6 @@ +# These bots are too stupid and too frequent. They download very large data files. User-agent: AhrefsBot -Disallow: / - User-agent: dotbot Disallow: / + +# If you are a human, feel free to ignore this robots.txt file. -- 2.47.3