From ce1b74f4e2f48628c8430520d8e60cbe2eb72975 Mon Sep 17 00:00:00 2001
From: Seirdy <seirdy@seirdy.one>
Date: Fri, 26 Jul 2024 23:03:30 -0400
Subject: [PATCH] update facebook GenAI crawler in robots.txt

---
 static/robots.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/static/robots.txt b/static/robots.txt
index 434dc0b..1ca14c3 100644
--- a/static/robots.txt
+++ b/static/robots.txt
@@ -110,8 +110,10 @@ Disallow: /
 
 # FacebookBot crawls public web pages to improve language models for our speech
 # recognition technology.
+# UPDATE 2024-07: The Meta-ExternalAgent crawler crawls the web for use cases such as training AI models or improving products by indexing content directly.
 # <https://developers.facebook.com/docs/sharing/bot/?_fb_noscript=1>
-User-Agent:  FacebookBot
+User-Agent: FacebookBot
+User-Agent: meta-externalagent
 Disallow: /
 
 # I'm not blocking CCBot for now. It publishes a free index for anyone to use.