From 198b03a0e59e5cab35b69bf224c5a2bb40d7f717 Mon Sep 17 00:00:00 2001
From: Alex Pott <alex.a.pott@googlemail.com>
Date: Wed, 5 Oct 2022 12:58:39 +0100
Subject: [PATCH] Issue #3271222 by Rajab Natshah, yogeshmpawar, BramDriesen:
 Include Disallow Oembed media links in the robots.txt file for better Drupal
 SEO

---
 core/assets/scaffold/files/robots.txt | 4 ++++
 robots.txt                            | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/core/assets/scaffold/files/robots.txt b/core/assets/scaffold/files/robots.txt
index 18f8df8e693c..ebcd04b96ca4 100644
--- a/core/assets/scaffold/files/robots.txt
+++ b/core/assets/scaffold/files/robots.txt
@@ -49,6 +49,8 @@ Disallow: /user/register
 Disallow: /user/password
 Disallow: /user/login
 Disallow: /user/logout
+Disallow: /media/oembed
+Disallow: /*/media/oembed
 # Paths (no clean URLs)
 Disallow: /index.php/admin/
 Disallow: /index.php/comment/reply/
@@ -59,3 +61,5 @@ Disallow: /index.php/user/password
 Disallow: /index.php/user/register
 Disallow: /index.php/user/login
 Disallow: /index.php/user/logout
+Disallow: /index.php/media/oembed
+Disallow: /index.php/*/media/oembed
diff --git a/robots.txt b/robots.txt
index 18f8df8e693c..ebcd04b96ca4 100644
--- a/robots.txt
+++ b/robots.txt
@@ -49,6 +49,8 @@ Disallow: /user/register
 Disallow: /user/password
 Disallow: /user/login
 Disallow: /user/logout
+Disallow: /media/oembed
+Disallow: /*/media/oembed
 # Paths (no clean URLs)
 Disallow: /index.php/admin/
 Disallow: /index.php/comment/reply/
@@ -59,3 +61,5 @@ Disallow: /index.php/user/password
 Disallow: /index.php/user/register
 Disallow: /index.php/user/login
 Disallow: /index.php/user/logout
+Disallow: /index.php/media/oembed
+Disallow: /index.php/*/media/oembed
-- 
GitLab