Verified Commit 198b03a0 authored by Alex Pott's avatar Alex Pott
Browse files

Issue #3271222 by Rajab Natshah, yogeshmpawar, BramDriesen: Include Disallow... 

Issue #3271222 by Rajab Natshah, yogeshmpawar, BramDriesen: Include Disallow Oembed media links in the robots.txt file for better Drupal SEO
parent f4b955cf

core/assets/scaffold/files/robots.txt

+4 −0
Original line number Diff line number Diff line
@@ -49,6 +49,8 @@ Disallow: /user/register 
Disallow: /user/password

Disallow: /user/login

Disallow: /user/logout

Disallow: /media/oembed

Disallow: /*/media/oembed

# Paths (no clean URLs)

Disallow: /index.php/admin/

Disallow: /index.php/comment/reply/
@@ -59,3 +61,5 @@ Disallow: /index.php/user/password 
Disallow: /index.php/user/register

Disallow: /index.php/user/login

Disallow: /index.php/user/logout

Disallow: /index.php/media/oembed

Disallow: /index.php/*/media/oembed

robots.txt

+4 −0
Original line number Diff line number Diff line
@@ -49,6 +49,8 @@ Disallow: /user/register 
Disallow: /user/password

Disallow: /user/login

Disallow: /user/logout

Disallow: /media/oembed

Disallow: /*/media/oembed

# Paths (no clean URLs)

Disallow: /index.php/admin/

Disallow: /index.php/comment/reply/
@@ -59,3 +61,5 @@ Disallow: /index.php/user/password 
Disallow: /index.php/user/register

Disallow: /index.php/user/login

Disallow: /index.php/user/logout

Disallow: /index.php/media/oembed

Disallow: /index.php/*/media/oembed