Update robots.txt, add ROBOTS_DISALLOWED and extra-robots.txt

CanastaWiki · Aug 2, 2024 · eef597c · eef597c
1 parent fa52300
commit eef597c
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/_sources/configs/robots.php b/_sources/configs/robots.php
@@ -5,6 +5,8 @@
 ini_set( 'display_errors', 0 );
 error_reporting( 0 );
 
+echo "# The following lines are generated by robots.php\n";
+
 header( 'Content-Type: text/plain' );
 
 $robotsDisallowed = getenv( 'ROBOTS_DISALLOWED' );
@@ -22,10 +24,7 @@
 
 	$siteMapUrl = "$server$script/sitemap$subdir/sitemap-index-$identifier.xml";
 
-  echo "# Add the sitemap url:\n";
 	echo "Sitemap: $siteMapUrl\n";
-
-  echo "\n# Content of the robots.txt file:\n";
 }
 
 readfile( 'robots-main.txt' );