Update robots.txt, add ROBOTS_DISALLOWED and extra-robots.txt

_sources/configs/robots.php

@@ -5,8 +5,15 @@
 ini_set( 'display_errors', 0 );
 error_reporting( 0 );
 
+echo "# It generated by the robots.php file\n";
+
 header( 'Content-Type: text/plain' );
 
+$robotsDisallowed = getenv( 'ROBOTS_DISALLOWED' );
+if ( !empty( $robotsDisallowed ) && in_array( strtolower($robotsDisallowed), [ 'true', '1' ] ) ) {
+	die( "User-agent: *\nDisallow: /\n" );
+}
+
 $enableSitemapEnv = getenv( 'MW_ENABLE_SITEMAP_GENERATOR');
 // match the value check to the isTrue function at _sources/scripts/functions.sh
 if ( !empty( $enableSitemapEnv ) && in_array( $enableSitemapEnv, [ 'true', 'True', 'TRUE', '1' ] ) ) {
@@ -25,3 +32,12 @@
 }
 
 readfile( 'robots-main.txt' );
+
+// If the file `extra-robots.txt` is created under the name
+// `/var/www/mediawiki/extra-robots.txt` then its contents get appended to the
+// default `robots.txt` - see MW-312
+if ( is_readable( 'extra-robots.txt' ) ) {
+	// Extra line to separate the files so that rules don't combine
+	echo "\n";
+	readfile( 'extra-robots.txt' );
+}