Add support for all ChatGPT crawlers

donatj · Aug 28, 2024 · 8781e03 · 8781e03
1 parent b0ffbe4
commit 8781e03
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 6 deletions.
diff --git a/src/UserAgent/Browsers.php b/src/UserAgent/Browsers.php
@@ -25,6 +25,7 @@ interface Browsers {
 	const GOOGLEBOT           = 'Googlebot';
 	const GOOGLEBOT_IMAGE     = 'Googlebot-Image';
 	const GOOGLEBOT_VIDEO     = 'Googlebot-Video';
+	const GPTBOT              = 'GPTBot';
 	const HEADLESSCHROME      = 'HeadlessChrome';
 	const IEMOBILE            = 'IEMobile';
 	const IMESSAGEBOT         = 'iMessageBot';
@@ -36,6 +37,7 @@ interface Browsers {
 	const MSNBOT_MEDIA        = 'msnbot-media';
 	const NETFRONT            = 'NetFront';
 	const NINTENDOBROWSER     = 'NintendoBrowser';
+	const OAI_SEARCHBOT       = 'OAI-SearchBot';
 	const OCULUSBROWSER       = 'OculusBrowser';
 	const OPERA               = 'Opera';
 	const PUFFIN              = 'Puffin';

diff --git a/src/UserAgentParser.php b/src/UserAgentParser.php
@@ -90,7 +90,7 @@ function parse_user_agent( $u_agent = null ) {
 		} elseif( $platform == 'Adr' ) {
 			$platform = 'Android';
 		} elseif( $platform === null ) {
-			if(preg_match_all('%(?P<platform>Android)[:/ ]%ix', $u_agent, $result)) {
+			if( preg_match_all('%(?P<platform>Android)[:/ ]%ix', $u_agent, $result) ) {
 				$platform = $result[PLATFORM][0];
 			}
 		}
@@ -99,7 +99,7 @@ function parse_user_agent( $u_agent = null ) {
 %(?P<browser>Camino|Kindle(\ Fire)?|Firefox|Iceweasel|IceCat|Safari|MSIE|Trident|AppleWebKit|
 TizenBrowser|(?:Headless)?Chrome|YaBrowser|Vivaldi|IEMobile|Opera|OPR|Silk|Midori|(?-i:Edge)|EdgA?|CriOS|UCBrowser|Puffin|
 OculusBrowser|SamsungBrowser|SailfishBrowser|XiaoMi/MiuiBrowser|YaApp_Android|
-Baiduspider|Applebot|Facebot|Googlebot|YandexBot|bingbot|Lynx|Version|Wget|curl|ChatGPT-User|
+Baiduspider|Applebot|Facebot|Googlebot|YandexBot|bingbot|Lynx|Version|Wget|curl|ChatGPT-User|GPTBot|OAI-SearchBot|
 Valve\ Steam\ Tenfoot|
 NintendoBrowser|PLAYSTATION\ (?:\d|Vita)+)
 \)?;?
@@ -126,7 +126,7 @@ function parse_user_agent( $u_agent = null ) {
 
 		$lowerBrowser = array_map('strtolower', $result[BROWSER]);
 
-		$find = function ( $search, &$key = null, &$value = null ) use ( $lowerBrowser ) {
+		$find = function( $search, &$key = null, &$value = null ) use ( $lowerBrowser ) {
 			$search = (array)$search;
 
 			foreach( $search as $val ) {
@@ -142,7 +142,7 @@ function parse_user_agent( $u_agent = null ) {
 			return false;
 		};
 
-		$findT = function ( array $search, &$key = null, &$value = null ) use ( $find ) {
+		$findT = function( array $search, &$key = null, &$value = null ) use ( $find ) {
 			$value2 = null;
 			if( $find(array_keys($search), $key, $value2) ) {
 				$value = $search[$value2];
@@ -169,7 +169,7 @@ function parse_user_agent( $u_agent = null ) {
 		} elseif( $find('NintendoBrowser', $key) || $platform == 'Nintendo 3DS' ) {
 			$browser = 'NintendoBrowser';
 			$version = $result[BROWSER_VERSION][$key];
-		} elseif( $find(['Kindle'], $key, $platform) ) {
+		} elseif( $find([ 'Kindle' ], $key, $platform) ) {
 			$browser = $result[BROWSER][$key];
 			$version = $result[BROWSER_VERSION][$key];
 		} elseif( $find('Opera', $key, $browser) ) {
@@ -188,7 +188,7 @@ function parse_user_agent( $u_agent = null ) {
 					}
 				}
 			}
-		} elseif( $find([ 'Applebot', 'IEMobile', 'Edge', 'Midori', 'Vivaldi', 'OculusBrowser', 'SamsungBrowser', 'Valve Steam Tenfoot', 'Chrome', 'HeadlessChrome', 'SailfishBrowser', 'ChatGPT-User' ], $key, $browser) ) {
+		} elseif( $find([ 'Applebot', 'IEMobile', 'Edge', 'Midori', 'Vivaldi', 'OculusBrowser', 'SamsungBrowser', 'Valve Steam Tenfoot', 'Chrome', 'HeadlessChrome', 'SailfishBrowser' ], $key, $browser) ) {
 			$version = $result[BROWSER_VERSION][$key];
 		} elseif( $rv_result && $find('Trident') ) {
 			$browser = 'MSIE';
@@ -203,6 +203,10 @@ function parse_user_agent( $u_agent = null ) {
 				$browser = 'BlackBerry Browser';
 			} elseif( $find('Safari', $key, $browser) || $find('TizenBrowser', $key, $browser) ) {
 				$version = $result[BROWSER_VERSION][$key];
+			} elseif( count($result[BROWSER]) ) {
+				$key     = count($result[BROWSER]) - 1;
+				$browser = $result[BROWSER][$key];
+				$version = $result[BROWSER_VERSION][$key];
 			}
 
 			if( $find('Version', $key) ) {

diff --git a/tests/user_agents.dist.json b/tests/user_agents.dist.json
@@ -1399,6 +1399,11 @@
 		"browser": "bingbot",
 		"version": "2.0"
 	},
+	"Mozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko); compatible; ChatGPT-User\/1.0; +https:\/\/openai.com\/bot": {
+		"platform": null,
+		"browser": "ChatGPT-User",
+		"version": "1.0"
+	},
 	"curl\/7.19.7 (universal-apple-darwin10.0) libcurl\/7.19.7 OpenSSL\/0.9.8r zlib\/1.2.3": {
 		"platform": null,
 		"browser": "curl",
@@ -1439,6 +1444,11 @@
 		"browser": "Googlebot-Video",
 		"version": "1.0"
 	},
+	"Mozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko); compatible; GPTBot\/1.1; +https:\/\/openai.com\/gptbot": {
+		"platform": null,
+		"browser": "GPTBot",
+		"version": "1.1"
+	},
 	"Lynx\/2.8.6rel.4 libwww-FM\/2.14 SSL-MM\/1.4.1 OpenSSL\/0.9.7l Lynxlet\/0.7.0": {
 		"platform": null,
 		"browser": "Lynx",
@@ -1449,6 +1459,11 @@
 		"browser": "msnbot-media",
 		"version": "1.1"
 	},
+	"Mozilla\/5.0 AppleWebKit\/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot\/1.0; +https:\/\/openai.com\/searchbot": {
+		"platform": null,
+		"browser": "OAI-SearchBot",
+		"version": "1.0"
+	},
 	"Slackbot 1.0 (+https:\/\/api.slack.com\/robots)": {
 		"platform": null,
 		"browser": "Slackbot",