From 399662d2afa58dfd631128e3eac033d90464f794 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 28 Aug 2012 09:56:59 +0200 Subject: [PATCH 01/11] [ticket/10897] Bot list updated 'Baidu [Spider]' updated. 'Bing [Bot]' fixed only identation. 'Exabot [Bot]' changed because there is used also 'Exabot-Thumbnails' user agent. 'NG-Search [Bot]' no longer exists. 'Nutch/CVS [Bot]' is the same of 'Nutch [Bot]'. 'OmniExplorer [Bot]' no longer exists. 'Seekport [Bot]' no longer exists. 'Synoo [Bot]' no longer exists. 'Voyager [Bot]' updated. 'W3C [Validator]' corrected (there was a *). 'WiseNut [Bot]' no longer exists. PHPBB3-10897 --- phpBB/install/install_install.php | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/phpBB/install/install_install.php b/phpBB/install/install_install.php index f1003b07d7..67ebb8b1c5 100644 --- a/phpBB/install/install_install.php +++ b/phpBB/install/install_install.php @@ -2103,9 +2103,9 @@ class install_install extends module 'Alexa [Bot]' => array('ia_archiver', ''), 'Alta Vista [Bot]' => array('Scooter/', ''), 'Ask Jeeves [Bot]' => array('Ask Jeeves', ''), - 'Baidu [Spider]' => array('Baiduspider+(', ''), - 'Bing [Bot]' => array('bingbot/', ''), - 'Exabot [Bot]' => array('Exabot/', ''), + 'Baidu [Spider]' => array('Baiduspider', ''), + 'Bing [Bot]' => array('bingbot/', ''), + 'Exabot [Bot]' => array('Exabot', ''), 'FAST Enterprise [Crawler]' => array('FAST Enterprise Crawler', ''), 'FAST WebCrawler [Crawler]' => array('FAST-WebCrawler/', ''), 'Francis [Bot]' => array('http://www.neomo.de/', ''), @@ -2124,27 +2124,21 @@ class install_install extends module 'MSN NewsBlogs' => array('msnbot-NewsBlogs/', ''), 'MSN [Bot]' => array('msnbot/', ''), 'MSNbot Media' => array('msnbot-media/', ''), - 'NG-Search [Bot]' => array('NG-Search/', ''), 'Nutch [Bot]' => array('http://lucene.apache.org/nutch/', ''), - 'Nutch/CVS [Bot]' => array('NutchCVS/', ''), - 'OmniExplorer [Bot]' => array('OmniExplorer_Bot/', ''), 'Online link [Validator]' => array('online link validator', ''), 'psbot [Picsearch]' => array('psbot/0', ''), - 'Seekport [Bot]' => array('Seekbot/', ''), 'Sensis [Crawler]' => array('Sensis Web Crawler', ''), 'SEO Crawler' => array('SEO search Crawler/', ''), 'Seoma [Crawler]' => array('Seoma [SEO Crawler]', ''), 'SEOSearch [Crawler]' => array('SEOsearch/', ''), 'Snappy [Bot]' => array('Snappy/1.1 ( http://www.urltrends.com/ )', ''), 'Steeler [Crawler]' => array('http://www.tkl.iis.u-tokyo.ac.jp/~crawler/', ''), - 'Synoo [Bot]' => array('SynooBot/', ''), 'Telekom [Bot]' => array('crawleradmin.t-info@telekom.de', ''), 'TurnitinBot [Bot]' => array('TurnitinBot/', ''), - 'Voyager [Bot]' => array('voyager/1.0', ''), + 'Voyager [Bot]' => array('voyager/', ''), 'W3 [Sitesearch]' => array('W3 SiteSearch Crawler', ''), 'W3C [Linkcheck]' => array('W3C-checklink/', ''), - 'W3C [Validator]' => array('W3C_*Validator', ''), - 'WiseNut [Bot]' => array('http://www.WISEnutbot.com', ''), + 'W3C [Validator]' => array('W3C_Validator', ''), 'YaCy [Bot]' => array('yacybot', ''), 'Yahoo MMCrawler [Bot]' => array('Yahoo-MMCrawler/', ''), 'Yahoo Slurp [Bot]' => array('Yahoo! DE Slurp', ''), From 53e9bab237470e827243fcf3743eb7ca3d9ee346 Mon Sep 17 00:00:00 2001 From: Carlo Date: Tue, 28 Aug 2012 10:02:27 +0200 Subject: [PATCH 02/11] [ticket/10897] Update bots during phpBB update PHPBB3-10897 --- phpBB/install/database_update.php | 70 +++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index 099ab6aeb5..0343e0dc86 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2119,6 +2119,76 @@ function change_database_data(&$no_updates, $version) AND module_basename = \'profile\' AND module_mode = \'signature\''; _sql($sql, $errored, $error_ary); + + $bots_to_update = array( + 'Baidu [Spider]' => array('Baiduspider', ''), + 'Exabot [Bot]' => array('Exabot', ''), + 'Voyager [Bot]' => array('voyager/', ''), + 'W3C [Validator]' => array('W3C_Validator', ''), + ); + + $bots_to_delete = array( + 'NG-Search [Bot]', + 'Nutch/CVS [Bot]', + 'OmniExplorer [Bot]', + 'Seekport [Bot]', + 'Synoo [Bot]', + 'WiseNut [Bot]', + ); + + foreach($bots_to_update as $bot_name => $bot_array) + { + list($bot_agent, $bot_ip) = $bot_array; + + $bot_name_clean = utf8_clean_string($bot_name); + + $sql = 'SELECT user_id + FROM ' . USERS_TABLE . " + WHERE username_clean = '" . $db->sql_escape($bot_name_clean) . "'"; + $result = $db->sql_query($sql); + $is_user = (bool) $db->sql_fetchfield('user_id'); + $db->sql_freeresult($result); + + if ($is_user) + { + $sql = 'UPDATE ' . BOTS_TABLE . ' SET ' . $db->sql_build_array('UPDATE', array( + 'bot_agent' => (string) $bot_agent, + 'bot_ip' => (string) $bot_ip, + )) . " WHERE bot_name = '" . $db->sql_escape($bot_name) . "'"; + + _sql($sql, $errored, $error_ary); + } + } + + if (!function_exists('user_delete')) + { + include($phpbb_root_path . 'includes/functions_user.' . $phpEx); + } + + foreach($bots_to_delete as $bot_name) + { + $bot_name_clean = utf8_clean_string($bot_name); + + $sql = 'SELECT user_id + FROM ' . USERS_TABLE . " + WHERE username_clean = '" . $db->sql_escape($bot_name_clean) . "'"; + $result = $db->sql_query($sql); + $bot_user_id = $db->sql_fetchfield('user_id'); + $is_user = (bool) $bot_user_id; + $db->sql_freeresult($result); + + if ($is_user) + { + $sql = 'DELETE FROM ' . BOTS_TABLE . " + WHERE bot_name = '" . $db->sql_escape($bot_name) . "'"; + + _sql($sql, $errored, $error_ary); + + user_delete('remove', $bot_user_id); + } + } + + $no_updates = false; break; } } From 07e7c475d510a221c9080529647ec432cca038c6 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:12:12 +0100 Subject: [PATCH 03/11] [ticket/10897] Add space after foreach. PHPBB3-10897 --- phpBB/install/database_update.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index 0343e0dc86..4c94cdf5bd 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2136,7 +2136,7 @@ function change_database_data(&$no_updates, $version) 'WiseNut [Bot]', ); - foreach($bots_to_update as $bot_name => $bot_array) + foreach ($bots_to_update as $bot_name => $bot_array) { list($bot_agent, $bot_ip) = $bot_array; @@ -2165,7 +2165,7 @@ function change_database_data(&$no_updates, $version) include($phpbb_root_path . 'includes/functions_user.' . $phpEx); } - foreach($bots_to_delete as $bot_name) + foreach ($bots_to_delete as $bot_name) { $bot_name_clean = utf8_clean_string($bot_name); From b2183c9b705dc5283f3a1c3abbc80331f74e4b72 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:12:27 +0100 Subject: [PATCH 04/11] [ticket/10897] Remove unnecessary string casting. PHPBB3-10897 --- phpBB/install/database_update.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index 4c94cdf5bd..bbb05e67ec 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2152,8 +2152,8 @@ function change_database_data(&$no_updates, $version) if ($is_user) { $sql = 'UPDATE ' . BOTS_TABLE . ' SET ' . $db->sql_build_array('UPDATE', array( - 'bot_agent' => (string) $bot_agent, - 'bot_ip' => (string) $bot_ip, + 'bot_agent' => $bot_agent, + 'bot_ip' => $bot_ip, )) . " WHERE bot_name = '" . $db->sql_escape($bot_name) . "'"; _sql($sql, $errored, $error_ary); From de6d741d96e57a704e7653e8c73a20b3c196a0d9 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:19:18 +0100 Subject: [PATCH 05/11] [ticket/10897] Add comment about what's going on. PHPBB3-10897 --- phpBB/install/database_update.php | 1 + 1 file changed, 1 insertion(+) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index bbb05e67ec..c8675fa95a 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2120,6 +2120,7 @@ function change_database_data(&$no_updates, $version) AND module_mode = \'signature\''; _sql($sql, $errored, $error_ary); + // Update bot list $bots_to_update = array( 'Baidu [Spider]' => array('Baiduspider', ''), 'Exabot [Bot]' => array('Exabot', ''), From 935bc33268bb062158d6421242335483fcd955bd Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:19:58 +0100 Subject: [PATCH 06/11] [ticket/10897] Move bot delete data to the relevant foreach loop. PHPBB3-10897 --- phpBB/install/database_update.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index c8675fa95a..0372b6e608 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2128,15 +2128,6 @@ function change_database_data(&$no_updates, $version) 'W3C [Validator]' => array('W3C_Validator', ''), ); - $bots_to_delete = array( - 'NG-Search [Bot]', - 'Nutch/CVS [Bot]', - 'OmniExplorer [Bot]', - 'Seekport [Bot]', - 'Synoo [Bot]', - 'WiseNut [Bot]', - ); - foreach ($bots_to_update as $bot_name => $bot_array) { list($bot_agent, $bot_ip) = $bot_array; @@ -2166,6 +2157,15 @@ function change_database_data(&$no_updates, $version) include($phpbb_root_path . 'includes/functions_user.' . $phpEx); } + $bots_to_delete = array( + 'NG-Search [Bot]', + 'Nutch/CVS [Bot]', + 'OmniExplorer [Bot]', + 'Seekport [Bot]', + 'Synoo [Bot]', + 'WiseNut [Bot]', + ); + foreach ($bots_to_delete as $bot_name) { $bot_name_clean = utf8_clean_string($bot_name); From 63c64694bc253358a50dce7dce99a7551b5e2b17 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:41:59 +0100 Subject: [PATCH 07/11] [ticket/10897] Update by user_id instead of bot_name. PHPBB3-10897 --- phpBB/install/database_update.php | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index 0372b6e608..6496069658 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2132,22 +2132,23 @@ function change_database_data(&$no_updates, $version) { list($bot_agent, $bot_ip) = $bot_array; - $bot_name_clean = utf8_clean_string($bot_name); - $sql = 'SELECT user_id FROM ' . USERS_TABLE . " - WHERE username_clean = '" . $db->sql_escape($bot_name_clean) . "'"; + WHERE username_clean = '" . $db->sql_escape(utf8_clean_string($bot_name)) . "'"; $result = $db->sql_query($sql); - $is_user = (bool) $db->sql_fetchfield('user_id'); + $bot_user_id = (int) $db->sql_fetchfield('user_id'); $db->sql_freeresult($result); - if ($is_user) + if ($bot_user_id) { - $sql = 'UPDATE ' . BOTS_TABLE . ' SET ' . $db->sql_build_array('UPDATE', array( + $update_array = array( 'bot_agent' => $bot_agent, 'bot_ip' => $bot_ip, - )) . " WHERE bot_name = '" . $db->sql_escape($bot_name) . "'"; + ); + $sql = 'UPDATE ' . BOTS_TABLE . ' + SET ' . $db->sql_build_array('UPDATE', $update_array) . " + WHERE user_id = $bot_user_id"; _sql($sql, $errored, $error_ary); } } From c054757f3caad66152eaafe09f2c98abcb9ef0e2 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:45:20 +0100 Subject: [PATCH 08/11] [ticket/10897] Do not handle IP address. There is no need. PHPBB3-10897 --- phpBB/install/database_update.php | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index 6496069658..154fc313a0 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2121,17 +2121,16 @@ function change_database_data(&$no_updates, $version) _sql($sql, $errored, $error_ary); // Update bot list + // Bot name to bot user agent map $bots_to_update = array( - 'Baidu [Spider]' => array('Baiduspider', ''), - 'Exabot [Bot]' => array('Exabot', ''), - 'Voyager [Bot]' => array('voyager/', ''), - 'W3C [Validator]' => array('W3C_Validator', ''), + 'Baidu [Spider]' => 'Baiduspider', + 'Exabot [Bot]' => 'Exabot', + 'Voyager [Bot]' => 'voyager/', + 'W3C [Validator]' => 'W3C_Validator', ); - foreach ($bots_to_update as $bot_name => $bot_array) + foreach ($bots_to_update as $bot_name => $bot_agent) { - list($bot_agent, $bot_ip) = $bot_array; - $sql = 'SELECT user_id FROM ' . USERS_TABLE . " WHERE username_clean = '" . $db->sql_escape(utf8_clean_string($bot_name)) . "'"; @@ -2141,13 +2140,8 @@ function change_database_data(&$no_updates, $version) if ($bot_user_id) { - $update_array = array( - 'bot_agent' => $bot_agent, - 'bot_ip' => $bot_ip, - ); - $sql = 'UPDATE ' . BOTS_TABLE . ' - SET ' . $db->sql_build_array('UPDATE', $update_array) . " + SET bot_agent = ' . $db->sql_escape($bot_agent) . " WHERE user_id = $bot_user_id"; _sql($sql, $errored, $error_ary); } From 520ffdf368bee84549a0b284bd2df4da1d633a8f Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:47:50 +0100 Subject: [PATCH 09/11] [ticket/10897] Make sure the user we're fetching is a bot. PHPBB3-10897 --- phpBB/install/database_update.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index 154fc313a0..bd12a90288 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2132,8 +2132,9 @@ function change_database_data(&$no_updates, $version) foreach ($bots_to_update as $bot_name => $bot_agent) { $sql = 'SELECT user_id - FROM ' . USERS_TABLE . " - WHERE username_clean = '" . $db->sql_escape(utf8_clean_string($bot_name)) . "'"; + FROM ' . USERS_TABLE . ' + WHERE user_type = ' . USER_IGNORE . " + AND username_clean = '" . $db->sql_escape(utf8_clean_string($bot_name)) . "'"; $result = $db->sql_query($sql); $bot_user_id = (int) $db->sql_fetchfield('user_id'); $db->sql_freeresult($result); From 54700f5ba24fe944616e017c3c2ff9c09c312445 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:53:33 +0100 Subject: [PATCH 10/11] [ticket/10897] Use same code/query for deleting. PHPBB3-10897 --- phpBB/install/database_update.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index bd12a90288..b8bf85e283 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2164,21 +2164,18 @@ function change_database_data(&$no_updates, $version) foreach ($bots_to_delete as $bot_name) { - $bot_name_clean = utf8_clean_string($bot_name); - $sql = 'SELECT user_id - FROM ' . USERS_TABLE . " - WHERE username_clean = '" . $db->sql_escape($bot_name_clean) . "'"; + FROM ' . USERS_TABLE . ' + WHERE user_type = ' . USER_IGNORE . " + AND username_clean = '" . $db->sql_escape(utf8_clean_string($bot_name)) . "'"; $result = $db->sql_query($sql); - $bot_user_id = $db->sql_fetchfield('user_id'); - $is_user = (bool) $bot_user_id; + $bot_user_id = (int) $db->sql_fetchfield('user_id'); $db->sql_freeresult($result); - if ($is_user) + if ($bot_user_id) { $sql = 'DELETE FROM ' . BOTS_TABLE . " - WHERE bot_name = '" . $db->sql_escape($bot_name) . "'"; - + WHERE user_id = $bot_user_id"; _sql($sql, $errored, $error_ary); user_delete('remove', $bot_user_id); From 60ea1c10ef0038fdb00fd38253d4b7407215e699 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 Nov 2012 12:59:05 +0100 Subject: [PATCH 11/11] [ticket/10897] Combine bot updates and bot delete. PHPBB3-10897 --- phpBB/install/database_update.php | 73 ++++++++++++++----------------- 1 file changed, 32 insertions(+), 41 deletions(-) diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index b8bf85e283..f7980ab1be 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2120,16 +2120,30 @@ function change_database_data(&$no_updates, $version) AND module_mode = \'signature\''; _sql($sql, $errored, $error_ary); - // Update bot list - // Bot name to bot user agent map - $bots_to_update = array( + // Update bots + if (!function_exists('user_delete')) + { + include($phpbb_root_path . 'includes/functions_user.' . $phpEx); + } + + $bots_updates = array( + // Bot Deletions + 'NG-Search [Bot]' => false, + 'Nutch/CVS [Bot]' => false, + 'OmniExplorer [Bot]' => false, + 'Seekport [Bot]' => false, + 'Synoo [Bot]' => false, + 'WiseNut [Bot]' => false, + + // Bot Updates + // Bot name to bot user agent map 'Baidu [Spider]' => 'Baiduspider', 'Exabot [Bot]' => 'Exabot', 'Voyager [Bot]' => 'voyager/', 'W3C [Validator]' => 'W3C_Validator', ); - foreach ($bots_to_update as $bot_name => $bot_agent) + foreach ($bots_updates as $bot_name => $bot_agent) { $sql = 'SELECT user_id FROM ' . USERS_TABLE . ' @@ -2141,44 +2155,21 @@ function change_database_data(&$no_updates, $version) if ($bot_user_id) { - $sql = 'UPDATE ' . BOTS_TABLE . ' - SET bot_agent = ' . $db->sql_escape($bot_agent) . " - WHERE user_id = $bot_user_id"; - _sql($sql, $errored, $error_ary); - } - } + if ($bot_agent === false) + { + $sql = 'DELETE FROM ' . BOTS_TABLE . " + WHERE user_id = $bot_user_id"; + _sql($sql, $errored, $error_ary); - if (!function_exists('user_delete')) - { - include($phpbb_root_path . 'includes/functions_user.' . $phpEx); - } - - $bots_to_delete = array( - 'NG-Search [Bot]', - 'Nutch/CVS [Bot]', - 'OmniExplorer [Bot]', - 'Seekport [Bot]', - 'Synoo [Bot]', - 'WiseNut [Bot]', - ); - - foreach ($bots_to_delete as $bot_name) - { - $sql = 'SELECT user_id - FROM ' . USERS_TABLE . ' - WHERE user_type = ' . USER_IGNORE . " - AND username_clean = '" . $db->sql_escape(utf8_clean_string($bot_name)) . "'"; - $result = $db->sql_query($sql); - $bot_user_id = (int) $db->sql_fetchfield('user_id'); - $db->sql_freeresult($result); - - if ($bot_user_id) - { - $sql = 'DELETE FROM ' . BOTS_TABLE . " - WHERE user_id = $bot_user_id"; - _sql($sql, $errored, $error_ary); - - user_delete('remove', $bot_user_id); + user_delete('remove', $bot_user_id); + } + else + { + $sql = 'UPDATE ' . BOTS_TABLE . " + SET bot_agent = '" . $db->sql_escape($bot_agent) . "' + WHERE user_id = $bot_user_id"; + _sql($sql, $errored, $error_ary); + } } }