From efe25a0b49c7445bef665ab28fc979e485e49289 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sat, 29 Oct 2011 15:31:29 +0200 Subject: [PATCH 1/8] [ticket/9813] Use SHOW TABLE STATUS to get search stats for native on MySQL. PHPBB3-9813 --- phpBB/includes/search/fulltext_native.php | 38 +++++++++++++++++------ 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/phpBB/includes/search/fulltext_native.php b/phpBB/includes/search/fulltext_native.php index b63205fd76..e5eee50fdb 100644 --- a/phpBB/includes/search/fulltext_native.php +++ b/phpBB/includes/search/fulltext_native.php @@ -1461,17 +1461,35 @@ class fulltext_native extends search_backend { global $db; - $sql = 'SELECT COUNT(*) as total_words - FROM ' . SEARCH_WORDLIST_TABLE; - $result = $db->sql_query($sql); - $this->stats['total_words'] = (int) $db->sql_fetchfield('total_words'); - $db->sql_freeresult($result); + switch ($db->sql_layer) + { + case 'mysql4': + case 'mysqli': + $sql = "SHOW TABLE STATUS LIKE '" . SEARCH_WORDLIST_TABLE . "'"; + $result = $db->sql_query($sql); + $this->stats['total_words'] = (int) $db->sql_fetchfield('Rows'); + $db->sql_freeresult($result); - $sql = 'SELECT COUNT(*) as total_matches - FROM ' . SEARCH_WORDMATCH_TABLE; - $result = $db->sql_query($sql); - $this->stats['total_matches'] = (int) $db->sql_fetchfield('total_matches'); - $db->sql_freeresult($result); + $sql = "SHOW TABLE STATUS LIKE '" . SEARCH_WORDMATCH_TABLE . "'"; + $result = $db->sql_query($sql); + $this->stats['total_matches'] = (int) $db->sql_fetchfield('Rows'); + $db->sql_freeresult($result); + break; + + default: + $sql = 'SELECT COUNT(*) as total_words + FROM ' . SEARCH_WORDLIST_TABLE; + $result = $db->sql_query($sql); + $this->stats['total_words'] = (int) $db->sql_fetchfield('total_words'); + $db->sql_freeresult($result); + + $sql = 'SELECT COUNT(*) as total_matches + FROM ' . SEARCH_WORDMATCH_TABLE; + $result = $db->sql_query($sql); + $this->stats['total_matches'] = (int) $db->sql_fetchfield('total_matches'); + $db->sql_freeresult($result); + break; + } } /** From 97cf433dea658127ed60c70ea47d1c3830964f25 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 12 Dec 2011 02:41:48 +0100 Subject: [PATCH 2/8] [ticket/9813] Use table status row count only if greater than 100000 or exact. PHPBB3-9813 --- phpBB/includes/search/fulltext_native.php | 67 ++++++++++++++--------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/phpBB/includes/search/fulltext_native.php b/phpBB/includes/search/fulltext_native.php index e5eee50fdb..69937ff9c9 100644 --- a/phpBB/includes/search/fulltext_native.php +++ b/phpBB/includes/search/fulltext_native.php @@ -1458,38 +1458,53 @@ class fulltext_native extends search_backend } function get_stats() + { + $this->stats['total_words'] = $this->get_table_row_count(SEARCH_WORDLIST_TABLE); + $this->stats['total_matches'] = $this->get_table_row_count(SEARCH_WORDMATCH_TABLE); + } + + /** + * Gets statistics for a specific database table. + * + * @param string $table_name Table name + * + * @return string Row count. Prefixed with ~ if estimated. + * + * @access protected + */ + function get_table_row_count($table_name) { global $db; - switch ($db->sql_layer) + if (stripos($db->sql_layer, 'mysql') === 0) { - case 'mysql4': - case 'mysqli': - $sql = "SHOW TABLE STATUS LIKE '" . SEARCH_WORDLIST_TABLE . "'"; - $result = $db->sql_query($sql); - $this->stats['total_words'] = (int) $db->sql_fetchfield('Rows'); - $db->sql_freeresult($result); + $sql = "SHOW TABLE STATUS + LIKE '" . $table_name . "'"; + $result = $db->sql_query($sql); + $table_status = $db->sql_fetchrow($result); + $db->sql_freeresult($result); - $sql = "SHOW TABLE STATUS LIKE '" . SEARCH_WORDMATCH_TABLE . "'"; - $result = $db->sql_query($sql); - $this->stats['total_matches'] = (int) $db->sql_fetchfield('Rows'); - $db->sql_freeresult($result); - break; - - default: - $sql = 'SELECT COUNT(*) as total_words - FROM ' . SEARCH_WORDLIST_TABLE; - $result = $db->sql_query($sql); - $this->stats['total_words'] = (int) $db->sql_fetchfield('total_words'); - $db->sql_freeresult($result); - - $sql = 'SELECT COUNT(*) as total_matches - FROM ' . SEARCH_WORDMATCH_TABLE; - $result = $db->sql_query($sql); - $this->stats['total_matches'] = (int) $db->sql_fetchfield('total_matches'); - $db->sql_freeresult($result); - break; + if (isset($table_status['Engine'])) + { + if ($table_status['Engine'] === 'MyISAM') + { + return $table_status['Rows']; + } + else if ($table_status['Engine'] === 'InnoDB' && $table_status['Rows'] > 100000) + { + return '~' . $table_status['Rows']; + } + } } + + // Get exact row count by actually counting rows + $sql = 'SELECT COUNT(*) AS rows_total + FROM ' . $table_name; + $result = $db->sql_query($sql); + $rows_total = $db->sql_fetchfield('rows_total'); + $db->sql_freeresult($result); + + return $rows_total; } /** From f9953fc3395e6b206c0789d40f89f9d2463348e7 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sat, 18 Feb 2012 19:29:32 +0100 Subject: [PATCH 3/8] [ticket/10653] Add ability to count table rows to database abstraction layer. PHPBB3-10653 --- phpBB/includes/db/dbal.php | 35 ++++++++++++ phpBB/includes/db/mysql.php | 70 +++++++++++++++++++++++ phpBB/includes/db/mysqli.php | 70 +++++++++++++++++++++++ phpBB/includes/search/fulltext_native.php | 46 +-------------- 4 files changed, 177 insertions(+), 44 deletions(-) diff --git a/phpBB/includes/db/dbal.php b/phpBB/includes/db/dbal.php index 230c9c8ed7..5d456c2ff0 100644 --- a/phpBB/includes/db/dbal.php +++ b/phpBB/includes/db/dbal.php @@ -900,6 +900,41 @@ class dbal return true; } + + /** + * Gets the estimated number of rows in a specified table. + * + * @param string $table_name Table name + * + * @return string Number of rows in $table_name. + * Prefixed with ~ if estimated (otherwise exact). + * + * @access public + */ + function get_estimated_row_count($table_name) + { + return $this->get_row_count($table_name); + } + + /** + * Gets the exact number of rows in a specified table. + * + * @param string $table_name Table name + * + * @return string Exact number of rows in $table_name. + * + * @access public + */ + function get_row_count($table_name) + { + $sql = 'SELECT COUNT(*) AS rows_total + FROM ' . $this->sql_escape($table_name); + $result = $this->sql_query($sql); + $rows_total = $this->sql_fetchfield('rows_total'); + $this->sql_freeresult($result); + + return $rows_total; + } } /** diff --git a/phpBB/includes/db/mysql.php b/phpBB/includes/db/mysql.php index 1e24c79577..87413e808d 100644 --- a/phpBB/includes/db/mysql.php +++ b/phpBB/includes/db/mysql.php @@ -318,6 +318,76 @@ class dbal_mysql extends dbal return @mysql_real_escape_string($msg, $this->db_connect_id); } + /** + * Gets the estimated number of rows in a specified table. + * + * @param string $table_name Table name + * + * @return string Number of rows in $table_name. + * Prefixed with ~ if estimated (otherwise exact). + * + * @access public + */ + function get_estimated_row_count($table_name) + { + $table_status = $this->get_table_status($table_name); + + if (isset($table_status['Engine'])) + { + if ($table_status['Engine'] === 'MyISAM') + { + return $table_status['Rows']; + } + else if ($table_status['Engine'] === 'InnoDB' && $table_status['Rows'] > 100000) + { + return '~' . $table_status['Rows']; + } + } + + return $this->get_row_count($table_name); + } + + /** + * Gets the exact number of rows in a specified table. + * + * @param string $table_name Table name + * + * @return string Exact number of rows in $table_name. + * + * @access public + */ + function get_row_count($table_name) + { + $table_status = $this->get_table_status($table_name); + + if (isset($table_status['Engine']) && $table_status['Engine'] === 'MyISAM') + { + return $table_status['Rows']; + } + + return parent::get_row_count($table_name); + } + + /** + * Gets some information about the specified table. + * + * @param string $table_name Table name + * + * @return array + * + * @access protected + */ + function get_table_status($table_name) + { + $sql = "SHOW TABLE STATUS + LIKE '" . $this->sql_escape($table_name) . "'"; + $result = $this->sql_query($sql); + $table_status = $this->sql_fetchrow($result); + $this->sql_freeresult($result); + + return $table_status; + } + /** * Build LIKE expression * @access private diff --git a/phpBB/includes/db/mysqli.php b/phpBB/includes/db/mysqli.php index 456ce906d0..49b2789ce6 100644 --- a/phpBB/includes/db/mysqli.php +++ b/phpBB/includes/db/mysqli.php @@ -315,6 +315,76 @@ class dbal_mysqli extends dbal return @mysqli_real_escape_string($this->db_connect_id, $msg); } + /** + * Gets the estimated number of rows in a specified table. + * + * @param string $table_name Table name + * + * @return string Number of rows in $table_name. + * Prefixed with ~ if estimated (otherwise exact). + * + * @access public + */ + function get_estimated_row_count($table_name) + { + $table_status = $this->get_table_status($table_name); + + if (isset($table_status['Engine'])) + { + if ($table_status['Engine'] === 'MyISAM') + { + return $table_status['Rows']; + } + else if ($table_status['Engine'] === 'InnoDB' && $table_status['Rows'] > 100000) + { + return '~' . $table_status['Rows']; + } + } + + return $this->get_row_count($table_name); + } + + /** + * Gets the exact number of rows in a specified table. + * + * @param string $table_name Table name + * + * @return string Exact number of rows in $table_name. + * + * @access public + */ + function get_row_count($table_name) + { + $table_status = $this->get_table_status($table_name); + + if (isset($table_status['Engine']) && $table_status['Engine'] === 'MyISAM') + { + return $table_status['Rows']; + } + + return parent::get_row_count($table_name); + } + + /** + * Gets some information about the specified table. + * + * @param string $table_name Table name + * + * @return array + * + * @access protected + */ + function get_table_status($table_name) + { + $sql = "SHOW TABLE STATUS + LIKE '" . $this->sql_escape($table_name) . "'"; + $result = $this->sql_query($sql); + $table_status = $this->sql_fetchrow($result); + $this->sql_freeresult($result); + + return $table_status; + } + /** * Build LIKE expression * @access private diff --git a/phpBB/includes/search/fulltext_native.php b/phpBB/includes/search/fulltext_native.php index 69937ff9c9..dc961f3c8a 100644 --- a/phpBB/includes/search/fulltext_native.php +++ b/phpBB/includes/search/fulltext_native.php @@ -1458,53 +1458,11 @@ class fulltext_native extends search_backend } function get_stats() - { - $this->stats['total_words'] = $this->get_table_row_count(SEARCH_WORDLIST_TABLE); - $this->stats['total_matches'] = $this->get_table_row_count(SEARCH_WORDMATCH_TABLE); - } - - /** - * Gets statistics for a specific database table. - * - * @param string $table_name Table name - * - * @return string Row count. Prefixed with ~ if estimated. - * - * @access protected - */ - function get_table_row_count($table_name) { global $db; - if (stripos($db->sql_layer, 'mysql') === 0) - { - $sql = "SHOW TABLE STATUS - LIKE '" . $table_name . "'"; - $result = $db->sql_query($sql); - $table_status = $db->sql_fetchrow($result); - $db->sql_freeresult($result); - - if (isset($table_status['Engine'])) - { - if ($table_status['Engine'] === 'MyISAM') - { - return $table_status['Rows']; - } - else if ($table_status['Engine'] === 'InnoDB' && $table_status['Rows'] > 100000) - { - return '~' . $table_status['Rows']; - } - } - } - - // Get exact row count by actually counting rows - $sql = 'SELECT COUNT(*) AS rows_total - FROM ' . $table_name; - $result = $db->sql_query($sql); - $rows_total = $db->sql_fetchfield('rows_total'); - $db->sql_freeresult($result); - - return $rows_total; + $this->stats['total_words'] = $db->get_estimated_row_count(SEARCH_WORDLIST_TABLE); + $this->stats['total_matches'] = $db->get_estimated_row_count(SEARCH_WORDMATCH_TABLE); } /** From 6df721b21528e4d9881fa191159b123bb2b6b33f Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 4 Mar 2012 01:59:40 +0100 Subject: [PATCH 4/8] [ticket/10653] Unit tests for get_row_count() and get_estimated_row_count(). PHPBB3-10653 --- tests/dbal/select_test.php | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/dbal/select_test.php b/tests/dbal/select_test.php index 05b0e68e29..9d46b47b17 100644 --- a/tests/dbal/select_test.php +++ b/tests/dbal/select_test.php @@ -357,4 +357,29 @@ class phpbb_dbal_select_test extends phpbb_database_test_case $this->assertSame(false, $row); } + + public function test_get_row_count() + { + $this->assertSame( + 3, + (int) $this->new_dbal()->get_row_count('phpbb_users'), + "Failed asserting that user table has exactly 3 rows." + ); + } + + public function test_get_estimated_row_count() + { + $actual = $this->new_dbal()->get_estimated_row_count('phpbb_users'); + + if (is_string($actual) && isset($actual[0]) && $actual[0] === '~') + { + $actual = substr($actual, -1); + } + + $this->assertGreaterThan( + 1, + $actual, + "Failed asserting that estimated row count of user table is greater than 1." + ); + } } From f3af5945e3675c4d799000908c88475ee7016ac4 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Sun, 4 Mar 2012 22:11:45 +0100 Subject: [PATCH 5/8] [ticket/10653] Fix parameter to substr() in unit tests. Should be 1, not -1. PHPBB3-10653 --- tests/dbal/select_test.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/dbal/select_test.php b/tests/dbal/select_test.php index 9d46b47b17..21b12777dc 100644 --- a/tests/dbal/select_test.php +++ b/tests/dbal/select_test.php @@ -373,7 +373,7 @@ class phpbb_dbal_select_test extends phpbb_database_test_case if (is_string($actual) && isset($actual[0]) && $actual[0] === '~') { - $actual = substr($actual, -1); + $actual = substr($actual, 1); } $this->assertGreaterThan( From 785c75254e763a24cf56765ca9a7e02140982497 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 5 Mar 2012 00:29:28 +0100 Subject: [PATCH 6/8] [ticket/9813] Also use estimated row count of posts table for fulltext mysql. Since this is 'only' for statistics anyway, using an estimated value does no harm. Also, if MyISAM is the underlying storage engine for the posts table, the value will actually be exact. PHPBB3-9813 --- phpBB/includes/search/fulltext_mysql.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/phpBB/includes/search/fulltext_mysql.php b/phpBB/includes/search/fulltext_mysql.php index 52372a14d8..54cc894f6a 100644 --- a/phpBB/includes/search/fulltext_mysql.php +++ b/phpBB/includes/search/fulltext_mysql.php @@ -896,11 +896,7 @@ class fulltext_mysql extends search_backend } $db->sql_freeresult($result); - $sql = 'SELECT COUNT(post_id) as total_posts - FROM ' . POSTS_TABLE; - $result = $db->sql_query($sql); - $this->stats['total_posts'] = (int) $db->sql_fetchfield('total_posts'); - $db->sql_freeresult($result); + $this->stats['total_posts'] = $db->get_estimated_row_count(POSTS_TABLE); } /** From 01d90e59a8c311778a832ba4920e4c5a85b1256f Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 5 Mar 2012 00:42:36 +0100 Subject: [PATCH 7/8] [ticket/9813] Only get posts table row count if we detected a fulltext index. PHPBB3-9813 --- phpBB/includes/search/fulltext_mysql.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/includes/search/fulltext_mysql.php b/phpBB/includes/search/fulltext_mysql.php index 54cc894f6a..779ec1d216 100644 --- a/phpBB/includes/search/fulltext_mysql.php +++ b/phpBB/includes/search/fulltext_mysql.php @@ -896,7 +896,7 @@ class fulltext_mysql extends search_backend } $db->sql_freeresult($result); - $this->stats['total_posts'] = $db->get_estimated_row_count(POSTS_TABLE); + $this->stats['total_posts'] = empty($this->stats) ? 0 : $db->get_estimated_row_count(POSTS_TABLE); } /** From c11607bdd1ad4209a7697f0dc8d46ea81795b14c Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 7 Mar 2012 12:11:18 +0100 Subject: [PATCH 8/8] [ticket/10653] Call get_row_count of base class in mysql get_estimated_row_count There is no point in fetching the table status again. PHPBB3-10653 --- phpBB/includes/db/mysql.php | 2 +- phpBB/includes/db/mysqli.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/phpBB/includes/db/mysql.php b/phpBB/includes/db/mysql.php index 87413e808d..1ccb785150 100644 --- a/phpBB/includes/db/mysql.php +++ b/phpBB/includes/db/mysql.php @@ -344,7 +344,7 @@ class dbal_mysql extends dbal } } - return $this->get_row_count($table_name); + return parent::get_row_count($table_name); } /** diff --git a/phpBB/includes/db/mysqli.php b/phpBB/includes/db/mysqli.php index 49b2789ce6..a311b8cda6 100644 --- a/phpBB/includes/db/mysqli.php +++ b/phpBB/includes/db/mysqli.php @@ -341,7 +341,7 @@ class dbal_mysqli extends dbal } } - return $this->get_row_count($table_name); + return parent::get_row_count($table_name); } /**