From 80a08d9c5401f9834fbe94918c63cfc77dec172c Mon Sep 17 00:00:00 2001 From: rxu Date: Sun, 12 Dec 2021 10:53:56 +0700 Subject: [PATCH 1/2] [ticket/16941] Add sphinx tests to 3.3.x Also adjust Sphinx keywords splitting to be consistent with other search backends when it comes to handling hyphen (like ignoring hyphen when it hasn't NOT meaning and ignoring hyphen wrapped with "plus" signs) PHPBB3-16941 --- .github/setup-sphinx.sh | 144 ++++++++++++++++++ .github/workflows/tests.yml | 4 + phpBB/phpbb/search/fulltext_sphinx.php | 42 ++++- tests/functional/search/base.php | 25 ++- tests/functional/search/sphinx_test.php | 25 ++- .../functional/visibility_softdelete_test.php | 11 +- .../phpbb_functional_test_case.php | 18 +++ 7 files changed, 263 insertions(+), 6 deletions(-) create mode 100755 .github/setup-sphinx.sh diff --git a/.github/setup-sphinx.sh b/.github/setup-sphinx.sh new file mode 100755 index 0000000000..8e1c861809 --- /dev/null +++ b/.github/setup-sphinx.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# +# This file is part of the phpBB Forum Software package. +# +# @copyright (c) phpBB Limited +# @license GNU General Public License, version 2 (GPL-2.0) +# +# For full copyright and license information, please see +# the docs/CREDITS.txt file. +# +set -e +set -x + +sudo apt-get update +sudo apt-get install -q -y sphinxsearch + +DIR=$(dirname "$0") + +SPHINX_DAEMON_HOST="localhost" +SPHINX_DAEMON_PORT="9312" +SPHINX_CONF="$DIR/sphinx.conf" +SPHINX_DATA_DIR="/var/run/sphinxsearch" +SPHINX_LOG="$SPHINX_DATA_DIR/log/searchd.log" +SPHINX_QUERY_LOG="$SPHINX_DATA_DIR/log/sphinx-query.log" +ID="gokw5rvjvvxp8kgj" # Randomly generated via phpBB unique_id() + +PHPBB_TEST_DBHOST="0.0.0.0" +PHPBB_TEST_DBNAME="phpbb_tests" +PHPBB_TEST_DBUSER="root" +PHPBB_TEST_DBPASSWD="" + +sudo service sphinxsearch stop +sudo mkdir -p "$SPHINX_DATA_DIR/log" +sudo chown "sphinxsearch" "$SPHINX_DATA_DIR/log" + +# Generate configuration file for Sphinx +echo " +source source_phpbb_${ID}_main +{ + type = mysql # mysql or pgsql + sql_host = $PHPBB_TEST_DBHOST + sql_user = $PHPBB_TEST_DBUSER + sql_pass = $PHPBB_TEST_DBPASSWD + sql_db = $PHPBB_TEST_DBNAME + sql_port = + sql_query_pre = SET NAMES 'utf8' + sql_query_pre = UPDATE phpbb_sphinx SET max_doc_id = (SELECT MAX(post_id) FROM phpbb_posts) WHERE counter_id = 1 + sql_query_range = SELECT MIN(post_id), MAX(post_id) FROM phpbb_posts + sql_range_step = 5000 + sql_query = SELECT \ + p.post_id AS id, \ + p.forum_id, \ + p.topic_id, \ + p.poster_id, \ + p.post_visibility, \ + CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post, \ + p.post_time, \ + p.post_subject, \ + p.post_subject as title, \ + p.post_text as data, \ + t.topic_last_post_time, \ + 0 as deleted \ + FROM phpbb_posts p, phpbb_topics t \ + WHERE \ + p.topic_id = t.topic_id \ + AND p.post_id >= \$start AND p.post_id <= \$end + sql_query_post = + sql_query_post_index = UPDATE phpbb_sphinx SET max_doc_id = \$maxid WHERE counter_id = 1 + sql_attr_uint = forum_id + sql_attr_uint = topic_id + sql_attr_uint = poster_id + sql_attr_uint = post_visibility + sql_attr_bool = topic_first_post + sql_attr_bool = deleted + sql_attr_timestamp = post_time + sql_attr_timestamp = topic_last_post_time + sql_attr_string = post_subject +} +source source_phpbb_${ID}_delta : source_phpbb_${ID}_main +{ + sql_query_pre = SET NAMES 'utf8' + sql_query_range = + sql_range_step = + sql_query = SELECT \ + p.post_id AS id, \ + p.forum_id, \ + p.topic_id, \ + p.poster_id, \ + p.post_visibility, \ + CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post, \ + p.post_time, \ + p.post_subject, \ + p.post_subject as title, \ + p.post_text as data, \ + t.topic_last_post_time, \ + 0 as deleted \ + FROM phpbb_posts p, phpbb_topics t \ + WHERE \ + p.topic_id = t.topic_id \ + AND p.post_id >= ( SELECT max_doc_id FROM phpbb_sphinx WHERE counter_id=1 ) + sql_query_post_index = +} +index index_phpbb_${ID}_main +{ + path = $SPHINX_DATA_DIR/index_phpbb_${ID}_main + source = source_phpbb_${ID}_main + docinfo = extern + morphology = none + stopwords = + wordforms = + exceptions = + min_word_len = 2 + charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF + ignore_chars = U+0027, U+002C + min_prefix_len = 3 + min_infix_len = 0 + html_strip = 1 + index_exact_words = 0 + blend_chars = U+23, U+24, U+25, U+26, U+40 +} +index index_phpbb_${ID}_delta : index_phpbb_${ID}_main +{ + path = $SPHINX_DATA_DIR/index_phpbb_${ID}_delta + source = source_phpbb_${ID}_delta +} +indexer +{ + mem_limit = 512M +} +searchd +{ + listen = $SPHINX_DAEMON_PORT + log = $SPHINX_LOG + query_log = $SPHINX_QUERY_LOG + read_timeout = 5 + max_children = 30 + pid_file = $SPHINX_DATA_DIR/searchd.pid + binlog_path = $SPHINX_DATA_DIR +} +" > $SPHINX_CONF + +sudo mv "$SPHINX_CONF" "/etc/sphinxsearch/sphinx.conf" +sudo sed -i "s/START=no/START=yes/g" "/etc/default/sphinxsearch" +sudo chmod 777 "/var/run/sphinxsearch" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e3d8ea8643..8def02d28e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -211,6 +211,10 @@ jobs: run: | .github/setup-ldap.sh + - name: Setup SPHINX + run: | + .github/setup-sphinx.sh + - name: Lint tests if: ${{ matrix.SLOWTESTS != 1 && steps.database-type.outputs.db == 'mysql' }} run: phpBB/vendor/bin/phpunit tests/lint_test.php diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index 4f3c6c4cc3..4ee40582c5 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -455,9 +455,47 @@ class fulltext_sphinx $this->sphinx->SetMatchMode(SPH_MATCH_ANY); } - if (strlen($keywords) > 0) + // Split words + $split_keywords = preg_replace('#([^\p{L}\p{N}\'*"()])#u', '$1$1', str_replace('\'\'', '\' \'', trim($keywords))); + $matches = array(); + preg_match_all('#(?:[^\p{L}\p{N}*"()]|^)([+\-|]?(?:[\p{L}\p{N}*"()]+\'?)*[\p{L}\p{N}*"()])(?:[^\p{L}\p{N}*"()]|$)#u', $split_keywords, $matches); + $this->split_words = $matches[1]; + + if ($terms == 'any') { - $this->search_query = str_replace('"', '"', $keywords); + $this->search_query = ''; + foreach ($this->split_words as $word) + { + if ((strpos($word, '+') === 0) || (strpos($word, '-') === 0) || (strpos($word, '|') === 0)) + { + $word = substr($word, 1); + } + $this->search_query .= $word . ' '; + } + } + else + { + $this->search_query = ''; + foreach ($this->split_words as $word) + { + if ((strpos($word, '+') === 0) || (strpos($word, '-') === 0)) + { + $this->search_query .= $word . ' '; + } + else if (strpos($word, '|') === 0) + { + $this->search_query .= substr($word, 1) . ' '; + } + else + { + $this->search_query .= '+' . $word . ' '; + } + } + } + + if ($this->search_query) + { + $this->search_query = str_replace('"', '"', $this->search_query); return true; } diff --git a/tests/functional/search/base.php b/tests/functional/search/base.php index e5ee1feee3..af5bd8066a 100644 --- a/tests/functional/search/base.php +++ b/tests/functional/search/base.php @@ -150,8 +150,19 @@ abstract class phpbb_functional_search_base extends phpbb_functional_test_case if ($values["config[search_type]"] != $this->search_backend) { $values["config[search_type]"] = $this->search_backend; + + if (strpos($this->search_backend, 'fulltext_sphinx')) + { + // Set board Sphinx id in according to respective setup-sphinx.sh $ID value + $sql = 'UPDATE ' . CONFIG_TABLE . " + SET config_value = '" . $this->db->sql_escape('gokw5rvjvvxp8kgj') . "' + WHERE config_name = '" . $this->db->sql_escape('fulltext_sphinx_id') . "'"; + $this->db->sql_query($sql); + } + $form->setValues($values); $crawler = self::submit($form); + $this->purge_cache(); $form = $crawler->selectButton($this->lang('YES'))->form(); $values = $form->getValues(); @@ -244,7 +255,12 @@ abstract class phpbb_functional_search_base extends phpbb_functional_test_case // Ensure search index has been actually created $crawler = self::request('GET', 'adm/index.php?i=acp_search&mode=index&sid=' . $this->sid); - $posts_indexed = (int) $crawler->filter('#acp_search_index_' . $search_type . ' td')->eq(1)->text(); + $posts_indexed = (int) $crawler->filter('#acp_search_index_' . $search_type . ' td')->reduce( + function ($node, $i) { + // Find the value of total posts indexed + return (strpos($node->text(), $this->lang('FULLTEXT_MYSQL_TOTAL_POSTS')) !== false || strpos($node->text(), $this->lang('TOTAL_WORDS')) !== false); + }) + ->nextAll()->eq(0)->text(); $this->assertTrue($posts_indexed > 0); } @@ -281,7 +297,12 @@ abstract class phpbb_functional_search_base extends phpbb_functional_test_case // Ensure search index has been actually removed $crawler = self::request('GET', 'adm/index.php?i=acp_search&mode=index&sid=' . $this->sid); - $posts_indexed = (int) $crawler->filter('#acp_search_index_' . $this->search_backend . ' td')->eq(1)->text(); + $posts_indexed = (int) $crawler->filter('#acp_search_index_' . $this->search_backend . ' td')->reduce( + function ($node, $i) { + // Find the value of total posts indexed + return (strpos($node->text(), $this->lang('FULLTEXT_MYSQL_TOTAL_POSTS')) !== false || strpos($node->text(), $this->lang('TOTAL_WORDS')) !== false); + }) + ->nextAll()->eq(0)->text(); $this->assertEquals(0, $posts_indexed); } } diff --git a/tests/functional/search/sphinx_test.php b/tests/functional/search/sphinx_test.php index 926f1c6424..f5fa893519 100644 --- a/tests/functional/search/sphinx_test.php +++ b/tests/functional/search/sphinx_test.php @@ -20,8 +20,31 @@ class phpbb_functional_search_sphinx_test extends phpbb_functional_search_base { protected $search_backend = '\phpbb\search\fulltext_sphinx'; + protected function create_search_index($backend = null) + { + parent::create_search_index($backend); + $this->purge_cache(); + + if (!$backend || $this->search_backend == $backend) + { + $output = $retval = null; + + // After creating phpBB search index, build Sphinx index + exec('sudo -S service sphinxsearch stop', $output, $retval); // Attempt to stop sphinxsearch service in case it's running + exec('sudo -S indexer --all', $output, $retval); // Run sphinxsearch indexer + exec('sudo -S service sphinxsearch start', $output, $retval); // Attempt to start sphinxsearch service again + } + } + public function test_search_backend() { - $this->markTestIncomplete('Sphinx Tests are not supported'); + if ($this->db->sql_layer != 'mysqli') // Sphinx test runs on MySQL/MariaDB only so far + { + $this->markTestIncomplete('Sphinx Tests are not supported'); + } + else + { + parent::test_search_backend(); + } } } diff --git a/tests/functional/visibility_softdelete_test.php b/tests/functional/visibility_softdelete_test.php index 5128bb6005..0275ea42ca 100644 --- a/tests/functional/visibility_softdelete_test.php +++ b/tests/functional/visibility_softdelete_test.php @@ -601,7 +601,16 @@ class phpbb_functional_visibility_softdelete_test extends phpbb_functional_test_ // Assert new topic title is indexed as well $this->add_lang('search'); self::request('GET', "search.php?keywords=bang&sid={$this->sid}"); - $this->assertStringContainsString(sprintf($this->lang['FOUND_SEARCH_MATCHES'][1], 1), self::get_content()); + + // Sphinx search doesn't apply to unapproved or softdeleted posts + if (strpos($this->get_search_type(), 'fulltext_sphinx')) + { + $this->assertStringContainsString(sprintf($this->lang['FOUND_SEARCH_MATCHES'][2], 0), self::get_content()); + } + else + { + $this->assertStringContainsString(sprintf($this->lang['FOUND_SEARCH_MATCHES'][1], 1), self::get_content()); + } } public function test_move_topic_back() diff --git a/tests/test_framework/phpbb_functional_test_case.php b/tests/test_framework/phpbb_functional_test_case.php index 6f8bc2d013..dd3b14e715 100644 --- a/tests/test_framework/phpbb_functional_test_case.php +++ b/tests/test_framework/phpbb_functional_test_case.php @@ -735,6 +735,24 @@ class phpbb_functional_test_case extends phpbb_test_case return $group_id; } + /** + * Get current board's search type + * + * @return string Current search type setting + */ + protected function get_search_type() + { + $db = $this->get_db(); + $sql = 'SELECT config_value as search_type + FROM ' . CONFIG_TABLE . " + WHERE config_name = '" . $db->sql_escape('search_type') . "'"; + $result = $db->sql_query($sql); + $search_type = $db->sql_fetchfield('search_type'); + $db->sql_freeresult($result); + + return $search_type; + } + protected function remove_user_group($group_name, $usernames) { global $db, $cache, $auth, $config, $phpbb_dispatcher, $phpbb_log, $phpbb_container, $user, $phpbb_root_path, $phpEx; From 1ae9a49811189859669726aa4fa06a7def4f8ebe Mon Sep 17 00:00:00 2001 From: rxu Date: Fri, 25 Apr 2025 21:04:26 +0700 Subject: [PATCH 2/2] [ticket/16941] Remove ending slash from binlog_path PHPBB3-14401 --- phpBB/phpbb/search/fulltext_sphinx.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/phpbb/search/fulltext_sphinx.php b/phpBB/phpbb/search/fulltext_sphinx.php index 4ee40582c5..8affc60b15 100644 --- a/phpBB/phpbb/search/fulltext_sphinx.php +++ b/phpBB/phpbb/search/fulltext_sphinx.php @@ -356,7 +356,7 @@ class fulltext_sphinx array('read_timeout', '5'), array('max_children', '30'), array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'), - array('binlog_path', $this->config['fulltext_sphinx_data_path']), + array('binlog_path', rtrim($this->config['fulltext_sphinx_data_path'], '/\\')), // Trim trailing slash ), );