makary/phpbb
1
0
Fork 0
mirror of https://github.com/phpbb/phpbb.git synced 2025-06-11 22:08:54 +00:00
Code Issues Projects Releases Packages Wiki Activity

[ticket/15540] Check method signatures, visibility, etc

Browse source 
PHPBB3-15540
This commit is contained in:
rubencm 2021-03-22 19:56:48 +01:00
parent fd45ce1e91
commit 9dac66071e
9 changed files with 585 additions and 707 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
phpBB/develop/create_search_index.php
View file

@ -1,140 +0,0 @@
<?php
/**
*
* This file is part of the phpBB Forum Software package.
*
* @copyright (c) phpBB Limited <https://www.phpbb.com>
* @license GNU General Public License, version 2 (GPL-2.0)
*
* For full copyright and license information, please see
* the docs/CREDITS.txt file.
*
*/
if (php_sapi_name() != 'cli')
{
die("This program must be run from the command line.\n");
}
if ($argc < 2)
{
echo 'Usage: php ' . basename(__FILE__) . " index_type [batch_size]\n";
exit(1);
}
$class_name = basename($argv[1]);
define('IN_PHPBB', true);
$phpbb_root_path = (defined('PHPBB_ROOT_PATH')) ? PHPBB_ROOT_PATH : '../';
$phpEx = substr(strrchr(__FILE__, '.'), 1);
require($phpbb_root_path . 'common.' . $phpEx);
require($phpbb_root_path . 'includes/acp/acp_search.' . $phpEx);
$user->session_begin();
$auth->acl($user->data);
$user->setup('acp/search');
$search_name = ucfirst(strtolower(str_replace('_', ' ', $class_name)));
$search_errors = array();
$search = new $class_name($search_errors);
$batch_size = isset($argv[2]) ? $argv[2] : 2000;
if (method_exists($search, 'create_index'))
{
if ($error = $search->create_index(null, ''))
{
var_dump($error);
exit(1);
}
}
else
{
$sql = 'SELECT forum_id, enable_indexing
FROM ' . FORUMS_TABLE;
$result = $db->sql_query($sql);
while ($row = $db->sql_fetchrow($result))
{
$forums[$row['forum_id']] = (bool) $row['enable_indexing'];
}
$db->sql_freeresult($result);
$sql = 'SELECT post_id
FROM ' . POSTS_TABLE . '
ORDER BY post_id DESC';
$result = $db->sql_query_limit($sql, 1);
$max_post_id = (int) $db->sql_fetchfield('post_id');
$post_counter = 0;
while ($post_counter <= $max_post_id)
{
$row_count = 0;
$time = time();
printf("Processing posts with %d <= post_id <= %d\n",
$post_counter + 1,
$post_counter + $batch_size
);
$sql = 'SELECT post_id, post_subject, post_text, poster_id, forum_id
FROM ' . POSTS_TABLE . '
WHERE post_id >= ' . (int) ($post_counter + 1) . '
AND post_id <= ' . (int) ($post_counter + $batch_size);
$result = $db->sql_query($sql);
$buffer = $db->sql_buffer_nested_transactions();
if ($buffer)
{
$rows = $db->sql_fetchrowset($result);
$rows[] = false; // indicate end of array for while loop below
$db->sql_freeresult($result);
}
$i = 0;
while ($row = ($buffer ? $rows[$i++] : $db->sql_fetchrow($result)))
{
// Indexing enabled for this forum or global announcement?
// Global announcements get indexed by default.
if (!$row['forum_id'] || !empty($forums[$row['forum_id']]))
{
++$row_count;
$search->index('post',
$row['post_id'],
$row['post_text'],
$row['post_subject'],
$row['poster_id'],
$row['forum_id']
);
if ($row_count % 10 == 0)
{
echo '.';
}
}
}
$delta = (time() - $time);
$delta = $delta <= 0 ? 1 : $delta;
printf(" %d posts/sec\n", $row_count / $delta);
if (!$buffer)
{
$db->sql_freeresult($result);
}
$post_counter += $batch_size;
}
}
$search->tidy();
$phpbb_log->add('admin', $user->data['user_id'], $user->ip, 'LOG_SEARCH_INDEX_CREATED', false, array($search_name));
echo $user->lang['SEARCH_INDEX_CREATED'] . "\n";
echo 'Peak Memory Usage: ' . get_formatted_filesize(memory_get_peak_usage()) . "\n";
exit(0);

5
phpBB/includes/mcp/mcp_post.php
View file

@ -635,10 +635,7 @@ function change_poster(&$post_info, $userdata)
$search_backend_factory = $phpbb_container->get('search.backend_factory');
$search = $search_backend_factory->get_active();
if (method_exists($search, 'destroy_cache'))
{
$search->destroy_cache(array(), array($post_info['user_id'], $userdata['user_id']));
}
$search->index_remove([], [$post_info['user_id'], $userdata['user_id']], []);
$from_username = $post_info['username'];
$to_username = $userdata['username'];

6
phpBB/phpbb/search/backend/base.php
View file

@ -35,7 +35,7 @@ abstract class base implements search_backend_interface
*
* @return int self::SEARCH_RESULT_NOT_IN_CACHE or self::SEARCH_RESULT_IN_CACHE or self::SEARCH_RESULT_INCOMPLETE
*/
public function obtain_ids(string $search_key, &$result_count, &$id_ary, &$start, $per_page, string $sort_dir): int
protected function obtain_ids(string $search_key, &$result_count, &$id_ary, &$start, $per_page, string $sort_dir): int
{
global $cache;
@ -116,7 +116,7 @@ abstract class base implements search_backend_interface
*
* @return null
*/
public function save_ids(string $search_key, string $keywords, $author_ary, int $result_count, &$id_ary, int $start, string $sort_dir)
protected function save_ids(string $search_key, string $keywords, $author_ary, int $result_count, &$id_ary, int $start, string $sort_dir)
{
global $cache, $config, $db, $user;
@ -226,7 +226,7 @@ abstract class base implements search_backend_interface
* @param array $words
* @param array|bool $authors
*/
public function destroy_cache($words, $authors = false): void
protected function destroy_cache($words, $authors = false): void
{
global $db, $cache, $config;

77
phpBB/phpbb/search/backend/fulltext_mysql.php
View file

@ -348,34 +348,6 @@ class fulltext_mysql extends base implements search_backend_interface
return false;
}
/**
* Turns text into an array of words
* @param string $text contains post text/subject
*
* @return array
*/
public function split_message($text): array
{
// Split words
$text = preg_replace('#([^\p{L}\p{N}\'*])#u', '$1$1', str_replace('\'\'', '\' \'', trim($text)));
$matches = array();
preg_match_all('#(?:[^\p{L}\p{N}*]|^)([+\-|]?(?:[\p{L}\p{N}*]+\'?)*[\p{L}\p{N}*])(?:[^\p{L}\p{N}*]|$)#u', $text, $matches);
$text = $matches[1];
// remove too short or too long words
$text = array_values($text);
for ($i = 0, $n = count($text); $i < $n; $i++)
{
$text[$i] = trim($text[$i]);
if (utf8_strlen($text[$i]) < $this->config['fulltext_mysql_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_mysql_max_word_len'])
{
unset($text[$i]);
}
}
return array_values($text);
}
/**
* {@inheritdoc}
*/
@ -871,14 +843,15 @@ class fulltext_mysql extends base implements search_backend_interface
}
/**
* Destroys cached search results, that contained one of the new words in a post so the results won't be outdated
*
* @param string $mode contains the post mode: edit, post, reply, quote ...
* @param int $post_id contains the post id of the post to index
* @param string $message contains the post text of the post
* @param string $subject contains the subject of the post to index
* @param int $poster_id contains the user id of the poster
* @param int $forum_id contains the forum id of parent forum of the post
* {@inheritdoc}
*/
public function supports_phrase_search(): bool
{
return false;
}
/**
* {@inheritdoc}
*/
public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
{
@ -930,7 +903,7 @@ class fulltext_mysql extends base implements search_backend_interface
*/
public function index_remove($post_ids, $author_ids, $forum_ids)
{
$this->destroy_cache(array(), array_unique($author_ids));
$this->destroy_cache([], array_unique($author_ids));
}
/**
@ -939,7 +912,7 @@ class fulltext_mysql extends base implements search_backend_interface
public function tidy()
{
// destroy too old cached search results
$this->destroy_cache(array());
$this->destroy_cache([]);
$this->config->set('search_last_gc', time(), false);
}
@ -1151,6 +1124,34 @@ class fulltext_mysql extends base implements search_backend_interface
$this->stats['total_posts'] = empty($this->stats) ? 0 : $this->db->get_estimated_row_count(POSTS_TABLE);
}
/**
* Turns text into an array of words
* @param string $text contains post text/subject
*
* @return array
*/
protected function split_message($text): array
{
// Split words
$text = preg_replace('#([^\p{L}\p{N}\'*])#u', '$1$1', str_replace('\'\'', '\' \'', trim($text)));
$matches = array();
preg_match_all('#(?:[^\p{L}\p{N}*]|^)([+\-|]?(?:[\p{L}\p{N}*]+\'?)*[\p{L}\p{N}*])(?:[^\p{L}\p{N}*]|$)#u', $text, $matches);
$text = $matches[1];
// remove too short or too long words
$text = array_values($text);
for ($i = 0, $n = count($text); $i < $n; $i++)
{
$text[$i] = trim($text[$i]);
if (utf8_strlen($text[$i]) < $this->config['fulltext_mysql_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_mysql_max_word_len'])
{
unset($text[$i]);
}
}
return array_values($text);
}
/**
* {@inheritdoc}
*/

154
phpBB/phpbb/search/backend/fulltext_native.php
View file

@ -1303,78 +1303,11 @@ class fulltext_native extends base implements search_backend_interface
}
/**
* Split a text into words of a given length
*
* The text is converted to UTF-8, cleaned up, and split. Then, words that
* conform to the defined length range are returned in an array.
*
* NOTE: duplicates are NOT removed from the return array
*
* @param string $text Text to split, encoded in UTF-8
* @return array Array of UTF-8 words
* {@inheritdoc}
*/
public function split_message($text)
public function supports_phrase_search(): bool
{
$match = $words = array();
/**
* Taken from the original code
*/
// Do not index code
$match[] = '#\[code(?:=.*?)?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is';
// BBcode
$match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#';
$min = $this->word_length['min'];
$isset_min = $min - 1;
/**
* Clean up the string, remove HTML tags, remove BBCodes
*/
$word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), -1), ' ');
while (strlen($word))
{
if (strlen($word) > 255 || strlen($word) <= $isset_min)
{
/**
* Words longer than 255 bytes are ignored. This will have to be
* changed whenever we change the length of search_wordlist.word_text
*
* Words shorter than $isset_min bytes are ignored, too
*/
$word = strtok(' ');
continue;
}
$len = utf8_strlen($word);
/**
* Test whether the word is too short to be indexed.
*
* Note that this limit does NOT apply to CJK and Hangul
*/
if ($len < $min)
{
/**
* Note: this could be optimized. If the codepoint is lower than Hangul's range
* we know that it will also be lower than CJK ranges
*/
if ((strncmp($word, self::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, self::UTF8_HANGUL_LAST, 3) > 0)
&& (strncmp($word, self::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, self::UTF8_CJK_LAST, 3) > 0)
&& (strncmp($word, self::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, self::UTF8_CJK_B_LAST, 4) > 0))
{
$word = strtok(' ');
continue;
}
}
$words[] = $word;
$word = strtok(' ');
}
return $words;
return false;
}
/**
@ -1686,6 +1619,8 @@ class fulltext_native extends base implements search_backend_interface
$this->config->set('search_last_gc', time(), false);
}
// create_index is inherited from base.php
/**
* {@inheritdoc}
*/
@ -1733,14 +1668,14 @@ class fulltext_native extends base implements search_backend_interface
/**
* {@inheritdoc}
*/
public function index_created()
public function index_created(): bool
{
if (!count($this->stats))
{
$this->get_stats();
}
return ($this->stats['total_words'] && $this->stats['total_matches']) ? true : false;
return $this->stats['total_words'] && $this->stats['total_matches'];
}
/**
@ -1767,6 +1702,81 @@ class fulltext_native extends base implements search_backend_interface
$this->stats['total_matches'] = $this->db->get_estimated_row_count(SEARCH_WORDMATCH_TABLE);
}
/**
* Split a text into words of a given length
*
* The text is converted to UTF-8, cleaned up, and split. Then, words that
* conform to the defined length range are returned in an array.
*
* NOTE: duplicates are NOT removed from the return array
*
* @param string $text Text to split, encoded in UTF-8
* @return array Array of UTF-8 words
*/
protected function split_message($text)
{
$match = $words = array();
/**
* Taken from the original code
*/
// Do not index code
$match[] = '#\[code(?:=.*?)?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is';
// BBcode
$match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#';
$min = $this->word_length['min'];
$isset_min = $min - 1;
/**
* Clean up the string, remove HTML tags, remove BBCodes
*/
$word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), -1), ' ');
while (strlen($word))
{
if (strlen($word) > 255 || strlen($word) <= $isset_min)
{
/**
* Words longer than 255 bytes are ignored. This will have to be
* changed whenever we change the length of search_wordlist.word_text
*
* Words shorter than $isset_min bytes are ignored, too
*/
$word = strtok(' ');
continue;
}
$len = utf8_strlen($word);
/**
* Test whether the word is too short to be indexed.
*
* Note that this limit does NOT apply to CJK and Hangul
*/
if ($len < $min)
{
/**
* Note: this could be optimized. If the codepoint is lower than Hangul's range
* we know that it will also be lower than CJK ranges
*/
if ((strncmp($word, self::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, self::UTF8_HANGUL_LAST, 3) > 0)
&& (strncmp($word, self::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, self::UTF8_CJK_LAST, 3) > 0)
&& (strncmp($word, self::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, self::UTF8_CJK_B_LAST, 4) > 0))
{
$word = strtok(' ');
continue;
}
}
$words[] = $word;
$word = strtok(' ');
}
return $words;
}
/**
* Clean up a text to remove non-alphanumeric characters
*

81
phpBB/phpbb/search/backend/fulltext_postgres.php
View file

@ -174,16 +174,6 @@ class fulltext_postgres extends base implements search_backend_interface
return $this->word_length;
}
/**
* Returns if phrase search is supported or not
*
* @return bool
*/
public function supports_phrase_search()
{
return $this->phrase_search;
}
/**
* {@inheritdoc}
*/
@ -274,31 +264,6 @@ class fulltext_postgres extends base implements search_backend_interface
return false;
}
/**
* Turns text into an array of words
* @param string $text contains post text/subject
*/
public function split_message($text)
{
// Split words
$text = preg_replace('#([^\p{L}\p{N}\'*])#u', '$1$1', str_replace('\'\'', '\' \'', trim($text)));
$matches = array();
preg_match_all('#(?:[^\p{L}\p{N}*]|^)([+\-|]?(?:[\p{L}\p{N}*]+\'?)*[\p{L}\p{N}*])(?:[^\p{L}\p{N}*]|$)#u', $text, $matches);
$text = $matches[1];
// remove too short or too long words
$text = array_values($text);
for ($i = 0, $n = count($text); $i < $n; $i++)
{
$text[$i] = trim($text[$i]);
if (utf8_strlen($text[$i]) < $this->config['fulltext_postgres_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_postgres_max_word_len'])
{
unset($text[$i]);
}
}
return array_values($text);
}
/**
* {@inheritdoc}
@ -833,14 +798,15 @@ class fulltext_postgres extends base implements search_backend_interface
}
/**
* Destroys cached search results, that contained one of the new words in a post so the results won't be outdated
*
* @param string $mode contains the post mode: edit, post, reply, quote ...
* @param int $post_id contains the post id of the post to index
* @param string $message contains the post text of the post
* @param string $subject contains the subject of the post to index
* @param int $poster_id contains the user id of the poster
* @param int $forum_id contains the forum id of parent forum of the post
* {@inheritdoc}
*/
public function supports_phrase_search(): bool
{
return $this->phrase_search;
}
/**
* {@inheritdoc}
*/
public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
{
@ -892,7 +858,7 @@ class fulltext_postgres extends base implements search_backend_interface
*/
public function index_remove($post_ids, $author_ids, $forum_ids)
{
$this->destroy_cache(array(), $author_ids);
$this->destroy_cache([], $author_ids);
}
/**
@ -1095,6 +1061,33 @@ class fulltext_postgres extends base implements search_backend_interface
$this->stats['total_posts'] = $this->config['num_posts'];
}
/**
* Turns text into an array of words
* @param string $text contains post text/subject
* @return array
*/
protected function split_message($text)
{
// Split words
$text = preg_replace('#([^\p{L}\p{N}\'*])#u', '$1$1', str_replace('\'\'', '\' \'', trim($text)));
$matches = array();
preg_match_all('#(?:[^\p{L}\p{N}*]|^)([+\-|]?(?:[\p{L}\p{N}*]+\'?)*[\p{L}\p{N}*])(?:[^\p{L}\p{N}*]|$)#u', $text, $matches);
$text = $matches[1];
// remove too short or too long words
$text = array_values($text);
for ($i = 0, $n = count($text); $i < $n; $i++)
{
$text[$i] = trim($text[$i]);
if (utf8_strlen($text[$i]) < $this->config['fulltext_postgres_min_word_len'] || utf8_strlen($text[$i]) > $this->config['fulltext_postgres_max_word_len'])
{
unset($text[$i]);
}
}
return array_values($text);
}
/**
* {@inheritdoc}
*/

544
phpBB/phpbb/search/backend/fulltext_sphinx.php
View file

@ -189,30 +189,6 @@ class fulltext_sphinx implements search_backend_interface
return ($this->db->get_sql_layer() == 'mysqli' || $this->db->get_sql_layer() == 'postgres') && class_exists('SphinxClient');
}
/**
* {@inheritdoc}
*/
public function get_search_query()
{
return $this->search_query;
}
/**
* {@inheritdoc}
*/
public function get_word_length()
{
return false;
}
/**
* {@inheritdoc}
*/
public function get_common_words()
{
return array();
}
/**
* {@inheritdoc}
*/
@ -230,203 +206,27 @@ class fulltext_sphinx implements search_backend_interface
}
/**
* Generates content of sphinx.conf
*
* @return bool True if sphinx.conf content is correctly generated, false otherwise
* {@inheritdoc}
*/
protected function config_generate()
public function get_search_query()
{
// Check if Database is supported by Sphinx
if ($this->db->get_sql_layer() == 'mysqli')
{
$this->dbtype = 'mysql';
return $this->search_query;
}
else if ($this->db->get_sql_layer() == 'postgres')
{
$this->dbtype = 'pgsql';
}
else
{
$this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_WRONG_DATABASE');
return false;
}
// Check if directory paths have been filled
if (!$this->config['fulltext_sphinx_data_path'])
{
$this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA');
return false;
}
include($this->phpbb_root_path . 'config.' . $this->php_ext);
/* Now that we're sure everything was entered correctly,
generate a config for the index. We use a config value
fulltext_sphinx_id for this, as it should be unique. */
$config_object = new \phpbb\search\sphinx\config($this->config_file_data);
$config_data = array(
'source source_phpbb_' . $this->id . '_main' => array(
array('type', $this->dbtype . ' # mysql or pgsql'),
// This config value sql_host needs to be changed incase sphinx and sql are on different servers
array('sql_host', $dbhost . ' # SQL server host sphinx connects to'),
array('sql_user', '[dbuser]'),
array('sql_pass', '[dbpassword]'),
array('sql_db', $dbname),
array('sql_port', $dbport . ' # optional, default is 3306 for mysql and 5432 for pgsql'),
array('sql_query_pre', 'SET NAMES \'utf8\''),
array('sql_query_pre', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = (SELECT MAX(post_id) FROM ' . POSTS_TABLE . ') WHERE counter_id = 1'),
array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''),
array('sql_range_step', '5000'),
array('sql_query', 'SELECT
p.post_id AS id,
p.forum_id,
p.topic_id,
p.poster_id,
p.post_visibility,
CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
p.post_time,
p.post_subject,
p.post_subject as title,
p.post_text as data,
t.topic_last_post_time,
0 as deleted
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
WHERE
p.topic_id = t.topic_id
AND p.post_id >= $start AND p.post_id <= $end'),
array('sql_query_post', ''),
array('sql_query_post_index', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = $maxid WHERE counter_id = 1'),
array('sql_attr_uint', 'forum_id'),
array('sql_attr_uint', 'topic_id'),
array('sql_attr_uint', 'poster_id'),
array('sql_attr_uint', 'post_visibility'),
array('sql_attr_bool', 'topic_first_post'),
array('sql_attr_bool', 'deleted'),
array('sql_attr_timestamp', 'post_time'),
array('sql_attr_timestamp', 'topic_last_post_time'),
array('sql_attr_string', 'post_subject'),
),
'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array(
array('sql_query_pre', 'SET NAMES \'utf8\''),
array('sql_query_range', ''),
array('sql_range_step', ''),
array('sql_query', 'SELECT
p.post_id AS id,
p.forum_id,
p.topic_id,
p.poster_id,
p.post_visibility,
CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
p.post_time,
p.post_subject,
p.post_subject as title,
p.post_text as data,
t.topic_last_post_time,
0 as deleted
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
WHERE
p.topic_id = t.topic_id
AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'),
array('sql_query_post_index', ''),
),
'index index_phpbb_' . $this->id . '_main' => array(
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'),
array('source', 'source_phpbb_' . $this->id . '_main'),
array('docinfo', 'extern'),
array('morphology', 'none'),
array('stopwords', ''),
array('wordforms', ' # optional, specify path to wordforms file. See ./docs/sphinx_wordforms.txt for example'),
array('exceptions', ' # optional, specify path to exceptions file. See ./docs/sphinx_exceptions.txt for example'),
array('min_word_len', '2'),
array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'),
array('ignore_chars', 'U+0027, U+002C'),
array('min_prefix_len', '3 # Minimum number of characters for wildcard searches by prefix (min 1). Default is 3. If specified, set min_infix_len to 0'),
array('min_infix_len', '0 # Minimum number of characters for wildcard searches by infix (min 2). If specified, set min_prefix_len to 0'),
array('html_strip', '1'),
array('index_exact_words', '0 # Set to 1 to enable exact search operator. Requires wordforms or morphology'),
array('blend_chars', 'U+23, U+24, U+25, U+26, U+40'),
),
'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array(
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'),
array('source', 'source_phpbb_' . $this->id . '_delta'),
),
'indexer' => array(
array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'),
),
'searchd' => array(
array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '9312')),
array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'),
array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'),
array('read_timeout', '5'),
array('max_children', '30'),
array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'),
array('binlog_path', $this->config['fulltext_sphinx_data_path']),
),
);
$non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true);
$delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true);
/**
* Allow adding/changing the Sphinx configuration data
*
* @event core.search_sphinx_modify_config_data
* @var array config_data Array with the Sphinx configuration data
* @var array non_unique Array with the Sphinx non-unique variables to delete
* @var array delete Array with the Sphinx variables to delete
* @since 3.1.7-RC1
* {@inheritdoc}
*/
$vars = array(
'config_data',
'non_unique',
'delete',
);
extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_modify_config_data', compact($vars)));
foreach ($config_data as $section_name => $section_data)
public function get_common_words()
{
$section = $config_object->get_section_by_name($section_name);
if (!$section)
{
$section = $config_object->add_section($section_name);
return array();
}
foreach ($delete as $key => $void)
/**
* {@inheritdoc}
*/
public function get_word_length()
{
$section->delete_variables_by_name($key);
}
foreach ($non_unique as $key => $void)
{
$section->delete_variables_by_name($key);
}
foreach ($section_data as $entry)
{
$key = $entry[0];
$value = $entry[1];
if (!isset($non_unique[$key]))
{
$variable = $section->get_variable_by_name($key);
if (!$variable)
{
$section->create_variable($key, $value);
}
else
{
$variable->set_value($value);
}
}
else
{
$section->create_variable($key, $value);
}
}
}
$this->config_file_data = $config_object->get_data();
return true;
return false;
}
/**
@ -463,62 +263,6 @@ class fulltext_sphinx implements search_backend_interface
return false;
}
/**
* Cleans search query passed into Sphinx search engine, as follows:
* 1. Hyphenated words are replaced with keyword search for either the exact phrase with spaces
* or as a single word without spaces eg search for "know-it-all" becomes ("know it all"|"knowitall*")
* 2. Words with apostrophes are contracted eg "it's" becomes "its"
* 3. <, >, " and & are decoded from HTML entities.
* 4. Following special characters used as search operators in Sphinx are preserved when used with correct syntax:
* (a) quorum matching: "the world is a wonderful place"/3
* Finds 3 of the words within the phrase. Number must be between 1 and 9.
* (b) proximity search: "hello world"~10
* Finds hello and world within 10 words of each other. Number can be between 1 and 99.
* (c) strict word order: aaa << bbb << ccc
* Finds "aaa" only where it appears before "bbb" and only where "bbb" appears before "ccc".
* (d) exact match operator: if lemmatizer or stemming enabled,
* search will find exact match only and ignore other grammatical forms of the same word stem.
* eg. raining =cats and =dogs
* will not return "raining cat and dog"
* eg. ="search this exact phrase"
* will not return "searched this exact phrase", "searching these exact phrases".
* 5. Special characters /, ~, << and = not complying with the correct syntax
* and other reserved operators are escaped and searched literally.
* Special characters not explicitly listed in charset_table or blend_chars in sphinx.conf
* will not be indexed and keywords containing them will be ignored by Sphinx.
* By default, only $, %, & and @ characters are indexed and searchable.
* String transformation is in backend only and not visible to the end user
* nor reflected in the results page URL or keyword highlighting.
*
* @param string $search_string
* @return string
*/
public function sphinx_clean_search_string($search_string)
{
$from = ['@', '^', '$', '!', '&lt;', '&gt;', '&quot;', '&amp;', '\''];
$to = ['\@', '\^', '\$', '\!', '<', '>', '"', '&', ''];
$search_string = str_replace($from, $to, $search_string);
$search_string = strrev($search_string);
$search_string = preg_replace(['#\/(?!"[^"]+")#', '#~(?!"[^"]+")#'], ['/\\', '~\\'], $search_string);
$search_string = strrev($search_string);
$match = ['#(/|\\\\/)(?![1-9](\s|$))#', '#(~|\\\\~)(?!\d{1,2}(\s|$))#', '#((?:\p{L}|\p{N})+)-((?:\p{L}|\p{N})+)(?:-((?:\p{L}|\p{N})+))?(?:-((?:\p{L}|\p{N})+))?#i', '#<<\s*$#', '#(\S\K=|=(?=\s)|=$)#'];
$replace = ['\/', '\~', '("$1 $2 $3 $4"|$1$2$3$4*)', '\<\<', '\='];
$search_string = preg_replace($match, $replace, $search_string);
$search_string = preg_replace('#\s+"\|#', '"|', $search_string);
/**
* OPTIONAL: Thousands separator stripped from numbers, eg search for '90,000' is queried as '90000'.
* By default commas are stripped from search index so that '90,000' is indexed as '90000'
*/
// $search_string = preg_replace('#[0-9]{1,3}\K,(?=[0-9]{3})#', '', $search_string);
return $search_string;
}
/**
* {@inheritdoc}
*/
@ -774,6 +518,14 @@ class fulltext_sphinx implements search_backend_interface
return $this->keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, $id_ary, $start, $per_page);
}
/**
* {@inheritdoc}
*/
public function supports_phrase_search(): bool
{
return false;
}
/**
* {@inheritdoc}
*/
@ -857,7 +609,7 @@ class fulltext_sphinx implements search_backend_interface
/**
* Nothing needs to be destroyed
*/
public function tidy($create = false)
public function tidy()
{
$this->config->set('search_last_gc', time(), false);
}
@ -968,6 +720,62 @@ class fulltext_sphinx implements search_backend_interface
}
}
/**
* Cleans search query passed into Sphinx search engine, as follows:
* 1. Hyphenated words are replaced with keyword search for either the exact phrase with spaces
* or as a single word without spaces eg search for "know-it-all" becomes ("know it all"|"knowitall*")
* 2. Words with apostrophes are contracted eg "it's" becomes "its"
* 3. <, >, " and & are decoded from HTML entities.
* 4. Following special characters used as search operators in Sphinx are preserved when used with correct syntax:
* (a) quorum matching: "the world is a wonderful place"/3
* Finds 3 of the words within the phrase. Number must be between 1 and 9.
* (b) proximity search: "hello world"~10
* Finds hello and world within 10 words of each other. Number can be between 1 and 99.
* (c) strict word order: aaa << bbb << ccc
* Finds "aaa" only where it appears before "bbb" and only where "bbb" appears before "ccc".
* (d) exact match operator: if lemmatizer or stemming enabled,
* search will find exact match only and ignore other grammatical forms of the same word stem.
* eg. raining =cats and =dogs
* will not return "raining cat and dog"
* eg. ="search this exact phrase"
* will not return "searched this exact phrase", "searching these exact phrases".
* 5. Special characters /, ~, << and = not complying with the correct syntax
* and other reserved operators are escaped and searched literally.
* Special characters not explicitly listed in charset_table or blend_chars in sphinx.conf
* will not be indexed and keywords containing them will be ignored by Sphinx.
* By default, only $, %, & and @ characters are indexed and searchable.
* String transformation is in backend only and not visible to the end user
* nor reflected in the results page URL or keyword highlighting.
*
* @param string $search_string
* @return string
*/
protected function sphinx_clean_search_string($search_string)
{
$from = ['@', '^', '$', '!', '&lt;', '&gt;', '&quot;', '&amp;', '\''];
$to = ['\@', '\^', '\$', '\!', '<', '>', '"', '&', ''];
$search_string = str_replace($from, $to, $search_string);
$search_string = strrev($search_string);
$search_string = preg_replace(['#\/(?!"[^"]+")#', '#~(?!"[^"]+")#'], ['/\\', '~\\'], $search_string);
$search_string = strrev($search_string);
$match = ['#(/|\\\\/)(?![1-9](\s|$))#', '#(~|\\\\~)(?!\d{1,2}(\s|$))#', '#((?:\p{L}|\p{N})+)-((?:\p{L}|\p{N})+)(?:-((?:\p{L}|\p{N})+))?(?:-((?:\p{L}|\p{N})+))?#i', '#<<\s*$#', '#(\S\K=|=(?=\s)|=$)#'];
$replace = ['\/', '\~', '("$1 $2 $3 $4"|$1$2$3$4*)', '\<\<', '\='];
$search_string = preg_replace($match, $replace, $search_string);
$search_string = preg_replace('#\s+"\|#', '"|', $search_string);
/**
* OPTIONAL: Thousands separator stripped from numbers, eg search for '90,000' is queried as '90000'.
* By default commas are stripped from search index so that '90,000' is indexed as '90000'
*/
// $search_string = preg_replace('#[0-9]{1,3}\K,(?=[0-9]{3})#', '', $search_string);
return $search_string;
}
/**
* {@inheritdoc}
*/
@ -1010,4 +818,204 @@ class fulltext_sphinx implements search_backend_interface
'config' => $config_vars
);
}
/**
* Generates content of sphinx.conf
*
* @return bool True if sphinx.conf content is correctly generated, false otherwise
*/
protected function config_generate()
{
// Check if Database is supported by Sphinx
if ($this->db->get_sql_layer() == 'mysqli')
{
$this->dbtype = 'mysql';
}
else if ($this->db->get_sql_layer() == 'postgres')
{
$this->dbtype = 'pgsql';
}
else
{
$this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_WRONG_DATABASE');
return false;
}
// Check if directory paths have been filled
if (!$this->config['fulltext_sphinx_data_path'])
{
$this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA');
return false;
}
include($this->phpbb_root_path . 'config.' . $this->php_ext);
/* Now that we're sure everything was entered correctly,
generate a config for the index. We use a config value
fulltext_sphinx_id for this, as it should be unique. */
$config_object = new \phpbb\search\sphinx\config($this->config_file_data);
$config_data = array(
'source source_phpbb_' . $this->id . '_main' => array(
array('type', $this->dbtype . ' # mysql or pgsql'),
// This config value sql_host needs to be changed incase sphinx and sql are on different servers
array('sql_host', $dbhost . ' # SQL server host sphinx connects to'),
array('sql_user', '[dbuser]'),
array('sql_pass', '[dbpassword]'),
array('sql_db', $dbname),
array('sql_port', $dbport . ' # optional, default is 3306 for mysql and 5432 for pgsql'),
array('sql_query_pre', 'SET NAMES \'utf8\''),
array('sql_query_pre', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = (SELECT MAX(post_id) FROM ' . POSTS_TABLE . ') WHERE counter_id = 1'),
array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''),
array('sql_range_step', '5000'),
array('sql_query', 'SELECT
p.post_id AS id,
p.forum_id,
p.topic_id,
p.poster_id,
p.post_visibility,
CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
p.post_time,
p.post_subject,
p.post_subject as title,
p.post_text as data,
t.topic_last_post_time,
0 as deleted
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
WHERE
p.topic_id = t.topic_id
AND p.post_id >= $start AND p.post_id <= $end'),
array('sql_query_post', ''),
array('sql_query_post_index', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = $maxid WHERE counter_id = 1'),
array('sql_attr_uint', 'forum_id'),
array('sql_attr_uint', 'topic_id'),
array('sql_attr_uint', 'poster_id'),
array('sql_attr_uint', 'post_visibility'),
array('sql_attr_bool', 'topic_first_post'),
array('sql_attr_bool', 'deleted'),
array('sql_attr_timestamp', 'post_time'),
array('sql_attr_timestamp', 'topic_last_post_time'),
array('sql_attr_string', 'post_subject'),
),
'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array(
array('sql_query_pre', 'SET NAMES \'utf8\''),
array('sql_query_range', ''),
array('sql_range_step', ''),
array('sql_query', 'SELECT
p.post_id AS id,
p.forum_id,
p.topic_id,
p.poster_id,
p.post_visibility,
CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post,
p.post_time,
p.post_subject,
p.post_subject as title,
p.post_text as data,
t.topic_last_post_time,
0 as deleted
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
WHERE
p.topic_id = t.topic_id
AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'),
array('sql_query_post_index', ''),
),
'index index_phpbb_' . $this->id . '_main' => array(
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'),
array('source', 'source_phpbb_' . $this->id . '_main'),
array('docinfo', 'extern'),
array('morphology', 'none'),
array('stopwords', ''),
array('wordforms', ' # optional, specify path to wordforms file. See ./docs/sphinx_wordforms.txt for example'),
array('exceptions', ' # optional, specify path to exceptions file. See ./docs/sphinx_exceptions.txt for example'),
array('min_word_len', '2'),
array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'),
array('ignore_chars', 'U+0027, U+002C'),
array('min_prefix_len', '3 # Minimum number of characters for wildcard searches by prefix (min 1). Default is 3. If specified, set min_infix_len to 0'),
array('min_infix_len', '0 # Minimum number of characters for wildcard searches by infix (min 2). If specified, set min_prefix_len to 0'),
array('html_strip', '1'),
array('index_exact_words', '0 # Set to 1 to enable exact search operator. Requires wordforms or morphology'),
array('blend_chars', 'U+23, U+24, U+25, U+26, U+40'),
),
'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array(
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'),
array('source', 'source_phpbb_' . $this->id . '_delta'),
),
'indexer' => array(
array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'),
),
'searchd' => array(
array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '9312')),
array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'),
array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'),
array('read_timeout', '5'),
array('max_children', '30'),
array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'),
array('binlog_path', $this->config['fulltext_sphinx_data_path']),
),
);
$non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true);
$delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true);
/**
* Allow adding/changing the Sphinx configuration data
*
* @event core.search_sphinx_modify_config_data
* @var array config_data Array with the Sphinx configuration data
* @var array non_unique Array with the Sphinx non-unique variables to delete
* @var array delete Array with the Sphinx variables to delete
* @since 3.1.7-RC1
*/
$vars = array(
'config_data',
'non_unique',
'delete',
);
extract($this->phpbb_dispatcher->trigger_event('core.search_sphinx_modify_config_data', compact($vars)));
foreach ($config_data as $section_name => $section_data)
{
$section = $config_object->get_section_by_name($section_name);
if (!$section)
{
$section = $config_object->add_section($section_name);
}
foreach ($delete as $key => $void)
{
$section->delete_variables_by_name($key);
}
foreach ($non_unique as $key => $void)
{
$section->delete_variables_by_name($key);
}
foreach ($section_data as $entry)
{
$key = $entry[0];
$value = $entry[1];
if (!isset($non_unique[$key]))
{
$variable = $section->get_variable_by_name($key);
if (!$variable)
{
$section->create_variable($key, $value);
}
else
{
$variable->set_value($value);
}
}
else
{
$section->create_variable($key, $value);
}
}
}
$this->config_file_data = $config_object->get_data();
return true;
}
}

13
phpBB/phpbb/search/backend/search_backend_interface.php
View file

@ -15,6 +15,9 @@ namespace phpbb\search\backend;
interface search_backend_interface
{
// TODO: comprobar los pasos pr referencia
// TODO: eliminar acp_module de create index y delete index
/**
* Returns the name of this search backend to be displayed to administrators
*
@ -59,8 +62,6 @@ interface search_backend_interface
*/
public function get_word_length();
//public function init();
/**
* Splits keywords entered by a user into an array of words stored in $this->split_words
* Stores the tidied search query in $this->search_query
@ -114,8 +115,16 @@ interface search_backend_interface
*/
public function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $post_visibility, $topic_id, $author_ary, $author_name, &$id_ary, &$start, $per_page);
/**
* Returns if phrase search is supported or not
*
* @return bool
*/
public function supports_phrase_search(): bool;
/**
* Updates wordlist and wordmatch tables when a message is posted or changed
* Destroys cached search results, that contained one of the new words in a post so the results won't be outdated
*
* @param string $mode contains the post mode: edit, post, reply, quote ...
* @param int $post_id contains the post id of the post to index

2
phpBB/search.php
View file

@ -1287,7 +1287,7 @@ if ($keywords || $author || $author_id || $search_id || $submit)
// Check if search backend supports phrase search or not
$phrase_search_disabled = '';
if (strpos(html_entity_decode($keywords), '"') !== false && method_exists($search, 'supports_phrase_search'))
if (strpos(html_entity_decode($keywords), '"') !== false)
{
$phrase_search_disabled = $search->supports_phrase_search() ? false : true;
}