mirror of
https://github.com/phpbb/phpbb.git
synced 2025-06-27 21:58:52 +00:00
[feature/sphinx-fulltext-search] modify config class
Sphinx config class is modified to return the configuration data instead of writing it to a file. Search backend property config_file_data stores the generated data. PHPBB3-10946
This commit is contained in:
Dhruv Goel
Dhruv
parent
01261179ce
commit
f0692bb9e8
2 changed files with 146 additions and 159 deletions
|
|
@ -45,6 +45,7 @@ class phpbb_search_fulltext_sphinx
|
||||||
private $db;
|
private $db;
|
||||||
private $db_tools;
|
private $db_tools;
|
||||||
private $user;
|
private $user;
|
||||||
|
private $config_file_data = '';
|
||||||
public $word_length = array();
|
public $word_length = array();
|
||||||
public $search_query;
|
public $search_query;
|
||||||
public $common_words = array();
|
public $common_words = array();
|
||||||
|
|
@ -133,151 +134,151 @@ class phpbb_search_fulltext_sphinx
|
||||||
{
|
{
|
||||||
global $phpbb_root_path, $phpEx;
|
global $phpbb_root_path, $phpEx;
|
||||||
|
|
||||||
include ($phpbb_root_path . 'config.' . $phpEx);
|
include ($phpbb_root_path . 'config.' . $phpEx);
|
||||||
|
|
||||||
/* Now that we're sure everything was entered correctly,
|
/* Now that we're sure everything was entered correctly,
|
||||||
generate a config for the index. We misuse the avatar_salt
|
generate a config for the index. We misuse the avatar_salt
|
||||||
for this, as it should be unique. */
|
for this, as it should be unique. */
|
||||||
$config_object = new phpbb_search_sphinx_config($this->config['fulltext_sphinx_config_path'] . 'sphinx.conf');
|
$config_object = new phpbb_search_sphinx_config($this->config_file_data);
|
||||||
|
|
||||||
$config_data = array(
|
$config_data = array(
|
||||||
'source source_phpbb_' . $this->id . '_main' => array(
|
'source source_phpbb_' . $this->id . '_main' => array(
|
||||||
array('type', 'mysql'),
|
array('type', 'mysql'),
|
||||||
array('sql_host', $dbhost),
|
array('sql_host', $dbhost),
|
||||||
array('sql_user', $dbuser),
|
array('sql_user', $dbuser),
|
||||||
array('sql_pass', $dbpasswd),
|
array('sql_pass', $dbpasswd),
|
||||||
array('sql_db', $dbname),
|
array('sql_db', $dbname),
|
||||||
array('sql_port', $dbport),
|
array('sql_port', $dbport),
|
||||||
array('sql_query_pre', 'SET NAMES utf8'),
|
array('sql_query_pre', 'SET NAMES utf8'),
|
||||||
array('sql_query_pre', 'REPLACE INTO ' . SPHINX_TABLE . ' SELECT 1, MAX(post_id) FROM ' . POSTS_TABLE . ''),
|
array('sql_query_pre', 'REPLACE INTO ' . SPHINX_TABLE . ' SELECT 1, MAX(post_id) FROM ' . POSTS_TABLE . ''),
|
||||||
array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''),
|
array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''),
|
||||||
array('sql_range_step', '5000'),
|
array('sql_range_step', '5000'),
|
||||||
array('sql_query', 'SELECT
|
array('sql_query', 'SELECT
|
||||||
p.post_id AS id,
|
p.post_id AS id,
|
||||||
p.forum_id,
|
p.forum_id,
|
||||||
p.topic_id,
|
p.topic_id,
|
||||||
p.poster_id,
|
p.poster_id,
|
||||||
IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post,
|
IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post,
|
||||||
p.post_time,
|
p.post_time,
|
||||||
p.post_subject,
|
p.post_subject,
|
||||||
p.post_subject as title,
|
p.post_subject as title,
|
||||||
p.post_text as data,
|
p.post_text as data,
|
||||||
t.topic_last_post_time,
|
t.topic_last_post_time,
|
||||||
0 as deleted
|
0 as deleted
|
||||||
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
|
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
|
||||||
WHERE
|
WHERE
|
||||||
p.topic_id = t.topic_id
|
p.topic_id = t.topic_id
|
||||||
AND p.post_id >= $start AND p.post_id <= $end'),
|
AND p.post_id >= $start AND p.post_id <= $end'),
|
||||||
array('sql_query_post', ''),
|
array('sql_query_post', ''),
|
||||||
array('sql_query_post_index', 'REPLACE INTO ' . SPHINX_TABLE . ' ( counter_id, max_doc_id ) VALUES ( 1, $maxid )'),
|
array('sql_query_post_index', 'REPLACE INTO ' . SPHINX_TABLE . ' ( counter_id, max_doc_id ) VALUES ( 1, $maxid )'),
|
||||||
array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'),
|
array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'),
|
||||||
array('sql_attr_uint', 'forum_id'),
|
array('sql_attr_uint', 'forum_id'),
|
||||||
array('sql_attr_uint', 'topic_id'),
|
array('sql_attr_uint', 'topic_id'),
|
||||||
array('sql_attr_uint', 'poster_id'),
|
array('sql_attr_uint', 'poster_id'),
|
||||||
array('sql_attr_bool', 'topic_first_post'),
|
array('sql_attr_bool', 'topic_first_post'),
|
||||||
array('sql_attr_bool', 'deleted'),
|
array('sql_attr_bool', 'deleted'),
|
||||||
array('sql_attr_timestamp' , 'post_time'),
|
array('sql_attr_timestamp' , 'post_time'),
|
||||||
array('sql_attr_timestamp' , 'topic_last_post_time'),
|
array('sql_attr_timestamp' , 'topic_last_post_time'),
|
||||||
array('sql_attr_str2ordinal', 'post_subject'),
|
array('sql_attr_str2ordinal', 'post_subject'),
|
||||||
),
|
),
|
||||||
'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array(
|
'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array(
|
||||||
array('sql_query_pre', ''),
|
array('sql_query_pre', ''),
|
||||||
array('sql_query_range', ''),
|
array('sql_query_range', ''),
|
||||||
array('sql_range_step', ''),
|
array('sql_range_step', ''),
|
||||||
array('sql_query', 'SELECT
|
array('sql_query', 'SELECT
|
||||||
p.post_id AS id,
|
p.post_id AS id,
|
||||||
p.forum_id,
|
p.forum_id,
|
||||||
p.topic_id,
|
p.topic_id,
|
||||||
p.poster_id,
|
p.poster_id,
|
||||||
IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post,
|
IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post,
|
||||||
p.post_time,
|
p.post_time,
|
||||||
p.post_subject,
|
p.post_subject,
|
||||||
p.post_subject as title,
|
p.post_subject as title,
|
||||||
p.post_text as data,
|
p.post_text as data,
|
||||||
t.topic_last_post_time,
|
t.topic_last_post_time,
|
||||||
0 as deleted
|
0 as deleted
|
||||||
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
|
FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t
|
||||||
WHERE
|
WHERE
|
||||||
p.topic_id = t.topic_id
|
p.topic_id = t.topic_id
|
||||||
AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'),
|
AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'),
|
||||||
),
|
),
|
||||||
'index index_phpbb_' . $this->id . '_main' => array(
|
'index index_phpbb_' . $this->id . '_main' => array(
|
||||||
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'),
|
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'),
|
||||||
array('source', 'source_phpbb_' . $this->id . '_main'),
|
array('source', 'source_phpbb_' . $this->id . '_main'),
|
||||||
array('docinfo', 'extern'),
|
array('docinfo', 'extern'),
|
||||||
array('morphology', 'none'),
|
array('morphology', 'none'),
|
||||||
array('stopwords', $this->config['fulltext_sphinx_stopwords'] ? $this->config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''),
|
array('stopwords', $this->config['fulltext_sphinx_stopwords'] ? $this->config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''),
|
||||||
array('min_word_len', '2'),
|
array('min_word_len', '2'),
|
||||||
array('charset_type', 'utf-8'),
|
array('charset_type', 'utf-8'),
|
||||||
array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'),
|
array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'),
|
||||||
array('min_prefix_len', '0'),
|
array('min_prefix_len', '0'),
|
||||||
array('min_infix_len', '0'),
|
array('min_infix_len', '0'),
|
||||||
),
|
),
|
||||||
'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array(
|
'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array(
|
||||||
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'),
|
array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'),
|
||||||
array('source', 'source_phpbb_' . $this->id . '_delta'),
|
array('source', 'source_phpbb_' . $this->id . '_delta'),
|
||||||
),
|
),
|
||||||
'indexer' => array(
|
'indexer' => array(
|
||||||
array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'),
|
array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'),
|
||||||
),
|
),
|
||||||
'searchd' => array(
|
'searchd' => array(
|
||||||
array('compat_sphinxql_magics' , '0'),
|
array('compat_sphinxql_magics' , '0'),
|
||||||
array('listen' , '127.0.0.1'),
|
array('listen' , '127.0.0.1'),
|
||||||
array('port', ($this->config['fulltext_sphinx_port']) ? $this->config['fulltext_sphinx_port'] : '3312'),
|
array('port', ($this->config['fulltext_sphinx_port']) ? $this->config['fulltext_sphinx_port'] : '3312'),
|
||||||
array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'),
|
array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'),
|
||||||
array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'),
|
array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'),
|
||||||
array('read_timeout', '5'),
|
array('read_timeout', '5'),
|
||||||
array('max_children', '30'),
|
array('max_children', '30'),
|
||||||
array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'),
|
array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'),
|
||||||
array('max_matches', (string) SPHINX_MAX_MATCHES),
|
array('max_matches', (string) SPHINX_MAX_MATCHES),
|
||||||
array('binlog_path', $this->config['fulltext_sphinx_data_path']),
|
array('binlog_path', $this->config['fulltext_sphinx_data_path']),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
$non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true);
|
$non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true);
|
||||||
$delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true);
|
$delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true);
|
||||||
|
foreach ($config_data as $section_name => $section_data)
|
||||||
foreach ($config_data as $section_name => $section_data)
|
{
|
||||||
|
$section = $config_object->get_section_by_name($section_name);
|
||||||
|
if (!$section)
|
||||||
{
|
{
|
||||||
$section = $config_object->get_section_by_name($section_name);
|
$section = $config_object->add_section($section_name);
|
||||||
if (!$section)
|
}
|
||||||
{
|
|
||||||
$section = $config_object->add_section($section_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($delete as $key => $void)
|
foreach ($delete as $key => $void)
|
||||||
{
|
{
|
||||||
$section->delete_variables_by_name($key);
|
$section->delete_variables_by_name($key);
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach ($non_unique as $key => $void)
|
foreach ($non_unique as $key => $void)
|
||||||
{
|
{
|
||||||
$section->delete_variables_by_name($key);
|
$section->delete_variables_by_name($key);
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach ($section_data as $entry)
|
foreach ($section_data as $entry)
|
||||||
{
|
{
|
||||||
$key = $entry[0];
|
$key = $entry[0];
|
||||||
$value = $entry[1];
|
$value = $entry[1];
|
||||||
|
|
||||||
if (!isset($non_unique[$key]))
|
if (!isset($non_unique[$key]))
|
||||||
{
|
{
|
||||||
$variable = $section->get_variable_by_name($key);
|
$variable = $section->get_variable_by_name($key);
|
||||||
if (!$variable)
|
if (!$variable)
|
||||||
{
|
|
||||||
$variable = $section->create_variable($key, $value);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
$variable->set_value($value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
{
|
||||||
$variable = $section->create_variable($key, $value);
|
$variable = $section->create_variable($key, $value);
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
$variable->set_value($value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
$variable = $section->create_variable($key, $value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
$this->config_file_data = $config_object->get_data();
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,15 +27,15 @@ class phpbb_search_sphinx_config
|
||||||
var $sections = array();
|
var $sections = array();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor which optionally loads data from a file
|
* Constructor which optionally loads data from a variable
|
||||||
*
|
*
|
||||||
* @param string $filename The path to a file containing the sphinx configuration
|
* @param string $config_data Variable containing the sphinx configuration data
|
||||||
*/
|
*/
|
||||||
function __construct($filename = false)
|
function __construct($config_data)
|
||||||
{
|
{
|
||||||
if ($filename !== false && file_exists($filename))
|
if ($config_data != '')
|
||||||
{
|
{
|
||||||
$this->read($filename);
|
$this->read($config_data);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -70,22 +70,19 @@ class phpbb_search_sphinx_config
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Parses the config file at the given path, which is stored in $this->loaded for later use
|
* Reads the config file data
|
||||||
*
|
*
|
||||||
* @param string $filename The path to the config file
|
* @param string $config_data The config file data
|
||||||
*/
|
*/
|
||||||
function read($filename)
|
function read($config_data)
|
||||||
{
|
{
|
||||||
// Split the file into lines, we'll process it line by line
|
|
||||||
$config_file = file($filename);
|
|
||||||
|
|
||||||
$this->sections = array();
|
$this->sections = array();
|
||||||
|
|
||||||
$section = null;
|
$section = null;
|
||||||
$found_opening_bracket = false;
|
$found_opening_bracket = false;
|
||||||
$in_value = false;
|
$in_value = false;
|
||||||
|
|
||||||
foreach ($config_file as $i => $line)
|
foreach ($config_data as $i => $line)
|
||||||
{
|
{
|
||||||
/* If the value of a variable continues to the next line because the line
|
/* If the value of a variable continues to the next line because the line
|
||||||
break was escaped then we don't trim leading space but treat it as a part of the value */
|
break was escaped then we don't trim leading space but treat it as a part of the value */
|
||||||
|
|
@ -262,32 +259,21 @@ class phpbb_search_sphinx_config
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keep the filename for later use
|
|
||||||
$this->loaded = $filename;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes the config data into a file
|
* Returns the config data
|
||||||
*
|
*
|
||||||
* @param string $filename The optional filename into which the config data shall be written.
|
* @return string $data The config data that is generated.
|
||||||
* If it's not specified it will be written into the file that the config
|
|
||||||
* was originally read from.
|
|
||||||
*/
|
*/
|
||||||
function write($filename = false)
|
function get_data()
|
||||||
{
|
{
|
||||||
if ($filename === false && $this->loaded)
|
|
||||||
{
|
|
||||||
$filename = $this->loaded;
|
|
||||||
}
|
|
||||||
|
|
||||||
$data = "";
|
$data = "";
|
||||||
foreach ($this->sections as $section)
|
foreach ($this->sections as $section)
|
||||||
{
|
{
|
||||||
$data .= $section->to_string();
|
$data .= $section->to_string();
|
||||||
}
|
}
|
||||||
|
|
||||||
$fp = fopen($filename, 'wb');
|
return $data;
|
||||||
fwrite($fp, $data);
|
|
||||||
fclose($fp);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue