From a50b0faf4abfb1bba68e03d843c58f07f842cf12 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Fri, 4 May 2012 18:39:41 +0530 Subject: [PATCH 01/57] [feature/sphinx-fulltext-search] MOD by naderman installed in phpbb PHPBB3-10946 --- phpBB/includes/functions_sphinx.php | 507 +++++++++ phpBB/includes/search/fulltext_sphinx.php | 1166 +++++++++++++++++++ phpBB/includes/sphinxapi-0.9.8.php | 1202 ++++++++++++++++++++ phpBB/language/en/mods/fulltext_sphinx.php | 65 ++ 4 files changed, 2940 insertions(+) create mode 100644 phpBB/includes/functions_sphinx.php create mode 100644 phpBB/includes/search/fulltext_sphinx.php create mode 100644 phpBB/includes/sphinxapi-0.9.8.php create mode 100644 phpBB/language/en/mods/fulltext_sphinx.php diff --git a/phpBB/includes/functions_sphinx.php b/phpBB/includes/functions_sphinx.php new file mode 100644 index 0000000000..976f93f77c --- /dev/null +++ b/phpBB/includes/functions_sphinx.php @@ -0,0 +1,507 @@ +read($filename); + } + } + + /** + * Get a section object by its name + * + * @param string $name The name of the section that shall be returned + * @return sphinx_config_section The section object or null if none was found + */ + function &get_section_by_name($name) + { + for ($i = 0, $n = sizeof($this->sections); $i < $n; $i++) + { + // make sure this is really a section object and not a comment + if (is_a($this->sections[$i], 'sphinx_config_section') && $this->sections[$i]->get_name() == $name) + { + return $this->sections[$i]; + } + } + $null = null; + return $null; + } + + /** + * Appends a new empty section to the end of the config + * + * @param string $name The name for the new section + * @return sphinx_config_section The newly created section object + */ + function &add_section($name) + { + $this->sections[] = new sphinx_config_section($name, ''); + return $this->sections[sizeof($this->sections) - 1]; + } + + /** + * Parses the config file at the given path, which is stored in $this->loaded for later use + * + * @param string $filename The path to the config file + */ + function read($filename) + { + // split the file into lines, we'll process it line by line + $config_file = file($filename); + + $this->sections = array(); + + $section = null; + $found_opening_bracket = false; + $in_value = false; + + foreach ($config_file as $i => $line) + { + // if the value of a variable continues to the next line because the line break was escaped + // then we don't trim leading space but treat it as a part of the value + if ($in_value) + { + $line = rtrim($line); + } + else + { + $line = trim($line); + } + + // if we're not inside a section look for one + if (!$section) + { + // add empty lines and comments as comment objects to the section list + // that way they're not deleted when reassembling the file from the sections + if (!$line || $line[0] == '#') + { + $this->sections[] = new sphinx_config_comment($config_file[$i]); + continue; + } + else + { + // otherwise we scan the line reading the section name until we find + // an opening curly bracket or a comment + $section_name = ''; + $section_name_comment = ''; + $found_opening_bracket = false; + for ($j = 0, $n = strlen($line); $j < $n; $j++) + { + if ($line[$j] == '#') + { + $section_name_comment = substr($line, $j); + break; + } + + if ($found_opening_bracket) + { + continue; + } + + if ($line[$j] == '{') + { + $found_opening_bracket = true; + continue; + } + + $section_name .= $line[$j]; + } + + // and then we create the new section object + $section_name = trim($section_name); + $section = new sphinx_config_section($section_name, $section_name_comment); + } + } + else // if we're looking for variables inside a section + { + $skip_first = false; + + // if we're not in a value continuing over the line feed + if (!$in_value) + { + // then add empty lines and comments as comment objects to the variable list + // of this section so they're not deleted on reassembly + if (!$line || $line[0] == '#') + { + $section->add_variable(new sphinx_config_comment($config_file[$i])); + continue; + } + + // as long as we haven't yet actually found an opening bracket for this section + // we treat everything as comments so it's not deleted either + if (!$found_opening_bracket) + { + if ($line[0] == '{') + { + $skip_first = true; + $line = substr($line, 1); + $found_opening_bracket = true; + } + else + { + $section->add_variable(new sphinx_config_comment($config_file[$i])); + continue; + } + } + } + + // if we did not find a comment in this line or still add to the previous line's value ... + if ($line || $in_value) + { + if (!$in_value) + { + $name = ''; + $value = ''; + $comment = ''; + $found_assignment = false; + } + $in_value = false; + $end_section = false; + + // ... then we should prase this line char by char: + // - first there's the variable name + // - then an equal sign + // - the variable value + // - possibly a backslash before the linefeed in this case we need to continue + // parsing the value in the next line + // - a # indicating that the rest of the line is a comment + // - a closing curly bracket indicating the end of this section + for ($j = 0, $n = strlen($line); $j < $n; $j++) + { + if ($line[$j] == '#') + { + $comment = substr($line, $j); + break; + } + else if ($line[$j] == '}') + { + $comment = substr($line, $j + 1); + $end_section = true; + break; + } + else if (!$found_assignment) + { + if ($line[$j] == '=') + { + $found_assignment = true; + } + else + { + $name .= $line[$j]; + } + } + else + { + if ($line[$j] == '\\' && $j == $n - 1) + { + $value .= "\n"; + $in_value = true; + continue 2; // go to the next line and keep processing the value in there + } + $value .= $line[$j]; + } + } + + // if a name and an equal sign were found then we have append a new variable object to the section + if ($name && $found_assignment) + { + $section->add_variable(new sphinx_config_variable(trim($name), trim($value), ($end_section) ? '' : $comment)); + continue; + } + + // if we found a closing curly bracket this section has been completed and we can append it to the section list + // and continue with looking for the next section + if ($end_section) + { + $section->set_end_comment($comment); + $this->sections[] = $section; + $section = null; + continue; + } + } + + // if we did not find anything meaningful up to here, then just treat it as a comment + $comment = ($skip_first) ? "\t" . substr(ltrim($config_file[$i]), 1) : $config_file[$i]; + $section->add_variable(new sphinx_config_comment($comment)); + } + } + + // keep the filename for later use + $this->loaded = $filename; + } + + /** + * Writes the config data into a file + * + * @param string $filename The optional filename into which the config data shall be written. + * If it's not specified it will be written into the file that the config + * was originally read from. + */ + function write($filename = false) + { + if ($filename === false && $this->loaded) + { + $filename = $this->loaded; + } + + $data = ""; + foreach ($this->sections as $section) + { + $data .= $section->to_string(); + } + + $fp = fopen($filename, 'wb'); + fwrite($fp, $data); + fclose($fp); + } +} + +/** +* sphinx_config_section +* Represents a single section inside the sphinx configuration +*/ +class sphinx_config_section +{ + var $name; + var $comment; + var $end_comment; + var $variables = array(); + + /** + * Construct a new section + * + * @param string $name Name of the section + * @param string $comment Comment that should be appended after the name in the + * textual format. + */ + function sphinx_config_section($name, $comment) + { + $this->name = $name; + $this->comment = $comment; + $this->end_comment = ''; + } + + /** + * Add a variable object to the list of variables in this section + * + * @param sphinx_config_variable $variable The variable object + */ + function add_variable($variable) + { + $this->variables[] = $variable; + } + + /** + * Adds a comment after the closing bracket in the textual representation + */ + function set_end_comment($end_comment) + { + $this->end_comment = $end_comment; + } + + /** + * Getter for the name of this section + * + * @return string Section's name + */ + function get_name() + { + return $this->name; + } + + /** + * Get a variable object by its name + * + * @param string $name The name of the variable that shall be returned + * @return sphinx_config_section The first variable object from this section with the + * given name or null if none was found + */ + function &get_variable_by_name($name) + { + for ($i = 0, $n = sizeof($this->variables); $i < $n; $i++) + { + // make sure this is a variable object and not a comment + if (is_a($this->variables[$i], 'sphinx_config_variable') && $this->variables[$i]->get_name() == $name) + { + return $this->variables[$i]; + } + } + $null = null; + return $null; + } + + /** + * Deletes all variables with the given name + * + * @param string $name The name of the variable objects that are supposed to be removed + */ + function delete_variables_by_name($name) + { + for ($i = 0; $i < sizeof($this->variables); $i++) + { + // make sure this is a variable object and not a comment + if (is_a($this->variables[$i], 'sphinx_config_variable') && $this->variables[$i]->get_name() == $name) + { + array_splice($this->variables, $i, 1); + $i--; + } + } + } + + /** + * Create a new variable object and append it to the variable list of this section + * + * @param string $name The name for the new variable + * @param string $value The value for the new variable + * @return sphinx_config_variable Variable object that was created + */ + function &create_variable($name, $value) + { + $this->variables[] = new sphinx_config_variable($name, $value, ''); + return $this->variables[sizeof($this->variables) - 1]; + } + + /** + * Turns this object into a string which can be written to a config file + * + * @return string Config data in textual form, parsable for sphinx + */ + function to_string() + { + $content = $this->name . " " . $this->comment . "\n{\n"; + + // make sure we don't get too many newlines after the opening bracket + while (trim($this->variables[0]->to_string()) == "") + { + array_shift($this->variables); + } + + foreach ($this->variables as $variable) + { + $content .= $variable->to_string(); + } + $content .= '}' . $this->end_comment . "\n"; + + return $content; + } +} + +/** +* sphinx_config_variable +* Represents a single variable inside the sphinx configuration +*/ +class sphinx_config_variable +{ + var $name; + var $value; + var $comment; + + /** + * Constructs a new variable object + * + * @param string $name Name of the variable + * @param string $value Value of the variable + * @param string $comment Optional comment after the variable in the + * config file + */ + function sphinx_config_variable($name, $value, $comment) + { + $this->name = $name; + $this->value = $value; + $this->comment = $comment; + } + + /** + * Getter for the variable's name + * + * @return string The variable object's name + */ + function get_name() + { + return $this->name; + } + + /** + * Allows changing the variable's value + * + * @param string $value New value for this variable + */ + function set_value($value) + { + $this->value = $value; + } + + /** + * Turns this object into a string readable by sphinx + * + * @return string Config data in textual form + */ + function to_string() + { + return "\t" . $this->name . ' = ' . str_replace("\n", "\\\n", $this->value) . ' ' . $this->comment . "\n"; + } +} + + +/** +* sphinx_config_comment +* Represents a comment inside the sphinx configuration +*/ +class sphinx_config_comment +{ + var $exact_string; + + /** + * Create a new comment + * + * @param string $exact_string The content of the comment including newlines, leading whitespace, etc. + */ + function sphinx_config_comment($exact_string) + { + $this->exact_string = $exact_string; + } + + /** + * Simply returns the comment as it was created + * + * @return string The exact string that was specified in the constructor + */ + function to_string() + { + return $this->exact_string; + } +} + +?> \ No newline at end of file diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php new file mode 100644 index 0000000000..e0c467df93 --- /dev/null +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -0,0 +1,1166 @@ +id = $config['avatar_salt']; + $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main'; + + $this->sphinx = new SphinxClient (); + + if (!empty($config['fulltext_sphinx_configured'])) + { + if ($config['fulltext_sphinx_autorun'] && !file_exists($config['fulltext_sphinx_data_path'] . 'searchd.pid') && $this->index_created(true)) + { + $this->shutdown_searchd(); +// $cwd = getcwd(); +// chdir($config['fulltext_sphinx_bin_path']); + exec($config['fulltext_sphinx_bin_path'] . SEARCHD_NAME . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf >> ' . $config['fulltext_sphinx_data_path'] . 'log/searchd-startup.log 2>&1 &'); +// chdir($cwd); + } + + // we only support localhost for now + $this->sphinx->SetServer('localhost', (isset($config['fulltext_sphinx_port']) && $config['fulltext_sphinx_port']) ? (int) $config['fulltext_sphinx_port'] : 3312); + } + + $config['fulltext_sphinx_min_word_len'] = 2; + $config['fulltext_sphinx_max_word_len'] = 400; + + $error = false; + } + + /** + * Checks permissions and paths, if everything is correct it generates the config file + */ + function init() + { + global $db, $user, $config; + + if ($db->sql_layer != 'mysql' && $db->sql_layer != 'mysql4' && $db->sql_layer != 'mysqli') + { + return $user->lang['FULLTEXT_SPHINX_WRONG_DATABASE']; + } + + if ($error = $this->config_updated()) + { + return $error; + } + + // move delta to main index each hour + set_config('search_gc', 3600); + + return false; + } + + function config_updated() + { + global $db, $user, $config, $phpbb_root_path, $phpEx; + + if ($config['fulltext_sphinx_autoconf']) + { + $paths = array('fulltext_sphinx_bin_path', 'fulltext_sphinx_config_path', 'fulltext_sphinx_data_path'); + + // check for completeness and add trailing slash if it's not present + foreach ($paths as $path) + { + if (empty($config[$path])) + { + return $user->lang['FULLTEXT_SPHINX_UNCONFIGURED']; + } + if ($config[$path] && substr($config[$path], -1) != '/') + { + set_config($path, $config[$path] . '/'); + } + } + } + + $executables = array( + $config['fulltext_sphinx_bin_path'] . INDEXER_NAME, + $config['fulltext_sphinx_bin_path'] . SEARCHD_NAME, + ); + + if ($config['fulltext_sphinx_autorun']) + { + foreach ($executables as $executable) + { + if (!file_exists($executable)) + { + return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_FOUND'], $executable); + } + + if (!function_exists('exec')) + { + return $user->lang['FULLTEXT_SPHINX_REQUIRES_EXEC']; + } + + $output = array(); + @exec($executable, $output); + + $output = implode("\n", $output); + if (strpos($output, 'Sphinx ') === false) + { + return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_EXECUTABLE'], $executable); + } + } + } + + $writable_paths = array( + $config['fulltext_sphinx_config_path'] => array('config' => 'fulltext_sphinx_autoconf', 'subdir' => false), + $config['fulltext_sphinx_data_path'] => array('config' => 'fulltext_sphinx_autorun', 'subdir' => 'log'), + $config['fulltext_sphinx_data_path'] . 'log/' => array('config' => 'fulltext_sphinx_autorun', 'subdir' => false), + ); + + foreach ($writable_paths as $path => $info) + { + if ($config[$info['config']]) + { + // make sure directory exists + // if we could drop the @ here and figure out whether the file really + // doesn't exist or whether open_basedir is in effect, would be nice + if (!@file_exists($path)) + { + return sprintf($user->lang['FULLTEXT_SPHINX_DIRECTORY_NOT_FOUND'], $path); + } + + // now check if it is writable by storing a simple file + $filename = $path . 'write_test'; + $fp = @fopen($filename, 'wb'); + if ($fp === false) + { + return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_WRITABLE'], $filename); + } + @fclose($fp); + + @unlink($filename); + + if ($info['subdir'] !== false) + { + if (!is_dir($path . $info['subdir'])) + { + mkdir($path . $info['subdir']); + } + } + } + } + + if ($config['fulltext_sphinx_autoconf']) + { + include ($phpbb_root_path . 'config.' . $phpEx); + + // now that we're sure everything was entered correctly, generate a config for the index + // we misuse the avatar_salt for this, as it should be unique ;-) + + if (!class_exists('sphinx_config')) + { + include($phpbb_root_path . 'includes/functions_sphinx.php'); + } + + if (!file_exists($config['fulltext_sphinx_config_path'] . 'sphinx.conf')) + { + $filename = $config['fulltext_sphinx_config_path'] . 'sphinx.conf'; + $fp = @fopen($filename, 'wb'); + if ($fp === false) + { + return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_WRITABLE'], $filename); + } + @fclose($fp); + } + + $config_object = new sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); + + $config_data = array( + "source source_phpbb_{$this->id}_main" => array( + array('type', 'mysql'), + array('sql_host', $dbhost), + array('sql_user', $dbuser), + array('sql_pass', $dbpasswd), + array('sql_db', $dbname), + array('sql_port', $dbport), + array('sql_query_pre', 'SET NAMES utf8'), + array('sql_query_pre', 'REPLACE INTO ' . SPHINX_TABLE . ' SELECT 1, MAX(post_id) FROM ' . POSTS_TABLE . ''), + array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''), + array('sql_range_step', '5000'), + array('sql_query', 'SELECT + p.post_id AS id, + p.forum_id, + p.topic_id, + p.poster_id, + IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, + p.post_time, + p.post_subject, + p.post_subject as title, + p.post_text as data, + t.topic_last_post_time, + 0 as deleted + FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t + WHERE + p.topic_id = t.topic_id + AND p.post_id >= $start AND p.post_id <= $end'), + array('sql_query_post', ''), + array('sql_query_post_index', 'REPLACE INTO ' . SPHINX_TABLE . ' ( counter_id, max_doc_id ) VALUES ( 1, $maxid )'), + array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'), + array('sql_attr_uint', 'forum_id'), + array('sql_attr_uint', 'topic_id'), + array('sql_attr_uint', 'poster_id'), + array('sql_attr_bool', 'topic_first_post'), + array('sql_attr_bool', 'deleted'), + array('sql_attr_timestamp' , 'post_time'), + array('sql_attr_timestamp' , 'topic_last_post_time'), + array('sql_attr_str2ordinal', 'post_subject'), + ), + "source source_phpbb_{$this->id}_delta : source_phpbb_{$this->id}_main" => array( + array('sql_query_pre', ''), + array('sql_query_range', ''), + array('sql_range_step', ''), + array('sql_query', 'SELECT + p.post_id AS id, + p.forum_id, + p.topic_id, + p.poster_id, + IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, + p.post_time, + p.post_subject, + p.post_subject as title, + p.post_text as data, + t.topic_last_post_time, + 0 as deleted + FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t + WHERE + p.topic_id = t.topic_id + AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'), + ), + "index index_phpbb_{$this->id}_main" => array( + array('path', $config['fulltext_sphinx_data_path'] . "index_phpbb_{$this->id}_main"), + array('source', "source_phpbb_{$this->id}_main"), + array('docinfo', 'extern'), + array('morphology', 'none'), + array('stopwords', (file_exists($config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt') && $config['fulltext_sphinx_stopwords']) ? $config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), + array('min_word_len', '2'), + array('charset_type', 'utf-8'), + array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+ß410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), + array('min_prefix_len', '0'), + array('min_infix_len', '0'), + ), + "index index_phpbb_{$this->id}_delta : index_phpbb_{$this->id}_main" => array( + array('path', $config['fulltext_sphinx_data_path'] . "index_phpbb_{$this->id}_delta"), + array('source', "source_phpbb_{$this->id}_delta"), + ), + 'indexer' => array( + array('mem_limit', $config['fulltext_sphinx_indexer_mem_limit'] . 'M'), + ), + 'searchd' => array( + array('address' , '127.0.0.1'), + array('port', ($config['fulltext_sphinx_port']) ? $config['fulltext_sphinx_port'] : '3312'), + array('log', $config['fulltext_sphinx_data_path'] . "log/searchd.log"), + array('query_log', $config['fulltext_sphinx_data_path'] . "log/sphinx-query.log"), + array('read_timeout', '5'), + array('max_children', '30'), + array('pid_file', $config['fulltext_sphinx_data_path'] . "searchd.pid"), + array('max_matches', (string) MAX_MATCHES), + ), + ); + + $non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true); + $delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true); + + foreach ($config_data as $section_name => $section_data) + { + $section = &$config_object->get_section_by_name($section_name); + if (!$section) + { + $section = &$config_object->add_section($section_name); + } + + foreach ($delete as $key => $void) + { + $section->delete_variables_by_name($key); + } + + foreach ($non_unique as $key => $void) + { + $section->delete_variables_by_name($key); + } + + foreach ($section_data as $entry) + { + $key = $entry[0]; + $value = $entry[1]; + + if (!isset($non_unique[$key])) + { + $variable = &$section->get_variable_by_name($key); + if (!$variable) + { + $variable = &$section->create_variable($key, $value); + } + else + { + $variable->set_value($value); + } + } + else + { + $variable = &$section->create_variable($key, $value); + } + } + } + + $config_object->write($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); + } + + set_config('fulltext_sphinx_configured', '1'); + + $this->shutdown_searchd(); + $this->tidy(); + + return false; + } + + /** + * Splits keywords entered by a user into an array of words stored in $this->split_words + * Stores the tidied search query in $this->search_query + * + * @param string $keywords Contains the keyword as entered by the user + * @param string $terms is either 'all' or 'any' + * @return false if no valid keywords were found and otherwise true + */ + function split_keywords(&$keywords, $terms) + { + global $config; + + if ($terms == 'all') + { + $match = array('#\sand\s#i', '#\sor\s#i', '#\snot\s#i', '#\+#', '#-#', '#\|#', '#@#'); + $replace = array(' & ', ' | ', ' - ', ' +', ' -', ' |', ''); + + $replacements = 0; + $keywords = preg_replace($match, $replace, $keywords); + $this->sphinx->SetMatchMode(SPH_MATCH_EXTENDED); + } + else + { + $this->sphinx->SetMatchMode(SPH_MATCH_ANY); + } + + $match = array(); + // Keep quotes + $match[] = "#"#"; + // KeepNew lines + $match[] = "#[\n]+#"; + + $replace = array('"', " "); + + $keywords = str_replace(array('"', "\n"), array('"', ' '), trim($keywords)); + + if (strlen($keywords) > 0) + { + $this->search_query = str_replace('"', '"', $keywords); + return true; + } + + return false; + } + + /** + * Performs a search on keywords depending on display specific params. You have to run split_keywords() first. + * + * @param string $type contains either posts or topics depending on what should be searched for + * @param string $fields contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched) + * @param string $terms is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words) + * @param array $sort_by_sql contains SQL code for the ORDER BY part of a query + * @param string $sort_key is the key of $sort_by_sql for the selected sorting + * @param string $sort_dir is either a or d representing ASC and DESC + * @param string $sort_days specifies the maximum amount of days a post may be old + * @param array $ex_fid_ary specifies an array of forum ids which should not be searched + * @param array $m_approve_fid_ary specifies an array of forum ids in which the searcher is allowed to view unapproved posts + * @param int $topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched + * @param array $author_ary an array of author ids if the author should be ignored during the search the array is empty + * @param string $author_name specifies the author match, when ANONYMOUS is also a search-match + * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered + * @param int $start indicates the first index of the page + * @param int $per_page number of ids each page is supposed to contain + * @return boolean|int total number of results + * + * @access public + */ + function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $m_approve_fid_ary, $topic_id, $author_ary, $author_name, &$id_ary, $start, $per_page) + { + global $config, $db, $auth; + + // No keywords? No posts. + if (!strlen($this->search_query) && !sizeof($author_ary)) + { + return false; + } + + $id_ary = array(); + + $join_topic = ($type == 'posts') ? false : true; + + // sorting + + if ($type == 'topics') + { + switch ($sort_key) + { + case 'a': + $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'poster_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); + break; + case 'f': + $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'forum_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); + break; + case 'i': + case 's': + $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'post_subject ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); + break; + case 't': + default: + $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'topic_last_post_time ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); + break; + } + } + else + { + switch ($sort_key) + { + case 'a': + $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'poster_id'); + break; + case 'f': + $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'forum_id'); + break; + case 'i': + case 's': + $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_subject'); + break; + case 't': + default: + $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_time'); + break; + } + } + + // most narrow filters first + if ($topic_id) + { + $this->sphinx->SetFilter('topic_id', array($topic_id)); + } + + $search_query_prefix = ''; + + switch($fields) + { + case 'titleonly': + // only search the title + if ($terms == 'all') + { + $search_query_prefix = '@title '; + } + $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); // weight for the title + $this->sphinx->SetFilter('topic_first_post', array(1)); // 1 is first_post, 0 is not first post + break; + + case 'msgonly': + // only search the body + if ($terms == 'all') + { + $search_query_prefix = '@data '; + } + $this->sphinx->SetFieldWeights(array("title" => 1, "data" => 5)); // weight for the body + break; + + case 'firstpost': + $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); // more relative weight for the title, also search the body + $this->sphinx->SetFilter('topic_first_post', array(1)); // 1 is first_post, 0 is not first post + break; + + default: + $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); // more relative weight for the title, also search the body + break; + } + + if (sizeof($author_ary)) + { + $this->sphinx->SetFilter('poster_id', $author_ary); + } + + if (sizeof($ex_fid_ary)) + { + // All forums that a user is allowed to access + $fid_ary = array_unique(array_intersect(array_keys($auth->acl_getf('f_read', true)), array_keys($auth->acl_getf('f_search', true)))); + // All forums that the user wants to and can search in + $search_forums = array_diff($fid_ary, $ex_fid_ary); + + if (sizeof($search_forums)) + { + $this->sphinx->SetFilter('forum_id', $search_forums); + } + } + + $this->sphinx->SetFilter('deleted', array(0)); + + $this->sphinx->SetLimits($start, (int) $per_page, MAX_MATCHES); + $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); + + // could be connection to localhost:3312 failed (errno=111, msg=Connection refused) during rotate, retry if so + $retries = CONNECT_RETRIES; + while (!$result && (strpos($this->sphinx->_error, "errno=111,") !== false) && $retries--) + { + usleep(CONNECT_WAIT_TIME); + $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); + } + + $id_ary = array(); + if (isset($result['matches'])) + { + if ($type == 'posts') + { + $id_ary = array_keys($result['matches']); + } + else + { + foreach($result['matches'] as $key => $value) + { + $id_ary[] = $value['attrs']['topic_id']; + } + } + } + else + { + return false; + } + + $result_count = $result['total_found']; + + $id_ary = array_slice($id_ary, 0, (int) $per_page); + + return $result_count; + } + + /** + * Performs a search on an author's posts without caring about message contents. Depends on display specific params + * + * @param string $type contains either posts or topics depending on what should be searched for + * @param boolean $firstpost_only if true, only topic starting posts will be considered + * @param array $sort_by_sql contains SQL code for the ORDER BY part of a query + * @param string $sort_key is the key of $sort_by_sql for the selected sorting + * @param string $sort_dir is either a or d representing ASC and DESC + * @param string $sort_days specifies the maximum amount of days a post may be old + * @param array $ex_fid_ary specifies an array of forum ids which should not be searched + * @param array $m_approve_fid_ary specifies an array of forum ids in which the searcher is allowed to view unapproved posts + * @param int $topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched + * @param array $author_ary an array of author ids + * @param string $author_name specifies the author match, when ANONYMOUS is also a search-match + * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered + * @param int $start indicates the first index of the page + * @param int $per_page number of ids each page is supposed to contain + * @return boolean|int total number of results + * + * @access public + */ + function author_search($type, $firstpost_only, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $m_approve_fid_ary, $topic_id, $author_ary, $author_name, &$id_ary, $start, $per_page) + { + $this->search_query = ''; + + $this->sphinx->SetMatchMode(SPH_MATCH_FULLSCAN); + $fields = ($firstpost_only) ? 'firstpost' : 'all'; + $terms = 'all'; + return $this->keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $m_approve_fid_ary, $topic_id, $author_ary, $author_name, $id_ary, $start, $per_page); + } + + /** + * Updates wordlist and wordmatch tables when a message is posted or changed + * + * @param string $mode Contains the post mode: edit, post, reply, quote + * @param int $post_id The id of the post which is modified/created + * @param string &$message New or updated post content + * @param string &$subject New or updated post subject + * @param int $poster_id Post author's user id + * @param int $forum_id The id of the forum in which the post is located + * + * @access public + */ + function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id) + { + global $config, $db; + + if ($mode == 'edit') + { + $this->sphinx->UpdateAttributes($this->indexes, array('forum_id', 'poster_id'), array((int)$post_id => array((int)$forum_id, (int)$poster_id))); + } + else if ($mode != 'post' && $post_id) + { + // update topic_last_post_time for full topic + $sql = 'SELECT p1.post_id + FROM ' . POSTS_TABLE . ' p1 + LEFT JOIN ' . POSTS_TABLE . ' p2 ON (p1.topic_id = p2.topic_id) + WHERE p2.post_id = ' . $post_id; + $result = $db->sql_query($sql); + + $post_updates = array(); + $post_time = time(); + while ($row = $db->sql_fetchrow($result)) + { + $post_updates[(int)$row['post_id']] = array((int) $post_time); + } + $db->sql_freeresult($result); + + if (sizeof($post_updates)) + { + $this->sphinx->UpdateAttributes($this->indexes, array('topic_last_post_time'), $post_updates); + } + } + + if ($config['fulltext_sphinx_autorun']) + { + if ($this->index_created()) + { + $rotate = ($this->searchd_running()) ? ' --rotate' : ''; + + $cwd = getcwd(); + chdir($config['fulltext_sphinx_bin_path']); + exec('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_delta >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); + var_dump('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_delta >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); + chdir($cwd); + } + } + } + + /** + * Delete a post from the index after it was deleted + */ + function index_remove($post_ids, $author_ids, $forum_ids) + { + $values = array(); + foreach ($post_ids as $post_id) + { + $values[$post_id] = array(1); + } + + $this->sphinx->UpdateAttributes($this->indexes, array('deleted'), $values); + } + + /** + * Destroy old cache entries + */ + function tidy($create = false) + { + global $config; + + if ($config['fulltext_sphinx_autorun']) + { + if ($this->index_created() || $create) + { + $rotate = ($this->searchd_running()) ? ' --rotate' : ''; + + $cwd = getcwd(); + chdir($config['fulltext_sphinx_bin_path']); + exec('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_main >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); + exec('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_delta >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); + chdir($cwd); + } + } + + set_config('search_last_gc', time(), true); + } + + /** + * Create sphinx table + */ + function create_index($acp_module, $u_action) + { + global $db, $user, $config; + + $this->shutdown_searchd(); + + if (!isset($config['fulltext_sphinx_configured']) || !$config['fulltext_sphinx_configured']) + { + $user->add_lang('mods/fulltext_sphinx'); + + return $user->lang['FULLTEXT_SPHINX_CONFIGURE_FIRST']; + } + + if (!$this->index_created()) + { + $sql = 'CREATE TABLE IF NOT EXISTS ' . SPHINX_TABLE . ' ( + counter_id INT NOT NULL PRIMARY KEY, + max_doc_id INT NOT NULL + )'; + $db->sql_query($sql); + + $sql = 'TRUNCATE TABLE ' . SPHINX_TABLE; + $db->sql_query($sql); + } + + // start indexing process + $this->tidy(true); + + $this->shutdown_searchd(); + + return false; + } + + /** + * Drop sphinx table + */ + function delete_index($acp_module, $u_action) + { + global $db, $config; + + $this->shutdown_searchd(); + + if ($config['fulltext_sphinx_autorun']) + { + sphinx_unlink_by_pattern($config['fulltext_sphinx_data_path'], '#^index_phpbb_' . $this->id . '.*$#'); + } + + if (!$this->index_created()) + { + return false; + } + + $sql = 'DROP TABLE ' . SPHINX_TABLE; + $db->sql_query($sql); + + $this->shutdown_searchd(); + + return false; + } + + /** + * Returns true if the sphinx table was created + */ + function index_created($allow_new_files = true) + { + global $db, $config; + + $sql = 'SHOW TABLES LIKE \'' . SPHINX_TABLE . '\''; + $result = $db->sql_query($sql); + $row = $db->sql_fetchrow($result); + $db->sql_freeresult($result); + + $created = false; + + if ($row) + { + if ($config['fulltext_sphinx_autorun']) + { + if ((file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main.spd') && file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta.spd')) || ($allow_new_files && file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main.new.spd') && file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta.new.spd'))) + { + $created = true; + } + } + else + { + $created = true; + } + } + + return $created; + } + + /** + * Kills the searchd process and makes sure there's no locks left over + */ + function shutdown_searchd() + { + global $config; + + if ($config['fulltext_sphinx_autorun']) + { + if (!function_exists('exec')) + { + set_config('fulltext_sphinx_autorun', '0'); + return; + } + + exec('killall -9 ' . SEARCHD_NAME . ' >> /dev/null 2>&1 &'); + + if (file_exists($config['fulltext_sphinx_data_path'] . 'searchd.pid')) + { + unlink($config['fulltext_sphinx_data_path'] . 'searchd.pid'); + } + + sphinx_unlink_by_pattern($config['fulltext_sphinx_data_path'], '#^.*\.spl$#'); + } + } + + /** + * Checks whether searchd is running, if it's not running it makes sure there's no left over + * files by calling shutdown_searchd. + * + * @return boolean Whether searchd is running or not + */ + function searchd_running() + { + global $config; + + // if we cannot manipulate the service assume it is running + if (!$config['fulltext_sphinx_autorun']) + { + return true; + } + + if (file_exists($config['fulltext_sphinx_data_path'] . 'searchd.pid')) + { + $pid = trim(file_get_contents($config['fulltext_sphinx_data_path'] . 'searchd.pid')); + + if ($pid) + { + $output = array(); + $pidof_command = 'pidof'; + + exec('whereis -b pidof', $output); + if (sizeof($output) > 1) + { + $output = explode(' ', trim($output[0])); + $pidof_command = $output[1]; // 0 is pidof: + } + + $output = array(); + exec($pidof_command . ' ' . SEARCHD_NAME, $output); + if (sizeof($output) && (trim($output[0]) == $pid || trim($output[1]) == $pid)) + { + return true; + } + } + } + + // make sure it's really not running + $this->shutdown_searchd(); + + return false; + } + + /** + * Returns an associative array containing information about the indexes + */ + function index_stats() + { + global $user; + + if (empty($this->stats)) + { + $this->get_stats(); + } + + $user->add_lang('mods/fulltext_sphinx'); + + return array( + $user->lang['FULLTEXT_SPHINX_MAIN_POSTS'] => ($this->index_created()) ? $this->stats['main_posts'] : 0, + $user->lang['FULLTEXT_SPHINX_DELTA_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] - $this->stats['main_posts'] : 0, + $user->lang['FULLTEXT_MYSQL_TOTAL_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] : 0, + $user->lang['FULLTEXT_SPHINX_LAST_SEARCHES'] => nl2br($this->stats['last_searches']), + ); + } + + /** + * Collects stats that can be displayed on the index maintenance page + */ + function get_stats() + { + global $db, $config; + + if ($this->index_created()) + { + $sql = 'SELECT COUNT(post_id) as total_posts + FROM ' . POSTS_TABLE; + $result = $db->sql_query($sql); + $this->stats['total_posts'] = (int) $db->sql_fetchfield('total_posts'); + $db->sql_freeresult($result); + + $sql = 'SELECT COUNT(p.post_id) as main_posts + FROM ' . POSTS_TABLE . ' p, ' . SPHINX_TABLE . ' m + WHERE p.post_id <= m.max_doc_id + AND m.counter_id = 1'; + $result = $db->sql_query($sql); + $this->stats['main_posts'] = (int) $db->sql_fetchfield('main_posts'); + $db->sql_freeresult($result); + } + + $this->stats['last_searches'] = ''; + if ($config['fulltext_sphinx_autorun']) + { + if (file_exists($config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log')) + { + $last_searches = explode("\n", utf8_htmlspecialchars(sphinx_read_last_lines($config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log', 3))); + + foreach($last_searches as $i => $search) + { + if (strpos($search, '[' . $this->indexes . ']') !== false) + { + $last_searches[$i] = str_replace('[' . $this->indexes . ']', '', $search); + } + else + { + $last_searches[$i] = ''; + } + } + $this->stats['last_searches'] = implode("\n", $last_searches); + } + } + } + + /** + * Returns a list of options for the ACP to display + */ + function acp() + { + global $user, $config; + + $user->add_lang('mods/fulltext_sphinx'); + + $config_vars = array( + 'fulltext_sphinx_autoconf' => 'bool', + 'fulltext_sphinx_autorun' => 'bool', + 'fulltext_sphinx_config_path' => 'string', + 'fulltext_sphinx_data_path' => 'string', + 'fulltext_sphinx_bin_path' => 'string', + 'fulltext_sphinx_port' => 'int', + 'fulltext_sphinx_stopwords' => 'bool', + 'fulltext_sphinx_indexer_mem_limit' => 'int', + ); + + $defaults = array( + 'fulltext_sphinx_autoconf' => '1', + 'fulltext_sphinx_autorun' => '1', + 'fulltext_sphinx_indexer_mem_limit' => '512', + ); + + foreach ($config_vars as $config_var => $type) + { + if (!isset($config[$config_var])) + { + $default = ''; + if (isset($defaults[$config_var])) + { + $default = $defaults[$config_var]; + } + set_config($config_var, $default); + } + } + + $no_autoconf = false; + $no_autorun = false; + $bin_path = $config['fulltext_sphinx_bin_path']; + + // try to guess the path if it is empty + if (empty($bin_path)) + { + if (@file_exists('/usr/local/bin/' . INDEXER_NAME) && @file_exists('/usr/local/bin/' . SEARCHD_NAME)) + { + $bin_path = '/usr/local/bin/'; + } + else if (@file_exists('/usr/bin/' . INDEXER_NAME) && @file_exists('/usr/bin/' . SEARCHD_NAME)) + { + $bin_path = '/usr/bin/'; + } + else + { + $output = array(); + if (!function_exists('exec') || null === @exec('whereis -b ' . INDEXER_NAME, $output)) + { + $no_autorun = true; + } + else if (sizeof($output)) + { + $output = explode(' ', $output[0]); + array_shift($output); // remove indexer: + + foreach ($output as $path) + { + $path = dirname($path) . '/'; + + if (file_exists($path . INDEXER_NAME) && file_exists($path . SEARCHD_NAME)) + { + $bin_path = $path; + break; + } + } + } + } + } + + if ($no_autorun) + { + set_config('fulltext_sphinx_autorun', '0'); + } + + if ($no_autoconf) + { + set_config('fulltext_sphinx_autoconf', '0'); + } + + // rewrite config if fulltext sphinx is enabled + if ($config['fulltext_sphinx_autoconf'] && isset($config['fulltext_sphinx_configured']) && $config['fulltext_sphinx_configured']) + { + $this->config_updated(); + } + + // check whether stopwords file is available and enabled + if (@file_exists($config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt')) + { + $stopwords_available = true; + $stopwords_active = $config['fulltext_sphinx_stopwords']; + } + else + { + $stopwords_available = false; + $stopwords_active = false; + set_config('fulltext_sphinx_stopwords', '0'); + } + + $tpl = ' + ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_BEFORE']. ' +
+

' . $user->lang['FULLTEXT_SPHINX_AUTOCONF_EXPLAIN'] . '
+
+
+
+

' . $user->lang['FULLTEXT_SPHINX_AUTORUN_EXPLAIN'] . '
+
+
+
+

' . $user->lang['FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN'] . '
+
+
+
+

' . $user->lang['FULLTEXT_SPHINX_BIN_PATH_EXPLAIN'] . '
+
+
+
+

' . $user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '
+
+
+ ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_AFTER']. ' +
+

' . $user->lang['FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN'] . '
+
+
+
+

' . $user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '
+
+
+
+

' . $user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN'] . '
+
' . $user->lang['MIB'] . '
+
+ '; + + // These are fields required in the config table + return array( + 'tpl' => $tpl, + 'config' => $config_vars + ); + } +} + +/** +* Deletes all files from a directory that match a certain pattern +* +* @param string $path Path from which files shall be deleted +* @param string $pattern PCRE pattern that a file needs to match in order to be deleted +*/ +function sphinx_unlink_by_pattern($path, $pattern) +{ + $dir = opendir($path); + while (false !== ($file = readdir($dir))) + { + if (is_file($path . $file) && preg_match($pattern, $file)) + { + unlink($path . $file); + } + } + closedir($dir); +} + +/** +* Reads the last from a file +* +* @param string $file The filename from which the lines shall be read +* @param int $amount The number of lines to be read from the end +* @return string Last lines of the file +*/ +function sphinx_read_last_lines($file, $amount) +{ + $fp = fopen($file, 'r'); + fseek($fp, 0, SEEK_END); + + $c = ''; + $i = 0; + + while ($i < $amount) + { + fseek($fp, -2, SEEK_CUR); + $c = fgetc($fp); + if ($c == "\n") + { + $i++; + } + if (feof($fp)) + { + break; + } + } + + $string = fread($fp, 8192); + fclose($fp); + + return $string; +} + +?> \ No newline at end of file diff --git a/phpBB/includes/sphinxapi-0.9.8.php b/phpBB/includes/sphinxapi-0.9.8.php new file mode 100644 index 0000000000..6a7ea17760 --- /dev/null +++ b/phpBB/includes/sphinxapi-0.9.8.php @@ -0,0 +1,1202 @@ +=8 ) + { + $i = (int)$v; + return pack ( "NN", $i>>32, $i&((1<<32)-1) ); + } + + // x32 route, bcmath + $x = "4294967296"; + if ( function_exists("bcmul") ) + { + $h = bcdiv ( $v, $x, 0 ); + $l = bcmod ( $v, $x ); + return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit + } + + // x32 route, 15 or less decimal digits + // we can use float, because its actually double and has 52 precision bits + if ( strlen($v)<=15 ) + { + $f = (float)$v; + $h = (int)($f/$x); + $l = (int)($f-$x*$h); + return pack ( "NN", $h, $l ); + } + + // x32 route, 16 or more decimal digits + // well, let me know if you *really* need this + die ( "INTERNAL ERROR: packing more than 15-digit numeric on 32-bit PHP is not implemented yet (contact support)" ); +} + + +/// portably unpack 64 unsigned bits, network order to numeric +function sphUnpack64 ( $v ) +{ + list($h,$l) = array_values ( unpack ( "N*N*", $v ) ); + + // x64 route + if ( PHP_INT_SIZE>=8 ) + { + if ( $h<0 ) $h += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again + if ( $l<0 ) $l += (1<<32); + return ($h<<32) + $l; + } + + // x32 route + $h = sprintf ( "%u", $h ); + $l = sprintf ( "%u", $l ); + $x = "4294967296"; + + // bcmath + if ( function_exists("bcmul") ) + return bcadd ( $l, bcmul ( $x, $h ) ); + + // no bcmath, 15 or less decimal digits + // we can use float, because its actually double and has 52 precision bits + if ( $h<1048576 ) + { + $f = ((float)$h)*$x + (float)$l; + return sprintf ( "%.0f", $f ); // builtin conversion is only about 39-40 bits precise! + } + + // x32 route, 16 or more decimal digits + // well, let me know if you *really* need this + die ( "INTERNAL ERROR: unpacking more than 15-digit numeric on 32-bit PHP is not implemented yet (contact support)" ); +} + + +/// sphinx searchd client class +class SphinxClient +{ + var $_host; ///< searchd host (default is "localhost") + var $_port; ///< searchd port (default is 3312) + var $_offset; ///< how many records to seek from result-set start (default is 0) + var $_limit; ///< how many records to return from result-set starting at offset (default is 20) + var $_mode; ///< query matching mode (default is SPH_MATCH_ALL) + var $_weights; ///< per-field weights (default is 1 for all fields) + var $_sort; ///< match sorting mode (default is SPH_SORT_RELEVANCE) + var $_sortby; ///< attribute to sort by (defualt is "") + var $_min_id; ///< min ID to match (default is 0, which means no limit) + var $_max_id; ///< max ID to match (default is 0, which means no limit) + var $_filters; ///< search filters + var $_groupby; ///< group-by attribute name + var $_groupfunc; ///< group-by function (to pre-process group-by attribute value with) + var $_groupsort; ///< group-by sorting clause (to sort groups in result set with) + var $_groupdistinct;///< group-by count-distinct attribute + var $_maxmatches; ///< max matches to retrieve + var $_cutoff; ///< cutoff to stop searching at (default is 0) + var $_retrycount; ///< distributed retries count + var $_retrydelay; ///< distributed retries delay + var $_anchor; ///< geographical anchor point + var $_indexweights; ///< per-index weights + var $_ranker; ///< ranking mode (default is SPH_RANK_PROXIMITY_BM25) + var $_maxquerytime; ///< max query time, milliseconds (default is 0, do not limit) + var $_fieldweights; ///< per-field-name weights + + var $_error; ///< last error message + var $_warning; ///< last warning message + + var $_reqs; ///< requests array for multi-query + var $_mbenc; ///< stored mbstring encoding + var $_arrayresult; ///< whether $result["matches"] should be a hash or an array + var $_timeout; ///< connect timeout + + ///////////////////////////////////////////////////////////////////////////// + // common stuff + ///////////////////////////////////////////////////////////////////////////// + + /// create a new client object and fill defaults + function SphinxClient () + { + // per-client-object settings + $this->_host = "localhost"; + $this->_port = 3312; + + // per-query settings + $this->_offset = 0; + $this->_limit = 20; + $this->_mode = SPH_MATCH_ALL; + $this->_weights = array (); + $this->_sort = SPH_SORT_RELEVANCE; + $this->_sortby = ""; + $this->_min_id = 0; + $this->_max_id = 0; + $this->_filters = array (); + $this->_groupby = ""; + $this->_groupfunc = SPH_GROUPBY_DAY; + $this->_groupsort = "@group desc"; + $this->_groupdistinct= ""; + $this->_maxmatches = 1000; + $this->_cutoff = 0; + $this->_retrycount = 0; + $this->_retrydelay = 0; + $this->_anchor = array (); + $this->_indexweights= array (); + $this->_ranker = SPH_RANK_PROXIMITY_BM25; + $this->_maxquerytime= 0; + $this->_fieldweights= array(); + + $this->_error = ""; // per-reply fields (for single-query case) + $this->_warning = ""; + $this->_reqs = array (); // requests storage (for multi-query case) + $this->_mbenc = ""; + $this->_arrayresult = false; + $this->_timeout = 0; + } + + /// get last error message (string) + function GetLastError () + { + return $this->_error; + } + + /// get last warning message (string) + function GetLastWarning () + { + return $this->_warning; + } + + /// set searchd host name (string) and port (integer) + function SetServer ( $host, $port ) + { + assert ( is_string($host) ); + assert ( is_int($port) ); + $this->_host = $host; + $this->_port = $port; + } + + /// set server connection timeout (0 to remove) + function SetConnectTimeout ( $timeout ) + { + assert ( is_numeric($timeout) ); + $this->_timeout = $timeout; + } + + ///////////////////////////////////////////////////////////////////////////// + + /// enter mbstring workaround mode + function _MBPush () + { + $this->_mbenc = ""; + if ( ini_get ( "mbstring.func_overload" ) & 2 ) + { + $this->_mbenc = mb_internal_encoding(); + mb_internal_encoding ( "latin1" ); + } + } + + /// leave mbstring workaround mode + function _MBPop () + { + if ( $this->_mbenc ) + mb_internal_encoding ( $this->_mbenc ); + } + + /// connect to searchd server + function _Connect ($allow_retry = true) + { + $errno = 0; + $errstr = ""; + if ( $this->_timeout<=0 ) + $fp = @fsockopen ( $this->_host, $this->_port, $errno, $errstr ); + else + $fp = @fsockopen ( $this->_host, $this->_port, $errno, $errstr, $this->_timeout ); + + if ( !$fp ) + { + $errstr = trim ( $errstr ); + $this->_error = "connection to {$this->_host}:{$this->_port} failed (errno=$errno, msg=$errstr)"; + return false; + } + + // check version + //list(,$v) = unpack ( "N*", fread ( $fp, 4 ) ); + $version_data = unpack ( "N*", fread ( $fp, 4 ) ); + if (!isset($version_data[1])) + { + // this should not happen, try to reconnect ONCE + if ($allow_retry) + { + return $this->_Connect(false); + } + else + { + $this->_error = "unexpected version data"; + return false; + } + } + $v = $version_data[1]; + $v = (int)$v; + if ( $v<1 ) + { + fclose ( $fp ); + $this->_error = "expected searchd protocol version 1+, got version '$v'"; + return false; + } + + // all ok, send my version + fwrite ( $fp, pack ( "N", 1 ) ); + return $fp; + } + + /// get and check response packet from searchd server + function _GetResponse ( $fp, $client_ver ) + { + $response = ""; + $len = 0; + + $header = fread ( $fp, 8 ); + if ( strlen($header)==8 ) + { + list ( $status, $ver, $len ) = array_values ( unpack ( "n2a/Nb", $header ) ); + $left = $len; + while ( $left>0 && !feof($fp) ) + { + $chunk = fread ( $fp, $left ); + if ( $chunk ) + { + $response .= $chunk; + $left -= strlen($chunk); + } + } + } + fclose ( $fp ); + + // check response + $read = strlen ( $response ); + if ( !$response || $read!=$len ) + { + $this->_error = $len + ? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=$read)" + : "received zero-sized searchd response"; + return false; + } + + // check status + if ( $status==SEARCHD_WARNING ) + { + list(,$wlen) = unpack ( "N*", substr ( $response, 0, 4 ) ); + $this->_warning = substr ( $response, 4, $wlen ); + return substr ( $response, 4+$wlen ); + } + if ( $status==SEARCHD_ERROR ) + { + $this->_error = "searchd error: " . substr ( $response, 4 ); + return false; + } + if ( $status==SEARCHD_RETRY ) + { + $this->_error = "temporary searchd error: " . substr ( $response, 4 ); + return false; + } + if ( $status!=SEARCHD_OK ) + { + $this->_error = "unknown status code '$status'"; + return false; + } + + // check version + if ( $ver<$client_ver ) + { + $this->_warning = sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work", + $ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ); + } + + return $response; + } + + ///////////////////////////////////////////////////////////////////////////// + // searching + ///////////////////////////////////////////////////////////////////////////// + + /// set offset and count into result set, + /// and optionally set max-matches and cutoff limits + function SetLimits ( $offset, $limit, $max=0, $cutoff=0 ) + { + assert ( is_int($offset) ); + assert ( is_int($limit) ); + assert ( $offset>=0 ); + assert ( $limit>0 ); + assert ( $max>=0 ); + $this->_offset = $offset; + $this->_limit = $limit; + if ( $max>0 ) + $this->_maxmatches = $max; + if ( $cutoff>0 ) + $this->_cutoff = $cutoff; + } + + /// set maximum query time, in milliseconds, per-index + /// integer, 0 means "do not limit" + function SetMaxQueryTime ( $max ) + { + assert ( is_int($max) ); + assert ( $max>=0 ); + $this->_maxquerytime = $max; + } + + /// set matching mode + function SetMatchMode ( $mode ) + { + assert ( $mode==SPH_MATCH_ALL + || $mode==SPH_MATCH_ANY + || $mode==SPH_MATCH_PHRASE + || $mode==SPH_MATCH_BOOLEAN + || $mode==SPH_MATCH_EXTENDED + || $mode==SPH_MATCH_FULLSCAN + || $mode==SPH_MATCH_EXTENDED2 ); + $this->_mode = $mode; + } + + /// set ranking mode + function SetRankingMode ( $ranker ) + { + assert ( $ranker==SPH_RANK_PROXIMITY_BM25 + || $ranker==SPH_RANK_BM25 + || $ranker==SPH_RANK_NONE + || $ranker==SPH_RANK_WORDCOUNT ); + $this->_ranker = $ranker; + } + + /// set matches sorting mode + function SetSortMode ( $mode, $sortby="" ) + { + assert ( + $mode==SPH_SORT_RELEVANCE || + $mode==SPH_SORT_ATTR_DESC || + $mode==SPH_SORT_ATTR_ASC || + $mode==SPH_SORT_TIME_SEGMENTS || + $mode==SPH_SORT_EXTENDED || + $mode==SPH_SORT_EXPR ); + assert ( is_string($sortby) ); + assert ( $mode==SPH_SORT_RELEVANCE || strlen($sortby)>0 ); + + $this->_sort = $mode; + $this->_sortby = $sortby; + } + + /// bind per-field weights by order + /// DEPRECATED; use SetFieldWeights() instead + function SetWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $weight ) + assert ( is_int($weight) ); + + $this->_weights = $weights; + } + + /// bind per-field weights by name + function SetFieldWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $name=>$weight ) + { + assert ( is_string($name) ); + assert ( is_int($weight) ); + } + $this->_fieldweights = $weights; + } + + /// bind per-index weights by name + function SetIndexWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $index=>$weight ) + { + assert ( is_string($index) ); + assert ( is_int($weight) ); + } + $this->_indexweights = $weights; + } + + /// set IDs range to match + /// only match records if document ID is beetwen $min and $max (inclusive) + function SetIDRange ( $min, $max ) + { + assert ( is_numeric($min) ); + assert ( is_numeric($max) ); + assert ( $min<=$max ); + $this->_min_id = $min; + $this->_max_id = $max; + } + + /// set values set filter + /// only match records where $attribute value is in given set + function SetFilter ( $attribute, $values, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_array($values) ); + assert ( count($values) ); + + if ( is_array($values) && count($values) ) + { + foreach ( $values as $value ) + assert ( is_numeric($value) ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_VALUES, "attr"=>$attribute, "exclude"=>$exclude, "values"=>$values ); + } + } + + /// set range filter + /// only match records if $attribute value is beetwen $min and $max (inclusive) + function SetFilterRange ( $attribute, $min, $max, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_int($min) ); + assert ( is_int($max) ); + assert ( $min<=$max ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_RANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); + } + + /// set float range filter + /// only match records if $attribute value is beetwen $min and $max (inclusive) + function SetFilterFloatRange ( $attribute, $min, $max, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_float($min) ); + assert ( is_float($max) ); + assert ( $min<=$max ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_FLOATRANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); + } + + /// setup anchor point for geosphere distance calculations + /// required to use @geodist in filters and sorting + /// latitude and longitude must be in radians + function SetGeoAnchor ( $attrlat, $attrlong, $lat, $long ) + { + assert ( is_string($attrlat) ); + assert ( is_string($attrlong) ); + assert ( is_float($lat) ); + assert ( is_float($long) ); + + $this->_anchor = array ( "attrlat"=>$attrlat, "attrlong"=>$attrlong, "lat"=>$lat, "long"=>$long ); + } + + /// set grouping attribute and function + function SetGroupBy ( $attribute, $func, $groupsort="@group desc" ) + { + assert ( is_string($attribute) ); + assert ( is_string($groupsort) ); + assert ( $func==SPH_GROUPBY_DAY + || $func==SPH_GROUPBY_WEEK + || $func==SPH_GROUPBY_MONTH + || $func==SPH_GROUPBY_YEAR + || $func==SPH_GROUPBY_ATTR + || $func==SPH_GROUPBY_ATTRPAIR ); + + $this->_groupby = $attribute; + $this->_groupfunc = $func; + $this->_groupsort = $groupsort; + } + + /// set count-distinct attribute for group-by queries + function SetGroupDistinct ( $attribute ) + { + assert ( is_string($attribute) ); + $this->_groupdistinct = $attribute; + } + + /// set distributed retries count and delay + function SetRetries ( $count, $delay=0 ) + { + assert ( is_int($count) && $count>=0 ); + assert ( is_int($delay) && $delay>=0 ); + $this->_retrycount = $count; + $this->_retrydelay = $delay; + } + + /// set result set format (hash or array; hash by default) + /// PHP specific; needed for group-by-MVA result sets that may contain duplicate IDs + function SetArrayResult ( $arrayresult ) + { + assert ( is_bool($arrayresult) ); + $this->_arrayresult = $arrayresult; + } + + ////////////////////////////////////////////////////////////////////////////// + + /// clear all filters (for multi-queries) + function ResetFilters () + { + $this->_filters = array(); + $this->_anchor = array(); + } + + /// clear groupby settings (for multi-queries) + function ResetGroupBy () + { + $this->_groupby = ""; + $this->_groupfunc = SPH_GROUPBY_DAY; + $this->_groupsort = "@group desc"; + $this->_groupdistinct= ""; + } + + ////////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, run given search query through given indexes, + /// and return the search results + function Query ( $query, $index="*", $comment="" ) + { + assert ( empty($this->_reqs) ); + + $this->AddQuery ( $query, $index, $comment ); + $results = $this->RunQueries (); + $this->_reqs = array (); // just in case it failed too early + + if ( !is_array($results) ) + return false; // probably network error; error message should be already filled + + $this->_error = $results[0]["error"]; + $this->_warning = $results[0]["warning"]; + if ( $results[0]["status"]==SEARCHD_ERROR ) + return false; + else + return $results[0]; + } + + /// helper to pack floats in network byte order + function _PackFloat ( $f ) + { + $t1 = pack ( "f", $f ); // machine order + list(,$t2) = unpack ( "L*", $t1 ); // int in machine order + return pack ( "N", $t2 ); + } + + /// add query to multi-query batch + /// returns index into results array from RunQueries() call + function AddQuery ( $query, $index="*", $comment="" ) + { + // mbstring workaround + $this->_MBPush (); + + // build request + $req = pack ( "NNNNN", $this->_offset, $this->_limit, $this->_mode, $this->_ranker, $this->_sort ); // mode and limits + $req .= pack ( "N", strlen($this->_sortby) ) . $this->_sortby; + $req .= pack ( "N", strlen($query) ) . $query; // query itself + $req .= pack ( "N", count($this->_weights) ); // weights + foreach ( $this->_weights as $weight ) + $req .= pack ( "N", (int)$weight ); + $req .= pack ( "N", strlen($index) ) . $index; // indexes + $req .= pack ( "N", 1 ); // id64 range marker + $req .= sphPack64 ( $this->_min_id ) . sphPack64 ( $this->_max_id ); // id64 range + + // filters + $req .= pack ( "N", count($this->_filters) ); + foreach ( $this->_filters as $filter ) + { + $req .= pack ( "N", strlen($filter["attr"]) ) . $filter["attr"]; + $req .= pack ( "N", $filter["type"] ); + switch ( $filter["type"] ) + { + case SPH_FILTER_VALUES: + $req .= pack ( "N", count($filter["values"]) ); + foreach ( $filter["values"] as $value ) + $req .= pack ( "N", floatval($value) ); // this uberhack is to workaround 32bit signed int limit on x32 platforms + break; + + case SPH_FILTER_RANGE: + $req .= pack ( "NN", $filter["min"], $filter["max"] ); + break; + + case SPH_FILTER_FLOATRANGE: + $req .= $this->_PackFloat ( $filter["min"] ) . $this->_PackFloat ( $filter["max"] ); + break; + + default: + assert ( 0 && "internal error: unhandled filter type" ); + } + $req .= pack ( "N", $filter["exclude"] ); + } + + // group-by clause, max-matches count, group-sort clause, cutoff count + $req .= pack ( "NN", $this->_groupfunc, strlen($this->_groupby) ) . $this->_groupby; + $req .= pack ( "N", $this->_maxmatches ); + $req .= pack ( "N", strlen($this->_groupsort) ) . $this->_groupsort; + $req .= pack ( "NNN", $this->_cutoff, $this->_retrycount, $this->_retrydelay ); + $req .= pack ( "N", strlen($this->_groupdistinct) ) . $this->_groupdistinct; + + // anchor point + if ( empty($this->_anchor) ) + { + $req .= pack ( "N", 0 ); + } else + { + $a =& $this->_anchor; + $req .= pack ( "N", 1 ); + $req .= pack ( "N", strlen($a["attrlat"]) ) . $a["attrlat"]; + $req .= pack ( "N", strlen($a["attrlong"]) ) . $a["attrlong"]; + $req .= $this->_PackFloat ( $a["lat"] ) . $this->_PackFloat ( $a["long"] ); + } + + // per-index weights + $req .= pack ( "N", count($this->_indexweights) ); + foreach ( $this->_indexweights as $idx=>$weight ) + $req .= pack ( "N", strlen($idx) ) . $idx . pack ( "N", $weight ); + + // max query time + $req .= pack ( "N", $this->_maxquerytime ); + + // per-field weights + $req .= pack ( "N", count($this->_fieldweights) ); + foreach ( $this->_fieldweights as $field=>$weight ) + $req .= pack ( "N", strlen($field) ) . $field . pack ( "N", $weight ); + + // comment + $req .= pack ( "N", strlen($comment) ) . $comment; + + // mbstring workaround + $this->_MBPop (); + + // store request to requests array + $this->_reqs[] = $req; + return count($this->_reqs)-1; + } + + /// connect to searchd, run queries batch, and return an array of result sets + function RunQueries () + { + if ( empty($this->_reqs) ) + { + $this->_error = "no queries defined, issue AddQuery() first"; + return false; + } + + // mbstring workaround + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop (); + return false; + } + + //////////////////////////// + // send query, get response + //////////////////////////// + + $nreqs = count($this->_reqs); + $req = join ( "", $this->_reqs ); + $len = 4+strlen($req); + $req = pack ( "nnNN", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $len, $nreqs ) . $req; // add header + + fwrite ( $fp, $req, $len+8 ); + if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_SEARCH ) )) + { + $this->_MBPop (); + return false; + } + + $this->_reqs = array (); + + ////////////////// + // parse response + ////////////////// + + $p = 0; // current position + $max = strlen($response); // max position for checks, to protect against broken responses + + $results = array (); + for ( $ires=0; $ires<$nreqs && $p<$max; $ires++ ) + { + $results[] = array(); + $result =& $results[$ires]; + + $result["error"] = ""; + $result["warning"] = ""; + + // extract status + list(,$status) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $result["status"] = $status; + if ( $status!=SEARCHD_OK ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $message = substr ( $response, $p, $len ); $p += $len; + + if ( $status==SEARCHD_WARNING ) + { + $result["warning"] = $message; + } else + { + $result["error"] = $message; + continue; + } + } + + // read schema + $fields = array (); + $attrs = array (); + + list(,$nfields) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + while ( $nfields-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $fields[] = substr ( $response, $p, $len ); $p += $len; + } + $result["fields"] = $fields; + + list(,$nattrs) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + while ( $nattrs-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attr = substr ( $response, $p, $len ); $p += $len; + list(,$type) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attrs[$attr] = $type; + } + $result["attrs"] = $attrs; + + // read match count + list(,$count) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + list(,$id64) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + + // read matches + $idx = -1; + while ( $count-->0 && $p<$max ) + { + // index into result array + $idx++; + + // parse document id and weight + if ( $id64 ) + { + $doc = sphUnpack64 ( substr ( $response, $p, 8 ) ); $p += 8; + list(,$weight) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + } else + { + list ( $doc, $weight ) = array_values ( unpack ( "N*N*", + substr ( $response, $p, 8 ) ) ); + $p += 8; + + if ( PHP_INT_SIZE>=8 ) + { + // x64 route, workaround broken unpack() in 5.2.2+ + if ( $doc<0 ) $doc += (1<<32); + } else + { + // x32 route, workaround php signed/unsigned braindamage + $doc = sprintf ( "%u", $doc ); + } + } + $weight = sprintf ( "%u", $weight ); + + // create match entry + if ( $this->_arrayresult ) + $result["matches"][$idx] = array ( "id"=>$doc, "weight"=>$weight ); + else + $result["matches"][$doc]["weight"] = $weight; + + // parse and create attributes + $attrvals = array (); + foreach ( $attrs as $attr=>$type ) + { + // handle floats + if ( $type==SPH_ATTR_FLOAT ) + { + list(,$uval) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + list(,$fval) = unpack ( "f*", pack ( "L", $uval ) ); + $attrvals[$attr] = $fval; + continue; + } + + // handle everything else as unsigned ints + list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + if ( $type & SPH_ATTR_MULTI ) + { + $attrvals[$attr] = array (); + $nvalues = $val; + while ( $nvalues-->0 && $p<$max ) + { + list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attrvals[$attr][] = sprintf ( "%u", $val ); + } + } else + { + $attrvals[$attr] = sprintf ( "%u", $val ); + } + } + + if ( $this->_arrayresult ) + $result["matches"][$idx]["attrs"] = $attrvals; + else + $result["matches"][$doc]["attrs"] = $attrvals; + } + + list ( $total, $total_found, $msecs, $words ) = + array_values ( unpack ( "N*N*N*N*", substr ( $response, $p, 16 ) ) ); + $result["total"] = sprintf ( "%u", $total ); + $result["total_found"] = sprintf ( "%u", $total_found ); + $result["time"] = sprintf ( "%.3f", $msecs/1000 ); + $p += 16; + + while ( $words-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $word = substr ( $response, $p, $len ); $p += $len; + list ( $docs, $hits ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; + $result["words"][$word] = array ( + "docs"=>sprintf ( "%u", $docs ), + "hits"=>sprintf ( "%u", $hits ) ); + } + } + + $this->_MBPop (); + return $results; + } + + ///////////////////////////////////////////////////////////////////////////// + // excerpts generation + ///////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, and generate exceprts (snippets) + /// of given documents for given query. returns false on failure, + /// an array of snippets on success + function BuildExcerpts ( $docs, $index, $words, $opts=array() ) + { + assert ( is_array($docs) ); + assert ( is_string($index) ); + assert ( is_string($words) ); + assert ( is_array($opts) ); + + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + ///////////////// + // fixup options + ///////////////// + + if ( !isset($opts["before_match"]) ) $opts["before_match"] = ""; + if ( !isset($opts["after_match"]) ) $opts["after_match"] = ""; + if ( !isset($opts["chunk_separator"]) ) $opts["chunk_separator"] = " ... "; + if ( !isset($opts["limit"]) ) $opts["limit"] = 256; + if ( !isset($opts["around"]) ) $opts["around"] = 5; + if ( !isset($opts["exact_phrase"]) ) $opts["exact_phrase"] = false; + if ( !isset($opts["single_passage"]) ) $opts["single_passage"] = false; + if ( !isset($opts["use_boundaries"]) ) $opts["use_boundaries"] = false; + if ( !isset($opts["weight_order"]) ) $opts["weight_order"] = false; + + ///////////////// + // build request + ///////////////// + + // v.1.0 req + $flags = 1; // remove spaces + if ( $opts["exact_phrase"] ) $flags |= 2; + if ( $opts["single_passage"] ) $flags |= 4; + if ( $opts["use_boundaries"] ) $flags |= 8; + if ( $opts["weight_order"] ) $flags |= 16; + $req = pack ( "NN", 0, $flags ); // mode=0, flags=$flags + $req .= pack ( "N", strlen($index) ) . $index; // req index + $req .= pack ( "N", strlen($words) ) . $words; // req words + + // options + $req .= pack ( "N", strlen($opts["before_match"]) ) . $opts["before_match"]; + $req .= pack ( "N", strlen($opts["after_match"]) ) . $opts["after_match"]; + $req .= pack ( "N", strlen($opts["chunk_separator"]) ) . $opts["chunk_separator"]; + $req .= pack ( "N", (int)$opts["limit"] ); + $req .= pack ( "N", (int)$opts["around"] ); + + // documents + $req .= pack ( "N", count($docs) ); + foreach ( $docs as $doc ) + { + assert ( is_string($doc) ); + $req .= pack ( "N", strlen($doc) ) . $doc; + } + + //////////////////////////// + // send query, get response + //////////////////////////// + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $len ) . $req; // add header + $wrote = fwrite ( $fp, $req, $len+8 ); + if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_EXCERPT ) )) + { + $this->_MBPop (); + return false; + } + + ////////////////// + // parse response + ////////////////// + + $pos = 0; + $res = array (); + $rlen = strlen($response); + for ( $i=0; $i $rlen ) + { + $this->_error = "incomplete reply"; + $this->_MBPop (); + return false; + } + $res[] = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + } + + $this->_MBPop (); + return $res; + } + + + ///////////////////////////////////////////////////////////////////////////// + // keyword generation + ///////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, and generate keyword list for a given query + /// returns false on failure, + /// an array of words on success + function BuildKeywords ( $query, $index, $hits ) + { + assert ( is_string($query) ); + assert ( is_string($index) ); + assert ( is_bool($hits) ); + + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + ///////////////// + // build request + ///////////////// + + // v.1.0 req + $req = pack ( "N", strlen($query) ) . $query; // req query + $req .= pack ( "N", strlen($index) ) . $index; // req index + $req .= pack ( "N", (int)$hits ); + + //////////////////////////// + // send query, get response + //////////////////////////// + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, $len ) . $req; // add header + $wrote = fwrite ( $fp, $req, $len+8 ); + if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_KEYWORDS ) )) + { + $this->_MBPop (); + return false; + } + + ////////////////// + // parse response + ////////////////// + + $pos = 0; + $res = array (); + $rlen = strlen($response); + list(,$nwords) = unpack ( "N*", substr ( $response, $pos, 4 ) ); + $pos += 4; + for ( $i=0; $i<$nwords; $i++ ) + { + list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; + $tokenized = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + + list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; + $normalized = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + + $res[] = array ( "tokenized"=>$tokenized, "normalized"=>$normalized ); + + if ( $hits ) + { + list($ndocs,$nhits) = array_values ( unpack ( "N*N*", substr ( $response, $pos, 8 ) ) ); + $pos += 8; + $res [$i]["docs"] = $ndocs; + $res [$i]["hits"] = $nhits; + } + + if ( $pos > $rlen ) + { + $this->_error = "incomplete reply"; + $this->_MBPop (); + return false; + } + } + + $this->_MBPop (); + return $res; + } + + function EscapeString ( $string ) + { + $from = array ( '(',')','|','-','!','@','~','"','&', '/' ); + $to = array ( '\(','\)','\|','\-','\!','\@','\~','\"', '\&', '\/' ); + + return str_replace ( $from, $to, $string ); + } + + ///////////////////////////////////////////////////////////////////////////// + // attribute updates + ///////////////////////////////////////////////////////////////////////////// + + /// update given attribute values on given documents in given indexes + /// returns amount of updated documents (0 or more) on success, or -1 on failure + function UpdateAttributes ( $index, $attrs, $values ) + { + // verify everything + assert ( is_string($index) ); + + assert ( is_array($attrs) ); + foreach ( $attrs as $attr ) + assert ( is_string($attr) ); + + assert ( is_array($values) ); + foreach ( $values as $id=>$entry ) + { + assert ( is_numeric($id) ); + assert ( is_array($entry) ); + assert ( count($entry)==count($attrs) ); + foreach ( $entry as $v ) + assert ( is_int($v) ); + } + + // build request + $req = pack ( "N", strlen($index) ) . $index; + + $req .= pack ( "N", count($attrs) ); + foreach ( $attrs as $attr ) + $req .= pack ( "N", strlen($attr) ) . $attr; + + $req .= pack ( "N", count($values) ); + foreach ( $values as $id=>$entry ) + { + $req .= sphPack64 ( $id ); + foreach ( $entry as $v ) + $req .= pack ( "N", $v ); + } + + // mbstring workaround + $this->_MBPush (); + + // connect, send query, get response + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop (); + return -1; + } + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, $len ) . $req; // add header + fwrite ( $fp, $req, $len+8 ); + + if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_UPDATE ) )) + { + $this->_MBPop (); + return -1; + } + + // parse response + list(,$updated) = unpack ( "N*", substr ( $response, 0, 4 ) ); + $this->_MBPop (); + return $updated; + } +} + +// +// $Id$ +// \ No newline at end of file diff --git a/phpBB/language/en/mods/fulltext_sphinx.php b/phpBB/language/en/mods/fulltext_sphinx.php new file mode 100644 index 0000000000..e06328afc8 --- /dev/null +++ b/phpBB/language/en/mods/fulltext_sphinx.php @@ -0,0 +1,65 @@ + 'Automatically configure Sphinx', + 'FULLTEXT_SPHINX_AUTOCONF_EXPLAIN' => 'This is the easiest way to install Sphinx, just select the settings here and a config file will be written for you. This requires write permissions on the configuration folder.', + 'FULLTEXT_SPHINX_AUTORUN' => 'Automatically run Sphinx', + 'FULLTEXT_SPHINX_AUTORUN_EXPLAIN' => 'This is the easiest way to run Sphinx. Select the paths in this dialogue and the Sphinx daemon will be started and stopped as needed. You can also create an index from the ACP. If your PHP installation forbids the use of exec you can disable this and run Sphinx manually.', + 'FULLTEXT_SPHINX_BIN_PATH' => 'Path to executables directory', + 'FULLTEXT_SPHINX_BIN_PATH_EXPLAIN' => 'Skip if autorun is disabled. If this path could not be determined automatically you have to enter the path to the directory in which the sphinx executables indexer and searchd reside.', + 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to configuration directory', + 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'Skip if autoconf is disabled. You should create this config directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody).', + 'FULLTEXT_SPHINX_CONFIGURE_FIRST' => 'Before you create an index you have to enable and configure sphinx under GENERAL -> SERVER CONFIGURATION -> Search settings.', + 'FULLTEXT_SPHINX_CONFIGURE_BEFORE' => 'Configure the following settings BEFORE activating Sphinx', + 'FULLTEXT_SPHINX_CONFIGURE_AFTER' => 'The following settings do not have to be configured before activating Sphinx', + 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', + 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'Skip if autorun is disabled. You should create this directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody). It will be used to store the indexes and log files.', + 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', + 'FULLTEXT_SPHINX_DIRECTORY_NOT_FOUND' => 'The directory %s does not exist. Please correct your path settings.', + 'FULLTEXT_SPHINX_FILE_NOT_EXECUTABLE' => 'The file %s is not executable for the webserver.', + 'FULLTEXT_SPHINX_FILE_NOT_FOUND' => 'The file %s does not exist. Please correct your path settings.', + 'FULLTEXT_SPHINX_FILE_NOT_WRITABLE' => 'The file %s cannot be written by the webserver.', + 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', + 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', + 'FULLTEXT_SPHINX_LAST_SEARCHES' => 'Recent search queries', + 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', + 'FULLTEXT_SPHINX_PORT' => 'Sphinx search deamon port', + 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search deamon on localhost listens. Leave empty to use the default 3312', + 'FULLTEXT_SPHINX_REQUIRES_EXEC' => 'The sphinx plugin for phpBB requires PHP’s exec function which is disabled on your system.', + 'FULLTEXT_SPHINX_UNCONFIGURED' => 'Please set all necessary options in the "Fulltext Sphinx" section of the previous page before you try to activate the sphinx plugin.', + 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx plugin for phpBB currently only supports MySQL', + 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', + 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'This setting only works with autoconf enabled. You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', +)); + +?> \ No newline at end of file From fcf0d04b20f1c862117a8ab962d692bd2b8b074f Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Wed, 9 May 2012 19:13:36 +0530 Subject: [PATCH 02/57] [feature/sphinx-fulltext-search] minor changes some minor code changes to make it working against current develop and comply with other search backend coding convetions. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index e0c467df93..ef357970a0 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -18,33 +18,27 @@ if (!defined('IN_PHPBB')) /** * @ignore */ +/** +* This statement is necessary as this file is sometimes included from within a +* function and the variables used are in global space. +*/ +global $phpbb_root_path, $phpEx, $table_prefix; require($phpbb_root_path . "includes/sphinxapi-0.9.8." . $phpEx); define('INDEXER_NAME', 'indexer'); define('SEARCHD_NAME', 'searchd'); -define('SPHINX_TABLE', table_prefix() . 'sphinx'); +define('SPHINX_TABLE', $table_prefix . 'sphinx'); define('MAX_MATCHES', 20000); define('CONNECT_RETRIES', 3); define('CONNECT_WAIT_TIME', 300); -/** -* Returns the global table prefix -* This function is necessary as this file is sometimes included from within a -* function and table_prefix is in global space. -*/ -function table_prefix() -{ - global $table_prefix; - return $table_prefix; -} - /** * fulltext_sphinx * Fulltext search based on the sphinx search deamon * @package search */ -class fulltext_sphinx +class phpbb_search_fulltext_sphinx { var $stats = array(); var $word_length = array(); @@ -53,7 +47,7 @@ class fulltext_sphinx var $common_words = array(); var $id; - function fulltext_sphinx(&$error) + public function __construct(&$error) { global $config; @@ -82,6 +76,16 @@ class fulltext_sphinx $error = false; } + + /** + * Returns the name of this search backend to be displayed to administrators + * + * @return string Name + */ + public function get_name() + { + return 'Sphinx Fulltext'; + } /** * Checks permissions and paths, if everything is correct it generates the config file From 99d4660df68d71ea56cccb150ae858c1dd7575b8 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Wed, 20 Jun 2012 05:11:53 +0530 Subject: [PATCH 03/57] [feature/sphinx-fulltext-search] update config file Sphinx config file updated according to new documentation. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index ef357970a0..6c5092f4aa 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -295,7 +295,7 @@ class phpbb_search_fulltext_sphinx array('stopwords', (file_exists($config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt') && $config['fulltext_sphinx_stopwords']) ? $config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), array('min_word_len', '2'), array('charset_type', 'utf-8'), - array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+ß410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), + array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), array('min_prefix_len', '0'), array('min_infix_len', '0'), ), @@ -307,7 +307,8 @@ class phpbb_search_fulltext_sphinx array('mem_limit', $config['fulltext_sphinx_indexer_mem_limit'] . 'M'), ), 'searchd' => array( - array('address' , '127.0.0.1'), + array('compat_sphinxql_magics' , '0'), + array('listen' , '127.0.0.1'), array('port', ($config['fulltext_sphinx_port']) ? $config['fulltext_sphinx_port'] : '3312'), array('log', $config['fulltext_sphinx_data_path'] . "log/searchd.log"), array('query_log', $config['fulltext_sphinx_data_path'] . "log/sphinx-query.log"), From 8d76bc45ee19186f40dd3b459a9bd33e5e4c23d9 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 26 Jun 2012 02:35:36 +0530 Subject: [PATCH 04/57] [feature/sphinx-fulltext-search] minor fixes in formatting Add a newline at the end of files. Update License information in package docbloc. PHPBB3-10946 --- phpBB/includes/functions_sphinx.php | 5 ++--- phpBB/includes/search/fulltext_sphinx.php | 5 +---- phpBB/includes/sphinxapi-0.9.8.php | 2 +- phpBB/language/en/mods/fulltext_sphinx.php | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/phpBB/includes/functions_sphinx.php b/phpBB/includes/functions_sphinx.php index 976f93f77c..0f83f8cfb5 100644 --- a/phpBB/includes/functions_sphinx.php +++ b/phpBB/includes/functions_sphinx.php @@ -2,9 +2,8 @@ /** * * @package search -* @version $Id$ * @copyright (c) 2005 phpBB Group -* @license http://opensource.org/licenses/gpl-license.php GNU Public License +* @license http://opensource.org/licenses/gpl-2.0.php GNU General Public License v2 * */ @@ -504,4 +503,4 @@ class sphinx_config_comment } } -?> \ No newline at end of file +?> diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 6c5092f4aa..9ae6438af2 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -2,9 +2,8 @@ /** * * @package search -* @version $Id$ * @copyright (c) 2005 phpBB Group -* @license http://opensource.org/licenses/gpl-license.php GNU Public License +* @license http://opensource.org/licenses/gpl-2.0.php GNU General Public License v2 * */ @@ -1167,5 +1166,3 @@ function sphinx_read_last_lines($file, $amount) return $string; } - -?> \ No newline at end of file diff --git a/phpBB/includes/sphinxapi-0.9.8.php b/phpBB/includes/sphinxapi-0.9.8.php index 6a7ea17760..816895d464 100644 --- a/phpBB/includes/sphinxapi-0.9.8.php +++ b/phpBB/includes/sphinxapi-0.9.8.php @@ -1199,4 +1199,4 @@ class SphinxClient // // $Id$ -// \ No newline at end of file +// diff --git a/phpBB/language/en/mods/fulltext_sphinx.php b/phpBB/language/en/mods/fulltext_sphinx.php index e06328afc8..f3fd68aa62 100644 --- a/phpBB/language/en/mods/fulltext_sphinx.php +++ b/phpBB/language/en/mods/fulltext_sphinx.php @@ -62,4 +62,4 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'This setting only works with autoconf enabled. You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', )); -?> \ No newline at end of file +?> From a3b2caf8416c687306b3c2e83b2fdc6e8708cce0 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Mon, 25 Jun 2012 00:03:46 +0530 Subject: [PATCH 05/57] [feature/sphinx-fulltext-search] include sample sphinx.conf in docs PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 96 +++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 phpBB/docs/sphinx.sample.conf diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf new file mode 100644 index 0000000000..d7e59a11fc --- /dev/null +++ b/phpBB/docs/sphinx.sample.conf @@ -0,0 +1,96 @@ +source source_phpbb_{AVATAR_SALT}_main +{ + type = mysql + sql_host = localhost + sql_user = username + sql_pass = password + sql_db = db_name + sql_port = 3306 #optional, default is 3306 + sql_query_range = SELECT MIN(post_id), MAX(post_id) FROM phpbb_posts + sql_range_step = 5000 + sql_query = SELECT \ + p.post_id AS id, \ + p.forum_id, \ + p.topic_id, \ + p.poster_id, \ + IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, \ + p.post_time, \ + p.post_subject, \ + p.post_subject as title, \ + p.post_text as data, \ + t.topic_last_post_time, \ + 0 as deleted \ + FROM phpbb_posts p, phpbb_topics t \ + WHERE \ + p.topic_id = t.topic_id \ + AND p.post_id >= $start AND p.post_id <= $end + sql_query_post = + sql_query_post_index = REPLACE INTO phpbb_sphinx ( counter_id, max_doc_id ) VALUES ( 1, $maxid ) + sql_query_info = SELECT * FROM phpbb_posts WHERE post_id = $id + sql_query_pre = SET NAMES utf8 + sql_query_pre = REPLACE INTO phpbb_sphinx SELECT 1, MAX(post_id) FROM phpbb_posts + sql_attr_uint = forum_id + sql_attr_uint = topic_id + sql_attr_uint = poster_id + sql_attr_bool = topic_first_post + sql_attr_bool = deleted + sql_attr_timestamp = post_time + sql_attr_timestamp = topic_last_post_time + sql_attr_str2ordinal = post_subject +} +source source_phpbb_{AVATAR_SALT}_delta : source_phpbb_{AVATAR_SALT}_main +{ + sql_query_range = + sql_range_step = + sql_query = SELECT \ + p.post_id AS id, \ + p.forum_id, \ + p.topic_id, \ + p.poster_id, \ + IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, \ + p.post_time, \ + p.post_subject, \ + p.post_subject as title, \ + p.post_text as data, \ + t.topic_last_post_time, \ + 0 as deleted \ + FROM phpbb_posts p, phpbb_topics t \ + WHERE \ + p.topic_id = t.topic_id \ + AND p.post_id >= ( SELECT max_doc_id FROM phpbb_sphinx WHERE counter_id=1 ) + sql_query_pre = +} +index index_phpbb_{AVATAR_SALT}_main +{ + path = {DATA_PATH}/index_phpbb_{AVATAR_SALT}_main + source = source_phpbb_{AVATAR_SALT}_main + docinfo = extern + morphology = none + stopwords = + min_word_len = 2 + charset_type = utf-8 + charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF + min_prefix_len = 0 + min_infix_len = 0 +} +index index_phpbb_{AVATAR_SALT}_delta : index_phpbb_{AVATAR_SALT}_main +{ + path = {DATA_PATH}/index_phpbb_{AVATAR_SALT}_delta + source = source_phpbb_{AVATAR_SALT}_delta +} +indexer +{ + mem_limit = 512M +} +searchd +{ + compat_sphinxql_magics = 0 + listen = 127.0.0.1 + port = 3312 + log = {DATA_PATH}/log/searchd.log + query_log = {DATA_PATH}/log/sphinx-query.log + read_timeout = 5 + max_children = 30 + pid_file = {DATA_PATH}/searchd.pid + max_matches = 20000 +} From 455a35d8361c93657874e140a2ad5b2e5c267757 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 26 Jun 2012 03:56:03 +0530 Subject: [PATCH 06/57] [feature/sphinx-fulltext-search] temporary commit to pull out sphinx-api also need to add the latest sphinx api instead of this. PHPBB3-10946 --- phpBB/includes/sphinxapi-0.9.8.php | 1202 ---------------------------- 1 file changed, 1202 deletions(-) delete mode 100644 phpBB/includes/sphinxapi-0.9.8.php diff --git a/phpBB/includes/sphinxapi-0.9.8.php b/phpBB/includes/sphinxapi-0.9.8.php deleted file mode 100644 index 816895d464..0000000000 --- a/phpBB/includes/sphinxapi-0.9.8.php +++ /dev/null @@ -1,1202 +0,0 @@ -=8 ) - { - $i = (int)$v; - return pack ( "NN", $i>>32, $i&((1<<32)-1) ); - } - - // x32 route, bcmath - $x = "4294967296"; - if ( function_exists("bcmul") ) - { - $h = bcdiv ( $v, $x, 0 ); - $l = bcmod ( $v, $x ); - return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit - } - - // x32 route, 15 or less decimal digits - // we can use float, because its actually double and has 52 precision bits - if ( strlen($v)<=15 ) - { - $f = (float)$v; - $h = (int)($f/$x); - $l = (int)($f-$x*$h); - return pack ( "NN", $h, $l ); - } - - // x32 route, 16 or more decimal digits - // well, let me know if you *really* need this - die ( "INTERNAL ERROR: packing more than 15-digit numeric on 32-bit PHP is not implemented yet (contact support)" ); -} - - -/// portably unpack 64 unsigned bits, network order to numeric -function sphUnpack64 ( $v ) -{ - list($h,$l) = array_values ( unpack ( "N*N*", $v ) ); - - // x64 route - if ( PHP_INT_SIZE>=8 ) - { - if ( $h<0 ) $h += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again - if ( $l<0 ) $l += (1<<32); - return ($h<<32) + $l; - } - - // x32 route - $h = sprintf ( "%u", $h ); - $l = sprintf ( "%u", $l ); - $x = "4294967296"; - - // bcmath - if ( function_exists("bcmul") ) - return bcadd ( $l, bcmul ( $x, $h ) ); - - // no bcmath, 15 or less decimal digits - // we can use float, because its actually double and has 52 precision bits - if ( $h<1048576 ) - { - $f = ((float)$h)*$x + (float)$l; - return sprintf ( "%.0f", $f ); // builtin conversion is only about 39-40 bits precise! - } - - // x32 route, 16 or more decimal digits - // well, let me know if you *really* need this - die ( "INTERNAL ERROR: unpacking more than 15-digit numeric on 32-bit PHP is not implemented yet (contact support)" ); -} - - -/// sphinx searchd client class -class SphinxClient -{ - var $_host; ///< searchd host (default is "localhost") - var $_port; ///< searchd port (default is 3312) - var $_offset; ///< how many records to seek from result-set start (default is 0) - var $_limit; ///< how many records to return from result-set starting at offset (default is 20) - var $_mode; ///< query matching mode (default is SPH_MATCH_ALL) - var $_weights; ///< per-field weights (default is 1 for all fields) - var $_sort; ///< match sorting mode (default is SPH_SORT_RELEVANCE) - var $_sortby; ///< attribute to sort by (defualt is "") - var $_min_id; ///< min ID to match (default is 0, which means no limit) - var $_max_id; ///< max ID to match (default is 0, which means no limit) - var $_filters; ///< search filters - var $_groupby; ///< group-by attribute name - var $_groupfunc; ///< group-by function (to pre-process group-by attribute value with) - var $_groupsort; ///< group-by sorting clause (to sort groups in result set with) - var $_groupdistinct;///< group-by count-distinct attribute - var $_maxmatches; ///< max matches to retrieve - var $_cutoff; ///< cutoff to stop searching at (default is 0) - var $_retrycount; ///< distributed retries count - var $_retrydelay; ///< distributed retries delay - var $_anchor; ///< geographical anchor point - var $_indexweights; ///< per-index weights - var $_ranker; ///< ranking mode (default is SPH_RANK_PROXIMITY_BM25) - var $_maxquerytime; ///< max query time, milliseconds (default is 0, do not limit) - var $_fieldweights; ///< per-field-name weights - - var $_error; ///< last error message - var $_warning; ///< last warning message - - var $_reqs; ///< requests array for multi-query - var $_mbenc; ///< stored mbstring encoding - var $_arrayresult; ///< whether $result["matches"] should be a hash or an array - var $_timeout; ///< connect timeout - - ///////////////////////////////////////////////////////////////////////////// - // common stuff - ///////////////////////////////////////////////////////////////////////////// - - /// create a new client object and fill defaults - function SphinxClient () - { - // per-client-object settings - $this->_host = "localhost"; - $this->_port = 3312; - - // per-query settings - $this->_offset = 0; - $this->_limit = 20; - $this->_mode = SPH_MATCH_ALL; - $this->_weights = array (); - $this->_sort = SPH_SORT_RELEVANCE; - $this->_sortby = ""; - $this->_min_id = 0; - $this->_max_id = 0; - $this->_filters = array (); - $this->_groupby = ""; - $this->_groupfunc = SPH_GROUPBY_DAY; - $this->_groupsort = "@group desc"; - $this->_groupdistinct= ""; - $this->_maxmatches = 1000; - $this->_cutoff = 0; - $this->_retrycount = 0; - $this->_retrydelay = 0; - $this->_anchor = array (); - $this->_indexweights= array (); - $this->_ranker = SPH_RANK_PROXIMITY_BM25; - $this->_maxquerytime= 0; - $this->_fieldweights= array(); - - $this->_error = ""; // per-reply fields (for single-query case) - $this->_warning = ""; - $this->_reqs = array (); // requests storage (for multi-query case) - $this->_mbenc = ""; - $this->_arrayresult = false; - $this->_timeout = 0; - } - - /// get last error message (string) - function GetLastError () - { - return $this->_error; - } - - /// get last warning message (string) - function GetLastWarning () - { - return $this->_warning; - } - - /// set searchd host name (string) and port (integer) - function SetServer ( $host, $port ) - { - assert ( is_string($host) ); - assert ( is_int($port) ); - $this->_host = $host; - $this->_port = $port; - } - - /// set server connection timeout (0 to remove) - function SetConnectTimeout ( $timeout ) - { - assert ( is_numeric($timeout) ); - $this->_timeout = $timeout; - } - - ///////////////////////////////////////////////////////////////////////////// - - /// enter mbstring workaround mode - function _MBPush () - { - $this->_mbenc = ""; - if ( ini_get ( "mbstring.func_overload" ) & 2 ) - { - $this->_mbenc = mb_internal_encoding(); - mb_internal_encoding ( "latin1" ); - } - } - - /// leave mbstring workaround mode - function _MBPop () - { - if ( $this->_mbenc ) - mb_internal_encoding ( $this->_mbenc ); - } - - /// connect to searchd server - function _Connect ($allow_retry = true) - { - $errno = 0; - $errstr = ""; - if ( $this->_timeout<=0 ) - $fp = @fsockopen ( $this->_host, $this->_port, $errno, $errstr ); - else - $fp = @fsockopen ( $this->_host, $this->_port, $errno, $errstr, $this->_timeout ); - - if ( !$fp ) - { - $errstr = trim ( $errstr ); - $this->_error = "connection to {$this->_host}:{$this->_port} failed (errno=$errno, msg=$errstr)"; - return false; - } - - // check version - //list(,$v) = unpack ( "N*", fread ( $fp, 4 ) ); - $version_data = unpack ( "N*", fread ( $fp, 4 ) ); - if (!isset($version_data[1])) - { - // this should not happen, try to reconnect ONCE - if ($allow_retry) - { - return $this->_Connect(false); - } - else - { - $this->_error = "unexpected version data"; - return false; - } - } - $v = $version_data[1]; - $v = (int)$v; - if ( $v<1 ) - { - fclose ( $fp ); - $this->_error = "expected searchd protocol version 1+, got version '$v'"; - return false; - } - - // all ok, send my version - fwrite ( $fp, pack ( "N", 1 ) ); - return $fp; - } - - /// get and check response packet from searchd server - function _GetResponse ( $fp, $client_ver ) - { - $response = ""; - $len = 0; - - $header = fread ( $fp, 8 ); - if ( strlen($header)==8 ) - { - list ( $status, $ver, $len ) = array_values ( unpack ( "n2a/Nb", $header ) ); - $left = $len; - while ( $left>0 && !feof($fp) ) - { - $chunk = fread ( $fp, $left ); - if ( $chunk ) - { - $response .= $chunk; - $left -= strlen($chunk); - } - } - } - fclose ( $fp ); - - // check response - $read = strlen ( $response ); - if ( !$response || $read!=$len ) - { - $this->_error = $len - ? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=$read)" - : "received zero-sized searchd response"; - return false; - } - - // check status - if ( $status==SEARCHD_WARNING ) - { - list(,$wlen) = unpack ( "N*", substr ( $response, 0, 4 ) ); - $this->_warning = substr ( $response, 4, $wlen ); - return substr ( $response, 4+$wlen ); - } - if ( $status==SEARCHD_ERROR ) - { - $this->_error = "searchd error: " . substr ( $response, 4 ); - return false; - } - if ( $status==SEARCHD_RETRY ) - { - $this->_error = "temporary searchd error: " . substr ( $response, 4 ); - return false; - } - if ( $status!=SEARCHD_OK ) - { - $this->_error = "unknown status code '$status'"; - return false; - } - - // check version - if ( $ver<$client_ver ) - { - $this->_warning = sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work", - $ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ); - } - - return $response; - } - - ///////////////////////////////////////////////////////////////////////////// - // searching - ///////////////////////////////////////////////////////////////////////////// - - /// set offset and count into result set, - /// and optionally set max-matches and cutoff limits - function SetLimits ( $offset, $limit, $max=0, $cutoff=0 ) - { - assert ( is_int($offset) ); - assert ( is_int($limit) ); - assert ( $offset>=0 ); - assert ( $limit>0 ); - assert ( $max>=0 ); - $this->_offset = $offset; - $this->_limit = $limit; - if ( $max>0 ) - $this->_maxmatches = $max; - if ( $cutoff>0 ) - $this->_cutoff = $cutoff; - } - - /// set maximum query time, in milliseconds, per-index - /// integer, 0 means "do not limit" - function SetMaxQueryTime ( $max ) - { - assert ( is_int($max) ); - assert ( $max>=0 ); - $this->_maxquerytime = $max; - } - - /// set matching mode - function SetMatchMode ( $mode ) - { - assert ( $mode==SPH_MATCH_ALL - || $mode==SPH_MATCH_ANY - || $mode==SPH_MATCH_PHRASE - || $mode==SPH_MATCH_BOOLEAN - || $mode==SPH_MATCH_EXTENDED - || $mode==SPH_MATCH_FULLSCAN - || $mode==SPH_MATCH_EXTENDED2 ); - $this->_mode = $mode; - } - - /// set ranking mode - function SetRankingMode ( $ranker ) - { - assert ( $ranker==SPH_RANK_PROXIMITY_BM25 - || $ranker==SPH_RANK_BM25 - || $ranker==SPH_RANK_NONE - || $ranker==SPH_RANK_WORDCOUNT ); - $this->_ranker = $ranker; - } - - /// set matches sorting mode - function SetSortMode ( $mode, $sortby="" ) - { - assert ( - $mode==SPH_SORT_RELEVANCE || - $mode==SPH_SORT_ATTR_DESC || - $mode==SPH_SORT_ATTR_ASC || - $mode==SPH_SORT_TIME_SEGMENTS || - $mode==SPH_SORT_EXTENDED || - $mode==SPH_SORT_EXPR ); - assert ( is_string($sortby) ); - assert ( $mode==SPH_SORT_RELEVANCE || strlen($sortby)>0 ); - - $this->_sort = $mode; - $this->_sortby = $sortby; - } - - /// bind per-field weights by order - /// DEPRECATED; use SetFieldWeights() instead - function SetWeights ( $weights ) - { - assert ( is_array($weights) ); - foreach ( $weights as $weight ) - assert ( is_int($weight) ); - - $this->_weights = $weights; - } - - /// bind per-field weights by name - function SetFieldWeights ( $weights ) - { - assert ( is_array($weights) ); - foreach ( $weights as $name=>$weight ) - { - assert ( is_string($name) ); - assert ( is_int($weight) ); - } - $this->_fieldweights = $weights; - } - - /// bind per-index weights by name - function SetIndexWeights ( $weights ) - { - assert ( is_array($weights) ); - foreach ( $weights as $index=>$weight ) - { - assert ( is_string($index) ); - assert ( is_int($weight) ); - } - $this->_indexweights = $weights; - } - - /// set IDs range to match - /// only match records if document ID is beetwen $min and $max (inclusive) - function SetIDRange ( $min, $max ) - { - assert ( is_numeric($min) ); - assert ( is_numeric($max) ); - assert ( $min<=$max ); - $this->_min_id = $min; - $this->_max_id = $max; - } - - /// set values set filter - /// only match records where $attribute value is in given set - function SetFilter ( $attribute, $values, $exclude=false ) - { - assert ( is_string($attribute) ); - assert ( is_array($values) ); - assert ( count($values) ); - - if ( is_array($values) && count($values) ) - { - foreach ( $values as $value ) - assert ( is_numeric($value) ); - - $this->_filters[] = array ( "type"=>SPH_FILTER_VALUES, "attr"=>$attribute, "exclude"=>$exclude, "values"=>$values ); - } - } - - /// set range filter - /// only match records if $attribute value is beetwen $min and $max (inclusive) - function SetFilterRange ( $attribute, $min, $max, $exclude=false ) - { - assert ( is_string($attribute) ); - assert ( is_int($min) ); - assert ( is_int($max) ); - assert ( $min<=$max ); - - $this->_filters[] = array ( "type"=>SPH_FILTER_RANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); - } - - /// set float range filter - /// only match records if $attribute value is beetwen $min and $max (inclusive) - function SetFilterFloatRange ( $attribute, $min, $max, $exclude=false ) - { - assert ( is_string($attribute) ); - assert ( is_float($min) ); - assert ( is_float($max) ); - assert ( $min<=$max ); - - $this->_filters[] = array ( "type"=>SPH_FILTER_FLOATRANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); - } - - /// setup anchor point for geosphere distance calculations - /// required to use @geodist in filters and sorting - /// latitude and longitude must be in radians - function SetGeoAnchor ( $attrlat, $attrlong, $lat, $long ) - { - assert ( is_string($attrlat) ); - assert ( is_string($attrlong) ); - assert ( is_float($lat) ); - assert ( is_float($long) ); - - $this->_anchor = array ( "attrlat"=>$attrlat, "attrlong"=>$attrlong, "lat"=>$lat, "long"=>$long ); - } - - /// set grouping attribute and function - function SetGroupBy ( $attribute, $func, $groupsort="@group desc" ) - { - assert ( is_string($attribute) ); - assert ( is_string($groupsort) ); - assert ( $func==SPH_GROUPBY_DAY - || $func==SPH_GROUPBY_WEEK - || $func==SPH_GROUPBY_MONTH - || $func==SPH_GROUPBY_YEAR - || $func==SPH_GROUPBY_ATTR - || $func==SPH_GROUPBY_ATTRPAIR ); - - $this->_groupby = $attribute; - $this->_groupfunc = $func; - $this->_groupsort = $groupsort; - } - - /// set count-distinct attribute for group-by queries - function SetGroupDistinct ( $attribute ) - { - assert ( is_string($attribute) ); - $this->_groupdistinct = $attribute; - } - - /// set distributed retries count and delay - function SetRetries ( $count, $delay=0 ) - { - assert ( is_int($count) && $count>=0 ); - assert ( is_int($delay) && $delay>=0 ); - $this->_retrycount = $count; - $this->_retrydelay = $delay; - } - - /// set result set format (hash or array; hash by default) - /// PHP specific; needed for group-by-MVA result sets that may contain duplicate IDs - function SetArrayResult ( $arrayresult ) - { - assert ( is_bool($arrayresult) ); - $this->_arrayresult = $arrayresult; - } - - ////////////////////////////////////////////////////////////////////////////// - - /// clear all filters (for multi-queries) - function ResetFilters () - { - $this->_filters = array(); - $this->_anchor = array(); - } - - /// clear groupby settings (for multi-queries) - function ResetGroupBy () - { - $this->_groupby = ""; - $this->_groupfunc = SPH_GROUPBY_DAY; - $this->_groupsort = "@group desc"; - $this->_groupdistinct= ""; - } - - ////////////////////////////////////////////////////////////////////////////// - - /// connect to searchd server, run given search query through given indexes, - /// and return the search results - function Query ( $query, $index="*", $comment="" ) - { - assert ( empty($this->_reqs) ); - - $this->AddQuery ( $query, $index, $comment ); - $results = $this->RunQueries (); - $this->_reqs = array (); // just in case it failed too early - - if ( !is_array($results) ) - return false; // probably network error; error message should be already filled - - $this->_error = $results[0]["error"]; - $this->_warning = $results[0]["warning"]; - if ( $results[0]["status"]==SEARCHD_ERROR ) - return false; - else - return $results[0]; - } - - /// helper to pack floats in network byte order - function _PackFloat ( $f ) - { - $t1 = pack ( "f", $f ); // machine order - list(,$t2) = unpack ( "L*", $t1 ); // int in machine order - return pack ( "N", $t2 ); - } - - /// add query to multi-query batch - /// returns index into results array from RunQueries() call - function AddQuery ( $query, $index="*", $comment="" ) - { - // mbstring workaround - $this->_MBPush (); - - // build request - $req = pack ( "NNNNN", $this->_offset, $this->_limit, $this->_mode, $this->_ranker, $this->_sort ); // mode and limits - $req .= pack ( "N", strlen($this->_sortby) ) . $this->_sortby; - $req .= pack ( "N", strlen($query) ) . $query; // query itself - $req .= pack ( "N", count($this->_weights) ); // weights - foreach ( $this->_weights as $weight ) - $req .= pack ( "N", (int)$weight ); - $req .= pack ( "N", strlen($index) ) . $index; // indexes - $req .= pack ( "N", 1 ); // id64 range marker - $req .= sphPack64 ( $this->_min_id ) . sphPack64 ( $this->_max_id ); // id64 range - - // filters - $req .= pack ( "N", count($this->_filters) ); - foreach ( $this->_filters as $filter ) - { - $req .= pack ( "N", strlen($filter["attr"]) ) . $filter["attr"]; - $req .= pack ( "N", $filter["type"] ); - switch ( $filter["type"] ) - { - case SPH_FILTER_VALUES: - $req .= pack ( "N", count($filter["values"]) ); - foreach ( $filter["values"] as $value ) - $req .= pack ( "N", floatval($value) ); // this uberhack is to workaround 32bit signed int limit on x32 platforms - break; - - case SPH_FILTER_RANGE: - $req .= pack ( "NN", $filter["min"], $filter["max"] ); - break; - - case SPH_FILTER_FLOATRANGE: - $req .= $this->_PackFloat ( $filter["min"] ) . $this->_PackFloat ( $filter["max"] ); - break; - - default: - assert ( 0 && "internal error: unhandled filter type" ); - } - $req .= pack ( "N", $filter["exclude"] ); - } - - // group-by clause, max-matches count, group-sort clause, cutoff count - $req .= pack ( "NN", $this->_groupfunc, strlen($this->_groupby) ) . $this->_groupby; - $req .= pack ( "N", $this->_maxmatches ); - $req .= pack ( "N", strlen($this->_groupsort) ) . $this->_groupsort; - $req .= pack ( "NNN", $this->_cutoff, $this->_retrycount, $this->_retrydelay ); - $req .= pack ( "N", strlen($this->_groupdistinct) ) . $this->_groupdistinct; - - // anchor point - if ( empty($this->_anchor) ) - { - $req .= pack ( "N", 0 ); - } else - { - $a =& $this->_anchor; - $req .= pack ( "N", 1 ); - $req .= pack ( "N", strlen($a["attrlat"]) ) . $a["attrlat"]; - $req .= pack ( "N", strlen($a["attrlong"]) ) . $a["attrlong"]; - $req .= $this->_PackFloat ( $a["lat"] ) . $this->_PackFloat ( $a["long"] ); - } - - // per-index weights - $req .= pack ( "N", count($this->_indexweights) ); - foreach ( $this->_indexweights as $idx=>$weight ) - $req .= pack ( "N", strlen($idx) ) . $idx . pack ( "N", $weight ); - - // max query time - $req .= pack ( "N", $this->_maxquerytime ); - - // per-field weights - $req .= pack ( "N", count($this->_fieldweights) ); - foreach ( $this->_fieldweights as $field=>$weight ) - $req .= pack ( "N", strlen($field) ) . $field . pack ( "N", $weight ); - - // comment - $req .= pack ( "N", strlen($comment) ) . $comment; - - // mbstring workaround - $this->_MBPop (); - - // store request to requests array - $this->_reqs[] = $req; - return count($this->_reqs)-1; - } - - /// connect to searchd, run queries batch, and return an array of result sets - function RunQueries () - { - if ( empty($this->_reqs) ) - { - $this->_error = "no queries defined, issue AddQuery() first"; - return false; - } - - // mbstring workaround - $this->_MBPush (); - - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop (); - return false; - } - - //////////////////////////// - // send query, get response - //////////////////////////// - - $nreqs = count($this->_reqs); - $req = join ( "", $this->_reqs ); - $len = 4+strlen($req); - $req = pack ( "nnNN", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $len, $nreqs ) . $req; // add header - - fwrite ( $fp, $req, $len+8 ); - if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_SEARCH ) )) - { - $this->_MBPop (); - return false; - } - - $this->_reqs = array (); - - ////////////////// - // parse response - ////////////////// - - $p = 0; // current position - $max = strlen($response); // max position for checks, to protect against broken responses - - $results = array (); - for ( $ires=0; $ires<$nreqs && $p<$max; $ires++ ) - { - $results[] = array(); - $result =& $results[$ires]; - - $result["error"] = ""; - $result["warning"] = ""; - - // extract status - list(,$status) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $result["status"] = $status; - if ( $status!=SEARCHD_OK ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $message = substr ( $response, $p, $len ); $p += $len; - - if ( $status==SEARCHD_WARNING ) - { - $result["warning"] = $message; - } else - { - $result["error"] = $message; - continue; - } - } - - // read schema - $fields = array (); - $attrs = array (); - - list(,$nfields) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - while ( $nfields-->0 && $p<$max ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $fields[] = substr ( $response, $p, $len ); $p += $len; - } - $result["fields"] = $fields; - - list(,$nattrs) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - while ( $nattrs-->0 && $p<$max ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attr = substr ( $response, $p, $len ); $p += $len; - list(,$type) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attrs[$attr] = $type; - } - $result["attrs"] = $attrs; - - // read match count - list(,$count) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - list(,$id64) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - - // read matches - $idx = -1; - while ( $count-->0 && $p<$max ) - { - // index into result array - $idx++; - - // parse document id and weight - if ( $id64 ) - { - $doc = sphUnpack64 ( substr ( $response, $p, 8 ) ); $p += 8; - list(,$weight) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - } else - { - list ( $doc, $weight ) = array_values ( unpack ( "N*N*", - substr ( $response, $p, 8 ) ) ); - $p += 8; - - if ( PHP_INT_SIZE>=8 ) - { - // x64 route, workaround broken unpack() in 5.2.2+ - if ( $doc<0 ) $doc += (1<<32); - } else - { - // x32 route, workaround php signed/unsigned braindamage - $doc = sprintf ( "%u", $doc ); - } - } - $weight = sprintf ( "%u", $weight ); - - // create match entry - if ( $this->_arrayresult ) - $result["matches"][$idx] = array ( "id"=>$doc, "weight"=>$weight ); - else - $result["matches"][$doc]["weight"] = $weight; - - // parse and create attributes - $attrvals = array (); - foreach ( $attrs as $attr=>$type ) - { - // handle floats - if ( $type==SPH_ATTR_FLOAT ) - { - list(,$uval) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - list(,$fval) = unpack ( "f*", pack ( "L", $uval ) ); - $attrvals[$attr] = $fval; - continue; - } - - // handle everything else as unsigned ints - list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - if ( $type & SPH_ATTR_MULTI ) - { - $attrvals[$attr] = array (); - $nvalues = $val; - while ( $nvalues-->0 && $p<$max ) - { - list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $attrvals[$attr][] = sprintf ( "%u", $val ); - } - } else - { - $attrvals[$attr] = sprintf ( "%u", $val ); - } - } - - if ( $this->_arrayresult ) - $result["matches"][$idx]["attrs"] = $attrvals; - else - $result["matches"][$doc]["attrs"] = $attrvals; - } - - list ( $total, $total_found, $msecs, $words ) = - array_values ( unpack ( "N*N*N*N*", substr ( $response, $p, 16 ) ) ); - $result["total"] = sprintf ( "%u", $total ); - $result["total_found"] = sprintf ( "%u", $total_found ); - $result["time"] = sprintf ( "%.3f", $msecs/1000 ); - $p += 16; - - while ( $words-->0 && $p<$max ) - { - list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; - $word = substr ( $response, $p, $len ); $p += $len; - list ( $docs, $hits ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; - $result["words"][$word] = array ( - "docs"=>sprintf ( "%u", $docs ), - "hits"=>sprintf ( "%u", $hits ) ); - } - } - - $this->_MBPop (); - return $results; - } - - ///////////////////////////////////////////////////////////////////////////// - // excerpts generation - ///////////////////////////////////////////////////////////////////////////// - - /// connect to searchd server, and generate exceprts (snippets) - /// of given documents for given query. returns false on failure, - /// an array of snippets on success - function BuildExcerpts ( $docs, $index, $words, $opts=array() ) - { - assert ( is_array($docs) ); - assert ( is_string($index) ); - assert ( is_string($words) ); - assert ( is_array($opts) ); - - $this->_MBPush (); - - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop(); - return false; - } - - ///////////////// - // fixup options - ///////////////// - - if ( !isset($opts["before_match"]) ) $opts["before_match"] = ""; - if ( !isset($opts["after_match"]) ) $opts["after_match"] = ""; - if ( !isset($opts["chunk_separator"]) ) $opts["chunk_separator"] = " ... "; - if ( !isset($opts["limit"]) ) $opts["limit"] = 256; - if ( !isset($opts["around"]) ) $opts["around"] = 5; - if ( !isset($opts["exact_phrase"]) ) $opts["exact_phrase"] = false; - if ( !isset($opts["single_passage"]) ) $opts["single_passage"] = false; - if ( !isset($opts["use_boundaries"]) ) $opts["use_boundaries"] = false; - if ( !isset($opts["weight_order"]) ) $opts["weight_order"] = false; - - ///////////////// - // build request - ///////////////// - - // v.1.0 req - $flags = 1; // remove spaces - if ( $opts["exact_phrase"] ) $flags |= 2; - if ( $opts["single_passage"] ) $flags |= 4; - if ( $opts["use_boundaries"] ) $flags |= 8; - if ( $opts["weight_order"] ) $flags |= 16; - $req = pack ( "NN", 0, $flags ); // mode=0, flags=$flags - $req .= pack ( "N", strlen($index) ) . $index; // req index - $req .= pack ( "N", strlen($words) ) . $words; // req words - - // options - $req .= pack ( "N", strlen($opts["before_match"]) ) . $opts["before_match"]; - $req .= pack ( "N", strlen($opts["after_match"]) ) . $opts["after_match"]; - $req .= pack ( "N", strlen($opts["chunk_separator"]) ) . $opts["chunk_separator"]; - $req .= pack ( "N", (int)$opts["limit"] ); - $req .= pack ( "N", (int)$opts["around"] ); - - // documents - $req .= pack ( "N", count($docs) ); - foreach ( $docs as $doc ) - { - assert ( is_string($doc) ); - $req .= pack ( "N", strlen($doc) ) . $doc; - } - - //////////////////////////// - // send query, get response - //////////////////////////// - - $len = strlen($req); - $req = pack ( "nnN", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $len ) . $req; // add header - $wrote = fwrite ( $fp, $req, $len+8 ); - if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_EXCERPT ) )) - { - $this->_MBPop (); - return false; - } - - ////////////////// - // parse response - ////////////////// - - $pos = 0; - $res = array (); - $rlen = strlen($response); - for ( $i=0; $i $rlen ) - { - $this->_error = "incomplete reply"; - $this->_MBPop (); - return false; - } - $res[] = $len ? substr ( $response, $pos, $len ) : ""; - $pos += $len; - } - - $this->_MBPop (); - return $res; - } - - - ///////////////////////////////////////////////////////////////////////////// - // keyword generation - ///////////////////////////////////////////////////////////////////////////// - - /// connect to searchd server, and generate keyword list for a given query - /// returns false on failure, - /// an array of words on success - function BuildKeywords ( $query, $index, $hits ) - { - assert ( is_string($query) ); - assert ( is_string($index) ); - assert ( is_bool($hits) ); - - $this->_MBPush (); - - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop(); - return false; - } - - ///////////////// - // build request - ///////////////// - - // v.1.0 req - $req = pack ( "N", strlen($query) ) . $query; // req query - $req .= pack ( "N", strlen($index) ) . $index; // req index - $req .= pack ( "N", (int)$hits ); - - //////////////////////////// - // send query, get response - //////////////////////////// - - $len = strlen($req); - $req = pack ( "nnN", SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, $len ) . $req; // add header - $wrote = fwrite ( $fp, $req, $len+8 ); - if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_KEYWORDS ) )) - { - $this->_MBPop (); - return false; - } - - ////////////////// - // parse response - ////////////////// - - $pos = 0; - $res = array (); - $rlen = strlen($response); - list(,$nwords) = unpack ( "N*", substr ( $response, $pos, 4 ) ); - $pos += 4; - for ( $i=0; $i<$nwords; $i++ ) - { - list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; - $tokenized = $len ? substr ( $response, $pos, $len ) : ""; - $pos += $len; - - list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; - $normalized = $len ? substr ( $response, $pos, $len ) : ""; - $pos += $len; - - $res[] = array ( "tokenized"=>$tokenized, "normalized"=>$normalized ); - - if ( $hits ) - { - list($ndocs,$nhits) = array_values ( unpack ( "N*N*", substr ( $response, $pos, 8 ) ) ); - $pos += 8; - $res [$i]["docs"] = $ndocs; - $res [$i]["hits"] = $nhits; - } - - if ( $pos > $rlen ) - { - $this->_error = "incomplete reply"; - $this->_MBPop (); - return false; - } - } - - $this->_MBPop (); - return $res; - } - - function EscapeString ( $string ) - { - $from = array ( '(',')','|','-','!','@','~','"','&', '/' ); - $to = array ( '\(','\)','\|','\-','\!','\@','\~','\"', '\&', '\/' ); - - return str_replace ( $from, $to, $string ); - } - - ///////////////////////////////////////////////////////////////////////////// - // attribute updates - ///////////////////////////////////////////////////////////////////////////// - - /// update given attribute values on given documents in given indexes - /// returns amount of updated documents (0 or more) on success, or -1 on failure - function UpdateAttributes ( $index, $attrs, $values ) - { - // verify everything - assert ( is_string($index) ); - - assert ( is_array($attrs) ); - foreach ( $attrs as $attr ) - assert ( is_string($attr) ); - - assert ( is_array($values) ); - foreach ( $values as $id=>$entry ) - { - assert ( is_numeric($id) ); - assert ( is_array($entry) ); - assert ( count($entry)==count($attrs) ); - foreach ( $entry as $v ) - assert ( is_int($v) ); - } - - // build request - $req = pack ( "N", strlen($index) ) . $index; - - $req .= pack ( "N", count($attrs) ); - foreach ( $attrs as $attr ) - $req .= pack ( "N", strlen($attr) ) . $attr; - - $req .= pack ( "N", count($values) ); - foreach ( $values as $id=>$entry ) - { - $req .= sphPack64 ( $id ); - foreach ( $entry as $v ) - $req .= pack ( "N", $v ); - } - - // mbstring workaround - $this->_MBPush (); - - // connect, send query, get response - if (!( $fp = $this->_Connect() )) - { - $this->_MBPop (); - return -1; - } - - $len = strlen($req); - $req = pack ( "nnN", SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, $len ) . $req; // add header - fwrite ( $fp, $req, $len+8 ); - - if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_UPDATE ) )) - { - $this->_MBPop (); - return -1; - } - - // parse response - list(,$updated) = unpack ( "N*", substr ( $response, 0, 4 ) ); - $this->_MBPop (); - return $updated; - } -} - -// -// $Id$ -// From 02588069f045ae48984d68c9948c8ecd1c78580d Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 26 Jun 2012 19:06:19 +0530 Subject: [PATCH 07/57] [feature/sphinx-fulltext-search] fix config variables config variables now use class property for unique id PHPBB3-10946 Conflicts: phpBB/includes/search/fulltext_sphinx.php --- phpBB/includes/search/fulltext_sphinx.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 9ae6438af2..fb16c5639b 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -226,7 +226,7 @@ class phpbb_search_fulltext_sphinx $config_object = new sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); $config_data = array( - "source source_phpbb_{$this->id}_main" => array( + 'source source_phpbb_' . $this->id . '_main' => array( array('type', 'mysql'), array('sql_host', $dbhost), array('sql_user', $dbuser), @@ -265,7 +265,7 @@ class phpbb_search_fulltext_sphinx array('sql_attr_timestamp' , 'topic_last_post_time'), array('sql_attr_str2ordinal', 'post_subject'), ), - "source source_phpbb_{$this->id}_delta : source_phpbb_{$this->id}_main" => array( + 'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array( array('sql_query_pre', ''), array('sql_query_range', ''), array('sql_range_step', ''), @@ -286,9 +286,9 @@ class phpbb_search_fulltext_sphinx p.topic_id = t.topic_id AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'), ), - "index index_phpbb_{$this->id}_main" => array( - array('path', $config['fulltext_sphinx_data_path'] . "index_phpbb_{$this->id}_main"), - array('source', "source_phpbb_{$this->id}_main"), + 'index index_phpbb_' . $this->id . '_main' => array( + array('path', $config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'), + array('source', 'source_phpbb_' . $this->id . '_main'), array('docinfo', 'extern'), array('morphology', 'none'), array('stopwords', (file_exists($config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt') && $config['fulltext_sphinx_stopwords']) ? $config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), @@ -298,9 +298,9 @@ class phpbb_search_fulltext_sphinx array('min_prefix_len', '0'), array('min_infix_len', '0'), ), - "index index_phpbb_{$this->id}_delta : index_phpbb_{$this->id}_main" => array( - array('path', $config['fulltext_sphinx_data_path'] . "index_phpbb_{$this->id}_delta"), - array('source', "source_phpbb_{$this->id}_delta"), + 'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array( + array('path', $config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'), + array('source', 'source_phpbb_' . $this->id . '_delta'), ), 'indexer' => array( array('mem_limit', $config['fulltext_sphinx_indexer_mem_limit'] . 'M'), From 74a7407927cd5b54328a3941a9926ee35caf17b4 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Wed, 27 Jun 2012 00:17:39 +0530 Subject: [PATCH 08/57] [feature/sphinx-fulltext-search] improve classes in functions-sphinx.php PHPBB3-10946 --- phpBB/includes/functions_sphinx.php | 38 +++++++++++++---------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/phpBB/includes/functions_sphinx.php b/phpBB/includes/functions_sphinx.php index 0f83f8cfb5..a4f0e41491 100644 --- a/phpBB/includes/functions_sphinx.php +++ b/phpBB/includes/functions_sphinx.php @@ -46,16 +46,14 @@ class sphinx_config */ function &get_section_by_name($name) { - for ($i = 0, $n = sizeof($this->sections); $i < $n; $i++) + for ($i = 0, $size = sizeof($this->sections); $i < $size; $i++) { // make sure this is really a section object and not a comment - if (is_a($this->sections[$i], 'sphinx_config_section') && $this->sections[$i]->get_name() == $name) + if (($this->sections[$i] instanceof sphinx_config_section) && $this->sections[$i]->get_name() == $name) { return $this->sections[$i]; } } - $null = null; - return $null; } /** @@ -116,7 +114,7 @@ class sphinx_config $section_name = ''; $section_name_comment = ''; $found_opening_bracket = false; - for ($j = 0, $n = strlen($line); $j < $n; $j++) + for ($j = 0, $length = strlen($line); $j < $length; $j++) { if ($line[$j] == '#') { @@ -189,15 +187,15 @@ class sphinx_config $in_value = false; $end_section = false; - // ... then we should prase this line char by char: - // - first there's the variable name - // - then an equal sign - // - the variable value - // - possibly a backslash before the linefeed in this case we need to continue - // parsing the value in the next line - // - a # indicating that the rest of the line is a comment - // - a closing curly bracket indicating the end of this section - for ($j = 0, $n = strlen($line); $j < $n; $j++) + /* ... then we should prase this line char by char: + - first there's the variable name + - then an equal sign + - the variable value + - possibly a backslash before the linefeed in this case we need to continue + parsing the value in the next line + - a # indicating that the rest of the line is a comment + - a closing curly bracket indicating the end of this section*/ + for ($j = 0, $length = strlen($line); $j < $length; $j++) { if ($line[$j] == '#') { @@ -223,7 +221,7 @@ class sphinx_config } else { - if ($line[$j] == '\\' && $j == $n - 1) + if ($line[$j] == '\\' && $j == $length - 1) { $value .= "\n"; $in_value = true; @@ -349,16 +347,14 @@ class sphinx_config_section */ function &get_variable_by_name($name) { - for ($i = 0, $n = sizeof($this->variables); $i < $n; $i++) + for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) { // make sure this is a variable object and not a comment - if (is_a($this->variables[$i], 'sphinx_config_variable') && $this->variables[$i]->get_name() == $name) + if (($this->variables[$i] instanceof sphinx_config_variable) && $this->variables[$i]->get_name() == $name) { return $this->variables[$i]; } } - $null = null; - return $null; } /** @@ -368,10 +364,10 @@ class sphinx_config_section */ function delete_variables_by_name($name) { - for ($i = 0; $i < sizeof($this->variables); $i++) + for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) { // make sure this is a variable object and not a comment - if (is_a($this->variables[$i], 'sphinx_config_variable') && $this->variables[$i]->get_name() == $name) + if (($this->variables[$i] instanceof sphinx_config_variable) && $this->variables[$i]->get_name() == $name) { array_splice($this->variables, $i, 1); $i--; From f609555b1ae335b5ea996bf26ee2846058e5256a Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Wed, 27 Jun 2012 00:28:31 +0530 Subject: [PATCH 09/57] [feature/sphinx-fulltext-search] integrate sphinx language keys with core Language keys removed from mods folder and added to language/en/acp/search.php PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 6 -- phpBB/language/en/acp/search.php | 30 ++++++++++ phpBB/language/en/mods/fulltext_sphinx.php | 65 ---------------------- 3 files changed, 30 insertions(+), 71 deletions(-) delete mode 100644 phpBB/language/en/mods/fulltext_sphinx.php diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index fb16c5639b..2e263c1b55 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -733,8 +733,6 @@ class phpbb_search_fulltext_sphinx if (!isset($config['fulltext_sphinx_configured']) || !$config['fulltext_sphinx_configured']) { - $user->add_lang('mods/fulltext_sphinx'); - return $user->lang['FULLTEXT_SPHINX_CONFIGURE_FIRST']; } @@ -902,8 +900,6 @@ class phpbb_search_fulltext_sphinx $this->get_stats(); } - $user->add_lang('mods/fulltext_sphinx'); - return array( $user->lang['FULLTEXT_SPHINX_MAIN_POSTS'] => ($this->index_created()) ? $this->stats['main_posts'] : 0, $user->lang['FULLTEXT_SPHINX_DELTA_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] - $this->stats['main_posts'] : 0, @@ -966,8 +962,6 @@ class phpbb_search_fulltext_sphinx { global $user, $config; - $user->add_lang('mods/fulltext_sphinx'); - $config_vars = array( 'fulltext_sphinx_autoconf' => 'bool', 'fulltext_sphinx_autorun' => 'bool', diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index cd319c66a9..3fa7f34c64 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -69,6 +69,36 @@ $lang = array_merge($lang, array( 'FULLTEXT_POSTGRES_MIN_WORD_LEN_EXPLAIN' => 'Words with at least this many characters will be included in the query to the database.', 'FULLTEXT_POSTGRES_MAX_WORD_LEN_EXPLAIN' => 'Words with no more than this many characters will be included in the query to the database.', + 'FULLTEXT_SPHINX_AUTOCONF' => 'Automatically configure Sphinx', + 'FULLTEXT_SPHINX_AUTOCONF_EXPLAIN' => 'This is the easiest way to install Sphinx, just select the settings here and a config file will be written for you. This requires write permissions on the configuration folder.', + 'FULLTEXT_SPHINX_AUTORUN' => 'Automatically run Sphinx', + 'FULLTEXT_SPHINX_AUTORUN_EXPLAIN' => 'This is the easiest way to run Sphinx. Select the paths in this dialogue and the Sphinx daemon will be started and stopped as needed. You can also create an index from the ACP. If your PHP installation forbids the use of exec you can disable this and run Sphinx manually.', + 'FULLTEXT_SPHINX_BIN_PATH' => 'Path to executables directory', + 'FULLTEXT_SPHINX_BIN_PATH_EXPLAIN' => 'Skip if autorun is disabled. If this path could not be determined automatically you have to enter the path to the directory in which the sphinx executables indexer and searchd reside.', + 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to configuration directory', + 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'Skip if autoconf is disabled. You should create this config directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody).', + 'FULLTEXT_SPHINX_CONFIGURE_FIRST' => 'Before you create an index you have to enable and configure sphinx under GENERAL -> SERVER CONFIGURATION -> Search settings.', + 'FULLTEXT_SPHINX_CONFIGURE_BEFORE' => 'Configure the following settings BEFORE activating Sphinx', + 'FULLTEXT_SPHINX_CONFIGURE_AFTER' => 'The following settings do not have to be configured before activating Sphinx', + 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', + 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'Skip if autorun is disabled. You should create this directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody). It will be used to store the indexes and log files.', + 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', + 'FULLTEXT_SPHINX_DIRECTORY_NOT_FOUND' => 'The directory %s does not exist. Please correct your path settings.', + 'FULLTEXT_SPHINX_FILE_NOT_EXECUTABLE' => 'The file %s is not executable for the webserver.', + 'FULLTEXT_SPHINX_FILE_NOT_FOUND' => 'The file %s does not exist. Please correct your path settings.', + 'FULLTEXT_SPHINX_FILE_NOT_WRITABLE' => 'The file %s cannot be written by the webserver.', + 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', + 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', + 'FULLTEXT_SPHINX_LAST_SEARCHES' => 'Recent search queries', + 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', + 'FULLTEXT_SPHINX_PORT' => 'Sphinx search deamon port', + 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search deamon on localhost listens. Leave empty to use the default 3312', + 'FULLTEXT_SPHINX_REQUIRES_EXEC' => 'The sphinx plugin for phpBB requires PHP’s exec function which is disabled on your system.', + 'FULLTEXT_SPHINX_UNCONFIGURED' => 'Please set all necessary options in the "Fulltext Sphinx" section of the previous page before you try to activate the sphinx plugin.', + 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx plugin for phpBB currently only supports MySQL', + 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', + 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'This setting only works with autoconf enabled. You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', + 'GENERAL_SEARCH_SETTINGS' => 'General search settings', 'GO_TO_SEARCH_INDEX' => 'Go to search index page', diff --git a/phpBB/language/en/mods/fulltext_sphinx.php b/phpBB/language/en/mods/fulltext_sphinx.php deleted file mode 100644 index f3fd68aa62..0000000000 --- a/phpBB/language/en/mods/fulltext_sphinx.php +++ /dev/null @@ -1,65 +0,0 @@ - 'Automatically configure Sphinx', - 'FULLTEXT_SPHINX_AUTOCONF_EXPLAIN' => 'This is the easiest way to install Sphinx, just select the settings here and a config file will be written for you. This requires write permissions on the configuration folder.', - 'FULLTEXT_SPHINX_AUTORUN' => 'Automatically run Sphinx', - 'FULLTEXT_SPHINX_AUTORUN_EXPLAIN' => 'This is the easiest way to run Sphinx. Select the paths in this dialogue and the Sphinx daemon will be started and stopped as needed. You can also create an index from the ACP. If your PHP installation forbids the use of exec you can disable this and run Sphinx manually.', - 'FULLTEXT_SPHINX_BIN_PATH' => 'Path to executables directory', - 'FULLTEXT_SPHINX_BIN_PATH_EXPLAIN' => 'Skip if autorun is disabled. If this path could not be determined automatically you have to enter the path to the directory in which the sphinx executables indexer and searchd reside.', - 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to configuration directory', - 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'Skip if autoconf is disabled. You should create this config directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody).', - 'FULLTEXT_SPHINX_CONFIGURE_FIRST' => 'Before you create an index you have to enable and configure sphinx under GENERAL -> SERVER CONFIGURATION -> Search settings.', - 'FULLTEXT_SPHINX_CONFIGURE_BEFORE' => 'Configure the following settings BEFORE activating Sphinx', - 'FULLTEXT_SPHINX_CONFIGURE_AFTER' => 'The following settings do not have to be configured before activating Sphinx', - 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', - 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'Skip if autorun is disabled. You should create this directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody). It will be used to store the indexes and log files.', - 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', - 'FULLTEXT_SPHINX_DIRECTORY_NOT_FOUND' => 'The directory %s does not exist. Please correct your path settings.', - 'FULLTEXT_SPHINX_FILE_NOT_EXECUTABLE' => 'The file %s is not executable for the webserver.', - 'FULLTEXT_SPHINX_FILE_NOT_FOUND' => 'The file %s does not exist. Please correct your path settings.', - 'FULLTEXT_SPHINX_FILE_NOT_WRITABLE' => 'The file %s cannot be written by the webserver.', - 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', - 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', - 'FULLTEXT_SPHINX_LAST_SEARCHES' => 'Recent search queries', - 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', - 'FULLTEXT_SPHINX_PORT' => 'Sphinx search deamon port', - 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search deamon on localhost listens. Leave empty to use the default 3312', - 'FULLTEXT_SPHINX_REQUIRES_EXEC' => 'The sphinx plugin for phpBB requires PHP’s exec function which is disabled on your system.', - 'FULLTEXT_SPHINX_UNCONFIGURED' => 'Please set all necessary options in the "Fulltext Sphinx" section of the previous page before you try to activate the sphinx plugin.', - 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx plugin for phpBB currently only supports MySQL', - 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', - 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'This setting only works with autoconf enabled. You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', -)); - -?> From bfd01f01877bcb9a9be9e2df5c6713c3e338579e Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Wed, 27 Jun 2012 01:03:04 +0530 Subject: [PATCH 10/57] [feature/sphinx-fulltext-search] remove all reference returns PHPBB3-10946 --- phpBB/includes/functions_sphinx.php | 8 ++++---- phpBB/includes/search/fulltext_sphinx.php | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/phpBB/includes/functions_sphinx.php b/phpBB/includes/functions_sphinx.php index a4f0e41491..a93a1d950f 100644 --- a/phpBB/includes/functions_sphinx.php +++ b/phpBB/includes/functions_sphinx.php @@ -44,7 +44,7 @@ class sphinx_config * @param string $name The name of the section that shall be returned * @return sphinx_config_section The section object or null if none was found */ - function &get_section_by_name($name) + function get_section_by_name($name) { for ($i = 0, $size = sizeof($this->sections); $i < $size; $i++) { @@ -62,7 +62,7 @@ class sphinx_config * @param string $name The name for the new section * @return sphinx_config_section The newly created section object */ - function &add_section($name) + function add_section($name) { $this->sections[] = new sphinx_config_section($name, ''); return $this->sections[sizeof($this->sections) - 1]; @@ -345,7 +345,7 @@ class sphinx_config_section * @return sphinx_config_section The first variable object from this section with the * given name or null if none was found */ - function &get_variable_by_name($name) + function get_variable_by_name($name) { for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) { @@ -382,7 +382,7 @@ class sphinx_config_section * @param string $value The value for the new variable * @return sphinx_config_variable Variable object that was created */ - function &create_variable($name, $value) + function create_variable($name, $value) { $this->variables[] = new sphinx_config_variable($name, $value, ''); return $this->variables[sizeof($this->variables) - 1]; diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 2e263c1b55..477b1646fb 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -323,10 +323,10 @@ class phpbb_search_fulltext_sphinx foreach ($config_data as $section_name => $section_data) { - $section = &$config_object->get_section_by_name($section_name); + $section = $config_object->get_section_by_name($section_name); if (!$section) { - $section = &$config_object->add_section($section_name); + $section = $config_object->add_section($section_name); } foreach ($delete as $key => $void) @@ -346,10 +346,10 @@ class phpbb_search_fulltext_sphinx if (!isset($non_unique[$key])) { - $variable = &$section->get_variable_by_name($key); + $variable = $section->get_variable_by_name($key); if (!$variable) { - $variable = &$section->create_variable($key, $value); + $variable = $section->create_variable($key, $value); } else { @@ -358,7 +358,7 @@ class phpbb_search_fulltext_sphinx } else { - $variable = &$section->create_variable($key, $value); + $variable = $section->create_variable($key, $value); } } } From 39f8a5fa9f71724d0abd98cdf7a7d82fc7e7bb0f Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Wed, 27 Jun 2012 03:44:03 +0530 Subject: [PATCH 11/57] [feature/sphinx-fulltext-search] use sql_build_query for query Uses sql_build_query for JOIN query. Remove casting to int and space for phpbb conventions to be followed PHPBB3-10946 Conflicts: phpBB/includes/search/fulltext_sphinx.php --- phpBB/includes/search/fulltext_sphinx.php | 24 ++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 477b1646fb..2690612b1a 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -53,7 +53,7 @@ class phpbb_search_fulltext_sphinx $this->id = $config['avatar_salt']; $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main'; - $this->sphinx = new SphinxClient (); + $this->sphinx = new SphinxClient(); if (!empty($config['fulltext_sphinx_configured'])) { @@ -648,18 +648,28 @@ class phpbb_search_fulltext_sphinx } else if ($mode != 'post' && $post_id) { - // update topic_last_post_time for full topic - $sql = 'SELECT p1.post_id - FROM ' . POSTS_TABLE . ' p1 - LEFT JOIN ' . POSTS_TABLE . ' p2 ON (p1.topic_id = p2.topic_id) - WHERE p2.post_id = ' . $post_id; + // Update topic_last_post_time for full topic + $sql_array = array( + 'SELECT' => 'p1.post_id', + 'FROM' => array( + POSTS_TABLE => 'p1', + ), + 'LEFT_JOIN' => array(array( + 'FROM' => array( + POSTS_TABLE => 'p2' + ), + 'ON' => 'p1.topic_id = p2.topic_id', + )), + ); + + $sql = $db->sql_build_query('SELECT', $sql_array); $result = $db->sql_query($sql); $post_updates = array(); $post_time = time(); while ($row = $db->sql_fetchrow($result)) { - $post_updates[(int)$row['post_id']] = array((int) $post_time); + $post_updates[(int)$row['post_id']] = array($post_time); } $db->sql_freeresult($result); From 10b706674e0fc100ff4e21d5fe100a9b532bb4bf Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 01:49:38 +0530 Subject: [PATCH 12/57] [feature/sphinx-fulltext-search] add binlog_path to config binlog files are now added to the data folder. PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 1 + phpBB/includes/search/fulltext_sphinx.php | 1 + 2 files changed, 2 insertions(+) diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf index d7e59a11fc..000d8157d6 100644 --- a/phpBB/docs/sphinx.sample.conf +++ b/phpBB/docs/sphinx.sample.conf @@ -93,4 +93,5 @@ searchd max_children = 30 pid_file = {DATA_PATH}/searchd.pid max_matches = 20000 + binlog_path = {DATA_PATH} } diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 2690612b1a..8514b9cabb 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -315,6 +315,7 @@ class phpbb_search_fulltext_sphinx array('max_children', '30'), array('pid_file', $config['fulltext_sphinx_data_path'] . "searchd.pid"), array('max_matches', (string) MAX_MATCHES), + array('binlog_path', $config['fulltext_sphinx_data_path']), ), ); From 97fda78e7d85444f21ba2b10b3d58c4639b85936 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Sun, 1 Jul 2012 01:23:57 +0530 Subject: [PATCH 13/57] [feature/sphinx-fulltext-search] Make different files for different classes Break the classes in functions-sphinx.php into different files with proper class names according to phpbb class auto loader conventions. PHPBB3-10946 Conflicts: phpBB/includes/search/sphinx/config.php --- phpBB/includes/search/fulltext_sphinx.php | 7 +- .../sphinx/config.php} | 251 ++---------------- .../includes/search/sphinx/config_comment.php | 45 ++++ .../includes/search/sphinx/config_section.php | 144 ++++++++++ .../search/sphinx/config_variable.php | 72 +++++ 5 files changed, 280 insertions(+), 239 deletions(-) rename phpBB/includes/{functions_sphinx.php => search/sphinx/config.php} (51%) create mode 100644 phpBB/includes/search/sphinx/config_comment.php create mode 100644 phpBB/includes/search/sphinx/config_section.php create mode 100644 phpBB/includes/search/sphinx/config_variable.php diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 8514b9cabb..6f3c688aed 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -207,11 +207,6 @@ class phpbb_search_fulltext_sphinx // now that we're sure everything was entered correctly, generate a config for the index // we misuse the avatar_salt for this, as it should be unique ;-) - if (!class_exists('sphinx_config')) - { - include($phpbb_root_path . 'includes/functions_sphinx.php'); - } - if (!file_exists($config['fulltext_sphinx_config_path'] . 'sphinx.conf')) { $filename = $config['fulltext_sphinx_config_path'] . 'sphinx.conf'; @@ -223,7 +218,7 @@ class phpbb_search_fulltext_sphinx @fclose($fp); } - $config_object = new sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); + $config_object = new phpbb_search_sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); $config_data = array( 'source source_phpbb_' . $this->id . '_main' => array( diff --git a/phpBB/includes/functions_sphinx.php b/phpBB/includes/search/sphinx/config.php similarity index 51% rename from phpBB/includes/functions_sphinx.php rename to phpBB/includes/search/sphinx/config.php index a93a1d950f..966cd0f284 100644 --- a/phpBB/includes/functions_sphinx.php +++ b/phpBB/includes/search/sphinx/config.php @@ -1,13 +1,14 @@ sections); $i < $size; $i++) { // make sure this is really a section object and not a comment - if (($this->sections[$i] instanceof sphinx_config_section) && $this->sections[$i]->get_name() == $name) + if (($this->sections[$i] instanceof phpbb_search_sphinx_config_section) && $this->sections[$i]->get_name() == $name) { return $this->sections[$i]; } @@ -59,12 +60,12 @@ class sphinx_config /** * Appends a new empty section to the end of the config * - * @param string $name The name for the new section - * @return sphinx_config_section The newly created section object + * @param string $name The name for the new section + * @return phpbb_search_sphinx_config_section The newly created section object */ function add_section($name) { - $this->sections[] = new sphinx_config_section($name, ''); + $this->sections[] = new phpbb_search_sphinx_config_section($name, ''); return $this->sections[sizeof($this->sections) - 1]; } @@ -104,7 +105,7 @@ class sphinx_config // that way they're not deleted when reassembling the file from the sections if (!$line || $line[0] == '#') { - $this->sections[] = new sphinx_config_comment($config_file[$i]); + $this->sections[] = new phpbb_search_sphinx_config_comment($config_file[$i]); continue; } else @@ -138,7 +139,7 @@ class sphinx_config // and then we create the new section object $section_name = trim($section_name); - $section = new sphinx_config_section($section_name, $section_name_comment); + $section = new phpbb_search_sphinx_config_section($section_name, $section_name_comment); } } else // if we're looking for variables inside a section @@ -152,7 +153,7 @@ class sphinx_config // of this section so they're not deleted on reassembly if (!$line || $line[0] == '#') { - $section->add_variable(new sphinx_config_comment($config_file[$i])); + $section->add_variable(new phpbb_search_sphinx_config_comment($config_file[$i])); continue; } @@ -168,7 +169,7 @@ class sphinx_config } else { - $section->add_variable(new sphinx_config_comment($config_file[$i])); + $section->add_variable(new phpbb_search_sphinx_config_comment($config_file[$i])); continue; } } @@ -234,7 +235,7 @@ class sphinx_config // if a name and an equal sign were found then we have append a new variable object to the section if ($name && $found_assignment) { - $section->add_variable(new sphinx_config_variable(trim($name), trim($value), ($end_section) ? '' : $comment)); + $section->add_variable(new phpbb_search_sphinx_config_variable(trim($name), trim($value), ($end_section) ? '' : $comment)); continue; } @@ -251,7 +252,7 @@ class sphinx_config // if we did not find anything meaningful up to here, then just treat it as a comment $comment = ($skip_first) ? "\t" . substr(ltrim($config_file[$i]), 1) : $config_file[$i]; - $section->add_variable(new sphinx_config_comment($comment)); + $section->add_variable(new phpbb_search_sphinx_config_comment($comment)); } } @@ -284,219 +285,3 @@ class sphinx_config fclose($fp); } } - -/** -* sphinx_config_section -* Represents a single section inside the sphinx configuration -*/ -class sphinx_config_section -{ - var $name; - var $comment; - var $end_comment; - var $variables = array(); - - /** - * Construct a new section - * - * @param string $name Name of the section - * @param string $comment Comment that should be appended after the name in the - * textual format. - */ - function sphinx_config_section($name, $comment) - { - $this->name = $name; - $this->comment = $comment; - $this->end_comment = ''; - } - - /** - * Add a variable object to the list of variables in this section - * - * @param sphinx_config_variable $variable The variable object - */ - function add_variable($variable) - { - $this->variables[] = $variable; - } - - /** - * Adds a comment after the closing bracket in the textual representation - */ - function set_end_comment($end_comment) - { - $this->end_comment = $end_comment; - } - - /** - * Getter for the name of this section - * - * @return string Section's name - */ - function get_name() - { - return $this->name; - } - - /** - * Get a variable object by its name - * - * @param string $name The name of the variable that shall be returned - * @return sphinx_config_section The first variable object from this section with the - * given name or null if none was found - */ - function get_variable_by_name($name) - { - for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) - { - // make sure this is a variable object and not a comment - if (($this->variables[$i] instanceof sphinx_config_variable) && $this->variables[$i]->get_name() == $name) - { - return $this->variables[$i]; - } - } - } - - /** - * Deletes all variables with the given name - * - * @param string $name The name of the variable objects that are supposed to be removed - */ - function delete_variables_by_name($name) - { - for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) - { - // make sure this is a variable object and not a comment - if (($this->variables[$i] instanceof sphinx_config_variable) && $this->variables[$i]->get_name() == $name) - { - array_splice($this->variables, $i, 1); - $i--; - } - } - } - - /** - * Create a new variable object and append it to the variable list of this section - * - * @param string $name The name for the new variable - * @param string $value The value for the new variable - * @return sphinx_config_variable Variable object that was created - */ - function create_variable($name, $value) - { - $this->variables[] = new sphinx_config_variable($name, $value, ''); - return $this->variables[sizeof($this->variables) - 1]; - } - - /** - * Turns this object into a string which can be written to a config file - * - * @return string Config data in textual form, parsable for sphinx - */ - function to_string() - { - $content = $this->name . " " . $this->comment . "\n{\n"; - - // make sure we don't get too many newlines after the opening bracket - while (trim($this->variables[0]->to_string()) == "") - { - array_shift($this->variables); - } - - foreach ($this->variables as $variable) - { - $content .= $variable->to_string(); - } - $content .= '}' . $this->end_comment . "\n"; - - return $content; - } -} - -/** -* sphinx_config_variable -* Represents a single variable inside the sphinx configuration -*/ -class sphinx_config_variable -{ - var $name; - var $value; - var $comment; - - /** - * Constructs a new variable object - * - * @param string $name Name of the variable - * @param string $value Value of the variable - * @param string $comment Optional comment after the variable in the - * config file - */ - function sphinx_config_variable($name, $value, $comment) - { - $this->name = $name; - $this->value = $value; - $this->comment = $comment; - } - - /** - * Getter for the variable's name - * - * @return string The variable object's name - */ - function get_name() - { - return $this->name; - } - - /** - * Allows changing the variable's value - * - * @param string $value New value for this variable - */ - function set_value($value) - { - $this->value = $value; - } - - /** - * Turns this object into a string readable by sphinx - * - * @return string Config data in textual form - */ - function to_string() - { - return "\t" . $this->name . ' = ' . str_replace("\n", "\\\n", $this->value) . ' ' . $this->comment . "\n"; - } -} - - -/** -* sphinx_config_comment -* Represents a comment inside the sphinx configuration -*/ -class sphinx_config_comment -{ - var $exact_string; - - /** - * Create a new comment - * - * @param string $exact_string The content of the comment including newlines, leading whitespace, etc. - */ - function sphinx_config_comment($exact_string) - { - $this->exact_string = $exact_string; - } - - /** - * Simply returns the comment as it was created - * - * @return string The exact string that was specified in the constructor - */ - function to_string() - { - return $this->exact_string; - } -} - -?> diff --git a/phpBB/includes/search/sphinx/config_comment.php b/phpBB/includes/search/sphinx/config_comment.php new file mode 100644 index 0000000000..63d3488aef --- /dev/null +++ b/phpBB/includes/search/sphinx/config_comment.php @@ -0,0 +1,45 @@ +exact_string = $exact_string; + } + + /** + * Simply returns the comment as it was created + * + * @return string The exact string that was specified in the constructor + */ + function to_string() + { + return $this->exact_string; + } +} diff --git a/phpBB/includes/search/sphinx/config_section.php b/phpBB/includes/search/sphinx/config_section.php new file mode 100644 index 0000000000..529254dd5a --- /dev/null +++ b/phpBB/includes/search/sphinx/config_section.php @@ -0,0 +1,144 @@ +name = $name; + $this->comment = $comment; + $this->end_comment = ''; + } + + /** + * Add a variable object to the list of variables in this section + * + * @param phpbb_search_sphinx_config_variable $variable The variable object + */ + function add_variable($variable) + { + $this->variables[] = $variable; + } + + /** + * Adds a comment after the closing bracket in the textual representation + */ + function set_end_comment($end_comment) + { + $this->end_comment = $end_comment; + } + + /** + * Getter for the name of this section + * + * @return string Section's name + */ + function get_name() + { + return $this->name; + } + + /** + * Get a variable object by its name + * + * @param string $name The name of the variable that shall be returned + * @return phpbb_search_sphinx_config_section The first variable object from this section with the + * given name or null if none was found + */ + function get_variable_by_name($name) + { + for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) + { + // make sure this is a variable object and not a comment + if (($this->variables[$i] instanceof phpbb_search_sphinx_config_variable) && $this->variables[$i]->get_name() == $name) + { + return $this->variables[$i]; + } + } + } + + /** + * Deletes all variables with the given name + * + * @param string $name The name of the variable objects that are supposed to be removed + */ + function delete_variables_by_name($name) + { + for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) + { + // make sure this is a variable object and not a comment + if (($this->variables[$i] instanceof phpbb_search_sphinx_config_variable) && $this->variables[$i]->get_name() == $name) + { + array_splice($this->variables, $i, 1); + $i--; + } + } + } + + /** + * Create a new variable object and append it to the variable list of this section + * + * @param string $name The name for the new variable + * @param string $value The value for the new variable + * @return phpbb_search_sphinx_config_variable Variable object that was created + */ + function create_variable($name, $value) + { + $this->variables[] = new phpbb_search_sphinx_config_variable($name, $value, ''); + return $this->variables[sizeof($this->variables) - 1]; + } + + /** + * Turns this object into a string which can be written to a config file + * + * @return string Config data in textual form, parsable for sphinx + */ + function to_string() + { + $content = $this->name . ' ' . $this->comment . "\n{\n"; + + // make sure we don't get too many newlines after the opening bracket + while (trim($this->variables[0]->to_string()) == '') + { + array_shift($this->variables); + } + + foreach ($this->variables as $variable) + { + $content .= $variable->to_string(); + } + $content .= '}' . $this->end_comment . "\n"; + + return $content; + } +} diff --git a/phpBB/includes/search/sphinx/config_variable.php b/phpBB/includes/search/sphinx/config_variable.php new file mode 100644 index 0000000000..dd7836f7c8 --- /dev/null +++ b/phpBB/includes/search/sphinx/config_variable.php @@ -0,0 +1,72 @@ +name = $name; + $this->value = $value; + $this->comment = $comment; + } + + /** + * Getter for the variable's name + * + * @return string The variable object's name + */ + function get_name() + { + return $this->name; + } + + /** + * Allows changing the variable's value + * + * @param string $value New value for this variable + */ + function set_value($value) + { + $this->value = $value; + } + + /** + * Turns this object into a string readable by sphinx + * + * @return string Config data in textual form + */ + function to_string() + { + return "\t" . $this->name . ' = ' . str_replace("\n", "\\\n", $this->value) . ' ' . $this->comment . "\n"; + } +} From 4a11a7b97027743b8239d31a4d51824bd807c5ac Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Sun, 1 Jul 2012 03:03:15 +0530 Subject: [PATCH 14/57] [feature/sphinx-fulltext-search] add sphinx_table constant to constants.php PHPBB3-10946 --- phpBB/includes/constants.php | 1 + phpBB/includes/search/fulltext_sphinx.php | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/includes/constants.php b/phpBB/includes/constants.php index 66d2a003c6..68af41ab20 100644 --- a/phpBB/includes/constants.php +++ b/phpBB/includes/constants.php @@ -260,6 +260,7 @@ define('SESSIONS_TABLE', $table_prefix . 'sessions'); define('SESSIONS_KEYS_TABLE', $table_prefix . 'sessions_keys'); define('SITELIST_TABLE', $table_prefix . 'sitelist'); define('SMILIES_TABLE', $table_prefix . 'smilies'); +define('SPHINX_TABLE', $table_prefix . 'sphinx'); define('STYLES_TABLE', $table_prefix . 'styles'); define('STYLES_TEMPLATE_TABLE', $table_prefix . 'styles_template'); define('STYLES_TEMPLATE_DATA_TABLE',$table_prefix . 'styles_template_data'); diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 6f3c688aed..53bff898eb 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -26,7 +26,6 @@ require($phpbb_root_path . "includes/sphinxapi-0.9.8." . $phpEx); define('INDEXER_NAME', 'indexer'); define('SEARCHD_NAME', 'searchd'); -define('SPHINX_TABLE', $table_prefix . 'sphinx'); define('MAX_MATCHES', 20000); define('CONNECT_RETRIES', 3); From 06eeed058df75c41496c5306bfa35725c45cf5f3 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Sun, 1 Jul 2012 03:17:45 +0530 Subject: [PATCH 15/57] [feature/sphinx-fulltext-search] remove unused arrays PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 53bff898eb..4c0adcd99e 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -395,14 +395,7 @@ class phpbb_search_fulltext_sphinx $this->sphinx->SetMatchMode(SPH_MATCH_ANY); } - $match = array(); - // Keep quotes - $match[] = "#"#"; - // KeepNew lines - $match[] = "#[\n]+#"; - - $replace = array('"', " "); - + // Keep quotes and new lines $keywords = str_replace(array('"', "\n"), array('"', ' '), trim($keywords)); if (strlen($keywords) > 0) From 0e9174d168a82bde16ec59d615e19b85a50cebcf Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Sun, 1 Jul 2012 05:06:51 +0530 Subject: [PATCH 16/57] [feature/sphinx-fulltext-search] use keywords_search instead of get_name using keyword_search method instead of get_name to distinguish between the search backend classes present in includes/search and other helper classes. PHPBB3-10946 --- phpBB/includes/acp/acp_search.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/includes/acp/acp_search.php b/phpBB/includes/acp/acp_search.php index 54a3e7aaa1..82d9b021fe 100644 --- a/phpBB/includes/acp/acp_search.php +++ b/phpBB/includes/acp/acp_search.php @@ -598,7 +598,7 @@ class acp_search { global $phpbb_root_path, $phpEx, $user; - if (!class_exists($type) || !method_exists($type, 'get_name')) + if (!class_exists($type) || !method_exists($type, 'keyword_search')) { $error = $user->lang['NO_SUCH_SEARCH_MODULE']; return $error; From 2503581cd562b39a108821da85cc0175735e24a5 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 02:33:02 +0530 Subject: [PATCH 17/57] [feature/sphinx-fulltext-search] add class properties indexes & sphinx PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 4c0adcd99e..4ace7c9753 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -44,6 +44,8 @@ class phpbb_search_fulltext_sphinx var $search_query; var $common_words = array(); var $id; + var $indexes; + var $sphinx; public function __construct(&$error) { From 8dcdf8a9732a570f26fee802af4bfcbd25f16ec2 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Sun, 1 Jul 2012 12:01:14 +0530 Subject: [PATCH 18/57] [feature/sphinx-fulltext-search] add docblock and access modifiers PHPBB3-10946 Conflicts: phpBB/includes/search/fulltext_sphinx.php --- phpBB/includes/search/fulltext_sphinx.php | 85 +++++++++++++++++++---- 1 file changed, 70 insertions(+), 15 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 4ace7c9753..82addca28a 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -38,15 +38,21 @@ define('CONNECT_WAIT_TIME', 300); */ class phpbb_search_fulltext_sphinx { - var $stats = array(); - var $word_length = array(); - var $split_words = array(); - var $search_query; - var $common_words = array(); - var $id; - var $indexes; - var $sphinx; + private $stats = array(); + private $split_words = array(); + private $id; + private $indexes; + private $sphinx; + public $word_length = array(); + public $search_query; + public $common_words = array(); + /** + * Constructor + * Creates a new phpbb_search_fulltext_postgres, which is used as a search backend. + * + * @param string|bool $error Any error that occurs is passed on through this reference variable otherwise false + */ public function __construct(&$error) { global $config; @@ -81,6 +87,8 @@ class phpbb_search_fulltext_sphinx * Returns the name of this search backend to be displayed to administrators * * @return string Name + * + * @access public */ public function get_name() { @@ -89,6 +97,10 @@ class phpbb_search_fulltext_sphinx /** * Checks permissions and paths, if everything is correct it generates the config file + * + * @return string|bool Language key of the error/incompatiblity occured + * + * @access public */ function init() { @@ -110,6 +122,13 @@ class phpbb_search_fulltext_sphinx return false; } + /** + * Updates the config file sphinx.conf and generates the same in case autoconf is selected + * + * @return string|bool Language key of the error/incompatiblity occured otherwise false + * + * @access private + */ function config_updated() { global $db, $user, $config, $phpbb_root_path, $phpEx; @@ -378,6 +397,8 @@ class phpbb_search_fulltext_sphinx * @param string $keywords Contains the keyword as entered by the user * @param string $terms is either 'all' or 'any' * @return false if no valid keywords were found and otherwise true + * + * @access public */ function split_keywords(&$keywords, $terms) { @@ -619,14 +640,14 @@ class phpbb_search_fulltext_sphinx /** * Updates wordlist and wordmatch tables when a message is posted or changed * - * @param string $mode Contains the post mode: edit, post, reply, quote - * @param int $post_id The id of the post which is modified/created - * @param string &$message New or updated post content - * @param string &$subject New or updated post subject - * @param int $poster_id Post author's user id - * @param int $forum_id The id of the forum in which the post is located + * @param string $mode Contains the post mode: edit, post, reply, quote + * @param int $post_id The id of the post which is modified/created + * @param string &$message New or updated post content + * @param string &$subject New or updated post subject + * @param int $poster_id Post author's user id + * @param int $forum_id The id of the forum in which the post is located * - * @access public + * @access public */ function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id) { @@ -686,6 +707,8 @@ class phpbb_search_fulltext_sphinx /** * Delete a post from the index after it was deleted + * + * @access public */ function index_remove($post_ids, $author_ids, $forum_ids) { @@ -700,6 +723,8 @@ class phpbb_search_fulltext_sphinx /** * Destroy old cache entries + * + * @access public */ function tidy($create = false) { @@ -724,6 +749,10 @@ class phpbb_search_fulltext_sphinx /** * Create sphinx table + * + * @return string|bool error string is returned incase of errors otherwise false + * + * @access public */ function create_index($acp_module, $u_action) { @@ -758,6 +787,10 @@ class phpbb_search_fulltext_sphinx /** * Drop sphinx table + * + * @return string|bool error string is returned incase of errors otherwise false + * + * @access public */ function delete_index($acp_module, $u_action) { @@ -785,6 +818,10 @@ class phpbb_search_fulltext_sphinx /** * Returns true if the sphinx table was created + * + * @return bool true if sphinx table was created + * + * @access public */ function index_created($allow_new_files = true) { @@ -817,6 +854,8 @@ class phpbb_search_fulltext_sphinx /** * Kills the searchd process and makes sure there's no locks left over + * + * @access private */ function shutdown_searchd() { @@ -846,6 +885,8 @@ class phpbb_search_fulltext_sphinx * files by calling shutdown_searchd. * * @return boolean Whether searchd is running or not + * + * @access private */ function searchd_running() { @@ -890,6 +931,10 @@ class phpbb_search_fulltext_sphinx /** * Returns an associative array containing information about the indexes + * + * @return string|bool Language string of error false otherwise + * + * @access public */ function index_stats() { @@ -910,6 +955,8 @@ class phpbb_search_fulltext_sphinx /** * Collects stats that can be displayed on the index maintenance page + * + * @access private */ function get_stats() { @@ -957,6 +1004,10 @@ class phpbb_search_fulltext_sphinx /** * Returns a list of options for the ACP to display + * + * @return associative array containing template and config variables + * + * @access public */ function acp() { @@ -1112,6 +1163,8 @@ class phpbb_search_fulltext_sphinx * * @param string $path Path from which files shall be deleted * @param string $pattern PCRE pattern that a file needs to match in order to be deleted +* +* @access private */ function sphinx_unlink_by_pattern($path, $pattern) { @@ -1132,6 +1185,8 @@ function sphinx_unlink_by_pattern($path, $pattern) * @param string $file The filename from which the lines shall be read * @param int $amount The number of lines to be read from the end * @return string Last lines of the file +* +* @access private */ function sphinx_read_last_lines($file, $amount) { From e486f4389c99c27cb723d4e9fd437130752f891e Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 03:51:40 +0530 Subject: [PATCH 19/57] [feature/sphinx-fulltext-search] remove autoconf Remove all code related to sphinx automatic configuration and all exec calls. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 386 +--------------------- 1 file changed, 1 insertion(+), 385 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 82addca28a..984bda68c2 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -24,9 +24,6 @@ if (!defined('IN_PHPBB')) global $phpbb_root_path, $phpEx, $table_prefix; require($phpbb_root_path . "includes/sphinxapi-0.9.8." . $phpEx); -define('INDEXER_NAME', 'indexer'); -define('SEARCHD_NAME', 'searchd'); - define('MAX_MATCHES', 20000); define('CONNECT_RETRIES', 3); define('CONNECT_WAIT_TIME', 300); @@ -64,15 +61,6 @@ class phpbb_search_fulltext_sphinx if (!empty($config['fulltext_sphinx_configured'])) { - if ($config['fulltext_sphinx_autorun'] && !file_exists($config['fulltext_sphinx_data_path'] . 'searchd.pid') && $this->index_created(true)) - { - $this->shutdown_searchd(); -// $cwd = getcwd(); -// chdir($config['fulltext_sphinx_bin_path']); - exec($config['fulltext_sphinx_bin_path'] . SEARCHD_NAME . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf >> ' . $config['fulltext_sphinx_data_path'] . 'log/searchd-startup.log 2>&1 &'); -// chdir($cwd); - } - // we only support localhost for now $this->sphinx->SetServer('localhost', (isset($config['fulltext_sphinx_port']) && $config['fulltext_sphinx_port']) ? (int) $config['fulltext_sphinx_port'] : 3312); } @@ -133,111 +121,10 @@ class phpbb_search_fulltext_sphinx { global $db, $user, $config, $phpbb_root_path, $phpEx; - if ($config['fulltext_sphinx_autoconf']) - { - $paths = array('fulltext_sphinx_bin_path', 'fulltext_sphinx_config_path', 'fulltext_sphinx_data_path'); - - // check for completeness and add trailing slash if it's not present - foreach ($paths as $path) - { - if (empty($config[$path])) - { - return $user->lang['FULLTEXT_SPHINX_UNCONFIGURED']; - } - if ($config[$path] && substr($config[$path], -1) != '/') - { - set_config($path, $config[$path] . '/'); - } - } - } - - $executables = array( - $config['fulltext_sphinx_bin_path'] . INDEXER_NAME, - $config['fulltext_sphinx_bin_path'] . SEARCHD_NAME, - ); - - if ($config['fulltext_sphinx_autorun']) - { - foreach ($executables as $executable) - { - if (!file_exists($executable)) - { - return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_FOUND'], $executable); - } - - if (!function_exists('exec')) - { - return $user->lang['FULLTEXT_SPHINX_REQUIRES_EXEC']; - } - - $output = array(); - @exec($executable, $output); - - $output = implode("\n", $output); - if (strpos($output, 'Sphinx ') === false) - { - return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_EXECUTABLE'], $executable); - } - } - } - - $writable_paths = array( - $config['fulltext_sphinx_config_path'] => array('config' => 'fulltext_sphinx_autoconf', 'subdir' => false), - $config['fulltext_sphinx_data_path'] => array('config' => 'fulltext_sphinx_autorun', 'subdir' => 'log'), - $config['fulltext_sphinx_data_path'] . 'log/' => array('config' => 'fulltext_sphinx_autorun', 'subdir' => false), - ); - - foreach ($writable_paths as $path => $info) - { - if ($config[$info['config']]) - { - // make sure directory exists - // if we could drop the @ here and figure out whether the file really - // doesn't exist or whether open_basedir is in effect, would be nice - if (!@file_exists($path)) - { - return sprintf($user->lang['FULLTEXT_SPHINX_DIRECTORY_NOT_FOUND'], $path); - } - - // now check if it is writable by storing a simple file - $filename = $path . 'write_test'; - $fp = @fopen($filename, 'wb'); - if ($fp === false) - { - return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_WRITABLE'], $filename); - } - @fclose($fp); - - @unlink($filename); - - if ($info['subdir'] !== false) - { - if (!is_dir($path . $info['subdir'])) - { - mkdir($path . $info['subdir']); - } - } - } - } - - if ($config['fulltext_sphinx_autoconf']) - { include ($phpbb_root_path . 'config.' . $phpEx); // now that we're sure everything was entered correctly, generate a config for the index // we misuse the avatar_salt for this, as it should be unique ;-) - - if (!file_exists($config['fulltext_sphinx_config_path'] . 'sphinx.conf')) - { - $filename = $config['fulltext_sphinx_config_path'] . 'sphinx.conf'; - $fp = @fopen($filename, 'wb'); - if ($fp === false) - { - return sprintf($user->lang['FULLTEXT_SPHINX_FILE_NOT_WRITABLE'], $filename); - } - @fclose($fp); - } - $config_object = new phpbb_search_sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); $config_data = array( @@ -379,12 +266,8 @@ class phpbb_search_fulltext_sphinx } } - $config_object->write($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); - } - set_config('fulltext_sphinx_configured', '1'); - $this->shutdown_searchd(); $this->tidy(); return false; @@ -689,20 +572,6 @@ class phpbb_search_fulltext_sphinx $this->sphinx->UpdateAttributes($this->indexes, array('topic_last_post_time'), $post_updates); } } - - if ($config['fulltext_sphinx_autorun']) - { - if ($this->index_created()) - { - $rotate = ($this->searchd_running()) ? ' --rotate' : ''; - - $cwd = getcwd(); - chdir($config['fulltext_sphinx_bin_path']); - exec('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_delta >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); - var_dump('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_delta >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); - chdir($cwd); - } - } } /** @@ -730,20 +599,6 @@ class phpbb_search_fulltext_sphinx { global $config; - if ($config['fulltext_sphinx_autorun']) - { - if ($this->index_created() || $create) - { - $rotate = ($this->searchd_running()) ? ' --rotate' : ''; - - $cwd = getcwd(); - chdir($config['fulltext_sphinx_bin_path']); - exec('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_main >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); - exec('./' . INDEXER_NAME . $rotate . ' --config ' . $config['fulltext_sphinx_config_path'] . 'sphinx.conf index_phpbb_' . $this->id . '_delta >> ' . $config['fulltext_sphinx_data_path'] . 'log/indexer.log 2>&1 &'); - chdir($cwd); - } - } - set_config('search_last_gc', time(), true); } @@ -758,8 +613,6 @@ class phpbb_search_fulltext_sphinx { global $db, $user, $config; - $this->shutdown_searchd(); - if (!isset($config['fulltext_sphinx_configured']) || !$config['fulltext_sphinx_configured']) { return $user->lang['FULLTEXT_SPHINX_CONFIGURE_FIRST']; @@ -780,8 +633,6 @@ class phpbb_search_fulltext_sphinx // start indexing process $this->tidy(true); - $this->shutdown_searchd(); - return false; } @@ -796,13 +647,6 @@ class phpbb_search_fulltext_sphinx { global $db, $config; - $this->shutdown_searchd(); - - if ($config['fulltext_sphinx_autorun']) - { - sphinx_unlink_by_pattern($config['fulltext_sphinx_data_path'], '#^index_phpbb_' . $this->id . '.*$#'); - } - if (!$this->index_created()) { return false; @@ -811,8 +655,6 @@ class phpbb_search_fulltext_sphinx $sql = 'DROP TABLE ' . SPHINX_TABLE; $db->sql_query($sql); - $this->shutdown_searchd(); - return false; } @@ -836,99 +678,12 @@ class phpbb_search_fulltext_sphinx if ($row) { - if ($config['fulltext_sphinx_autorun']) - { - if ((file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main.spd') && file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta.spd')) || ($allow_new_files && file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main.new.spd') && file_exists($config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta.new.spd'))) - { - $created = true; - } - } - else - { - $created = true; - } + $created = true; } return $created; } - /** - * Kills the searchd process and makes sure there's no locks left over - * - * @access private - */ - function shutdown_searchd() - { - global $config; - - if ($config['fulltext_sphinx_autorun']) - { - if (!function_exists('exec')) - { - set_config('fulltext_sphinx_autorun', '0'); - return; - } - - exec('killall -9 ' . SEARCHD_NAME . ' >> /dev/null 2>&1 &'); - - if (file_exists($config['fulltext_sphinx_data_path'] . 'searchd.pid')) - { - unlink($config['fulltext_sphinx_data_path'] . 'searchd.pid'); - } - - sphinx_unlink_by_pattern($config['fulltext_sphinx_data_path'], '#^.*\.spl$#'); - } - } - - /** - * Checks whether searchd is running, if it's not running it makes sure there's no left over - * files by calling shutdown_searchd. - * - * @return boolean Whether searchd is running or not - * - * @access private - */ - function searchd_running() - { - global $config; - - // if we cannot manipulate the service assume it is running - if (!$config['fulltext_sphinx_autorun']) - { - return true; - } - - if (file_exists($config['fulltext_sphinx_data_path'] . 'searchd.pid')) - { - $pid = trim(file_get_contents($config['fulltext_sphinx_data_path'] . 'searchd.pid')); - - if ($pid) - { - $output = array(); - $pidof_command = 'pidof'; - - exec('whereis -b pidof', $output); - if (sizeof($output) > 1) - { - $output = explode(' ', trim($output[0])); - $pidof_command = $output[1]; // 0 is pidof: - } - - $output = array(); - exec($pidof_command . ' ' . SEARCHD_NAME, $output); - if (sizeof($output) && (trim($output[0]) == $pid || trim($output[1]) == $pid)) - { - return true; - } - } - } - - // make sure it's really not running - $this->shutdown_searchd(); - - return false; - } - /** * Returns an associative array containing information about the indexes * @@ -980,26 +735,6 @@ class phpbb_search_fulltext_sphinx } $this->stats['last_searches'] = ''; - if ($config['fulltext_sphinx_autorun']) - { - if (file_exists($config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log')) - { - $last_searches = explode("\n", utf8_htmlspecialchars(sphinx_read_last_lines($config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log', 3))); - - foreach($last_searches as $i => $search) - { - if (strpos($search, '[' . $this->indexes . ']') !== false) - { - $last_searches[$i] = str_replace('[' . $this->indexes . ']', '', $search); - } - else - { - $last_searches[$i] = ''; - } - } - $this->stats['last_searches'] = implode("\n", $last_searches); - } - } } /** @@ -1014,8 +749,6 @@ class phpbb_search_fulltext_sphinx global $user, $config; $config_vars = array( - 'fulltext_sphinx_autoconf' => 'bool', - 'fulltext_sphinx_autorun' => 'bool', 'fulltext_sphinx_config_path' => 'string', 'fulltext_sphinx_data_path' => 'string', 'fulltext_sphinx_bin_path' => 'string', @@ -1025,8 +758,6 @@ class phpbb_search_fulltext_sphinx ); $defaults = array( - 'fulltext_sphinx_autoconf' => '1', - 'fulltext_sphinx_autorun' => '1', 'fulltext_sphinx_indexer_mem_limit' => '512', ); @@ -1043,57 +774,8 @@ class phpbb_search_fulltext_sphinx } } - $no_autoconf = false; - $no_autorun = false; $bin_path = $config['fulltext_sphinx_bin_path']; - // try to guess the path if it is empty - if (empty($bin_path)) - { - if (@file_exists('/usr/local/bin/' . INDEXER_NAME) && @file_exists('/usr/local/bin/' . SEARCHD_NAME)) - { - $bin_path = '/usr/local/bin/'; - } - else if (@file_exists('/usr/bin/' . INDEXER_NAME) && @file_exists('/usr/bin/' . SEARCHD_NAME)) - { - $bin_path = '/usr/bin/'; - } - else - { - $output = array(); - if (!function_exists('exec') || null === @exec('whereis -b ' . INDEXER_NAME, $output)) - { - $no_autorun = true; - } - else if (sizeof($output)) - { - $output = explode(' ', $output[0]); - array_shift($output); // remove indexer: - - foreach ($output as $path) - { - $path = dirname($path) . '/'; - - if (file_exists($path . INDEXER_NAME) && file_exists($path . SEARCHD_NAME)) - { - $bin_path = $path; - break; - } - } - } - } - } - - if ($no_autorun) - { - set_config('fulltext_sphinx_autorun', '0'); - } - - if ($no_autoconf) - { - set_config('fulltext_sphinx_autoconf', '0'); - } - // rewrite config if fulltext sphinx is enabled if ($config['fulltext_sphinx_autoconf'] && isset($config['fulltext_sphinx_configured']) && $config['fulltext_sphinx_configured']) { @@ -1115,14 +797,6 @@ class phpbb_search_fulltext_sphinx $tpl = ' ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_BEFORE']. ' -
-

' . $user->lang['FULLTEXT_SPHINX_AUTOCONF_EXPLAIN'] . '
-
-
-
-

' . $user->lang['FULLTEXT_SPHINX_AUTORUN_EXPLAIN'] . '
-
-

' . $user->lang['FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN'] . '
@@ -1157,61 +831,3 @@ class phpbb_search_fulltext_sphinx ); } } - -/** -* Deletes all files from a directory that match a certain pattern -* -* @param string $path Path from which files shall be deleted -* @param string $pattern PCRE pattern that a file needs to match in order to be deleted -* -* @access private -*/ -function sphinx_unlink_by_pattern($path, $pattern) -{ - $dir = opendir($path); - while (false !== ($file = readdir($dir))) - { - if (is_file($path . $file) && preg_match($pattern, $file)) - { - unlink($path . $file); - } - } - closedir($dir); -} - -/** -* Reads the last from a file -* -* @param string $file The filename from which the lines shall be read -* @param int $amount The number of lines to be read from the end -* @return string Last lines of the file -* -* @access private -*/ -function sphinx_read_last_lines($file, $amount) -{ - $fp = fopen($file, 'r'); - fseek($fp, 0, SEEK_END); - - $c = ''; - $i = 0; - - while ($i < $amount) - { - fseek($fp, -2, SEEK_CUR); - $c = fgetc($fp); - if ($c == "\n") - { - $i++; - } - if (feof($fp)) - { - break; - } - } - - $string = fread($fp, 8192); - fclose($fp); - - return $string; -} From 88089194e570edb77240138695034358062ffa58 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 03:55:23 +0530 Subject: [PATCH 20/57] [feature/sphinx-fulltext-search] prefix sphinx with constant names All constant names are prefixed with SPHINX_ PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 984bda68c2..7386833151 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -24,9 +24,9 @@ if (!defined('IN_PHPBB')) global $phpbb_root_path, $phpEx, $table_prefix; require($phpbb_root_path . "includes/sphinxapi-0.9.8." . $phpEx); -define('MAX_MATCHES', 20000); -define('CONNECT_RETRIES', 3); -define('CONNECT_WAIT_TIME', 300); +define('SPHINX_MAX_MATCHES', 20000); +define('SPHINX_CONNECT_RETRIES', 3); +define('SPHINX_CONNECT_WAIT_TIME', 300); /** * fulltext_sphinx @@ -216,7 +216,7 @@ class phpbb_search_fulltext_sphinx array('read_timeout', '5'), array('max_children', '30'), array('pid_file', $config['fulltext_sphinx_data_path'] . "searchd.pid"), - array('max_matches', (string) MAX_MATCHES), + array('max_matches', (string) SPHINX_MAX_MATCHES), array('binlog_path', $config['fulltext_sphinx_data_path']), ), ); @@ -451,14 +451,14 @@ class phpbb_search_fulltext_sphinx $this->sphinx->SetFilter('deleted', array(0)); - $this->sphinx->SetLimits($start, (int) $per_page, MAX_MATCHES); + $this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES); $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); // could be connection to localhost:3312 failed (errno=111, msg=Connection refused) during rotate, retry if so - $retries = CONNECT_RETRIES; + $retries = SPHINX_CONNECT_RETRIES; while (!$result && (strpos($this->sphinx->_error, "errno=111,") !== false) && $retries--) { - usleep(CONNECT_WAIT_TIME); + usleep(SPHINX_CONNECT_WAIT_TIME); $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); } From 569db1471b3000512232732a790d8653250e8012 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 04:15:35 +0530 Subject: [PATCH 21/57] [feature/sphinx-fulltext-search] fix stopwords option Stopwords option can be configured in ACP to generate correct sphinx config file. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 28 ++++------------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 7386833151..710c3d56be 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -193,7 +193,7 @@ class phpbb_search_fulltext_sphinx array('source', 'source_phpbb_' . $this->id . '_main'), array('docinfo', 'extern'), array('morphology', 'none'), - array('stopwords', (file_exists($config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt') && $config['fulltext_sphinx_stopwords']) ? $config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), + array('stopwords', $config['fulltext_sphinx_stopwords'] ? $config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), array('min_word_len', '2'), array('charset_type', 'utf-8'), array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), @@ -759,6 +759,7 @@ class phpbb_search_fulltext_sphinx $defaults = array( 'fulltext_sphinx_indexer_mem_limit' => '512', + 'fulltext_sphinx_stopwords' => '0', ); foreach ($config_vars as $config_var => $type) @@ -774,27 +775,6 @@ class phpbb_search_fulltext_sphinx } } - $bin_path = $config['fulltext_sphinx_bin_path']; - - // rewrite config if fulltext sphinx is enabled - if ($config['fulltext_sphinx_autoconf'] && isset($config['fulltext_sphinx_configured']) && $config['fulltext_sphinx_configured']) - { - $this->config_updated(); - } - - // check whether stopwords file is available and enabled - if (@file_exists($config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt')) - { - $stopwords_available = true; - $stopwords_active = $config['fulltext_sphinx_stopwords']; - } - else - { - $stopwords_available = false; - $stopwords_active = false; - set_config('fulltext_sphinx_stopwords', '0'); - } - $tpl = ' ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_BEFORE']. '
@@ -809,11 +789,11 @@ class phpbb_search_fulltext_sphinx

' . $user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '
- ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_AFTER']. '

' . $user->lang['FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN'] . '
-
+
+ ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_AFTER']. '

' . $user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '
From d2e42d7d619100695e0efe8d472c71f61cbfcb45 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 04:53:51 +0530 Subject: [PATCH 22/57] [feature/sphinx-fulltext-search] remove unnecessary code Some extra conditions and variables used in autoconf are removed. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 25 +++++------------------ 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 710c3d56be..48855ef7d8 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -59,11 +59,8 @@ class phpbb_search_fulltext_sphinx $this->sphinx = new SphinxClient(); - if (!empty($config['fulltext_sphinx_configured'])) - { - // we only support localhost for now - $this->sphinx->SetServer('localhost', (isset($config['fulltext_sphinx_port']) && $config['fulltext_sphinx_port']) ? (int) $config['fulltext_sphinx_port'] : 3312); - } + // We only support localhost for now + $this->sphinx->SetServer('localhost', (isset($config['fulltext_sphinx_port']) && $config['fulltext_sphinx_port']) ? (int) $config['fulltext_sphinx_port'] : 3312); $config['fulltext_sphinx_min_word_len'] = 2; $config['fulltext_sphinx_max_word_len'] = 400; @@ -125,7 +122,7 @@ class phpbb_search_fulltext_sphinx // now that we're sure everything was entered correctly, generate a config for the index // we misuse the avatar_salt for this, as it should be unique ;-) - $config_object = new phpbb_search_sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); + $config_object = new phpbb_search_sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); $config_data = array( 'source source_phpbb_' . $this->id . '_main' => array( @@ -266,10 +263,6 @@ class phpbb_search_fulltext_sphinx } } - set_config('fulltext_sphinx_configured', '1'); - - $this->tidy(); - return false; } @@ -613,11 +606,6 @@ class phpbb_search_fulltext_sphinx { global $db, $user, $config; - if (!isset($config['fulltext_sphinx_configured']) || !$config['fulltext_sphinx_configured']) - { - return $user->lang['FULLTEXT_SPHINX_CONFIGURE_FIRST']; - } - if (!$this->index_created()) { $sql = 'CREATE TABLE IF NOT EXISTS ' . SPHINX_TABLE . ' ( @@ -630,9 +618,6 @@ class phpbb_search_fulltext_sphinx $db->sql_query($sql); } - // start indexing process - $this->tidy(true); - return false; } @@ -645,7 +630,7 @@ class phpbb_search_fulltext_sphinx */ function delete_index($acp_module, $u_action) { - global $db, $config; + global $db; if (!$this->index_created()) { @@ -715,7 +700,7 @@ class phpbb_search_fulltext_sphinx */ function get_stats() { - global $db, $config; + global $db; if ($this->index_created()) { From 9711da2763f707408efde160357d51330fd17681 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 05:04:14 +0530 Subject: [PATCH 23/57] [feature/sphinx-fulltext-search] adds default config values Default config values are added to config table in new install as well as database_update. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 18 ------------------ phpBB/install/database_update.php | 10 ++++++++++ phpBB/install/schemas/schema_data.sql | 2 ++ 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 48855ef7d8..36c5c68a3b 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -742,24 +742,6 @@ class phpbb_search_fulltext_sphinx 'fulltext_sphinx_indexer_mem_limit' => 'int', ); - $defaults = array( - 'fulltext_sphinx_indexer_mem_limit' => '512', - 'fulltext_sphinx_stopwords' => '0', - ); - - foreach ($config_vars as $config_var => $type) - { - if (!isset($config[$config_var])) - { - $default = ''; - if (isset($defaults[$config_var])) - { - $default = $defaults[$config_var]; - } - set_config($config_var, $default); - } - } - $tpl = ' ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_BEFORE']. '
diff --git a/phpBB/install/database_update.php b/phpBB/install/database_update.php index 594397c815..0ffab8e413 100644 --- a/phpBB/install/database_update.php +++ b/phpBB/install/database_update.php @@ -2260,6 +2260,16 @@ function change_database_data(&$no_updates, $version) set_config('fulltext_postgres_max_word_len', 254); } + if (!isset($config['fulltext_sphinx_stopwords'])) + { + set_config('fulltext_sphinx_stopwords', 0); + } + + if (!isset($config['fulltext_sphinx_indexer_mem_limit'])) + { + set_config('fulltext_sphinx_indexer_mem_limit', 512); + } + if (!isset($config['load_jquery_cdn'])) { set_config('load_jquery_cdn', 0); diff --git a/phpBB/install/schemas/schema_data.sql b/phpBB/install/schemas/schema_data.sql index deefdafbc4..797f78e889 100644 --- a/phpBB/install/schemas/schema_data.sql +++ b/phpBB/install/schemas/schema_data.sql @@ -128,6 +128,8 @@ INSERT INTO phpbb_config (config_name, config_value) VALUES ('fulltext_native_mi INSERT INTO phpbb_config (config_name, config_value) VALUES ('fulltext_postgres_max_word_len', '254'); INSERT INTO phpbb_config (config_name, config_value) VALUES ('fulltext_postgres_min_word_len', '4'); INSERT INTO phpbb_config (config_name, config_value) VALUES ('fulltext_postgres_ts_name', 'simple'); +INSERT INTO phpbb_config (config_name, config_value) VALUES ('fulltext_sphinx_indexer_mem_limit', '512'); +INSERT INTO phpbb_config (config_name, config_value) VALUES ('fulltext_sphinx_stopwords', '0'); INSERT INTO phpbb_config (config_name, config_value) VALUES ('gzip_compress', '0'); INSERT INTO phpbb_config (config_name, config_value) VALUES ('hot_threshold', '25'); INSERT INTO phpbb_config (config_name, config_value) VALUES ('icons_path', 'images/icons'); From 79432aa4a0698a5a4a518521613aad4adbb40584 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 05:16:42 +0530 Subject: [PATCH 24/57] [feature/sphinx-fulltext-search] assign all globals to class properties PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 126 ++++++++++------------ 1 file changed, 56 insertions(+), 70 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 36c5c68a3b..f858b199b2 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -40,6 +40,10 @@ class phpbb_search_fulltext_sphinx private $id; private $indexes; private $sphinx; + private $auth; + private $config; + private $db; + private $user; public $word_length = array(); public $search_query; public $common_words = array(); @@ -52,7 +56,11 @@ class phpbb_search_fulltext_sphinx */ public function __construct(&$error) { - global $config; + global $config, $db, $user, $auth; + $this->config = $config; + $this->user = $user; + $this->db = $db; + $this->auth = $auth; $this->id = $config['avatar_salt']; $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main'; @@ -89,11 +97,9 @@ class phpbb_search_fulltext_sphinx */ function init() { - global $db, $user, $config; - - if ($db->sql_layer != 'mysql' && $db->sql_layer != 'mysql4' && $db->sql_layer != 'mysqli') + if ($this->db->sql_layer != 'mysql' && $this->db->sql_layer != 'mysql4' && $this->db->sql_layer != 'mysqli') { - return $user->lang['FULLTEXT_SPHINX_WRONG_DATABASE']; + return $this->user->lang['FULLTEXT_SPHINX_WRONG_DATABASE']; } if ($error = $this->config_updated()) @@ -116,13 +122,13 @@ class phpbb_search_fulltext_sphinx */ function config_updated() { - global $db, $user, $config, $phpbb_root_path, $phpEx; + global $phpbb_root_path, $phpEx; include ($phpbb_root_path . 'config.' . $phpEx); // now that we're sure everything was entered correctly, generate a config for the index // we misuse the avatar_salt for this, as it should be unique ;-) - $config_object = new phpbb_search_sphinx_config($config['fulltext_sphinx_config_path'] . 'sphinx.conf'); + $config_object = new phpbb_search_sphinx_config($this->config['fulltext_sphinx_config_path'] . 'sphinx.conf'); $config_data = array( 'source source_phpbb_' . $this->id . '_main' => array( @@ -186,11 +192,11 @@ class phpbb_search_fulltext_sphinx AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'), ), 'index index_phpbb_' . $this->id . '_main' => array( - array('path', $config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'), + array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'), array('source', 'source_phpbb_' . $this->id . '_main'), array('docinfo', 'extern'), array('morphology', 'none'), - array('stopwords', $config['fulltext_sphinx_stopwords'] ? $config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), + array('stopwords', $this->config['fulltext_sphinx_stopwords'] ? $this->config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), array('min_word_len', '2'), array('charset_type', 'utf-8'), array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), @@ -198,23 +204,23 @@ class phpbb_search_fulltext_sphinx array('min_infix_len', '0'), ), 'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array( - array('path', $config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'), + array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'), array('source', 'source_phpbb_' . $this->id . '_delta'), ), 'indexer' => array( - array('mem_limit', $config['fulltext_sphinx_indexer_mem_limit'] . 'M'), + array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'), ), 'searchd' => array( array('compat_sphinxql_magics' , '0'), array('listen' , '127.0.0.1'), - array('port', ($config['fulltext_sphinx_port']) ? $config['fulltext_sphinx_port'] : '3312'), - array('log', $config['fulltext_sphinx_data_path'] . "log/searchd.log"), - array('query_log', $config['fulltext_sphinx_data_path'] . "log/sphinx-query.log"), + array('port', ($this->config['fulltext_sphinx_port']) ? $this->config['fulltext_sphinx_port'] : '3312'), + array('log', $this->config['fulltext_sphinx_data_path'] . "log/searchd.log"), + array('query_log', $this->config['fulltext_sphinx_data_path'] . "log/sphinx-query.log"), array('read_timeout', '5'), array('max_children', '30'), - array('pid_file', $config['fulltext_sphinx_data_path'] . "searchd.pid"), + array('pid_file', $this->config['fulltext_sphinx_data_path'] . "searchd.pid"), array('max_matches', (string) SPHINX_MAX_MATCHES), - array('binlog_path', $config['fulltext_sphinx_data_path']), + array('binlog_path', $this->config['fulltext_sphinx_data_path']), ), ); @@ -278,8 +284,6 @@ class phpbb_search_fulltext_sphinx */ function split_keywords(&$keywords, $terms) { - global $config; - if ($terms == 'all') { $match = array('#\sand\s#i', '#\sor\s#i', '#\snot\s#i', '#\+#', '#-#', '#\|#', '#@#'); @@ -330,8 +334,6 @@ class phpbb_search_fulltext_sphinx */ function keyword_search($type, $fields, $terms, $sort_by_sql, $sort_key, $sort_dir, $sort_days, $ex_fid_ary, $m_approve_fid_ary, $topic_id, $author_ary, $author_name, &$id_ary, $start, $per_page) { - global $config, $db, $auth; - // No keywords? No posts. if (!strlen($this->search_query) && !sizeof($author_ary)) { @@ -432,7 +434,7 @@ class phpbb_search_fulltext_sphinx if (sizeof($ex_fid_ary)) { // All forums that a user is allowed to access - $fid_ary = array_unique(array_intersect(array_keys($auth->acl_getf('f_read', true)), array_keys($auth->acl_getf('f_search', true)))); + $fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($auth->acl_getf('f_search', true)))); // All forums that the user wants to and can search in $search_forums = array_diff($fid_ary, $ex_fid_ary); @@ -527,8 +529,6 @@ class phpbb_search_fulltext_sphinx */ function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id) { - global $config, $db; - if ($mode == 'edit') { $this->sphinx->UpdateAttributes($this->indexes, array('forum_id', 'poster_id'), array((int)$post_id => array((int)$forum_id, (int)$poster_id))); @@ -549,16 +549,16 @@ class phpbb_search_fulltext_sphinx )), ); - $sql = $db->sql_build_query('SELECT', $sql_array); - $result = $db->sql_query($sql); + $sql = $this->db->sql_build_query('SELECT', $sql_array); + $result = $this->db->sql_query($sql); $post_updates = array(); $post_time = time(); - while ($row = $db->sql_fetchrow($result)) + while ($row = $this->db->sql_fetchrow($result)) { $post_updates[(int)$row['post_id']] = array($post_time); } - $db->sql_freeresult($result); + $this->db->sql_freeresult($result); if (sizeof($post_updates)) { @@ -590,8 +590,6 @@ class phpbb_search_fulltext_sphinx */ function tidy($create = false) { - global $config; - set_config('search_last_gc', time(), true); } @@ -604,18 +602,16 @@ class phpbb_search_fulltext_sphinx */ function create_index($acp_module, $u_action) { - global $db, $user, $config; - if (!$this->index_created()) { $sql = 'CREATE TABLE IF NOT EXISTS ' . SPHINX_TABLE . ' ( counter_id INT NOT NULL PRIMARY KEY, max_doc_id INT NOT NULL )'; - $db->sql_query($sql); + $this->db->sql_query($sql); $sql = 'TRUNCATE TABLE ' . SPHINX_TABLE; - $db->sql_query($sql); + $this->db->sql_query($sql); } return false; @@ -630,15 +626,13 @@ class phpbb_search_fulltext_sphinx */ function delete_index($acp_module, $u_action) { - global $db; - if (!$this->index_created()) { return false; } $sql = 'DROP TABLE ' . SPHINX_TABLE; - $db->sql_query($sql); + $this->db->sql_query($sql); return false; } @@ -652,12 +646,10 @@ class phpbb_search_fulltext_sphinx */ function index_created($allow_new_files = true) { - global $db, $config; - $sql = 'SHOW TABLES LIKE \'' . SPHINX_TABLE . '\''; - $result = $db->sql_query($sql); - $row = $db->sql_fetchrow($result); - $db->sql_freeresult($result); + $result = $this->db->sql_query($sql); + $row = $this->db->sql_fetchrow($result); + $this->db->sql_freeresult($result); $created = false; @@ -678,18 +670,16 @@ class phpbb_search_fulltext_sphinx */ function index_stats() { - global $user; - if (empty($this->stats)) { $this->get_stats(); } return array( - $user->lang['FULLTEXT_SPHINX_MAIN_POSTS'] => ($this->index_created()) ? $this->stats['main_posts'] : 0, - $user->lang['FULLTEXT_SPHINX_DELTA_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] - $this->stats['main_posts'] : 0, - $user->lang['FULLTEXT_MYSQL_TOTAL_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] : 0, - $user->lang['FULLTEXT_SPHINX_LAST_SEARCHES'] => nl2br($this->stats['last_searches']), + $this->user->lang['FULLTEXT_SPHINX_MAIN_POSTS'] => ($this->index_created()) ? $this->stats['main_posts'] : 0, + $this->user->lang['FULLTEXT_SPHINX_DELTA_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] - $this->stats['main_posts'] : 0, + $this->user->lang['FULLTEXT_MYSQL_TOTAL_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] : 0, + $this->user->lang['FULLTEXT_SPHINX_LAST_SEARCHES'] => nl2br($this->stats['last_searches']), ); } @@ -700,23 +690,21 @@ class phpbb_search_fulltext_sphinx */ function get_stats() { - global $db; - if ($this->index_created()) { $sql = 'SELECT COUNT(post_id) as total_posts FROM ' . POSTS_TABLE; - $result = $db->sql_query($sql); - $this->stats['total_posts'] = (int) $db->sql_fetchfield('total_posts'); - $db->sql_freeresult($result); + $result = $this->db->sql_query($sql); + $this->stats['total_posts'] = (int) $this->db->sql_fetchfield('total_posts'); + $this->db->sql_freeresult($result); $sql = 'SELECT COUNT(p.post_id) as main_posts FROM ' . POSTS_TABLE . ' p, ' . SPHINX_TABLE . ' m WHERE p.post_id <= m.max_doc_id AND m.counter_id = 1'; - $result = $db->sql_query($sql); - $this->stats['main_posts'] = (int) $db->sql_fetchfield('main_posts'); - $db->sql_freeresult($result); + $result = $this->db->sql_query($sql); + $this->stats['main_posts'] = (int) $this->db->sql_fetchfield('main_posts'); + $this->db->sql_freeresult($result); } $this->stats['last_searches'] = ''; @@ -731,8 +719,6 @@ class phpbb_search_fulltext_sphinx */ function acp() { - global $user, $config; - $config_vars = array( 'fulltext_sphinx_config_path' => 'string', 'fulltext_sphinx_data_path' => 'string', @@ -743,31 +729,31 @@ class phpbb_search_fulltext_sphinx ); $tpl = ' - ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_BEFORE']. ' + ' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE_BEFORE']. '
-

' . $user->lang['FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN'] . '
-
+

' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN'] . '
+
-

' . $user->lang['FULLTEXT_SPHINX_BIN_PATH_EXPLAIN'] . '
+

' . $this->user->lang['FULLTEXT_SPHINX_BIN_PATH_EXPLAIN'] . '
-

' . $user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '
-
+

' . $this->user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '
+
-

' . $user->lang['FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN'] . '
-
+

' . $this->user->lang['FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN'] . '
+
- ' . $user->lang['FULLTEXT_SPHINX_CONFIGURE_AFTER']. ' + ' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE_AFTER']. '
-

' . $user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '
-
+

' . $this->user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '
+
-

' . $user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN'] . '
-
' . $user->lang['MIB'] . '
+

' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN'] . '
+
' . $this->user->lang['MIB'] . '
'; From 45c0956bcf72e59138c3b05e26c73b657298f562 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 05:23:23 +0530 Subject: [PATCH 25/57] [feature/sphinx-fulltext-search] implementing db_tools Use db_tools class for creating/dropping sphinx table. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 27 ++++++++++++++++------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index f858b199b2..317a35937d 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -43,6 +43,7 @@ class phpbb_search_fulltext_sphinx private $auth; private $config; private $db; + private $db_tools; private $user; public $word_length = array(); public $search_query; @@ -56,12 +57,20 @@ class phpbb_search_fulltext_sphinx */ public function __construct(&$error) { - global $config, $db, $user, $auth; + global $config, $db, $user, $auth, $phpbb_root_path, $phpEx; $this->config = $config; $this->user = $user; $this->db = $db; $this->auth = $auth; + if (!class_exists('phpbb_db_tools')) + { + require($phpbb_root_path . 'includes/db/db_tools.' . $phpEx); + } + + // Initialize phpbb_db_tools object + $this->db_tools = new phpbb_db_tools($this->db); + $this->id = $config['avatar_salt']; $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main'; @@ -604,11 +613,14 @@ class phpbb_search_fulltext_sphinx { if (!$this->index_created()) { - $sql = 'CREATE TABLE IF NOT EXISTS ' . SPHINX_TABLE . ' ( - counter_id INT NOT NULL PRIMARY KEY, - max_doc_id INT NOT NULL - )'; - $this->db->sql_query($sql); + $table_data = array( + 'COLUMNS' => array( + 'counter_id' => array('UINT', 0), + 'max_doc_id' => array('UINT', 0), + ), + 'PRIMARY_KEY' => 'counter_id', + ); + $this->db_tools->sql_create_table(SPHINX_TABLE, $table_data); $sql = 'TRUNCATE TABLE ' . SPHINX_TABLE; $this->db->sql_query($sql); @@ -631,8 +643,7 @@ class phpbb_search_fulltext_sphinx return false; } - $sql = 'DROP TABLE ' . SPHINX_TABLE; - $this->db->sql_query($sql); + $this->db_tools->sql_table_drop(SPHINX_TABLE); return false; } From 537a16220ea9561332a17004eec79f2854c68df4 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 05:33:17 +0530 Subject: [PATCH 26/57] [feature/sphinx-fulltext-search] remove recent search queries remove recent search queries from the stats as they can't be retreived and remove other language keys being used no more. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 3 --- phpBB/language/en/acp/search.php | 12 +----------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 317a35937d..1ff6ab21b5 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -690,7 +690,6 @@ class phpbb_search_fulltext_sphinx $this->user->lang['FULLTEXT_SPHINX_MAIN_POSTS'] => ($this->index_created()) ? $this->stats['main_posts'] : 0, $this->user->lang['FULLTEXT_SPHINX_DELTA_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] - $this->stats['main_posts'] : 0, $this->user->lang['FULLTEXT_MYSQL_TOTAL_POSTS'] => ($this->index_created()) ? $this->stats['total_posts'] : 0, - $this->user->lang['FULLTEXT_SPHINX_LAST_SEARCHES'] => nl2br($this->stats['last_searches']), ); } @@ -717,8 +716,6 @@ class phpbb_search_fulltext_sphinx $this->stats['main_posts'] = (int) $this->db->sql_fetchfield('main_posts'); $this->db->sql_freeresult($result); } - - $this->stats['last_searches'] = ''; } /** diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 3fa7f34c64..14701459eb 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -57,6 +57,7 @@ $lang = array_merge($lang, array( 'FULLTEXT_MYSQL_MIN_SEARCH_CHARS_EXPLAIN' => 'Words with at least this many characters will be indexed for searching. You or your host can only change this setting by changing the mysql configuration.', 'FULLTEXT_MYSQL_MAX_SEARCH_CHARS_EXPLAIN' => 'Words with no more than this many characters will be indexed for searching. You or your host can only change this setting by changing the mysql configuration.', +<<<<<<< HEAD 'FULLTEXT_POSTGRES_INCOMPATIBLE_DATABASE' => 'The PostgreSQL fulltext backend can only be used with PostgreSQL.', 'FULLTEXT_POSTGRES_TS_NOT_USABLE' => 'The PostgreSQL fulltext backend can only be used with PostgreSQL 8.3 and above.', 'FULLTEXT_POSTGRES_TOTAL_POSTS' => 'Total number of indexed posts', @@ -69,10 +70,6 @@ $lang = array_merge($lang, array( 'FULLTEXT_POSTGRES_MIN_WORD_LEN_EXPLAIN' => 'Words with at least this many characters will be included in the query to the database.', 'FULLTEXT_POSTGRES_MAX_WORD_LEN_EXPLAIN' => 'Words with no more than this many characters will be included in the query to the database.', - 'FULLTEXT_SPHINX_AUTOCONF' => 'Automatically configure Sphinx', - 'FULLTEXT_SPHINX_AUTOCONF_EXPLAIN' => 'This is the easiest way to install Sphinx, just select the settings here and a config file will be written for you. This requires write permissions on the configuration folder.', - 'FULLTEXT_SPHINX_AUTORUN' => 'Automatically run Sphinx', - 'FULLTEXT_SPHINX_AUTORUN_EXPLAIN' => 'This is the easiest way to run Sphinx. Select the paths in this dialogue and the Sphinx daemon will be started and stopped as needed. You can also create an index from the ACP. If your PHP installation forbids the use of exec you can disable this and run Sphinx manually.', 'FULLTEXT_SPHINX_BIN_PATH' => 'Path to executables directory', 'FULLTEXT_SPHINX_BIN_PATH_EXPLAIN' => 'Skip if autorun is disabled. If this path could not be determined automatically you have to enter the path to the directory in which the sphinx executables indexer and searchd reside.', 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to configuration directory', @@ -83,18 +80,11 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'Skip if autorun is disabled. You should create this directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody). It will be used to store the indexes and log files.', 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', - 'FULLTEXT_SPHINX_DIRECTORY_NOT_FOUND' => 'The directory %s does not exist. Please correct your path settings.', - 'FULLTEXT_SPHINX_FILE_NOT_EXECUTABLE' => 'The file %s is not executable for the webserver.', - 'FULLTEXT_SPHINX_FILE_NOT_FOUND' => 'The file %s does not exist. Please correct your path settings.', - 'FULLTEXT_SPHINX_FILE_NOT_WRITABLE' => 'The file %s cannot be written by the webserver.', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', - 'FULLTEXT_SPHINX_LAST_SEARCHES' => 'Recent search queries', 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', 'FULLTEXT_SPHINX_PORT' => 'Sphinx search deamon port', 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search deamon on localhost listens. Leave empty to use the default 3312', - 'FULLTEXT_SPHINX_REQUIRES_EXEC' => 'The sphinx plugin for phpBB requires PHP’s exec function which is disabled on your system.', - 'FULLTEXT_SPHINX_UNCONFIGURED' => 'Please set all necessary options in the "Fulltext Sphinx" section of the previous page before you try to activate the sphinx plugin.', 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx plugin for phpBB currently only supports MySQL', 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'This setting only works with autoconf enabled. You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', From fdb7e64e29d7b6286b505cf3a75fe53cfdfc7503 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 05:51:52 +0530 Subject: [PATCH 27/57] [feature/sphinx-fulltext-search] fix comments PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 34 +++++++----- phpBB/includes/search/sphinx/config.php | 52 +++++++++++-------- .../includes/search/sphinx/config_section.php | 6 +-- 3 files changed, 53 insertions(+), 39 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 1ff6ab21b5..c72db1862e 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -116,7 +116,7 @@ class phpbb_search_fulltext_sphinx return $error; } - // move delta to main index each hour + // Move delta to main index each hour set_config('search_gc', 3600); return false; @@ -135,8 +135,9 @@ class phpbb_search_fulltext_sphinx include ($phpbb_root_path . 'config.' . $phpEx); - // now that we're sure everything was entered correctly, generate a config for the index - // we misuse the avatar_salt for this, as it should be unique ;-) + /* Now that we're sure everything was entered correctly, + generate a config for the index. We misuse the avatar_salt + for this, as it should be unique. */ $config_object = new phpbb_search_sphinx_config($this->config['fulltext_sphinx_config_path'] . 'sphinx.conf'); $config_data = array( @@ -396,7 +397,7 @@ class phpbb_search_fulltext_sphinx } } - // most narrow filters first + // Most narrow filters first if ($topic_id) { $this->sphinx->SetFilter('topic_id', array($topic_id)); @@ -407,31 +408,37 @@ class phpbb_search_fulltext_sphinx switch($fields) { case 'titleonly': - // only search the title + // Only search the title if ($terms == 'all') { $search_query_prefix = '@title '; } - $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); // weight for the title - $this->sphinx->SetFilter('topic_first_post', array(1)); // 1 is first_post, 0 is not first post + // Weight for the title + $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); + // 1 is first_post, 0 is not first post + $this->sphinx->SetFilter('topic_first_post', array(1)); break; case 'msgonly': - // only search the body + // Only search the body if ($terms == 'all') { $search_query_prefix = '@data '; } - $this->sphinx->SetFieldWeights(array("title" => 1, "data" => 5)); // weight for the body + // Weight for the body + $this->sphinx->SetFieldWeights(array("title" => 1, "data" => 5)); break; case 'firstpost': - $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); // more relative weight for the title, also search the body - $this->sphinx->SetFilter('topic_first_post', array(1)); // 1 is first_post, 0 is not first post + // More relative weight for the title, also search the body + $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); + // 1 is first_post, 0 is not first post + $this->sphinx->SetFilter('topic_first_post', array(1)); break; default: - $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); // more relative weight for the title, also search the body + // More relative weight for the title, also search the body + $this->sphinx->SetFieldWeights(array("title" => 5, "data" => 1)); break; } @@ -458,7 +465,8 @@ class phpbb_search_fulltext_sphinx $this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES); $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); - // could be connection to localhost:3312 failed (errno=111, msg=Connection refused) during rotate, retry if so + /* Could be connection to localhost:3312 failed (errno=111, + msg=Connection refused) during rotate, retry if so */ $retries = SPHINX_CONNECT_RETRIES; while (!$result && (strpos($this->sphinx->_error, "errno=111,") !== false) && $retries--) { diff --git a/phpBB/includes/search/sphinx/config.php b/phpBB/includes/search/sphinx/config.php index 966cd0f284..3820eff178 100644 --- a/phpBB/includes/search/sphinx/config.php +++ b/phpBB/includes/search/sphinx/config.php @@ -49,7 +49,7 @@ class phpbb_search_sphinx_config { for ($i = 0, $size = sizeof($this->sections); $i < $size; $i++) { - // make sure this is really a section object and not a comment + // Make sure this is really a section object and not a comment if (($this->sections[$i] instanceof phpbb_search_sphinx_config_section) && $this->sections[$i]->get_name() == $name) { return $this->sections[$i]; @@ -76,7 +76,7 @@ class phpbb_search_sphinx_config */ function read($filename) { - // split the file into lines, we'll process it line by line + // Split the file into lines, we'll process it line by line $config_file = file($filename); $this->sections = array(); @@ -87,8 +87,8 @@ class phpbb_search_sphinx_config foreach ($config_file as $i => $line) { - // if the value of a variable continues to the next line because the line break was escaped - // then we don't trim leading space but treat it as a part of the value + /* If the value of a variable continues to the next line because the line + break was escaped then we don't trim leading space but treat it as a part of the value */ if ($in_value) { $line = rtrim($line); @@ -98,11 +98,11 @@ class phpbb_search_sphinx_config $line = trim($line); } - // if we're not inside a section look for one + // If we're not inside a section look for one if (!$section) { - // add empty lines and comments as comment objects to the section list - // that way they're not deleted when reassembling the file from the sections + /* add empty lines and comments as comment objects to the section list + that way they're not deleted when reassembling the file from the sections*/ if (!$line || $line[0] == '#') { $this->sections[] = new phpbb_search_sphinx_config_comment($config_file[$i]); @@ -110,8 +110,8 @@ class phpbb_search_sphinx_config } else { - // otherwise we scan the line reading the section name until we find - // an opening curly bracket or a comment + /* otherwise we scan the line reading the section name until we find + an opening curly bracket or a comment */ $section_name = ''; $section_name_comment = ''; $found_opening_bracket = false; @@ -137,28 +137,29 @@ class phpbb_search_sphinx_config $section_name .= $line[$j]; } - // and then we create the new section object + // And then we create the new section object $section_name = trim($section_name); $section = new phpbb_search_sphinx_config_section($section_name, $section_name_comment); } } - else // if we're looking for variables inside a section + else { + // If we're looking for variables inside a section $skip_first = false; - // if we're not in a value continuing over the line feed + // If we're not in a value continuing over the line feed if (!$in_value) { - // then add empty lines and comments as comment objects to the variable list - // of this section so they're not deleted on reassembly + /* then add empty lines and comments as comment objects to the variable list + of this section so they're not deleted on reassembly */ if (!$line || $line[0] == '#') { $section->add_variable(new phpbb_search_sphinx_config_comment($config_file[$i])); continue; } - // as long as we haven't yet actually found an opening bracket for this section - // we treat everything as comments so it's not deleted either + /* As long as we haven't yet actually found an opening bracket for this section + we treat everything as comments so it's not deleted either */ if (!$found_opening_bracket) { if ($line[0] == '{') @@ -175,7 +176,8 @@ class phpbb_search_sphinx_config } } - // if we did not find a comment in this line or still add to the previous line's value ... + /* If we did not find a comment in this line or still add to the previous + line's value ... */ if ($line || $in_value) { if (!$in_value) @@ -226,21 +228,24 @@ class phpbb_search_sphinx_config { $value .= "\n"; $in_value = true; - continue 2; // go to the next line and keep processing the value in there + // Go to the next line and keep processing the value in there + continue 2; } $value .= $line[$j]; } } - // if a name and an equal sign were found then we have append a new variable object to the section + /* If a name and an equal sign were found then we have append a + new variable object to the section */ if ($name && $found_assignment) { $section->add_variable(new phpbb_search_sphinx_config_variable(trim($name), trim($value), ($end_section) ? '' : $comment)); continue; } - // if we found a closing curly bracket this section has been completed and we can append it to the section list - // and continue with looking for the next section + /* if we found a closing curly bracket this section has been completed + and we can append it to the section list and continue with looking for + the next section */ if ($end_section) { $section->set_end_comment($comment); @@ -250,13 +255,14 @@ class phpbb_search_sphinx_config } } - // if we did not find anything meaningful up to here, then just treat it as a comment + /* If we did not find anything meaningful up to here, then just treat it + as a comment */ $comment = ($skip_first) ? "\t" . substr(ltrim($config_file[$i]), 1) : $config_file[$i]; $section->add_variable(new phpbb_search_sphinx_config_comment($comment)); } } - // keep the filename for later use + // Keep the filename for later use $this->loaded = $filename; } diff --git a/phpBB/includes/search/sphinx/config_section.php b/phpBB/includes/search/sphinx/config_section.php index 529254dd5a..ed20dba279 100644 --- a/phpBB/includes/search/sphinx/config_section.php +++ b/phpBB/includes/search/sphinx/config_section.php @@ -79,7 +79,7 @@ class phpbb_search_sphinx_config_section { for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) { - // make sure this is a variable object and not a comment + // Make sure this is a variable object and not a comment if (($this->variables[$i] instanceof phpbb_search_sphinx_config_variable) && $this->variables[$i]->get_name() == $name) { return $this->variables[$i]; @@ -96,7 +96,7 @@ class phpbb_search_sphinx_config_section { for ($i = 0, $size = sizeof($this->variables); $i < $size; $i++) { - // make sure this is a variable object and not a comment + // Make sure this is a variable object and not a comment if (($this->variables[$i] instanceof phpbb_search_sphinx_config_variable) && $this->variables[$i]->get_name() == $name) { array_splice($this->variables, $i, 1); @@ -127,7 +127,7 @@ class phpbb_search_sphinx_config_section { $content = $this->name . ' ' . $this->comment . "\n{\n"; - // make sure we don't get too many newlines after the opening bracket + // Make sure we don't get too many newlines after the opening bracket while (trim($this->variables[0]->to_string()) == '') { array_shift($this->variables); From 01261179ce71ff0699ae598828ae82ec98751037 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 06:07:42 +0530 Subject: [PATCH 28/57] [feature/sphinx-fulltext-search] improve formatting PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index c72db1862e..5d92c1ff01 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -22,7 +22,7 @@ if (!defined('IN_PHPBB')) * function and the variables used are in global space. */ global $phpbb_root_path, $phpEx, $table_prefix; -require($phpbb_root_path . "includes/sphinxapi-0.9.8." . $phpEx); +require($phpbb_root_path . 'includes/sphinxapi-0.9.8.' . $phpEx); define('SPHINX_MAX_MATCHES', 20000); define('SPHINX_CONNECT_RETRIES', 3); @@ -224,11 +224,11 @@ class phpbb_search_fulltext_sphinx array('compat_sphinxql_magics' , '0'), array('listen' , '127.0.0.1'), array('port', ($this->config['fulltext_sphinx_port']) ? $this->config['fulltext_sphinx_port'] : '3312'), - array('log', $this->config['fulltext_sphinx_data_path'] . "log/searchd.log"), - array('query_log', $this->config['fulltext_sphinx_data_path'] . "log/sphinx-query.log"), + array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), + array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), array('read_timeout', '5'), array('max_children', '30'), - array('pid_file', $this->config['fulltext_sphinx_data_path'] . "searchd.pid"), + array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'), array('max_matches', (string) SPHINX_MAX_MATCHES), array('binlog_path', $this->config['fulltext_sphinx_data_path']), ), @@ -352,9 +352,9 @@ class phpbb_search_fulltext_sphinx $id_ary = array(); - $join_topic = ($type == 'posts') ? false : true; + $join_topic = ($type != 'posts'); - // sorting + // Sorting if ($type == 'topics') { @@ -405,7 +405,7 @@ class phpbb_search_fulltext_sphinx $search_query_prefix = ''; - switch($fields) + switch ($fields) { case 'titleonly': // Only search the title @@ -483,7 +483,7 @@ class phpbb_search_fulltext_sphinx } else { - foreach($result['matches'] as $key => $value) + foreach ($result['matches'] as $key => $value) { $id_ary[] = $value['attrs']['topic_id']; } From f0692bb9e83f6af9725023028f07cba636d04a4b Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 06:38:36 +0530 Subject: [PATCH 29/57] [feature/sphinx-fulltext-search] modify config class Sphinx config class is modified to return the configuration data instead of writing it to a file. Search backend property config_file_data stores the generated data. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 265 +++++++++++----------- phpBB/includes/search/sphinx/config.php | 40 ++-- 2 files changed, 146 insertions(+), 159 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 5d92c1ff01..a54ebe1a59 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -45,6 +45,7 @@ class phpbb_search_fulltext_sphinx private $db; private $db_tools; private $user; + private $config_file_data = ''; public $word_length = array(); public $search_query; public $common_words = array(); @@ -133,151 +134,151 @@ class phpbb_search_fulltext_sphinx { global $phpbb_root_path, $phpEx; - include ($phpbb_root_path . 'config.' . $phpEx); + include ($phpbb_root_path . 'config.' . $phpEx); - /* Now that we're sure everything was entered correctly, - generate a config for the index. We misuse the avatar_salt - for this, as it should be unique. */ - $config_object = new phpbb_search_sphinx_config($this->config['fulltext_sphinx_config_path'] . 'sphinx.conf'); + /* Now that we're sure everything was entered correctly, + generate a config for the index. We misuse the avatar_salt + for this, as it should be unique. */ + $config_object = new phpbb_search_sphinx_config($this->config_file_data); - $config_data = array( - 'source source_phpbb_' . $this->id . '_main' => array( - array('type', 'mysql'), - array('sql_host', $dbhost), - array('sql_user', $dbuser), - array('sql_pass', $dbpasswd), - array('sql_db', $dbname), - array('sql_port', $dbport), - array('sql_query_pre', 'SET NAMES utf8'), - array('sql_query_pre', 'REPLACE INTO ' . SPHINX_TABLE . ' SELECT 1, MAX(post_id) FROM ' . POSTS_TABLE . ''), - array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''), - array('sql_range_step', '5000'), - array('sql_query', 'SELECT - p.post_id AS id, - p.forum_id, - p.topic_id, - p.poster_id, - IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, - p.post_time, - p.post_subject, - p.post_subject as title, - p.post_text as data, - t.topic_last_post_time, - 0 as deleted - FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t - WHERE - p.topic_id = t.topic_id - AND p.post_id >= $start AND p.post_id <= $end'), - array('sql_query_post', ''), - array('sql_query_post_index', 'REPLACE INTO ' . SPHINX_TABLE . ' ( counter_id, max_doc_id ) VALUES ( 1, $maxid )'), - array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'), - array('sql_attr_uint', 'forum_id'), - array('sql_attr_uint', 'topic_id'), - array('sql_attr_uint', 'poster_id'), - array('sql_attr_bool', 'topic_first_post'), - array('sql_attr_bool', 'deleted'), - array('sql_attr_timestamp' , 'post_time'), - array('sql_attr_timestamp' , 'topic_last_post_time'), - array('sql_attr_str2ordinal', 'post_subject'), - ), - 'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array( - array('sql_query_pre', ''), - array('sql_query_range', ''), - array('sql_range_step', ''), - array('sql_query', 'SELECT - p.post_id AS id, - p.forum_id, - p.topic_id, - p.poster_id, - IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, - p.post_time, - p.post_subject, - p.post_subject as title, - p.post_text as data, - t.topic_last_post_time, - 0 as deleted - FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t - WHERE - p.topic_id = t.topic_id - AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'), - ), - 'index index_phpbb_' . $this->id . '_main' => array( - array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'), - array('source', 'source_phpbb_' . $this->id . '_main'), - array('docinfo', 'extern'), - array('morphology', 'none'), - array('stopwords', $this->config['fulltext_sphinx_stopwords'] ? $this->config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), - array('min_word_len', '2'), - array('charset_type', 'utf-8'), - array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), - array('min_prefix_len', '0'), - array('min_infix_len', '0'), - ), - 'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array( - array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'), - array('source', 'source_phpbb_' . $this->id . '_delta'), - ), - 'indexer' => array( - array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'), - ), - 'searchd' => array( - array('compat_sphinxql_magics' , '0'), - array('listen' , '127.0.0.1'), - array('port', ($this->config['fulltext_sphinx_port']) ? $this->config['fulltext_sphinx_port'] : '3312'), - array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), - array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), - array('read_timeout', '5'), - array('max_children', '30'), - array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'), - array('max_matches', (string) SPHINX_MAX_MATCHES), - array('binlog_path', $this->config['fulltext_sphinx_data_path']), - ), - ); + $config_data = array( + 'source source_phpbb_' . $this->id . '_main' => array( + array('type', 'mysql'), + array('sql_host', $dbhost), + array('sql_user', $dbuser), + array('sql_pass', $dbpasswd), + array('sql_db', $dbname), + array('sql_port', $dbport), + array('sql_query_pre', 'SET NAMES utf8'), + array('sql_query_pre', 'REPLACE INTO ' . SPHINX_TABLE . ' SELECT 1, MAX(post_id) FROM ' . POSTS_TABLE . ''), + array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''), + array('sql_range_step', '5000'), + array('sql_query', 'SELECT + p.post_id AS id, + p.forum_id, + p.topic_id, + p.poster_id, + IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, + p.post_time, + p.post_subject, + p.post_subject as title, + p.post_text as data, + t.topic_last_post_time, + 0 as deleted + FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t + WHERE + p.topic_id = t.topic_id + AND p.post_id >= $start AND p.post_id <= $end'), + array('sql_query_post', ''), + array('sql_query_post_index', 'REPLACE INTO ' . SPHINX_TABLE . ' ( counter_id, max_doc_id ) VALUES ( 1, $maxid )'), + array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'), + array('sql_attr_uint', 'forum_id'), + array('sql_attr_uint', 'topic_id'), + array('sql_attr_uint', 'poster_id'), + array('sql_attr_bool', 'topic_first_post'), + array('sql_attr_bool', 'deleted'), + array('sql_attr_timestamp' , 'post_time'), + array('sql_attr_timestamp' , 'topic_last_post_time'), + array('sql_attr_str2ordinal', 'post_subject'), + ), + 'source source_phpbb_' . $this->id . '_delta : source_phpbb_' . $this->id . '_main' => array( + array('sql_query_pre', ''), + array('sql_query_range', ''), + array('sql_range_step', ''), + array('sql_query', 'SELECT + p.post_id AS id, + p.forum_id, + p.topic_id, + p.poster_id, + IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, + p.post_time, + p.post_subject, + p.post_subject as title, + p.post_text as data, + t.topic_last_post_time, + 0 as deleted + FROM ' . POSTS_TABLE . ' p, ' . TOPICS_TABLE . ' t + WHERE + p.topic_id = t.topic_id + AND p.post_id >= ( SELECT max_doc_id FROM ' . SPHINX_TABLE . ' WHERE counter_id=1 )'), + ), + 'index index_phpbb_' . $this->id . '_main' => array( + array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_main'), + array('source', 'source_phpbb_' . $this->id . '_main'), + array('docinfo', 'extern'), + array('morphology', 'none'), + array('stopwords', $this->config['fulltext_sphinx_stopwords'] ? $this->config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), + array('min_word_len', '2'), + array('charset_type', 'utf-8'), + array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), + array('min_prefix_len', '0'), + array('min_infix_len', '0'), + ), + 'index index_phpbb_' . $this->id . '_delta : index_phpbb_' . $this->id . '_main' => array( + array('path', $this->config['fulltext_sphinx_data_path'] . 'index_phpbb_' . $this->id . '_delta'), + array('source', 'source_phpbb_' . $this->id . '_delta'), + ), + 'indexer' => array( + array('mem_limit', $this->config['fulltext_sphinx_indexer_mem_limit'] . 'M'), + ), + 'searchd' => array( + array('compat_sphinxql_magics' , '0'), + array('listen' , '127.0.0.1'), + array('port', ($this->config['fulltext_sphinx_port']) ? $this->config['fulltext_sphinx_port'] : '3312'), + array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), + array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), + array('read_timeout', '5'), + array('max_children', '30'), + array('pid_file', $this->config['fulltext_sphinx_data_path'] . 'searchd.pid'), + array('max_matches', (string) SPHINX_MAX_MATCHES), + array('binlog_path', $this->config['fulltext_sphinx_data_path']), + ), + ); - $non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true); - $delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true); - - foreach ($config_data as $section_name => $section_data) + $non_unique = array('sql_query_pre' => true, 'sql_attr_uint' => true, 'sql_attr_timestamp' => true, 'sql_attr_str2ordinal' => true, 'sql_attr_bool' => true); + $delete = array('sql_group_column' => true, 'sql_date_column' => true, 'sql_str2ordinal_column' => true); + foreach ($config_data as $section_name => $section_data) + { + $section = $config_object->get_section_by_name($section_name); + if (!$section) { - $section = $config_object->get_section_by_name($section_name); - if (!$section) - { - $section = $config_object->add_section($section_name); - } + $section = $config_object->add_section($section_name); + } - foreach ($delete as $key => $void) - { - $section->delete_variables_by_name($key); - } + foreach ($delete as $key => $void) + { + $section->delete_variables_by_name($key); + } - foreach ($non_unique as $key => $void) - { - $section->delete_variables_by_name($key); - } + foreach ($non_unique as $key => $void) + { + $section->delete_variables_by_name($key); + } - foreach ($section_data as $entry) - { - $key = $entry[0]; - $value = $entry[1]; + foreach ($section_data as $entry) + { + $key = $entry[0]; + $value = $entry[1]; - if (!isset($non_unique[$key])) - { - $variable = $section->get_variable_by_name($key); - if (!$variable) - { - $variable = $section->create_variable($key, $value); - } - else - { - $variable->set_value($value); - } - } - else + if (!isset($non_unique[$key])) + { + $variable = $section->get_variable_by_name($key); + if (!$variable) { $variable = $section->create_variable($key, $value); } + else + { + $variable->set_value($value); + } + } + else + { + $variable = $section->create_variable($key, $value); } } + } + $this->config_file_data = $config_object->get_data(); return false; } diff --git a/phpBB/includes/search/sphinx/config.php b/phpBB/includes/search/sphinx/config.php index 3820eff178..e0ad667fb6 100644 --- a/phpBB/includes/search/sphinx/config.php +++ b/phpBB/includes/search/sphinx/config.php @@ -27,15 +27,15 @@ class phpbb_search_sphinx_config var $sections = array(); /** - * Constructor which optionally loads data from a file + * Constructor which optionally loads data from a variable * - * @param string $filename The path to a file containing the sphinx configuration + * @param string $config_data Variable containing the sphinx configuration data */ - function __construct($filename = false) + function __construct($config_data) { - if ($filename !== false && file_exists($filename)) + if ($config_data != '') { - $this->read($filename); + $this->read($config_data); } } @@ -70,22 +70,19 @@ class phpbb_search_sphinx_config } /** - * Parses the config file at the given path, which is stored in $this->loaded for later use + * Reads the config file data * - * @param string $filename The path to the config file + * @param string $config_data The config file data */ - function read($filename) + function read($config_data) { - // Split the file into lines, we'll process it line by line - $config_file = file($filename); - $this->sections = array(); $section = null; $found_opening_bracket = false; $in_value = false; - foreach ($config_file as $i => $line) + foreach ($config_data as $i => $line) { /* If the value of a variable continues to the next line because the line break was escaped then we don't trim leading space but treat it as a part of the value */ @@ -262,32 +259,21 @@ class phpbb_search_sphinx_config } } - // Keep the filename for later use - $this->loaded = $filename; } /** - * Writes the config data into a file + * Returns the config data * - * @param string $filename The optional filename into which the config data shall be written. - * If it's not specified it will be written into the file that the config - * was originally read from. + * @return string $data The config data that is generated. */ - function write($filename = false) + function get_data() { - if ($filename === false && $this->loaded) - { - $filename = $this->loaded; - } - $data = ""; foreach ($this->sections as $section) { $data .= $section->to_string(); } - $fp = fopen($filename, 'wb'); - fwrite($fp, $data); - fclose($fp); + return $data; } } From b16e70ae1d03587c7d7d7e106299a4e576491751 Mon Sep 17 00:00:00 2001 From: Dhruv Goel Date: Tue, 10 Jul 2012 12:32:42 +0530 Subject: [PATCH 30/57] [feature/sphinx-fulltext-search] remove bin_path fulltext_sphinx_bin_path from ACP as it is no longer required. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 5 ----- phpBB/language/en/acp/search.php | 3 --- 2 files changed, 8 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index a54ebe1a59..6488cbcd40 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -739,7 +739,6 @@ class phpbb_search_fulltext_sphinx $config_vars = array( 'fulltext_sphinx_config_path' => 'string', 'fulltext_sphinx_data_path' => 'string', - 'fulltext_sphinx_bin_path' => 'string', 'fulltext_sphinx_port' => 'int', 'fulltext_sphinx_stopwords' => 'bool', 'fulltext_sphinx_indexer_mem_limit' => 'int', @@ -751,10 +750,6 @@ class phpbb_search_fulltext_sphinx

' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN'] . '
-
-

' . $this->user->lang['FULLTEXT_SPHINX_BIN_PATH_EXPLAIN'] . '
-
-

' . $this->user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '
diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 14701459eb..9618bb90c5 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -57,7 +57,6 @@ $lang = array_merge($lang, array( 'FULLTEXT_MYSQL_MIN_SEARCH_CHARS_EXPLAIN' => 'Words with at least this many characters will be indexed for searching. You or your host can only change this setting by changing the mysql configuration.', 'FULLTEXT_MYSQL_MAX_SEARCH_CHARS_EXPLAIN' => 'Words with no more than this many characters will be indexed for searching. You or your host can only change this setting by changing the mysql configuration.', -<<<<<<< HEAD 'FULLTEXT_POSTGRES_INCOMPATIBLE_DATABASE' => 'The PostgreSQL fulltext backend can only be used with PostgreSQL.', 'FULLTEXT_POSTGRES_TS_NOT_USABLE' => 'The PostgreSQL fulltext backend can only be used with PostgreSQL 8.3 and above.', 'FULLTEXT_POSTGRES_TOTAL_POSTS' => 'Total number of indexed posts', @@ -70,8 +69,6 @@ $lang = array_merge($lang, array( 'FULLTEXT_POSTGRES_MIN_WORD_LEN_EXPLAIN' => 'Words with at least this many characters will be included in the query to the database.', 'FULLTEXT_POSTGRES_MAX_WORD_LEN_EXPLAIN' => 'Words with no more than this many characters will be included in the query to the database.', - 'FULLTEXT_SPHINX_BIN_PATH' => 'Path to executables directory', - 'FULLTEXT_SPHINX_BIN_PATH_EXPLAIN' => 'Skip if autorun is disabled. If this path could not be determined automatically you have to enter the path to the directory in which the sphinx executables indexer and searchd reside.', 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to configuration directory', 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'Skip if autoconf is disabled. You should create this config directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody).', 'FULLTEXT_SPHINX_CONFIGURE_FIRST' => 'Before you create an index you have to enable and configure sphinx under GENERAL -> SERVER CONFIGURATION -> Search settings.', From 4b40f0d3c6d14adc2b20b866cbeb42586cf8d874 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Wed, 11 Jul 2012 16:25:19 +0530 Subject: [PATCH 31/57] [feature/sphinx-fulltext-search] display config file in ACP sphinx config file is generated and displayed in the ACP for user to use it to start sphinx search daemon. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 22 +++++++++++++--------- phpBB/language/en/acp/search.php | 3 +++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 6488cbcd40..6e554eec00 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -112,11 +112,6 @@ class phpbb_search_fulltext_sphinx return $this->user->lang['FULLTEXT_SPHINX_WRONG_DATABASE']; } - if ($error = $this->config_updated()) - { - return $error; - } - // Move delta to main index each hour set_config('search_gc', 3600); @@ -124,16 +119,21 @@ class phpbb_search_fulltext_sphinx } /** - * Updates the config file sphinx.conf and generates the same in case autoconf is selected + * Generates content of sphinx.conf * - * @return string|bool Language key of the error/incompatiblity occured otherwise false + * @return bool True if sphinx.conf content is correctly generated, false otherwise * * @access private */ - function config_updated() + function config_generate() { global $phpbb_root_path, $phpEx; + if (!$this->config['fulltext_sphinx_data_path'] || !$this->config['fulltext_sphinx_config_path']) + { + return false; + } + include ($phpbb_root_path . 'config.' . $phpEx); /* Now that we're sure everything was entered correctly, @@ -280,7 +280,7 @@ class phpbb_search_fulltext_sphinx } $this->config_file_data = $config_object->get_data(); - return false; + return true; } /** @@ -767,6 +767,10 @@ class phpbb_search_fulltext_sphinx

' . $this->user->lang['FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN'] . '
' . $this->user->lang['MIB'] . '
+
+

' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN'] . '
+
' . (($this->config_generate()) ? '' : $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA')) . '
+
'; // These are fields required in the config table diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 9618bb90c5..778f9ddec5 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -85,6 +85,9 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx plugin for phpBB currently only supports MySQL', 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'This setting only works with autoconf enabled. You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', + 'FULLTEXT_SPHINX_CONFIG_FILE' => 'Sphinx config file', + 'FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN' => 'The generated content of the sphinx config file. This data needs to be pasted into the sphinx.conf which is used by sphinx search daemon.', + 'FULLTEXT_SPHINX_NO_CONFIG_DATA' => 'The sphinx data and config directory paths are not defined. Please define them to generate the config file.', 'GENERAL_SEARCH_SETTINGS' => 'General search settings', 'GO_TO_SEARCH_INDEX' => 'Go to search index page', From 172c583f1941a8b162f1a7bf258bb3e38149606d Mon Sep 17 00:00:00 2001 From: Dhruv Date: Wed, 11 Jul 2012 16:57:18 +0530 Subject: [PATCH 32/57] [feature/sphinx-fulltext-search] use new unique id instead of salt a new unique id is generated by sphinx and stored in the config table instead of using avatar_salt. PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 16 ++++++++-------- phpBB/includes/search/fulltext_sphinx.php | 15 ++++++++------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf index 000d8157d6..06595a766f 100644 --- a/phpBB/docs/sphinx.sample.conf +++ b/phpBB/docs/sphinx.sample.conf @@ -1,4 +1,4 @@ -source source_phpbb_{AVATAR_SALT}_main +source source_phpbb_{SPHINX_ID}_main { type = mysql sql_host = localhost @@ -38,7 +38,7 @@ source source_phpbb_{AVATAR_SALT}_main sql_attr_timestamp = topic_last_post_time sql_attr_str2ordinal = post_subject } -source source_phpbb_{AVATAR_SALT}_delta : source_phpbb_{AVATAR_SALT}_main +source source_phpbb_{SPHINX_ID}_delta : source_phpbb_{SPHINX_ID}_main { sql_query_range = sql_range_step = @@ -60,10 +60,10 @@ source source_phpbb_{AVATAR_SALT}_delta : source_phpbb_{AVATAR_SALT}_main AND p.post_id >= ( SELECT max_doc_id FROM phpbb_sphinx WHERE counter_id=1 ) sql_query_pre = } -index index_phpbb_{AVATAR_SALT}_main +index index_phpbb_{SPHINX_ID}_main { - path = {DATA_PATH}/index_phpbb_{AVATAR_SALT}_main - source = source_phpbb_{AVATAR_SALT}_main + path = {DATA_PATH}/index_phpbb_{SPHINX_ID}_main + source = source_phpbb_{SPHINX_ID}_main docinfo = extern morphology = none stopwords = @@ -73,10 +73,10 @@ index index_phpbb_{AVATAR_SALT}_main min_prefix_len = 0 min_infix_len = 0 } -index index_phpbb_{AVATAR_SALT}_delta : index_phpbb_{AVATAR_SALT}_main +index index_phpbb_{SPHINX_ID}_delta : index_phpbb_{SPHINX_ID}_main { - path = {DATA_PATH}/index_phpbb_{AVATAR_SALT}_delta - source = source_phpbb_{AVATAR_SALT}_delta + path = {DATA_PATH}/index_phpbb_{SPHINX_ID}_delta + source = source_phpbb_{SPHINX_ID}_delta } indexer { diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 6e554eec00..5bdbbff119 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -72,16 +72,17 @@ class phpbb_search_fulltext_sphinx // Initialize phpbb_db_tools object $this->db_tools = new phpbb_db_tools($this->db); - $this->id = $config['avatar_salt']; + if(!$this->config['fulltext_sphinx_id']) + { + set_config('fulltext_sphinx_id', unique_id()); + } + $this->id = $this->config['fulltext_sphinx_id']; $this->indexes = 'index_phpbb_' . $this->id . '_delta;index_phpbb_' . $this->id . '_main'; $this->sphinx = new SphinxClient(); // We only support localhost for now - $this->sphinx->SetServer('localhost', (isset($config['fulltext_sphinx_port']) && $config['fulltext_sphinx_port']) ? (int) $config['fulltext_sphinx_port'] : 3312); - - $config['fulltext_sphinx_min_word_len'] = 2; - $config['fulltext_sphinx_max_word_len'] = 400; + $this->sphinx->SetServer('localhost', (isset($this->config['fulltext_sphinx_port']) && $this->config['fulltext_sphinx_port']) ? (int) $this->config['fulltext_sphinx_port'] : 3312); $error = false; } @@ -137,8 +138,8 @@ class phpbb_search_fulltext_sphinx include ($phpbb_root_path . 'config.' . $phpEx); /* Now that we're sure everything was entered correctly, - generate a config for the index. We misuse the avatar_salt - for this, as it should be unique. */ + generate a config for the index. We use a config value + fulltext_sphinx_id for this, as it should be unique. */ $config_object = new phpbb_search_sphinx_config($this->config_file_data); $config_data = array( From b8103c5c31cbb42a46a40ac10c34ff09dc5efc60 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Wed, 11 Jul 2012 17:10:51 +0530 Subject: [PATCH 33/57] [feature/sphinx-fulltext-search] fix comments and indentation PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 2 +- phpBB/includes/search/sphinx/config.php | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 5bdbbff119..d942d0f027 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -135,7 +135,7 @@ class phpbb_search_fulltext_sphinx return false; } - include ($phpbb_root_path . 'config.' . $phpEx); + include($phpbb_root_path . 'config.' . $phpEx); /* Now that we're sure everything was entered correctly, generate a config for the index. We use a config value diff --git a/phpBB/includes/search/sphinx/config.php b/phpBB/includes/search/sphinx/config.php index e0ad667fb6..173193adc5 100644 --- a/phpBB/includes/search/sphinx/config.php +++ b/phpBB/includes/search/sphinx/config.php @@ -98,7 +98,7 @@ class phpbb_search_sphinx_config // If we're not inside a section look for one if (!$section) { - /* add empty lines and comments as comment objects to the section list + /* Add empty lines and comments as comment objects to the section list that way they're not deleted when reassembling the file from the sections*/ if (!$line || $line[0] == '#') { @@ -107,7 +107,7 @@ class phpbb_search_sphinx_config } else { - /* otherwise we scan the line reading the section name until we find + /* Otherwise we scan the line reading the section name until we find an opening curly bracket or a comment */ $section_name = ''; $section_name_comment = ''; @@ -147,7 +147,7 @@ class phpbb_search_sphinx_config // If we're not in a value continuing over the line feed if (!$in_value) { - /* then add empty lines and comments as comment objects to the variable list + /* Then add empty lines and comments as comment objects to the variable list of this section so they're not deleted on reassembly */ if (!$line || $line[0] == '#') { @@ -240,7 +240,7 @@ class phpbb_search_sphinx_config continue; } - /* if we found a closing curly bracket this section has been completed + /* If we found a closing curly bracket this section has been completed and we can append it to the section list and continue with looking for the next section */ if ($end_section) From 78e7f2a5290dc152cf2e386553e6308c74e2d005 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Wed, 11 Jul 2012 17:32:31 +0530 Subject: [PATCH 34/57] [feature/sphinx-fulltext-search] improve sphinx helper classes add access modifiers and docblocks to properties and methods of sphinx helper classes. PHPBB3-10946 --- phpBB/includes/search/sphinx/config.php | 15 ++++++++--- .../includes/search/sphinx/config_comment.php | 6 ++++- .../includes/search/sphinx/config_section.php | 26 ++++++++++++++++--- .../search/sphinx/config_variable.php | 14 +++++++--- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/phpBB/includes/search/sphinx/config.php b/phpBB/includes/search/sphinx/config.php index 173193adc5..795dff07ed 100644 --- a/phpBB/includes/search/sphinx/config.php +++ b/phpBB/includes/search/sphinx/config.php @@ -23,13 +23,14 @@ if (!defined('IN_PHPBB')) */ class phpbb_search_sphinx_config { - var $loaded = false; - var $sections = array(); + private $sections = array(); /** * Constructor which optionally loads data from a variable * * @param string $config_data Variable containing the sphinx configuration data + * + * @access public */ function __construct($config_data) { @@ -44,6 +45,8 @@ class phpbb_search_sphinx_config * * @param string $name The name of the section that shall be returned * @return phpbb_search_sphinx_config_section The section object or null if none was found + * + * @access public */ function get_section_by_name($name) { @@ -62,6 +65,8 @@ class phpbb_search_sphinx_config * * @param string $name The name for the new section * @return phpbb_search_sphinx_config_section The newly created section object + * + * @access public */ function add_section($name) { @@ -73,6 +78,8 @@ class phpbb_search_sphinx_config * Reads the config file data * * @param string $config_data The config file data + * + * @access private */ function read($config_data) { @@ -264,7 +271,9 @@ class phpbb_search_sphinx_config /** * Returns the config data * - * @return string $data The config data that is generated. + * @return string $data The config data that is generated + * + * @access public */ function get_data() { diff --git a/phpBB/includes/search/sphinx/config_comment.php b/phpBB/includes/search/sphinx/config_comment.php index 63d3488aef..7f695dbf0c 100644 --- a/phpBB/includes/search/sphinx/config_comment.php +++ b/phpBB/includes/search/sphinx/config_comment.php @@ -21,12 +21,14 @@ if (!defined('IN_PHPBB')) */ class phpbb_search_sphinx_config_comment { - var $exact_string; + private $exact_string; /** * Create a new comment * * @param string $exact_string The content of the comment including newlines, leading whitespace, etc. + * + * @access public */ function __construct($exact_string) { @@ -37,6 +39,8 @@ class phpbb_search_sphinx_config_comment * Simply returns the comment as it was created * * @return string The exact string that was specified in the constructor + * + * @access public */ function to_string() { diff --git a/phpBB/includes/search/sphinx/config_section.php b/phpBB/includes/search/sphinx/config_section.php index ed20dba279..79c9c8563d 100644 --- a/phpBB/includes/search/sphinx/config_section.php +++ b/phpBB/includes/search/sphinx/config_section.php @@ -21,10 +21,10 @@ if (!defined('IN_PHPBB')) */ class phpbb_search_sphinx_config_section { - var $name; - var $comment; - var $end_comment; - var $variables = array(); + private $name; + private $comment; + private $end_comment; + private $variables = array(); /** * Construct a new section @@ -32,6 +32,8 @@ class phpbb_search_sphinx_config_section * @param string $name Name of the section * @param string $comment Comment that should be appended after the name in the * textual format. + * + * @access public */ function __construct($name, $comment) { @@ -44,6 +46,8 @@ class phpbb_search_sphinx_config_section * Add a variable object to the list of variables in this section * * @param phpbb_search_sphinx_config_variable $variable The variable object + * + * @access public */ function add_variable($variable) { @@ -52,6 +56,10 @@ class phpbb_search_sphinx_config_section /** * Adds a comment after the closing bracket in the textual representation + * + * @param string $end_comment + * + * @access public */ function set_end_comment($end_comment) { @@ -62,6 +70,8 @@ class phpbb_search_sphinx_config_section * Getter for the name of this section * * @return string Section's name + * + * @access public */ function get_name() { @@ -74,6 +84,8 @@ class phpbb_search_sphinx_config_section * @param string $name The name of the variable that shall be returned * @return phpbb_search_sphinx_config_section The first variable object from this section with the * given name or null if none was found + * + * @access public */ function get_variable_by_name($name) { @@ -91,6 +103,8 @@ class phpbb_search_sphinx_config_section * Deletes all variables with the given name * * @param string $name The name of the variable objects that are supposed to be removed + * + * @access public */ function delete_variables_by_name($name) { @@ -111,6 +125,8 @@ class phpbb_search_sphinx_config_section * @param string $name The name for the new variable * @param string $value The value for the new variable * @return phpbb_search_sphinx_config_variable Variable object that was created + * + * @access public */ function create_variable($name, $value) { @@ -122,6 +138,8 @@ class phpbb_search_sphinx_config_section * Turns this object into a string which can be written to a config file * * @return string Config data in textual form, parsable for sphinx + * + * @access public */ function to_string() { diff --git a/phpBB/includes/search/sphinx/config_variable.php b/phpBB/includes/search/sphinx/config_variable.php index dd7836f7c8..35abe281cb 100644 --- a/phpBB/includes/search/sphinx/config_variable.php +++ b/phpBB/includes/search/sphinx/config_variable.php @@ -21,9 +21,9 @@ if (!defined('IN_PHPBB')) */ class phpbb_search_sphinx_config_variable { - var $name; - var $value; - var $comment; + private $name; + private $value; + private $comment; /** * Constructs a new variable object @@ -32,6 +32,8 @@ class phpbb_search_sphinx_config_variable * @param string $value Value of the variable * @param string $comment Optional comment after the variable in the * config file + * + * @access public */ function __construct($name, $value, $comment) { @@ -44,6 +46,8 @@ class phpbb_search_sphinx_config_variable * Getter for the variable's name * * @return string The variable object's name + * + * @access public */ function get_name() { @@ -54,6 +58,8 @@ class phpbb_search_sphinx_config_variable * Allows changing the variable's value * * @param string $value New value for this variable + * + * @access public */ function set_value($value) { @@ -64,6 +70,8 @@ class phpbb_search_sphinx_config_variable * Turns this object into a string readable by sphinx * * @return string Config data in textual form + * + * @access public */ function to_string() { From f40da411c389cb7718d31f1ee20f8487f25969f0 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Wed, 11 Jul 2012 17:46:58 +0530 Subject: [PATCH 35/57] [feature/sphinx-fulltext-search] modify language keys Modify language keys according to what the config setting actually does. Remove references to autoconf. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 3 +-- phpBB/language/en/acp/search.php | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index d942d0f027..08948803ba 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -746,7 +746,7 @@ class phpbb_search_fulltext_sphinx ); $tpl = ' - ' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE_BEFORE']. ' + ' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE']. '

' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN'] . '
@@ -759,7 +759,6 @@ class phpbb_search_fulltext_sphinx

' . $this->user->lang['FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN'] . '
- ' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE_AFTER']. '

' . $this->user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '
diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 778f9ddec5..970d9cd41b 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -70,12 +70,10 @@ $lang = array_merge($lang, array( 'FULLTEXT_POSTGRES_MAX_WORD_LEN_EXPLAIN' => 'Words with no more than this many characters will be included in the query to the database.', 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to configuration directory', - 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'Skip if autoconf is disabled. You should create this config directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody).', - 'FULLTEXT_SPHINX_CONFIGURE_FIRST' => 'Before you create an index you have to enable and configure sphinx under GENERAL -> SERVER CONFIGURATION -> Search settings.', - 'FULLTEXT_SPHINX_CONFIGURE_BEFORE' => 'Configure the following settings BEFORE activating Sphinx', - 'FULLTEXT_SPHINX_CONFIGURE_AFTER' => 'The following settings do not have to be configured before activating Sphinx', + 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'You should put the sphinx.conf file in this directory. This config directory should be outside the web accessable directories.', + 'FULLTEXT_SPHINX_CONFIGURE' => 'Configure the following settings to generate sphinx config file', 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', - 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'Skip if autorun is disabled. You should create this directory outside the web accessable directories. It has to be writable by the user as which your webserver is running (often www-data or nobody). It will be used to store the indexes and log files.', + 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'It will be used to store the indexes and log files. You should create this directory outside the web accessable directories.', 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', @@ -84,7 +82,7 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search deamon on localhost listens. Leave empty to use the default 3312', 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx plugin for phpBB currently only supports MySQL', 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', - 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'This setting only works with autoconf enabled. You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', + 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', 'FULLTEXT_SPHINX_CONFIG_FILE' => 'Sphinx config file', 'FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN' => 'The generated content of the sphinx config file. This data needs to be pasted into the sphinx.conf which is used by sphinx search daemon.', 'FULLTEXT_SPHINX_NO_CONFIG_DATA' => 'The sphinx data and config directory paths are not defined. Please define them to generate the config file.', From 13c451ca2e9a6717a8a98943ba022a6f41dcdd9c Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 12 Jul 2012 04:13:34 +0530 Subject: [PATCH 36/57] [feature/sphinx-fulltext-search] use sql_table_exists Use sql_table_exists( ) method in db_tools to support all database types. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 08948803ba..747c22a3ef 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -667,14 +667,9 @@ class phpbb_search_fulltext_sphinx */ function index_created($allow_new_files = true) { - $sql = 'SHOW TABLES LIKE \'' . SPHINX_TABLE . '\''; - $result = $this->db->sql_query($sql); - $row = $this->db->sql_fetchrow($result); - $this->db->sql_freeresult($result); - $created = false; - if ($row) + if ($this->db_tools->sql_table_exists(SPHINX_TABLE)) { $created = true; } From 118b57f71d9f563f8eeda8e5925d482c38ab1af8 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 12 Jul 2012 04:28:55 +0530 Subject: [PATCH 37/57] [feature/sphinx-fulltext-search] minor changes in sphinx.conf PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 4 ++-- phpBB/includes/search/fulltext_sphinx.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf index 06595a766f..eaef081aa1 100644 --- a/phpBB/docs/sphinx.sample.conf +++ b/phpBB/docs/sphinx.sample.conf @@ -6,6 +6,8 @@ source source_phpbb_{SPHINX_ID}_main sql_pass = password sql_db = db_name sql_port = 3306 #optional, default is 3306 + sql_query_pre = SET NAMES 'utf8' + sql_query_pre = REPLACE INTO phpbb_sphinx SELECT 1, MAX(post_id) FROM phpbb_posts sql_query_range = SELECT MIN(post_id), MAX(post_id) FROM phpbb_posts sql_range_step = 5000 sql_query = SELECT \ @@ -27,8 +29,6 @@ source source_phpbb_{SPHINX_ID}_main sql_query_post = sql_query_post_index = REPLACE INTO phpbb_sphinx ( counter_id, max_doc_id ) VALUES ( 1, $maxid ) sql_query_info = SELECT * FROM phpbb_posts WHERE post_id = $id - sql_query_pre = SET NAMES utf8 - sql_query_pre = REPLACE INTO phpbb_sphinx SELECT 1, MAX(post_id) FROM phpbb_posts sql_attr_uint = forum_id sql_attr_uint = topic_id sql_attr_uint = poster_id diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 747c22a3ef..c223284c72 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -150,7 +150,7 @@ class phpbb_search_fulltext_sphinx array('sql_pass', $dbpasswd), array('sql_db', $dbname), array('sql_port', $dbport), - array('sql_query_pre', 'SET NAMES utf8'), + array('sql_query_pre', 'SET NAMES \'utf8\''), array('sql_query_pre', 'REPLACE INTO ' . SPHINX_TABLE . ' SELECT 1, MAX(post_id) FROM ' . POSTS_TABLE . ''), array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''), array('sql_range_step', '5000'), From b81941a997760eca4f209cc100fe2baec3ef4468 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 12 Jul 2012 16:30:45 +0530 Subject: [PATCH 38/57] [feature/sphinx-fulltext-search] use CASE instead of IF IF is not supported in pgsql, use CASE instead supported in both mysql and pgsql. PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 4 ++-- phpBB/includes/search/fulltext_sphinx.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf index eaef081aa1..3ab2552096 100644 --- a/phpBB/docs/sphinx.sample.conf +++ b/phpBB/docs/sphinx.sample.conf @@ -15,7 +15,7 @@ source source_phpbb_{SPHINX_ID}_main p.forum_id, \ p.topic_id, \ p.poster_id, \ - IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, \ + CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post, \ p.post_time, \ p.post_subject, \ p.post_subject as title, \ @@ -47,7 +47,7 @@ source source_phpbb_{SPHINX_ID}_delta : source_phpbb_{SPHINX_ID}_main p.forum_id, \ p.topic_id, \ p.poster_id, \ - IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, \ + CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post, \ p.post_time, \ p.post_subject, \ p.post_subject as title, \ diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index c223284c72..f505703c09 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -159,7 +159,7 @@ class phpbb_search_fulltext_sphinx p.forum_id, p.topic_id, p.poster_id, - IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, + CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post, p.post_time, p.post_subject, p.post_subject as title, @@ -191,7 +191,7 @@ class phpbb_search_fulltext_sphinx p.forum_id, p.topic_id, p.poster_id, - IF(p.post_id = t.topic_first_post_id, 1, 0) as topic_first_post, + CASE WHEN p.post_id = t.topic_first_post_id THEN 1 ELSE 0 END as topic_first_post, p.post_time, p.post_subject, p.post_subject as title, From 81959927e53ebc62765ff075d23feeaf9b40a95d Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 12 Jul 2012 17:22:03 +0530 Subject: [PATCH 39/57] [feature/sphinx-fulltext-search] use Update in sphinx query Instead of REPLACE use UPDATE since pgsql does not support REPLACE. A row is inserted at time of creating table so REPLACE is no longer needed. PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 4 ++-- phpBB/includes/search/fulltext_sphinx.php | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf index 3ab2552096..8ffd54a880 100644 --- a/phpBB/docs/sphinx.sample.conf +++ b/phpBB/docs/sphinx.sample.conf @@ -7,7 +7,7 @@ source source_phpbb_{SPHINX_ID}_main sql_db = db_name sql_port = 3306 #optional, default is 3306 sql_query_pre = SET NAMES 'utf8' - sql_query_pre = REPLACE INTO phpbb_sphinx SELECT 1, MAX(post_id) FROM phpbb_posts + sql_query_pre = UPDATE phpbb_sphinx SET max_doc_id = MAX(post_id) WHERE counter_id = 1 sql_query_range = SELECT MIN(post_id), MAX(post_id) FROM phpbb_posts sql_range_step = 5000 sql_query = SELECT \ @@ -27,7 +27,7 @@ source source_phpbb_{SPHINX_ID}_main p.topic_id = t.topic_id \ AND p.post_id >= $start AND p.post_id <= $end sql_query_post = - sql_query_post_index = REPLACE INTO phpbb_sphinx ( counter_id, max_doc_id ) VALUES ( 1, $maxid ) + sql_query_post_index = UPDATE phpbb_sphinx SET max_doc_id = $maxid WHERE counter_id = 1 sql_query_info = SELECT * FROM phpbb_posts WHERE post_id = $id sql_attr_uint = forum_id sql_attr_uint = topic_id diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index f505703c09..d82a56c6f4 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -151,7 +151,7 @@ class phpbb_search_fulltext_sphinx array('sql_db', $dbname), array('sql_port', $dbport), array('sql_query_pre', 'SET NAMES \'utf8\''), - array('sql_query_pre', 'REPLACE INTO ' . SPHINX_TABLE . ' SELECT 1, MAX(post_id) FROM ' . POSTS_TABLE . ''), + array('sql_query_pre', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = (SELECT MAX(post_id) FROM ' . POSTS_TABLE . ') WHERE counter_id = 1'), array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''), array('sql_range_step', '5000'), array('sql_query', 'SELECT @@ -171,7 +171,7 @@ class phpbb_search_fulltext_sphinx p.topic_id = t.topic_id AND p.post_id >= $start AND p.post_id <= $end'), array('sql_query_post', ''), - array('sql_query_post_index', 'REPLACE INTO ' . SPHINX_TABLE . ' ( counter_id, max_doc_id ) VALUES ( 1, $maxid )'), + array('sql_query_post_index', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = $maxid WHERE counter_id = 1'), array('sql_query_info', 'SELECT * FROM ' . POSTS_TABLE . ' WHERE post_id = $id'), array('sql_attr_uint', 'forum_id'), array('sql_attr_uint', 'topic_id'), @@ -634,6 +634,13 @@ class phpbb_search_fulltext_sphinx $sql = 'TRUNCATE TABLE ' . SPHINX_TABLE; $this->db->sql_query($sql); + + $data = array( + 'counter_id' => '1', + 'max_doc_id' => '0', + ); + $sql = 'INSERT INTO ' . SPHINX_TABLE . ' ' . $this->db->sql_build_array('INSERT', $data); + $this->db->sql_query($sql); } return false; From 609ce3ae8fb55e717ff188d2ec9c10c6ae252b7a Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 12 Jul 2012 17:48:17 +0530 Subject: [PATCH 40/57] [feature/sphinx-fulltext-search] add pgsql functionality PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 2 +- phpBB/language/en/acp/search.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index d82a56c6f4..76caf9ae8c 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -108,7 +108,7 @@ class phpbb_search_fulltext_sphinx */ function init() { - if ($this->db->sql_layer != 'mysql' && $this->db->sql_layer != 'mysql4' && $this->db->sql_layer != 'mysqli') + if ($this->db->sql_layer != 'mysql' && $this->db->sql_layer != 'mysql4' && $this->db->sql_layer != 'mysqli' && $this->db->sql_layer != 'postgres') { return $this->user->lang['FULLTEXT_SPHINX_WRONG_DATABASE']; } diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 970d9cd41b..4aa9c89cde 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -80,7 +80,7 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', 'FULLTEXT_SPHINX_PORT' => 'Sphinx search deamon port', 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search deamon on localhost listens. Leave empty to use the default 3312', - 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx plugin for phpBB currently only supports MySQL', + 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx search for phpBB supports MySQL and PostgreSQL only.', 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', 'FULLTEXT_SPHINX_CONFIG_FILE' => 'Sphinx config file', From a3d103c9c03c79fe67963b9db5a5471c766fa401 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 12 Jul 2012 18:08:50 +0530 Subject: [PATCH 41/57] [feature/sphinx-fulltext-search] add support for postgres Don't generate sphinx config file if database is not supported. Add property $dbtype to write into sphinx config file according to sql_layer. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 76caf9ae8c..a4341f46e0 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -44,6 +44,7 @@ class phpbb_search_fulltext_sphinx private $config; private $db; private $db_tools; + private $dbtype; private $user; private $config_file_data = ''; public $word_length = array(); @@ -130,8 +131,25 @@ class phpbb_search_fulltext_sphinx { global $phpbb_root_path, $phpEx; + // Check if Database is supported by Sphinx + if ($this->db->sql_layer =='mysql' || $this->db->sql_layer == 'mysql4' || $this->db->sql_layer == 'mysqli') + { + $this->dbtype = 'mysql'; + } + else if ($this->db->sql_layer == 'postgres') + { + $this->dbtype = 'pgsql'; + } + else + { + $this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_WRONG_DATABASE'); + return false; + } + + // Check if directory paths have been filled if (!$this->config['fulltext_sphinx_data_path'] || !$this->config['fulltext_sphinx_config_path']) { + $this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA'); return false; } @@ -141,10 +159,9 @@ class phpbb_search_fulltext_sphinx generate a config for the index. We use a config value fulltext_sphinx_id for this, as it should be unique. */ $config_object = new phpbb_search_sphinx_config($this->config_file_data); - $config_data = array( 'source source_phpbb_' . $this->id . '_main' => array( - array('type', 'mysql'), + array('type', $this->dbtype), array('sql_host', $dbhost), array('sql_user', $dbuser), array('sql_pass', $dbpasswd), @@ -771,7 +788,7 @@ class phpbb_search_fulltext_sphinx

' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN'] . '
-
' . (($this->config_generate()) ? '' : $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA')) . '
+
' . (($this->config_generate()) ? '' : $this->config_file_data) . '
'; From 3ecc81f853bb1ec6262fc0615bb0ab8704616db9 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Sat, 21 Jul 2012 14:14:19 +0530 Subject: [PATCH 42/57] [feature/sphinx-fulltext-search] remove note from db_tools Note saying db_tools not being used currently is remove from db_tools.php We utilize db_tools in sphinx search. PHPBB3-10946 --- phpBB/includes/db/db_tools.php | 1 - 1 file changed, 1 deletion(-) diff --git a/phpBB/includes/db/db_tools.php b/phpBB/includes/db/db_tools.php index 73eae4e967..6df3aac9ce 100644 --- a/phpBB/includes/db/db_tools.php +++ b/phpBB/includes/db/db_tools.php @@ -20,7 +20,6 @@ if (!defined('IN_PHPBB')) * Currently not supported is returning SQL for creating tables. * * @package dbal -* @note currently not used within phpBB3, but may be utilized later. */ class phpbb_db_tools { From 1f77b95fe71e727238212ea4632220ae9cab99d7 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Sun, 22 Jul 2012 01:49:30 +0530 Subject: [PATCH 43/57] [feature/sphinx-fulltext-search] fix language keys' typo PHPBB3-10946 --- phpBB/language/en/acp/search.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 4aa9c89cde..37c403f43d 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -69,8 +69,8 @@ $lang = array_merge($lang, array( 'FULLTEXT_POSTGRES_MIN_WORD_LEN_EXPLAIN' => 'Words with at least this many characters will be included in the query to the database.', 'FULLTEXT_POSTGRES_MAX_WORD_LEN_EXPLAIN' => 'Words with no more than this many characters will be included in the query to the database.', - 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to configuration directory', - 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'You should put the sphinx.conf file in this directory. This config directory should be outside the web accessable directories.', + 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to config directory', + 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'You should put the sphinx.conf file in this directory. This config directory should be outside the web accessible directories.', 'FULLTEXT_SPHINX_CONFIGURE' => 'Configure the following settings to generate sphinx config file', 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'It will be used to store the indexes and log files. You should create this directory outside the web accessable directories.', @@ -78,8 +78,8 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', - 'FULLTEXT_SPHINX_PORT' => 'Sphinx search deamon port', - 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search deamon on localhost listens. Leave empty to use the default 3312', + 'FULLTEXT_SPHINX_PORT' => 'Sphinx search daemon port', + 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default 3312', 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx search for phpBB supports MySQL and PostgreSQL only.', 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', From 0e9eb9401a38fab3139a1df33fa7e0903ccfb18f Mon Sep 17 00:00:00 2001 From: Dhruv Date: Sun, 22 Jul 2012 01:53:04 +0530 Subject: [PATCH 44/57] [feature/sphinx-fulltext-search] use readonly instead of disabled PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index a4341f46e0..1888057db4 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -788,7 +788,7 @@ class phpbb_search_fulltext_sphinx

' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN'] . '
-
' . (($this->config_generate()) ? '' : $this->config_file_data) . '
+
' . (($this->config_generate()) ? '' : $this->config_file_data) . '
'; From 161e469b5a67b2911089ec0dfdb70bef355ed07e Mon Sep 17 00:00:00 2001 From: Dhruv Date: Sun, 22 Jul 2012 02:50:53 +0530 Subject: [PATCH 45/57] [feature/sphinx-fulltext-search] makes sql host configurable The SQL server host which sphinx connects to index the posts is now configurable via ACP. PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 4 ++-- phpBB/includes/search/fulltext_sphinx.php | 9 +++++++-- phpBB/language/en/acp/search.php | 2 ++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf index 8ffd54a880..d0a897e0cc 100644 --- a/phpBB/docs/sphinx.sample.conf +++ b/phpBB/docs/sphinx.sample.conf @@ -1,7 +1,7 @@ source source_phpbb_{SPHINX_ID}_main { - type = mysql - sql_host = localhost + type = mysql #mysql or pgsql + sql_host = localhost #SQL server host sphinx connects to sql_user = username sql_pass = password sql_db = db_name diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 1888057db4..3bdce5dfb9 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -162,11 +162,11 @@ class phpbb_search_fulltext_sphinx $config_data = array( 'source source_phpbb_' . $this->id . '_main' => array( array('type', $this->dbtype), - array('sql_host', $dbhost), + array('sql_host', $this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : $dbhost), array('sql_user', $dbuser), array('sql_pass', $dbpasswd), array('sql_db', $dbname), - array('sql_port', $dbport), + array('sql_port', $this->config['fulltext_sphinx_port']), array('sql_query_pre', 'SET NAMES \'utf8\''), array('sql_query_pre', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = (SELECT MAX(post_id) FROM ' . POSTS_TABLE . ') WHERE counter_id = 1'), array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''), @@ -759,6 +759,7 @@ class phpbb_search_fulltext_sphinx $config_vars = array( 'fulltext_sphinx_config_path' => 'string', 'fulltext_sphinx_data_path' => 'string', + 'fulltext_sphinx_host' => 'string', 'fulltext_sphinx_port' => 'int', 'fulltext_sphinx_stopwords' => 'bool', 'fulltext_sphinx_indexer_mem_limit' => 'int', @@ -778,6 +779,10 @@ class phpbb_search_fulltext_sphinx

' . $this->user->lang['FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN'] . '
+
+

' . $this->user->lang['FULLTEXT_SPHINX_HOST_EXPLAIN'] . '
+
+

' . $this->user->lang['FULLTEXT_SPHINX_PORT_EXPLAIN'] . '
diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 37c403f43d..99fbbac07e 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -75,6 +75,8 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'It will be used to store the indexes and log files. You should create this directory outside the web accessable directories.', 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', + 'FULLTEXT_SPHINX_HOST' => 'SQL server host', + 'FULLTEXT_SPHINX_HOST_EXPLAIN' => 'SQL server host, which the sphinx search daemon (searchd) connects to. Leave empty to use the default SQL server host', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', From e40758db84a23bc7a2c5324dbedcd7f0911abeea Mon Sep 17 00:00:00 2001 From: Dhruv Date: Sun, 22 Jul 2012 03:16:03 +0530 Subject: [PATCH 46/57] [feature/sphinx-fulltext-search] remove stopwords and config path Remove stopwords and config_path options from ACP. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 14 ++------------ phpBB/language/en/acp/search.php | 4 ---- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 3bdce5dfb9..e1052ee7da 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -147,7 +147,7 @@ class phpbb_search_fulltext_sphinx } // Check if directory paths have been filled - if (!$this->config['fulltext_sphinx_data_path'] || !$this->config['fulltext_sphinx_config_path']) + if (!$this->config['fulltext_sphinx_data_path']) { $this->config_file_data = $this->user->lang('FULLTEXT_SPHINX_NO_CONFIG_DATA'); return false; @@ -225,7 +225,7 @@ class phpbb_search_fulltext_sphinx array('source', 'source_phpbb_' . $this->id . '_main'), array('docinfo', 'extern'), array('morphology', 'none'), - array('stopwords', $this->config['fulltext_sphinx_stopwords'] ? $this->config['fulltext_sphinx_config_path'] . 'sphinx_stopwords.txt' : ''), + array('stopwords', ''), array('min_word_len', '2'), array('charset_type', 'utf-8'), array('charset_table', 'U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z, A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6, U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101, U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109, U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F, U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117, U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D, U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135, U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C, U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144, U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B, U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153, U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159, U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161, U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167, U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F, U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175, U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C, U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F, U+4E00..U+9FFF'), @@ -757,28 +757,18 @@ class phpbb_search_fulltext_sphinx function acp() { $config_vars = array( - 'fulltext_sphinx_config_path' => 'string', 'fulltext_sphinx_data_path' => 'string', 'fulltext_sphinx_host' => 'string', 'fulltext_sphinx_port' => 'int', - 'fulltext_sphinx_stopwords' => 'bool', 'fulltext_sphinx_indexer_mem_limit' => 'int', ); $tpl = ' ' . $this->user->lang['FULLTEXT_SPHINX_CONFIGURE']. ' -
-

' . $this->user->lang['FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN'] . '
-
-

' . $this->user->lang['FULLTEXT_SPHINX_DATA_PATH_EXPLAIN'] . '
-
-

' . $this->user->lang['FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN'] . '
-
-

' . $this->user->lang['FULLTEXT_SPHINX_HOST_EXPLAIN'] . '
diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 99fbbac07e..6d9201fb11 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -69,8 +69,6 @@ $lang = array_merge($lang, array( 'FULLTEXT_POSTGRES_MIN_WORD_LEN_EXPLAIN' => 'Words with at least this many characters will be included in the query to the database.', 'FULLTEXT_POSTGRES_MAX_WORD_LEN_EXPLAIN' => 'Words with no more than this many characters will be included in the query to the database.', - 'FULLTEXT_SPHINX_CONFIG_PATH' => 'Path to config directory', - 'FULLTEXT_SPHINX_CONFIG_PATH_EXPLAIN' => 'You should put the sphinx.conf file in this directory. This config directory should be outside the web accessible directories.', 'FULLTEXT_SPHINX_CONFIGURE' => 'Configure the following settings to generate sphinx config file', 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'It will be used to store the indexes and log files. You should create this directory outside the web accessable directories.', @@ -83,8 +81,6 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_PORT' => 'Sphinx search daemon port', 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default 3312', 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx search for phpBB supports MySQL and PostgreSQL only.', - 'FULLTEXT_SPHINX_STOPWORDS_FILE' => 'Stopwords activated', - 'FULLTEXT_SPHINX_STOPWORDS_FILE_EXPLAIN' => 'You can place a file called sphinx_stopwords.txt containing one word in each line in your config directory. If this file is present these words will be excluded from the indexing process.', 'FULLTEXT_SPHINX_CONFIG_FILE' => 'Sphinx config file', 'FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN' => 'The generated content of the sphinx config file. This data needs to be pasted into the sphinx.conf which is used by sphinx search daemon.', 'FULLTEXT_SPHINX_NO_CONFIG_DATA' => 'The sphinx data and config directory paths are not defined. Please define them to generate the config file.', From 39bac86f7db881a1035bebad56507145103218d5 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Sun, 22 Jul 2012 03:43:50 +0530 Subject: [PATCH 47/57] [feature/sphinx-fulltext-search] improve port option Use listen instead of deprecated port value in sphinx config file. sqlhost uses default $dbhost. PHPBB3-10946 --- phpBB/docs/sphinx.sample.conf | 5 ++--- phpBB/includes/search/fulltext_sphinx.php | 7 +++---- phpBB/language/en/acp/search.php | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/phpBB/docs/sphinx.sample.conf b/phpBB/docs/sphinx.sample.conf index d0a897e0cc..aa0e8d905d 100644 --- a/phpBB/docs/sphinx.sample.conf +++ b/phpBB/docs/sphinx.sample.conf @@ -5,7 +5,7 @@ source source_phpbb_{SPHINX_ID}_main sql_user = username sql_pass = password sql_db = db_name - sql_port = 3306 #optional, default is 3306 + sql_port = 3306 #optional, default is 3306 for mysql and 5432 for pgsql sql_query_pre = SET NAMES 'utf8' sql_query_pre = UPDATE phpbb_sphinx SET max_doc_id = MAX(post_id) WHERE counter_id = 1 sql_query_range = SELECT MIN(post_id), MAX(post_id) FROM phpbb_posts @@ -85,8 +85,7 @@ indexer searchd { compat_sphinxql_magics = 0 - listen = 127.0.0.1 - port = 3312 + listen = localhost:9312 log = {DATA_PATH}/log/searchd.log query_log = {DATA_PATH}/log/sphinx-query.log read_timeout = 5 diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index e1052ee7da..18037a2be0 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -166,7 +166,7 @@ class phpbb_search_fulltext_sphinx array('sql_user', $dbuser), array('sql_pass', $dbpasswd), array('sql_db', $dbname), - array('sql_port', $this->config['fulltext_sphinx_port']), + array('sql_port', $dbport), array('sql_query_pre', 'SET NAMES \'utf8\''), array('sql_query_pre', 'UPDATE ' . SPHINX_TABLE . ' SET max_doc_id = (SELECT MAX(post_id) FROM ' . POSTS_TABLE . ') WHERE counter_id = 1'), array('sql_query_range', 'SELECT MIN(post_id), MAX(post_id) FROM ' . POSTS_TABLE . ''), @@ -241,8 +241,7 @@ class phpbb_search_fulltext_sphinx ), 'searchd' => array( array('compat_sphinxql_magics' , '0'), - array('listen' , '127.0.0.1'), - array('port', ($this->config['fulltext_sphinx_port']) ? $this->config['fulltext_sphinx_port'] : '3312'), + array('listen' , 'localhost' . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '3312')), array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), array('read_timeout', '5'), @@ -759,7 +758,7 @@ class phpbb_search_fulltext_sphinx $config_vars = array( 'fulltext_sphinx_data_path' => 'string', 'fulltext_sphinx_host' => 'string', - 'fulltext_sphinx_port' => 'int', + 'fulltext_sphinx_port' => 'string', 'fulltext_sphinx_indexer_mem_limit' => 'int', ); diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 6d9201fb11..394d408fdb 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -79,7 +79,7 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', 'FULLTEXT_SPHINX_PORT' => 'Sphinx search daemon port', - 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default 3312', + 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default Sphinx API port 3312 ', 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx search for phpBB supports MySQL and PostgreSQL only.', 'FULLTEXT_SPHINX_CONFIG_FILE' => 'Sphinx config file', 'FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN' => 'The generated content of the sphinx config file. This data needs to be pasted into the sphinx.conf which is used by sphinx search daemon.', From 9f2faaa8f1799cf794388604dbf7946488be2376 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Sun, 22 Jul 2012 04:09:59 +0530 Subject: [PATCH 48/57] [feature/sphinx-fulltext-search] add trailing slash in language PHPBB3-10946 --- phpBB/language/en/acp/search.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 394d408fdb..c52448743c 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -71,7 +71,7 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_CONFIGURE' => 'Configure the following settings to generate sphinx config file', 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', - 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'It will be used to store the indexes and log files. You should create this directory outside the web accessable directories.', + 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'It will be used to store the indexes and log files. You should create this directory outside the web accessible directories. (should have a trailing slash)', 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', 'FULLTEXT_SPHINX_HOST' => 'SQL server host', 'FULLTEXT_SPHINX_HOST_EXPLAIN' => 'SQL server host, which the sphinx search daemon (searchd) connects to. Leave empty to use the default SQL server host', From 747af894a0b8d47079e704c66dcfce8ce00a7251 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 26 Jul 2012 16:18:53 +0530 Subject: [PATCH 49/57] [feature/sphinx-fulltext-search] fixing comments Use // for two liners in comments. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 4 +-- phpBB/includes/search/sphinx/config.php | 32 +++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 18037a2be0..9319a57236 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -483,8 +483,8 @@ class phpbb_search_fulltext_sphinx $this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES); $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); - /* Could be connection to localhost:3312 failed (errno=111, - msg=Connection refused) during rotate, retry if so */ + // Could be connection to localhost:3312 failed (errno=111, + // msg=Connection refused) during rotate, retry if so $retries = SPHINX_CONNECT_RETRIES; while (!$result && (strpos($this->sphinx->_error, "errno=111,") !== false) && $retries--) { diff --git a/phpBB/includes/search/sphinx/config.php b/phpBB/includes/search/sphinx/config.php index 795dff07ed..f1864f0c8c 100644 --- a/phpBB/includes/search/sphinx/config.php +++ b/phpBB/includes/search/sphinx/config.php @@ -91,8 +91,8 @@ class phpbb_search_sphinx_config foreach ($config_data as $i => $line) { - /* If the value of a variable continues to the next line because the line - break was escaped then we don't trim leading space but treat it as a part of the value */ + // If the value of a variable continues to the next line because the line + // break was escaped then we don't trim leading space but treat it as a part of the value if ($in_value) { $line = rtrim($line); @@ -105,8 +105,8 @@ class phpbb_search_sphinx_config // If we're not inside a section look for one if (!$section) { - /* Add empty lines and comments as comment objects to the section list - that way they're not deleted when reassembling the file from the sections*/ + // Add empty lines and comments as comment objects to the section list + // that way they're not deleted when reassembling the file from the sections if (!$line || $line[0] == '#') { $this->sections[] = new phpbb_search_sphinx_config_comment($config_file[$i]); @@ -114,8 +114,8 @@ class phpbb_search_sphinx_config } else { - /* Otherwise we scan the line reading the section name until we find - an opening curly bracket or a comment */ + // Otherwise we scan the line reading the section name until we find + // an opening curly bracket or a comment $section_name = ''; $section_name_comment = ''; $found_opening_bracket = false; @@ -154,16 +154,16 @@ class phpbb_search_sphinx_config // If we're not in a value continuing over the line feed if (!$in_value) { - /* Then add empty lines and comments as comment objects to the variable list - of this section so they're not deleted on reassembly */ + // Then add empty lines and comments as comment objects to the variable list + // of this section so they're not deleted on reassembly if (!$line || $line[0] == '#') { $section->add_variable(new phpbb_search_sphinx_config_comment($config_file[$i])); continue; } - /* As long as we haven't yet actually found an opening bracket for this section - we treat everything as comments so it's not deleted either */ + // As long as we haven't yet actually found an opening bracket for this section + // we treat everything as comments so it's not deleted either if (!$found_opening_bracket) { if ($line[0] == '{') @@ -180,8 +180,8 @@ class phpbb_search_sphinx_config } } - /* If we did not find a comment in this line or still add to the previous - line's value ... */ + // If we did not find a comment in this line or still add to the previous + // line's value ... if ($line || $in_value) { if (!$in_value) @@ -239,8 +239,8 @@ class phpbb_search_sphinx_config } } - /* If a name and an equal sign were found then we have append a - new variable object to the section */ + // If a name and an equal sign were found then we have append a + // new variable object to the section if ($name && $found_assignment) { $section->add_variable(new phpbb_search_sphinx_config_variable(trim($name), trim($value), ($end_section) ? '' : $comment)); @@ -259,8 +259,8 @@ class phpbb_search_sphinx_config } } - /* If we did not find anything meaningful up to here, then just treat it - as a comment */ + // If we did not find anything meaningful up to here, then just treat it + // as a comment $comment = ($skip_first) ? "\t" . substr(ltrim($config_file[$i]), 1) : $config_file[$i]; $section->add_variable(new phpbb_search_sphinx_config_comment($comment)); } From eb4298c646d2b41a46dfef5ad2a72ea520b13539 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 26 Jul 2012 16:55:11 +0530 Subject: [PATCH 50/57] [feature/sphinx-fulltext-search] coding changes acc to phbb conventions Add a new line after break. Change docblocks to be more informative. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 9319a57236..303b6feae8 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -103,7 +103,7 @@ class phpbb_search_fulltext_sphinx /** * Checks permissions and paths, if everything is correct it generates the config file * - * @return string|bool Language key of the error/incompatiblity occured + * @return string|bool Language key of the error/incompatiblity encountered, or false if successful * * @access public */ @@ -381,14 +381,19 @@ class phpbb_search_fulltext_sphinx case 'a': $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'poster_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); break; + case 'f': $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'forum_id ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); break; + case 'i': + case 's': $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'post_subject ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); break; + case 't': + default: $this->sphinx->SetGroupBy('topic_id', SPH_GROUPBY_ATTR, 'topic_last_post_time ' . (($sort_dir == 'a') ? 'ASC' : 'DESC')); break; @@ -401,14 +406,19 @@ class phpbb_search_fulltext_sphinx case 'a': $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'poster_id'); break; + case 'f': $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'forum_id'); break; + case 'i': + case 's': $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_subject'); break; + case 't': + default: $this->sphinx->SetSortMode(($sort_dir == 'a') ? SPH_SORT_ATTR_ASC : SPH_SORT_ATTR_DESC, 'post_time'); break; @@ -619,7 +629,7 @@ class phpbb_search_fulltext_sphinx } /** - * Destroy old cache entries + * Nothing needs to be destroyed * * @access public */ From a6b5b2784fc45764bce077481c40572b49c8b60d Mon Sep 17 00:00:00 2001 From: Dhruv Date: Thu, 26 Jul 2012 17:32:06 +0530 Subject: [PATCH 51/57] [feature/sphinx-fulltext-search] fix sphinx for arbitary host PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 303b6feae8..3d16438ab8 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -82,8 +82,7 @@ class phpbb_search_fulltext_sphinx $this->sphinx = new SphinxClient(); - // We only support localhost for now - $this->sphinx->SetServer('localhost', (isset($this->config['fulltext_sphinx_port']) && $this->config['fulltext_sphinx_port']) ? (int) $this->config['fulltext_sphinx_port'] : 3312); + $this->sphinx->SetServer(($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost'), ($this->config['fulltext_sphinx_port'] ? (int) $this->config['fulltext_sphinx_port'] : 3312)); $error = false; } @@ -162,7 +161,8 @@ class phpbb_search_fulltext_sphinx $config_data = array( 'source source_phpbb_' . $this->id . '_main' => array( array('type', $this->dbtype), - array('sql_host', $this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : $dbhost), + // This config value sql_host needs to be changed incase sphinx and sql are on different servers + array('sql_host', $dbhost), array('sql_user', $dbuser), array('sql_pass', $dbpasswd), array('sql_db', $dbname), @@ -241,7 +241,7 @@ class phpbb_search_fulltext_sphinx ), 'searchd' => array( array('compat_sphinxql_magics' , '0'), - array('listen' , 'localhost' . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '3312')), + array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '3312')), array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), array('read_timeout', '5'), From 3f4afedad34e8bc9a25d0636dee0cf182dd8a5eb Mon Sep 17 00:00:00 2001 From: Dhruv Date: Fri, 27 Jul 2012 02:40:05 +0530 Subject: [PATCH 52/57] [feature/sphinx-fulltext-search] fix language of host config PHPBB3-10946 --- phpBB/language/en/acp/search.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index c52448743c..038e77c190 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -73,13 +73,13 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_DATA_PATH' => 'Path to data directory', 'FULLTEXT_SPHINX_DATA_PATH_EXPLAIN' => 'It will be used to store the indexes and log files. You should create this directory outside the web accessible directories. (should have a trailing slash)', 'FULLTEXT_SPHINX_DELTA_POSTS' => 'Number of posts in frequently updated delta index', - 'FULLTEXT_SPHINX_HOST' => 'SQL server host', - 'FULLTEXT_SPHINX_HOST_EXPLAIN' => 'SQL server host, which the sphinx search daemon (searchd) connects to. Leave empty to use the default SQL server host', + 'FULLTEXT_SPHINX_HOST' => 'Sphinx search daemon host', + 'FULLTEXT_SPHINX_HOST_EXPLAIN' => 'Host on which the sphinx search daemon (searchd) listens. Leave empty to use the default localhost', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT' => 'Indexer memory limit', 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', 'FULLTEXT_SPHINX_PORT' => 'Sphinx search daemon port', - 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default Sphinx API port 3312 ', + 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default Sphinx API port 3312', 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx search for phpBB supports MySQL and PostgreSQL only.', 'FULLTEXT_SPHINX_CONFIG_FILE' => 'Sphinx config file', 'FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN' => 'The generated content of the sphinx config file. This data needs to be pasted into the sphinx.conf which is used by sphinx search daemon.', From 654798922574ef086d618b50a8a8d16eceaa5320 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Fri, 27 Jul 2012 02:48:25 +0530 Subject: [PATCH 53/57] [feature/sphinx-fulltext-search] use 9312 as default port Uses 9312 instead of 3312 as default port for searchd to listen on according to latest sphinx documentation. Use filename sphinxapi.php instead of old one. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 8 ++++---- phpBB/language/en/acp/search.php | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 3d16438ab8..e9772239bf 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -22,7 +22,7 @@ if (!defined('IN_PHPBB')) * function and the variables used are in global space. */ global $phpbb_root_path, $phpEx, $table_prefix; -require($phpbb_root_path . 'includes/sphinxapi-0.9.8.' . $phpEx); +require($phpbb_root_path . 'includes/sphinxapi.' . $phpEx); define('SPHINX_MAX_MATCHES', 20000); define('SPHINX_CONNECT_RETRIES', 3); @@ -82,7 +82,7 @@ class phpbb_search_fulltext_sphinx $this->sphinx = new SphinxClient(); - $this->sphinx->SetServer(($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost'), ($this->config['fulltext_sphinx_port'] ? (int) $this->config['fulltext_sphinx_port'] : 3312)); + $this->sphinx->SetServer(($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost'), ($this->config['fulltext_sphinx_port'] ? (int) $this->config['fulltext_sphinx_port'] : 9312)); $error = false; } @@ -241,7 +241,7 @@ class phpbb_search_fulltext_sphinx ), 'searchd' => array( array('compat_sphinxql_magics' , '0'), - array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '3312')), + array('listen' , ($this->config['fulltext_sphinx_host'] ? $this->config['fulltext_sphinx_host'] : 'localhost') . ':' . ($this->config['fulltext_sphinx_port'] ? $this->config['fulltext_sphinx_port'] : '9312')), array('log', $this->config['fulltext_sphinx_data_path'] . 'log/searchd.log'), array('query_log', $this->config['fulltext_sphinx_data_path'] . 'log/sphinx-query.log'), array('read_timeout', '5'), @@ -493,7 +493,7 @@ class phpbb_search_fulltext_sphinx $this->sphinx->SetLimits($start, (int) $per_page, SPHINX_MAX_MATCHES); $result = $this->sphinx->Query($search_query_prefix . str_replace('"', '"', $this->search_query), $this->indexes); - // Could be connection to localhost:3312 failed (errno=111, + // Could be connection to localhost:9312 failed (errno=111, // msg=Connection refused) during rotate, retry if so $retries = SPHINX_CONNECT_RETRIES; while (!$result && (strpos($this->sphinx->_error, "errno=111,") !== false) && $retries--) diff --git a/phpBB/language/en/acp/search.php b/phpBB/language/en/acp/search.php index 038e77c190..9f947dc816 100644 --- a/phpBB/language/en/acp/search.php +++ b/phpBB/language/en/acp/search.php @@ -79,7 +79,7 @@ $lang = array_merge($lang, array( 'FULLTEXT_SPHINX_INDEXER_MEM_LIMIT_EXPLAIN' => 'This number should at all times be lower than the RAM available on your machine. If you experience periodic performance problems this might be due to the indexer consuming too many resources. It might help to lower the amount of memory available to the indexer.', 'FULLTEXT_SPHINX_MAIN_POSTS' => 'Number of posts in main index', 'FULLTEXT_SPHINX_PORT' => 'Sphinx search daemon port', - 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default Sphinx API port 3312', + 'FULLTEXT_SPHINX_PORT_EXPLAIN' => 'Port on which the sphinx search daemon (searchd) listens. Leave empty to use the default Sphinx API port 9312', 'FULLTEXT_SPHINX_WRONG_DATABASE' => 'The sphinx search for phpBB supports MySQL and PostgreSQL only.', 'FULLTEXT_SPHINX_CONFIG_FILE' => 'Sphinx config file', 'FULLTEXT_SPHINX_CONFIG_FILE_EXPLAIN' => 'The generated content of the sphinx config file. This data needs to be pasted into the sphinx.conf which is used by sphinx search daemon.', From cec9f7d54e360bc2b6b4b12c3123d9c5216d2b62 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Fri, 27 Jul 2012 02:49:44 +0530 Subject: [PATCH 54/57] [feature/sphinx-fulltext-search] remove unused property Removes unused property $word_length PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 1 - 1 file changed, 1 deletion(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index e9772239bf..9cf6f41fa0 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -47,7 +47,6 @@ class phpbb_search_fulltext_sphinx private $dbtype; private $user; private $config_file_data = ''; - public $word_length = array(); public $search_query; public $common_words = array(); From fe8a0d3bc6767f43e7df0b6c964a7f19fa3a5ccc Mon Sep 17 00:00:00 2001 From: Dhruv Date: Fri, 27 Jul 2012 10:50:20 +0530 Subject: [PATCH 55/57] [feature/sphinx-fulltext-search] fix auth bug $this->auth replaces $auth as at other occurences of auth. PHPBB3-10946 --- phpBB/includes/search/fulltext_sphinx.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/includes/search/fulltext_sphinx.php b/phpBB/includes/search/fulltext_sphinx.php index 9cf6f41fa0..8371f6b377 100644 --- a/phpBB/includes/search/fulltext_sphinx.php +++ b/phpBB/includes/search/fulltext_sphinx.php @@ -477,7 +477,7 @@ class phpbb_search_fulltext_sphinx if (sizeof($ex_fid_ary)) { // All forums that a user is allowed to access - $fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($auth->acl_getf('f_search', true)))); + $fid_ary = array_unique(array_intersect(array_keys($this->auth->acl_getf('f_read', true)), array_keys($this->auth->acl_getf('f_search', true)))); // All forums that the user wants to and can search in $search_forums = array_diff($fid_ary, $ex_fid_ary); From 033a2328c4f4451d6ec1f3eb310b5ad4e846e28e Mon Sep 17 00:00:00 2001 From: Dhruv Date: Fri, 27 Jul 2012 11:27:25 +0530 Subject: [PATCH 56/57] [feature/sphinx-fulltext-search] add sphinxapi.php file PHPBB3-10946 --- phpBB/includes/sphinxapi.php | 1712 ++++++++++++++++++++++++++++++++++ 1 file changed, 1712 insertions(+) create mode 100644 phpBB/includes/sphinxapi.php diff --git a/phpBB/includes/sphinxapi.php b/phpBB/includes/sphinxapi.php new file mode 100644 index 0000000000..bd83b1d2e0 --- /dev/null +++ b/phpBB/includes/sphinxapi.php @@ -0,0 +1,1712 @@ +=8 ) + { + $v = (int)$v; + return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); + } + + // x32, int + if ( is_int($v) ) + return pack ( "NN", $v < 0 ? -1 : 0, $v ); + + // x32, bcmath + if ( function_exists("bcmul") ) + { + if ( bccomp ( $v, 0 ) == -1 ) + $v = bcadd ( "18446744073709551616", $v ); + $h = bcdiv ( $v, "4294967296", 0 ); + $l = bcmod ( $v, "4294967296" ); + return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit + } + + // x32, no-bcmath + $p = max(0, strlen($v) - 13); + $lo = abs((float)substr($v, $p)); + $hi = abs((float)substr($v, 0, $p)); + + $m = $lo + $hi*1316134912.0; // (10 ^ 13) % (1 << 32) = 1316134912 + $q = floor($m/4294967296.0); + $l = $m - ($q*4294967296.0); + $h = $hi*2328.0 + $q; // (10 ^ 13) / (1 << 32) = 2328 + + if ( $v<0 ) + { + if ( $l==0 ) + $h = 4294967296.0 - $h; + else + { + $h = 4294967295.0 - $h; + $l = 4294967296.0 - $l; + } + } + return pack ( "NN", $h, $l ); +} + +/// pack 64-bit unsigned +function sphPackU64 ( $v ) +{ + assert ( is_numeric($v) ); + + // x64 + if ( PHP_INT_SIZE>=8 ) + { + assert ( $v>=0 ); + + // x64, int + if ( is_int($v) ) + return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); + + // x64, bcmath + if ( function_exists("bcmul") ) + { + $h = bcdiv ( $v, 4294967296, 0 ); + $l = bcmod ( $v, 4294967296 ); + return pack ( "NN", $h, $l ); + } + + // x64, no-bcmath + $p = max ( 0, strlen($v) - 13 ); + $lo = (int)substr ( $v, $p ); + $hi = (int)substr ( $v, 0, $p ); + + $m = $lo + $hi*1316134912; + $l = $m % 4294967296; + $h = $hi*2328 + (int)($m/4294967296); + + return pack ( "NN", $h, $l ); + } + + // x32, int + if ( is_int($v) ) + return pack ( "NN", 0, $v ); + + // x32, bcmath + if ( function_exists("bcmul") ) + { + $h = bcdiv ( $v, "4294967296", 0 ); + $l = bcmod ( $v, "4294967296" ); + return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit + } + + // x32, no-bcmath + $p = max(0, strlen($v) - 13); + $lo = (float)substr($v, $p); + $hi = (float)substr($v, 0, $p); + + $m = $lo + $hi*1316134912.0; + $q = floor($m / 4294967296.0); + $l = $m - ($q * 4294967296.0); + $h = $hi*2328.0 + $q; + + return pack ( "NN", $h, $l ); +} + +// unpack 64-bit unsigned +function sphUnpackU64 ( $v ) +{ + list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); + + if ( PHP_INT_SIZE>=8 ) + { + if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again + if ( $lo<0 ) $lo += (1<<32); + + // x64, int + if ( $hi<=2147483647 ) + return ($hi<<32) + $lo; + + // x64, bcmath + if ( function_exists("bcmul") ) + return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); + + // x64, no-bcmath + $C = 100000; + $h = ((int)($hi / $C) << 32) + (int)($lo / $C); + $l = (($hi % $C) << 32) + ($lo % $C); + if ( $l>$C ) + { + $h += (int)($l / $C); + $l = $l % $C; + } + + if ( $h==0 ) + return $l; + return sprintf ( "%d%05d", $h, $l ); + } + + // x32, int + if ( $hi==0 ) + { + if ( $lo>0 ) + return $lo; + return sprintf ( "%u", $lo ); + } + + $hi = sprintf ( "%u", $hi ); + $lo = sprintf ( "%u", $lo ); + + // x32, bcmath + if ( function_exists("bcmul") ) + return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); + + // x32, no-bcmath + $hi = (float)$hi; + $lo = (float)$lo; + + $q = floor($hi/10000000.0); + $r = $hi - $q*10000000.0; + $m = $lo + $r*4967296.0; + $mq = floor($m/10000000.0); + $l = $m - $mq*10000000.0; + $h = $q*4294967296.0 + $r*429.0 + $mq; + + $h = sprintf ( "%.0f", $h ); + $l = sprintf ( "%07.0f", $l ); + if ( $h=="0" ) + return sprintf( "%.0f", (float)$l ); + return $h . $l; +} + +// unpack 64-bit signed +function sphUnpackI64 ( $v ) +{ + list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); + + // x64 + if ( PHP_INT_SIZE>=8 ) + { + if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again + if ( $lo<0 ) $lo += (1<<32); + + return ($hi<<32) + $lo; + } + + // x32, int + if ( $hi==0 ) + { + if ( $lo>0 ) + return $lo; + return sprintf ( "%u", $lo ); + } + // x32, int + elseif ( $hi==-1 ) + { + if ( $lo<0 ) + return $lo; + return sprintf ( "%.0f", $lo - 4294967296.0 ); + } + + $neg = ""; + $c = 0; + if ( $hi<0 ) + { + $hi = ~$hi; + $lo = ~$lo; + $c = 1; + $neg = "-"; + } + + $hi = sprintf ( "%u", $hi ); + $lo = sprintf ( "%u", $lo ); + + // x32, bcmath + if ( function_exists("bcmul") ) + return $neg . bcadd ( bcadd ( $lo, bcmul ( $hi, "4294967296" ) ), $c ); + + // x32, no-bcmath + $hi = (float)$hi; + $lo = (float)$lo; + + $q = floor($hi/10000000.0); + $r = $hi - $q*10000000.0; + $m = $lo + $r*4967296.0; + $mq = floor($m/10000000.0); + $l = $m - $mq*10000000.0 + $c; + $h = $q*4294967296.0 + $r*429.0 + $mq; + if ( $l==10000000 ) + { + $l = 0; + $h += 1; + } + + $h = sprintf ( "%.0f", $h ); + $l = sprintf ( "%07.0f", $l ); + if ( $h=="0" ) + return $neg . sprintf( "%.0f", (float)$l ); + return $neg . $h . $l; +} + + +function sphFixUint ( $value ) +{ + if ( PHP_INT_SIZE>=8 ) + { + // x64 route, workaround broken unpack() in 5.2.2+ + if ( $value<0 ) $value += (1<<32); + return $value; + } + else + { + // x32 route, workaround php signed/unsigned braindamage + return sprintf ( "%u", $value ); + } +} + + +/// sphinx searchd client class +class SphinxClient +{ + var $_host; ///< searchd host (default is "localhost") + var $_port; ///< searchd port (default is 9312) + var $_offset; ///< how many records to seek from result-set start (default is 0) + var $_limit; ///< how many records to return from result-set starting at offset (default is 20) + var $_mode; ///< query matching mode (default is SPH_MATCH_ALL) + var $_weights; ///< per-field weights (default is 1 for all fields) + var $_sort; ///< match sorting mode (default is SPH_SORT_RELEVANCE) + var $_sortby; ///< attribute to sort by (defualt is "") + var $_min_id; ///< min ID to match (default is 0, which means no limit) + var $_max_id; ///< max ID to match (default is 0, which means no limit) + var $_filters; ///< search filters + var $_groupby; ///< group-by attribute name + var $_groupfunc; ///< group-by function (to pre-process group-by attribute value with) + var $_groupsort; ///< group-by sorting clause (to sort groups in result set with) + var $_groupdistinct;///< group-by count-distinct attribute + var $_maxmatches; ///< max matches to retrieve + var $_cutoff; ///< cutoff to stop searching at (default is 0) + var $_retrycount; ///< distributed retries count + var $_retrydelay; ///< distributed retries delay + var $_anchor; ///< geographical anchor point + var $_indexweights; ///< per-index weights + var $_ranker; ///< ranking mode (default is SPH_RANK_PROXIMITY_BM25) + var $_rankexpr; ///< ranking mode expression (for SPH_RANK_EXPR) + var $_maxquerytime; ///< max query time, milliseconds (default is 0, do not limit) + var $_fieldweights; ///< per-field-name weights + var $_overrides; ///< per-query attribute values overrides + var $_select; ///< select-list (attributes or expressions, with optional aliases) + + var $_error; ///< last error message + var $_warning; ///< last warning message + var $_connerror; ///< connection error vs remote error flag + + var $_reqs; ///< requests array for multi-query + var $_mbenc; ///< stored mbstring encoding + var $_arrayresult; ///< whether $result["matches"] should be a hash or an array + var $_timeout; ///< connect timeout + + ///////////////////////////////////////////////////////////////////////////// + // common stuff + ///////////////////////////////////////////////////////////////////////////// + + /// create a new client object and fill defaults + function SphinxClient () + { + // per-client-object settings + $this->_host = "localhost"; + $this->_port = 9312; + $this->_path = false; + $this->_socket = false; + + // per-query settings + $this->_offset = 0; + $this->_limit = 20; + $this->_mode = SPH_MATCH_ALL; + $this->_weights = array (); + $this->_sort = SPH_SORT_RELEVANCE; + $this->_sortby = ""; + $this->_min_id = 0; + $this->_max_id = 0; + $this->_filters = array (); + $this->_groupby = ""; + $this->_groupfunc = SPH_GROUPBY_DAY; + $this->_groupsort = "@group desc"; + $this->_groupdistinct= ""; + $this->_maxmatches = 1000; + $this->_cutoff = 0; + $this->_retrycount = 0; + $this->_retrydelay = 0; + $this->_anchor = array (); + $this->_indexweights= array (); + $this->_ranker = SPH_RANK_PROXIMITY_BM25; + $this->_rankexpr = ""; + $this->_maxquerytime= 0; + $this->_fieldweights= array(); + $this->_overrides = array(); + $this->_select = "*"; + + $this->_error = ""; // per-reply fields (for single-query case) + $this->_warning = ""; + $this->_connerror = false; + + $this->_reqs = array (); // requests storage (for multi-query case) + $this->_mbenc = ""; + $this->_arrayresult = false; + $this->_timeout = 0; + } + + function __destruct() + { + if ( $this->_socket !== false ) + fclose ( $this->_socket ); + } + + /// get last error message (string) + function GetLastError () + { + return $this->_error; + } + + /// get last warning message (string) + function GetLastWarning () + { + return $this->_warning; + } + + /// get last error flag (to tell network connection errors from searchd errors or broken responses) + function IsConnectError() + { + return $this->_connerror; + } + + /// set searchd host name (string) and port (integer) + function SetServer ( $host, $port = 0 ) + { + assert ( is_string($host) ); + if ( $host[0] == '/') + { + $this->_path = 'unix://' . $host; + return; + } + if ( substr ( $host, 0, 7 )=="unix://" ) + { + $this->_path = $host; + return; + } + + assert ( is_int($port) ); + $this->_host = $host; + $this->_port = $port; + $this->_path = ''; + + } + + /// set server connection timeout (0 to remove) + function SetConnectTimeout ( $timeout ) + { + assert ( is_numeric($timeout) ); + $this->_timeout = $timeout; + } + + + function _Send ( $handle, $data, $length ) + { + if ( feof($handle) || fwrite ( $handle, $data, $length ) !== $length ) + { + $this->_error = 'connection unexpectedly closed (timed out?)'; + $this->_connerror = true; + return false; + } + return true; + } + + ///////////////////////////////////////////////////////////////////////////// + + /// enter mbstring workaround mode + function _MBPush () + { + $this->_mbenc = ""; + if ( ini_get ( "mbstring.func_overload" ) & 2 ) + { + $this->_mbenc = mb_internal_encoding(); + mb_internal_encoding ( "latin1" ); + } + } + + /// leave mbstring workaround mode + function _MBPop () + { + if ( $this->_mbenc ) + mb_internal_encoding ( $this->_mbenc ); + } + + /// connect to searchd server + function _Connect () + { + if ( $this->_socket!==false ) + { + // we are in persistent connection mode, so we have a socket + // however, need to check whether it's still alive + if ( !@feof ( $this->_socket ) ) + return $this->_socket; + + // force reopen + $this->_socket = false; + } + + $errno = 0; + $errstr = ""; + $this->_connerror = false; + + if ( $this->_path ) + { + $host = $this->_path; + $port = 0; + } + else + { + $host = $this->_host; + $port = $this->_port; + } + + if ( $this->_timeout<=0 ) + $fp = @fsockopen ( $host, $port, $errno, $errstr ); + else + $fp = @fsockopen ( $host, $port, $errno, $errstr, $this->_timeout ); + + if ( !$fp ) + { + if ( $this->_path ) + $location = $this->_path; + else + $location = "{$this->_host}:{$this->_port}"; + + $errstr = trim ( $errstr ); + $this->_error = "connection to $location failed (errno=$errno, msg=$errstr)"; + $this->_connerror = true; + return false; + } + + // send my version + // this is a subtle part. we must do it before (!) reading back from searchd. + // because otherwise under some conditions (reported on FreeBSD for instance) + // TCP stack could throttle write-write-read pattern because of Nagle. + if ( !$this->_Send ( $fp, pack ( "N", 1 ), 4 ) ) + { + fclose ( $fp ); + $this->_error = "failed to send client protocol version"; + return false; + } + + // check version + list(,$v) = unpack ( "N*", fread ( $fp, 4 ) ); + $v = (int)$v; + if ( $v<1 ) + { + fclose ( $fp ); + $this->_error = "expected searchd protocol version 1+, got version '$v'"; + return false; + } + + return $fp; + } + + /// get and check response packet from searchd server + function _GetResponse ( $fp, $client_ver ) + { + $response = ""; + $len = 0; + + $header = fread ( $fp, 8 ); + if ( strlen($header)==8 ) + { + list ( $status, $ver, $len ) = array_values ( unpack ( "n2a/Nb", $header ) ); + $left = $len; + while ( $left>0 && !feof($fp) ) + { + $chunk = fread ( $fp, min ( 8192, $left ) ); + if ( $chunk ) + { + $response .= $chunk; + $left -= strlen($chunk); + } + } + } + if ( $this->_socket === false ) + fclose ( $fp ); + + // check response + $read = strlen ( $response ); + if ( !$response || $read!=$len ) + { + $this->_error = $len + ? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=$read)" + : "received zero-sized searchd response"; + return false; + } + + // check status + if ( $status==SEARCHD_WARNING ) + { + list(,$wlen) = unpack ( "N*", substr ( $response, 0, 4 ) ); + $this->_warning = substr ( $response, 4, $wlen ); + return substr ( $response, 4+$wlen ); + } + if ( $status==SEARCHD_ERROR ) + { + $this->_error = "searchd error: " . substr ( $response, 4 ); + return false; + } + if ( $status==SEARCHD_RETRY ) + { + $this->_error = "temporary searchd error: " . substr ( $response, 4 ); + return false; + } + if ( $status!=SEARCHD_OK ) + { + $this->_error = "unknown status code '$status'"; + return false; + } + + // check version + if ( $ver<$client_ver ) + { + $this->_warning = sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work", + $ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ); + } + + return $response; + } + + ///////////////////////////////////////////////////////////////////////////// + // searching + ///////////////////////////////////////////////////////////////////////////// + + /// set offset and count into result set, + /// and optionally set max-matches and cutoff limits + function SetLimits ( $offset, $limit, $max=0, $cutoff=0 ) + { + assert ( is_int($offset) ); + assert ( is_int($limit) ); + assert ( $offset>=0 ); + assert ( $limit>0 ); + assert ( $max>=0 ); + $this->_offset = $offset; + $this->_limit = $limit; + if ( $max>0 ) + $this->_maxmatches = $max; + if ( $cutoff>0 ) + $this->_cutoff = $cutoff; + } + + /// set maximum query time, in milliseconds, per-index + /// integer, 0 means "do not limit" + function SetMaxQueryTime ( $max ) + { + assert ( is_int($max) ); + assert ( $max>=0 ); + $this->_maxquerytime = $max; + } + + /// set matching mode + function SetMatchMode ( $mode ) + { + assert ( $mode==SPH_MATCH_ALL + || $mode==SPH_MATCH_ANY + || $mode==SPH_MATCH_PHRASE + || $mode==SPH_MATCH_BOOLEAN + || $mode==SPH_MATCH_EXTENDED + || $mode==SPH_MATCH_FULLSCAN + || $mode==SPH_MATCH_EXTENDED2 ); + $this->_mode = $mode; + } + + /// set ranking mode + function SetRankingMode ( $ranker, $rankexpr="" ) + { + assert ( $ranker>=0 && $ranker_ranker = $ranker; + $this->_rankexpr = $rankexpr; + } + + /// set matches sorting mode + function SetSortMode ( $mode, $sortby="" ) + { + assert ( + $mode==SPH_SORT_RELEVANCE || + $mode==SPH_SORT_ATTR_DESC || + $mode==SPH_SORT_ATTR_ASC || + $mode==SPH_SORT_TIME_SEGMENTS || + $mode==SPH_SORT_EXTENDED || + $mode==SPH_SORT_EXPR ); + assert ( is_string($sortby) ); + assert ( $mode==SPH_SORT_RELEVANCE || strlen($sortby)>0 ); + + $this->_sort = $mode; + $this->_sortby = $sortby; + } + + /// bind per-field weights by order + /// DEPRECATED; use SetFieldWeights() instead + function SetWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $weight ) + assert ( is_int($weight) ); + + $this->_weights = $weights; + } + + /// bind per-field weights by name + function SetFieldWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $name=>$weight ) + { + assert ( is_string($name) ); + assert ( is_int($weight) ); + } + $this->_fieldweights = $weights; + } + + /// bind per-index weights by name + function SetIndexWeights ( $weights ) + { + assert ( is_array($weights) ); + foreach ( $weights as $index=>$weight ) + { + assert ( is_string($index) ); + assert ( is_int($weight) ); + } + $this->_indexweights = $weights; + } + + /// set IDs range to match + /// only match records if document ID is beetwen $min and $max (inclusive) + function SetIDRange ( $min, $max ) + { + assert ( is_numeric($min) ); + assert ( is_numeric($max) ); + assert ( $min<=$max ); + $this->_min_id = $min; + $this->_max_id = $max; + } + + /// set values set filter + /// only match records where $attribute value is in given set + function SetFilter ( $attribute, $values, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_array($values) ); + assert ( count($values) ); + + if ( is_array($values) && count($values) ) + { + foreach ( $values as $value ) + assert ( is_numeric($value) ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_VALUES, "attr"=>$attribute, "exclude"=>$exclude, "values"=>$values ); + } + } + + /// set range filter + /// only match records if $attribute value is beetwen $min and $max (inclusive) + function SetFilterRange ( $attribute, $min, $max, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_numeric($min) ); + assert ( is_numeric($max) ); + assert ( $min<=$max ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_RANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); + } + + /// set float range filter + /// only match records if $attribute value is beetwen $min and $max (inclusive) + function SetFilterFloatRange ( $attribute, $min, $max, $exclude=false ) + { + assert ( is_string($attribute) ); + assert ( is_float($min) ); + assert ( is_float($max) ); + assert ( $min<=$max ); + + $this->_filters[] = array ( "type"=>SPH_FILTER_FLOATRANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); + } + + /// setup anchor point for geosphere distance calculations + /// required to use @geodist in filters and sorting + /// latitude and longitude must be in radians + function SetGeoAnchor ( $attrlat, $attrlong, $lat, $long ) + { + assert ( is_string($attrlat) ); + assert ( is_string($attrlong) ); + assert ( is_float($lat) ); + assert ( is_float($long) ); + + $this->_anchor = array ( "attrlat"=>$attrlat, "attrlong"=>$attrlong, "lat"=>$lat, "long"=>$long ); + } + + /// set grouping attribute and function + function SetGroupBy ( $attribute, $func, $groupsort="@group desc" ) + { + assert ( is_string($attribute) ); + assert ( is_string($groupsort) ); + assert ( $func==SPH_GROUPBY_DAY + || $func==SPH_GROUPBY_WEEK + || $func==SPH_GROUPBY_MONTH + || $func==SPH_GROUPBY_YEAR + || $func==SPH_GROUPBY_ATTR + || $func==SPH_GROUPBY_ATTRPAIR ); + + $this->_groupby = $attribute; + $this->_groupfunc = $func; + $this->_groupsort = $groupsort; + } + + /// set count-distinct attribute for group-by queries + function SetGroupDistinct ( $attribute ) + { + assert ( is_string($attribute) ); + $this->_groupdistinct = $attribute; + } + + /// set distributed retries count and delay + function SetRetries ( $count, $delay=0 ) + { + assert ( is_int($count) && $count>=0 ); + assert ( is_int($delay) && $delay>=0 ); + $this->_retrycount = $count; + $this->_retrydelay = $delay; + } + + /// set result set format (hash or array; hash by default) + /// PHP specific; needed for group-by-MVA result sets that may contain duplicate IDs + function SetArrayResult ( $arrayresult ) + { + assert ( is_bool($arrayresult) ); + $this->_arrayresult = $arrayresult; + } + + /// set attribute values override + /// there can be only one override per attribute + /// $values must be a hash that maps document IDs to attribute values + function SetOverride ( $attrname, $attrtype, $values ) + { + assert ( is_string ( $attrname ) ); + assert ( in_array ( $attrtype, array ( SPH_ATTR_INTEGER, SPH_ATTR_TIMESTAMP, SPH_ATTR_BOOL, SPH_ATTR_FLOAT, SPH_ATTR_BIGINT ) ) ); + assert ( is_array ( $values ) ); + + $this->_overrides[$attrname] = array ( "attr"=>$attrname, "type"=>$attrtype, "values"=>$values ); + } + + /// set select-list (attributes or expressions), SQL-like syntax + function SetSelect ( $select ) + { + assert ( is_string ( $select ) ); + $this->_select = $select; + } + + ////////////////////////////////////////////////////////////////////////////// + + /// clear all filters (for multi-queries) + function ResetFilters () + { + $this->_filters = array(); + $this->_anchor = array(); + } + + /// clear groupby settings (for multi-queries) + function ResetGroupBy () + { + $this->_groupby = ""; + $this->_groupfunc = SPH_GROUPBY_DAY; + $this->_groupsort = "@group desc"; + $this->_groupdistinct= ""; + } + + /// clear all attribute value overrides (for multi-queries) + function ResetOverrides () + { + $this->_overrides = array (); + } + + ////////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, run given search query through given indexes, + /// and return the search results + function Query ( $query, $index="*", $comment="" ) + { + assert ( empty($this->_reqs) ); + + $this->AddQuery ( $query, $index, $comment ); + $results = $this->RunQueries (); + $this->_reqs = array (); // just in case it failed too early + + if ( !is_array($results) ) + return false; // probably network error; error message should be already filled + + $this->_error = $results[0]["error"]; + $this->_warning = $results[0]["warning"]; + if ( $results[0]["status"]==SEARCHD_ERROR ) + return false; + else + return $results[0]; + } + + /// helper to pack floats in network byte order + function _PackFloat ( $f ) + { + $t1 = pack ( "f", $f ); // machine order + list(,$t2) = unpack ( "L*", $t1 ); // int in machine order + return pack ( "N", $t2 ); + } + + /// add query to multi-query batch + /// returns index into results array from RunQueries() call + function AddQuery ( $query, $index="*", $comment="" ) + { + // mbstring workaround + $this->_MBPush (); + + // build request + $req = pack ( "NNNN", $this->_offset, $this->_limit, $this->_mode, $this->_ranker ); + if ( $this->_ranker==SPH_RANK_EXPR ) + $req .= pack ( "N", strlen($this->_rankexpr) ) . $this->_rankexpr; + $req .= pack ( "N", $this->_sort ); // (deprecated) sort mode + $req .= pack ( "N", strlen($this->_sortby) ) . $this->_sortby; + $req .= pack ( "N", strlen($query) ) . $query; // query itself + $req .= pack ( "N", count($this->_weights) ); // weights + foreach ( $this->_weights as $weight ) + $req .= pack ( "N", (int)$weight ); + $req .= pack ( "N", strlen($index) ) . $index; // indexes + $req .= pack ( "N", 1 ); // id64 range marker + $req .= sphPackU64 ( $this->_min_id ) . sphPackU64 ( $this->_max_id ); // id64 range + + // filters + $req .= pack ( "N", count($this->_filters) ); + foreach ( $this->_filters as $filter ) + { + $req .= pack ( "N", strlen($filter["attr"]) ) . $filter["attr"]; + $req .= pack ( "N", $filter["type"] ); + switch ( $filter["type"] ) + { + case SPH_FILTER_VALUES: + $req .= pack ( "N", count($filter["values"]) ); + foreach ( $filter["values"] as $value ) + $req .= sphPackI64 ( $value ); + break; + + case SPH_FILTER_RANGE: + $req .= sphPackI64 ( $filter["min"] ) . sphPackI64 ( $filter["max"] ); + break; + + case SPH_FILTER_FLOATRANGE: + $req .= $this->_PackFloat ( $filter["min"] ) . $this->_PackFloat ( $filter["max"] ); + break; + + default: + assert ( 0 && "internal error: unhandled filter type" ); + } + $req .= pack ( "N", $filter["exclude"] ); + } + + // group-by clause, max-matches count, group-sort clause, cutoff count + $req .= pack ( "NN", $this->_groupfunc, strlen($this->_groupby) ) . $this->_groupby; + $req .= pack ( "N", $this->_maxmatches ); + $req .= pack ( "N", strlen($this->_groupsort) ) . $this->_groupsort; + $req .= pack ( "NNN", $this->_cutoff, $this->_retrycount, $this->_retrydelay ); + $req .= pack ( "N", strlen($this->_groupdistinct) ) . $this->_groupdistinct; + + // anchor point + if ( empty($this->_anchor) ) + { + $req .= pack ( "N", 0 ); + } else + { + $a =& $this->_anchor; + $req .= pack ( "N", 1 ); + $req .= pack ( "N", strlen($a["attrlat"]) ) . $a["attrlat"]; + $req .= pack ( "N", strlen($a["attrlong"]) ) . $a["attrlong"]; + $req .= $this->_PackFloat ( $a["lat"] ) . $this->_PackFloat ( $a["long"] ); + } + + // per-index weights + $req .= pack ( "N", count($this->_indexweights) ); + foreach ( $this->_indexweights as $idx=>$weight ) + $req .= pack ( "N", strlen($idx) ) . $idx . pack ( "N", $weight ); + + // max query time + $req .= pack ( "N", $this->_maxquerytime ); + + // per-field weights + $req .= pack ( "N", count($this->_fieldweights) ); + foreach ( $this->_fieldweights as $field=>$weight ) + $req .= pack ( "N", strlen($field) ) . $field . pack ( "N", $weight ); + + // comment + $req .= pack ( "N", strlen($comment) ) . $comment; + + // attribute overrides + $req .= pack ( "N", count($this->_overrides) ); + foreach ( $this->_overrides as $key => $entry ) + { + $req .= pack ( "N", strlen($entry["attr"]) ) . $entry["attr"]; + $req .= pack ( "NN", $entry["type"], count($entry["values"]) ); + foreach ( $entry["values"] as $id=>$val ) + { + assert ( is_numeric($id) ); + assert ( is_numeric($val) ); + + $req .= sphPackU64 ( $id ); + switch ( $entry["type"] ) + { + case SPH_ATTR_FLOAT: $req .= $this->_PackFloat ( $val ); break; + case SPH_ATTR_BIGINT: $req .= sphPackI64 ( $val ); break; + default: $req .= pack ( "N", $val ); break; + } + } + } + + // select-list + $req .= pack ( "N", strlen($this->_select) ) . $this->_select; + + // mbstring workaround + $this->_MBPop (); + + // store request to requests array + $this->_reqs[] = $req; + return count($this->_reqs)-1; + } + + /// connect to searchd, run queries batch, and return an array of result sets + function RunQueries () + { + if ( empty($this->_reqs) ) + { + $this->_error = "no queries defined, issue AddQuery() first"; + return false; + } + + // mbstring workaround + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop (); + return false; + } + + // send query, get response + $nreqs = count($this->_reqs); + $req = join ( "", $this->_reqs ); + $len = 8+strlen($req); + $req = pack ( "nnNNN", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $len, 0, $nreqs ) . $req; // add header + + if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_SEARCH ) ) ) + { + $this->_MBPop (); + return false; + } + + // query sent ok; we can reset reqs now + $this->_reqs = array (); + + // parse and return response + return $this->_ParseSearchResponse ( $response, $nreqs ); + } + + /// parse and return search query (or queries) response + function _ParseSearchResponse ( $response, $nreqs ) + { + $p = 0; // current position + $max = strlen($response); // max position for checks, to protect against broken responses + + $results = array (); + for ( $ires=0; $ires<$nreqs && $p<$max; $ires++ ) + { + $results[] = array(); + $result =& $results[$ires]; + + $result["error"] = ""; + $result["warning"] = ""; + + // extract status + list(,$status) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $result["status"] = $status; + if ( $status!=SEARCHD_OK ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $message = substr ( $response, $p, $len ); $p += $len; + + if ( $status==SEARCHD_WARNING ) + { + $result["warning"] = $message; + } else + { + $result["error"] = $message; + continue; + } + } + + // read schema + $fields = array (); + $attrs = array (); + + list(,$nfields) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + while ( $nfields-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $fields[] = substr ( $response, $p, $len ); $p += $len; + } + $result["fields"] = $fields; + + list(,$nattrs) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + while ( $nattrs-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attr = substr ( $response, $p, $len ); $p += $len; + list(,$type) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attrs[$attr] = $type; + } + $result["attrs"] = $attrs; + + // read match count + list(,$count) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + list(,$id64) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + + // read matches + $idx = -1; + while ( $count-->0 && $p<$max ) + { + // index into result array + $idx++; + + // parse document id and weight + if ( $id64 ) + { + $doc = sphUnpackU64 ( substr ( $response, $p, 8 ) ); $p += 8; + list(,$weight) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + } + else + { + list ( $doc, $weight ) = array_values ( unpack ( "N*N*", + substr ( $response, $p, 8 ) ) ); + $p += 8; + $doc = sphFixUint($doc); + } + $weight = sprintf ( "%u", $weight ); + + // create match entry + if ( $this->_arrayresult ) + $result["matches"][$idx] = array ( "id"=>$doc, "weight"=>$weight ); + else + $result["matches"][$doc]["weight"] = $weight; + + // parse and create attributes + $attrvals = array (); + foreach ( $attrs as $attr=>$type ) + { + // handle 64bit ints + if ( $type==SPH_ATTR_BIGINT ) + { + $attrvals[$attr] = sphUnpackI64 ( substr ( $response, $p, 8 ) ); $p += 8; + continue; + } + + // handle floats + if ( $type==SPH_ATTR_FLOAT ) + { + list(,$uval) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + list(,$fval) = unpack ( "f*", pack ( "L", $uval ) ); + $attrvals[$attr] = $fval; + continue; + } + + // handle everything else as unsigned ints + list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + if ( $type==SPH_ATTR_MULTI ) + { + $attrvals[$attr] = array (); + $nvalues = $val; + while ( $nvalues-->0 && $p<$max ) + { + list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $attrvals[$attr][] = sphFixUint($val); + } + } else if ( $type==SPH_ATTR_MULTI64 ) + { + $attrvals[$attr] = array (); + $nvalues = $val; + while ( $nvalues>0 && $p<$max ) + { + $attrvals[$attr][] = sphUnpackU64 ( substr ( $response, $p, 8 ) ); $p += 8; + $nvalues -= 2; + } + } else if ( $type==SPH_ATTR_STRING ) + { + $attrvals[$attr] = substr ( $response, $p, $val ); + $p += $val; + } else + { + $attrvals[$attr] = sphFixUint($val); + } + } + + if ( $this->_arrayresult ) + $result["matches"][$idx]["attrs"] = $attrvals; + else + $result["matches"][$doc]["attrs"] = $attrvals; + } + + list ( $total, $total_found, $msecs, $words ) = + array_values ( unpack ( "N*N*N*N*", substr ( $response, $p, 16 ) ) ); + $result["total"] = sprintf ( "%u", $total ); + $result["total_found"] = sprintf ( "%u", $total_found ); + $result["time"] = sprintf ( "%.3f", $msecs/1000 ); + $p += 16; + + while ( $words-->0 && $p<$max ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $word = substr ( $response, $p, $len ); $p += $len; + list ( $docs, $hits ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; + $result["words"][$word] = array ( + "docs"=>sprintf ( "%u", $docs ), + "hits"=>sprintf ( "%u", $hits ) ); + } + } + + $this->_MBPop (); + return $results; + } + + ///////////////////////////////////////////////////////////////////////////// + // excerpts generation + ///////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, and generate exceprts (snippets) + /// of given documents for given query. returns false on failure, + /// an array of snippets on success + function BuildExcerpts ( $docs, $index, $words, $opts=array() ) + { + assert ( is_array($docs) ); + assert ( is_string($index) ); + assert ( is_string($words) ); + assert ( is_array($opts) ); + + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + ///////////////// + // fixup options + ///////////////// + + if ( !isset($opts["before_match"]) ) $opts["before_match"] = ""; + if ( !isset($opts["after_match"]) ) $opts["after_match"] = ""; + if ( !isset($opts["chunk_separator"]) ) $opts["chunk_separator"] = " ... "; + if ( !isset($opts["limit"]) ) $opts["limit"] = 256; + if ( !isset($opts["limit_passages"]) ) $opts["limit_passages"] = 0; + if ( !isset($opts["limit_words"]) ) $opts["limit_words"] = 0; + if ( !isset($opts["around"]) ) $opts["around"] = 5; + if ( !isset($opts["exact_phrase"]) ) $opts["exact_phrase"] = false; + if ( !isset($opts["single_passage"]) ) $opts["single_passage"] = false; + if ( !isset($opts["use_boundaries"]) ) $opts["use_boundaries"] = false; + if ( !isset($opts["weight_order"]) ) $opts["weight_order"] = false; + if ( !isset($opts["query_mode"]) ) $opts["query_mode"] = false; + if ( !isset($opts["force_all_words"]) ) $opts["force_all_words"] = false; + if ( !isset($opts["start_passage_id"]) ) $opts["start_passage_id"] = 1; + if ( !isset($opts["load_files"]) ) $opts["load_files"] = false; + if ( !isset($opts["html_strip_mode"]) ) $opts["html_strip_mode"] = "index"; + if ( !isset($opts["allow_empty"]) ) $opts["allow_empty"] = false; + if ( !isset($opts["passage_boundary"]) ) $opts["passage_boundary"] = "none"; + if ( !isset($opts["emit_zones"]) ) $opts["emit_zones"] = false; + if ( !isset($opts["load_files_scattered"]) ) $opts["load_files_scattered"] = false; + + + ///////////////// + // build request + ///////////////// + + // v.1.2 req + $flags = 1; // remove spaces + if ( $opts["exact_phrase"] ) $flags |= 2; + if ( $opts["single_passage"] ) $flags |= 4; + if ( $opts["use_boundaries"] ) $flags |= 8; + if ( $opts["weight_order"] ) $flags |= 16; + if ( $opts["query_mode"] ) $flags |= 32; + if ( $opts["force_all_words"] ) $flags |= 64; + if ( $opts["load_files"] ) $flags |= 128; + if ( $opts["allow_empty"] ) $flags |= 256; + if ( $opts["emit_zones"] ) $flags |= 512; + if ( $opts["load_files_scattered"] ) $flags |= 1024; + $req = pack ( "NN", 0, $flags ); // mode=0, flags=$flags + $req .= pack ( "N", strlen($index) ) . $index; // req index + $req .= pack ( "N", strlen($words) ) . $words; // req words + + // options + $req .= pack ( "N", strlen($opts["before_match"]) ) . $opts["before_match"]; + $req .= pack ( "N", strlen($opts["after_match"]) ) . $opts["after_match"]; + $req .= pack ( "N", strlen($opts["chunk_separator"]) ) . $opts["chunk_separator"]; + $req .= pack ( "NN", (int)$opts["limit"], (int)$opts["around"] ); + $req .= pack ( "NNN", (int)$opts["limit_passages"], (int)$opts["limit_words"], (int)$opts["start_passage_id"] ); // v.1.2 + $req .= pack ( "N", strlen($opts["html_strip_mode"]) ) . $opts["html_strip_mode"]; + $req .= pack ( "N", strlen($opts["passage_boundary"]) ) . $opts["passage_boundary"]; + + // documents + $req .= pack ( "N", count($docs) ); + foreach ( $docs as $doc ) + { + assert ( is_string($doc) ); + $req .= pack ( "N", strlen($doc) ) . $doc; + } + + //////////////////////////// + // send query, get response + //////////////////////////// + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $len ) . $req; // add header + if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_EXCERPT ) ) ) + { + $this->_MBPop (); + return false; + } + + ////////////////// + // parse response + ////////////////// + + $pos = 0; + $res = array (); + $rlen = strlen($response); + for ( $i=0; $i $rlen ) + { + $this->_error = "incomplete reply"; + $this->_MBPop (); + return false; + } + $res[] = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + } + + $this->_MBPop (); + return $res; + } + + + ///////////////////////////////////////////////////////////////////////////// + // keyword generation + ///////////////////////////////////////////////////////////////////////////// + + /// connect to searchd server, and generate keyword list for a given query + /// returns false on failure, + /// an array of words on success + function BuildKeywords ( $query, $index, $hits ) + { + assert ( is_string($query) ); + assert ( is_string($index) ); + assert ( is_bool($hits) ); + + $this->_MBPush (); + + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + ///////////////// + // build request + ///////////////// + + // v.1.0 req + $req = pack ( "N", strlen($query) ) . $query; // req query + $req .= pack ( "N", strlen($index) ) . $index; // req index + $req .= pack ( "N", (int)$hits ); + + //////////////////////////// + // send query, get response + //////////////////////////// + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, $len ) . $req; // add header + if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_KEYWORDS ) ) ) + { + $this->_MBPop (); + return false; + } + + ////////////////// + // parse response + ////////////////// + + $pos = 0; + $res = array (); + $rlen = strlen($response); + list(,$nwords) = unpack ( "N*", substr ( $response, $pos, 4 ) ); + $pos += 4; + for ( $i=0; $i<$nwords; $i++ ) + { + list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; + $tokenized = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + + list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; + $normalized = $len ? substr ( $response, $pos, $len ) : ""; + $pos += $len; + + $res[] = array ( "tokenized"=>$tokenized, "normalized"=>$normalized ); + + if ( $hits ) + { + list($ndocs,$nhits) = array_values ( unpack ( "N*N*", substr ( $response, $pos, 8 ) ) ); + $pos += 8; + $res [$i]["docs"] = $ndocs; + $res [$i]["hits"] = $nhits; + } + + if ( $pos > $rlen ) + { + $this->_error = "incomplete reply"; + $this->_MBPop (); + return false; + } + } + + $this->_MBPop (); + return $res; + } + + function EscapeString ( $string ) + { + $from = array ( '\\', '(',')','|','-','!','@','~','"','&', '/', '^', '$', '=' ); + $to = array ( '\\\\', '\(','\)','\|','\-','\!','\@','\~','\"', '\&', '\/', '\^', '\$', '\=' ); + + return str_replace ( $from, $to, $string ); + } + + ///////////////////////////////////////////////////////////////////////////// + // attribute updates + ///////////////////////////////////////////////////////////////////////////// + + /// batch update given attributes in given rows in given indexes + /// returns amount of updated documents (0 or more) on success, or -1 on failure + function UpdateAttributes ( $index, $attrs, $values, $mva=false ) + { + // verify everything + assert ( is_string($index) ); + assert ( is_bool($mva) ); + + assert ( is_array($attrs) ); + foreach ( $attrs as $attr ) + assert ( is_string($attr) ); + + assert ( is_array($values) ); + foreach ( $values as $id=>$entry ) + { + assert ( is_numeric($id) ); + assert ( is_array($entry) ); + assert ( count($entry)==count($attrs) ); + foreach ( $entry as $v ) + { + if ( $mva ) + { + assert ( is_array($v) ); + foreach ( $v as $vv ) + assert ( is_int($vv) ); + } else + assert ( is_int($v) ); + } + } + + // build request + $this->_MBPush (); + $req = pack ( "N", strlen($index) ) . $index; + + $req .= pack ( "N", count($attrs) ); + foreach ( $attrs as $attr ) + { + $req .= pack ( "N", strlen($attr) ) . $attr; + $req .= pack ( "N", $mva ? 1 : 0 ); + } + + $req .= pack ( "N", count($values) ); + foreach ( $values as $id=>$entry ) + { + $req .= sphPackU64 ( $id ); + foreach ( $entry as $v ) + { + $req .= pack ( "N", $mva ? count($v) : $v ); + if ( $mva ) + foreach ( $v as $vv ) + $req .= pack ( "N", $vv ); + } + } + + // connect, send query, get response + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop (); + return -1; + } + + $len = strlen($req); + $req = pack ( "nnN", SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, $len ) . $req; // add header + if ( !$this->_Send ( $fp, $req, $len+8 ) ) + { + $this->_MBPop (); + return -1; + } + + if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_UPDATE ) )) + { + $this->_MBPop (); + return -1; + } + + // parse response + list(,$updated) = unpack ( "N*", substr ( $response, 0, 4 ) ); + $this->_MBPop (); + return $updated; + } + + ///////////////////////////////////////////////////////////////////////////// + // persistent connections + ///////////////////////////////////////////////////////////////////////////// + + function Open() + { + if ( $this->_socket !== false ) + { + $this->_error = 'already connected'; + return false; + } + if ( !$fp = $this->_Connect() ) + return false; + + // command, command version = 0, body length = 4, body = 1 + $req = pack ( "nnNN", SEARCHD_COMMAND_PERSIST, 0, 4, 1 ); + if ( !$this->_Send ( $fp, $req, 12 ) ) + return false; + + $this->_socket = $fp; + return true; + } + + function Close() + { + if ( $this->_socket === false ) + { + $this->_error = 'not connected'; + return false; + } + + fclose ( $this->_socket ); + $this->_socket = false; + + return true; + } + + ////////////////////////////////////////////////////////////////////////// + // status + ////////////////////////////////////////////////////////////////////////// + + function Status () + { + $this->_MBPush (); + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return false; + } + + $req = pack ( "nnNN", SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, 1 ); // len=4, body=1 + if ( !( $this->_Send ( $fp, $req, 12 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_STATUS ) ) ) + { + $this->_MBPop (); + return false; + } + + $res = substr ( $response, 4 ); // just ignore length, error handling, etc + $p = 0; + list ( $rows, $cols ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; + + $res = array(); + for ( $i=0; $i<$rows; $i++ ) + for ( $j=0; $j<$cols; $j++ ) + { + list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; + $res[$i][] = substr ( $response, $p, $len ); $p += $len; + } + + $this->_MBPop (); + return $res; + } + + ////////////////////////////////////////////////////////////////////////// + // flush + ////////////////////////////////////////////////////////////////////////// + + function FlushAttributes () + { + $this->_MBPush (); + if (!( $fp = $this->_Connect() )) + { + $this->_MBPop(); + return -1; + } + + $req = pack ( "nnN", SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ); // len=0 + if ( !( $this->_Send ( $fp, $req, 8 ) ) || + !( $response = $this->_GetResponse ( $fp, VER_COMMAND_FLUSHATTRS ) ) ) + { + $this->_MBPop (); + return -1; + } + + $tag = -1; + if ( strlen($response)==4 ) + list(,$tag) = unpack ( "N*", $response ); + else + $this->_error = "unexpected response length"; + + $this->_MBPop (); + return $tag; + } +} + +// +// $Id: sphinxapi.php 3087 2012-01-30 23:07:35Z shodan $ +// From f1729281e6f69202730c0926f3799da516fd3ae9 Mon Sep 17 00:00:00 2001 From: Dhruv Date: Fri, 27 Jul 2012 11:28:14 +0530 Subject: [PATCH 57/57] [feature/sphinx-fulltext-search] add sphinx to Authors file PHPBB3-10946 --- phpBB/docs/AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/phpBB/docs/AUTHORS b/phpBB/docs/AUTHORS index f0b4e25549..85653f04e2 100644 --- a/phpBB/docs/AUTHORS +++ b/phpBB/docs/AUTHORS @@ -75,6 +75,7 @@ Jabber Class (c) 2006 Flyspray.org, http://www.flyspray.org/ Chora (c) 2000-2006, The Horde Project. http://horde.org/chora/ Horde Project (c) 2000-2006, The Horde Project. http://horde.org/ jQuery (c) 2011, John Resig. http://jquery.com/ +Sphinx Technologies Inc (c) 2001-2012 Andrew Aksyonoff, http://sphinxsearch.com/ PHP License, version 3.0: Pear (c) 2001-2004 PHP Group, http://pear.php.net