From f1a2558cfe8b0ead453dcb556440719462c1af40 Mon Sep 17 00:00:00 2001 From: rubencm Date: Sun, 28 Mar 2021 06:18:14 +0200 Subject: [PATCH] [ticket/16738] Multiple changes Add post helper to get last post id Launch exception if an action is in progress Launch an exception if the action is already done When listing search backend in cli, if the active backend is not indexed an error is launched Add state to commands Fix small error in sphinx search backend PHPBB3-16738 --- phpBB/config/default/container/services.yml | 1 + .../default/container/services_console.yml | 2 + .../default/container/services_post.yml | 5 ++ phpBB/language/en/cli.php | 4 ++ .../console/command/searchindex/create.php | 65 ++++++++++++++++--- .../console/command/searchindex/delete.php | 65 ++++++++++++++++--- .../console/command/searchindex/list_all.php | 11 +++- phpBB/phpbb/post/post_helper.php | 44 +++++++++++++ phpBB/phpbb/search/backend/base.php | 12 ++++ phpBB/phpbb/search/backend/fulltext_mysql.php | 12 ++++ .../phpbb/search/backend/fulltext_native.php | 6 ++ .../search/backend/fulltext_postgres.php | 12 ++++ .../phpbb/search/backend/fulltext_sphinx.php | 42 +++++++----- .../exception/index_created_exception.php | 20 ++++++ .../exception/index_empty_exception.php | 20 ++++++ 15 files changed, 282 insertions(+), 39 deletions(-) create mode 100644 phpBB/config/default/container/services_post.yml create mode 100644 phpBB/phpbb/post/post_helper.php create mode 100644 phpBB/phpbb/search/exception/index_created_exception.php create mode 100644 phpBB/phpbb/search/exception/index_empty_exception.php diff --git a/phpBB/config/default/container/services.yml b/phpBB/config/default/container/services.yml index 73260f7a92..547053631b 100644 --- a/phpBB/config/default/container/services.yml +++ b/phpBB/config/default/container/services.yml @@ -23,6 +23,7 @@ imports: - { resource: services_notification.yml } - { resource: services_password.yml } - { resource: services_php.yml } + - { resource: services_post.yml } - { resource: services_profilefield.yml } - { resource: services_report.yml } - { resource: services_routing.yml } diff --git a/phpBB/config/default/container/services_console.yml b/phpBB/config/default/container/services_console.yml index 422a99992f..a7e3f7b136 100644 --- a/phpBB/config/default/container/services_console.yml +++ b/phpBB/config/default/container/services_console.yml @@ -262,6 +262,7 @@ services: - '@config' - '@language' - '@log' + - '@post.helper' - '@search.backend_factory' - '@user' tags: @@ -273,6 +274,7 @@ services: - '@config' - '@language' - '@log' + - '@post.helper' - '@search.backend_factory' - '@user' tags: diff --git a/phpBB/config/default/container/services_post.yml b/phpBB/config/default/container/services_post.yml new file mode 100644 index 0000000000..f6ad5ba80f --- /dev/null +++ b/phpBB/config/default/container/services_post.yml @@ -0,0 +1,5 @@ +services: + post.helper: + class: phpbb\post\post_helper + arguments: + - '@dbal.conn' diff --git a/phpBB/language/en/cli.php b/phpBB/language/en/cli.php index 168d7f36e1..82c1c3d62a 100644 --- a/phpBB/language/en/cli.php +++ b/phpBB/language/en/cli.php @@ -152,6 +152,10 @@ $lang = array_merge($lang, array( 'CLI_SEARCHINDEX_CREATE_FAILURE' => 'Error creating search index', 'CLI_SEARCHINDEX_DELETE_SUCCESS' => 'Search index deleted successfully', 'CLI_SEARCHINDEX_DELETE_FAILURE' => 'Error deleting search index', + 'CLI_SEARCHINDEX_ALREADY_CREATED' => 'Search index is already created, try removing it first', + 'CLI_SEARCHINDEX_NO_CREATED' => 'Search index is already empty, try creating it first', + 'CLI_SEARCHINDEX_ACTION_IN_PROGRESS' => 'There is an action currently in progress. CLI doesn\'t support incomplete index/delete actions, please solve it from the ACP', + 'CLI_SEARCHINDEX_ACTIVE_NOT_INDEXED' => 'Active search backend isn\'t indexed', // In all the case %1$s is the logical name of the file and %2$s the real name on the filesystem // eg: big_image.png (2_a51529ae7932008cf8454a95af84cacd) generated. diff --git a/phpBB/phpbb/console/command/searchindex/create.php b/phpBB/phpbb/console/command/searchindex/create.php index e77f833ffb..9985373d20 100644 --- a/phpBB/phpbb/console/command/searchindex/create.php +++ b/phpBB/phpbb/console/command/searchindex/create.php @@ -17,6 +17,8 @@ use phpbb\config\config; use phpbb\console\command\command; use phpbb\language\language; use phpbb\log\log; +use phpbb\post\post_helper; +use phpbb\search\exception\index_created_exception; use phpbb\search\exception\no_search_backend_found_exception; use phpbb\search\search_backend_factory; use phpbb\user; @@ -27,6 +29,10 @@ use Symfony\Component\Console\Style\SymfonyStyle; class create extends command { + protected const STATE_SEARCH_TYPE = 0; + protected const STATE_ACTION = 1; + protected const STATE_POST_COUNTER = 2; + /** @var config */ protected $config; @@ -36,23 +42,28 @@ class create extends command /** @var log */ protected $log; + /** @var post_helper */ + protected $post_helper; + /** @var search_backend_factory */ protected $search_backend_factory; /** * Construct method * - * @param config $config - * @param language $language - * @param log $log - * @param search_backend_factory $search_backend_factory - * @param user $user + * @param config $config + * @param language $language + * @param log $log + * @param post_helper $post_helper + * @param search_backend_factory $search_backend_factory + * @param user $user */ - public function __construct(config $config, language $language, log $log, search_backend_factory $search_backend_factory, user $user) + public function __construct(config $config, language $language, log $log, post_helper $post_helper, search_backend_factory $search_backend_factory, user $user) { $this->config = $config; $this->language = $language; $this->log = $log; + $this->post_helper = $post_helper; $this->search_backend_factory = $search_backend_factory; parent::__construct($user); @@ -61,7 +72,7 @@ class create extends command /** * Sets the command name and description * - * @return null + * @return void */ protected function configure() { @@ -105,15 +116,31 @@ class create extends command return command::FAILURE; } + if (!empty($this->config['search_indexing_state'])) + { + var_dump($this->config['search_indexing_state']); + $io->error($this->language->lang('CLI_SEARCHINDEX_ACTION_IN_PROGRESS', $search_backend)); + return command::FAILURE; + } + try { - $progress = $this->create_progress_bar(1, $io, $output, true); + $progress = $this->create_progress_bar($this->post_helper->get_max_post_id(), $io, $output, true); $progress->setMessage(''); $progress->start(); - $counter = 0; + $state = [ + self::STATE_SEARCH_TYPE => $search->get_type(), + self::STATE_ACTION => 'create', + self::STATE_POST_COUNTER => 0 + ]; + $this->save_state($state); + + $counter = &$state[self::STATE_POST_COUNTER]; while (($status = $search->create_index($counter)) !== null) { + $this->save_state($state); + $progress->setMaxSteps($status['max_post_id']); $progress->setProgress($status['post_counter']); $progress->setMessage(round($status['rows_per_second'], 2) . ' rows/s'); @@ -123,15 +150,35 @@ class create extends command $io->newLine(2); } + catch(index_created_exception $e) + { + $this->save_state([]); + $io->error($this->language->lang('CLI_SEARCHINDEX_ALREADY_CREATED', $name)); + return command::FAILURE; + } catch (\Exception $e) { $io->error($this->language->lang('CLI_SEARCHINDEX_CREATE_FAILURE', $name)); return command::FAILURE; } + $search->tidy(); + + $this->save_state([]); + $this->log->add('admin', ANONYMOUS, '', 'LOG_SEARCH_INDEX_CREATED', false, array($name)); $io->success($this->language->lang('CLI_SEARCHINDEX_CREATE_SUCCESS', $name)); return command::SUCCESS; } + + /** + * @param array $state + */ + private function save_state(array $state = []): void + { + ksort($state); + + $this->config->set('search_indexing_state', implode(',', $state), true); + } } diff --git a/phpBB/phpbb/console/command/searchindex/delete.php b/phpBB/phpbb/console/command/searchindex/delete.php index 45176f785a..ec101ac084 100644 --- a/phpBB/phpbb/console/command/searchindex/delete.php +++ b/phpBB/phpbb/console/command/searchindex/delete.php @@ -17,6 +17,8 @@ use phpbb\config\config; use phpbb\console\command\command; use phpbb\language\language; use phpbb\log\log; +use phpbb\post\post_helper; +use phpbb\search\exception\index_empty_exception; use phpbb\search\exception\no_search_backend_found_exception; use phpbb\search\search_backend_factory; use phpbb\user; @@ -27,6 +29,10 @@ use Symfony\Component\Console\Style\SymfonyStyle; class delete extends command { + protected const STATE_SEARCH_TYPE = 0; + protected const STATE_ACTION = 1; + protected const STATE_POST_COUNTER = 2; + /** @var config */ protected $config; @@ -36,23 +42,28 @@ class delete extends command /** @var log */ protected $log; + /** @var post_helper */ + protected $post_helper; + /** @var search_backend_factory */ protected $search_backend_factory; /** * Construct method * - * @param config $config - * @param language $language - * @param log $log - * @param search_backend_factory $search_backend_factory - * @param user $user + * @param config $config + * @param language $language + * @param log $log + * @param post_helper $post_helper + * @param search_backend_factory $search_backend_factory + * @param user $user */ - public function __construct(config $config, language $language, log $log, search_backend_factory $search_backend_factory, user $user) + public function __construct(config $config, language $language, log $log, post_helper $post_helper, search_backend_factory $search_backend_factory, user $user) { $this->config = $config; $this->language = $language; $this->log = $log; + $this->post_helper = $post_helper; $this->search_backend_factory = $search_backend_factory; parent::__construct($user); @@ -61,7 +72,7 @@ class delete extends command /** * Sets the command name and description * - * @return null + * @return void */ protected function configure() { @@ -105,16 +116,30 @@ class delete extends command return command::FAILURE; } + if (!empty($this->config['search_indexing_state'])) + { + $io->error($this->language->lang('CLI_SEARCHINDEX_ACTION_IN_PROGRESS', $search_backend)); + return command::FAILURE; + } + try { - // TODO: Read the max_post_id from db because the bucle is not always executed - $progress = $this->create_progress_bar(1, $io, $output, true); + $progress = $this->create_progress_bar($this->post_helper->get_max_post_id(), $io, $output, true); $progress->setMessage(''); $progress->start(); - $counter = 0; + $state = [ + self::STATE_SEARCH_TYPE => $search->get_type(), + self::STATE_ACTION => 'delete', + self::STATE_POST_COUNTER => 0 + ]; + $this->save_state($state); + + $counter = &$state[self::STATE_POST_COUNTER]; while (($status = $search->delete_index($counter)) !== null) { + $this->save_state($state); + $progress->setMaxSteps($status['max_post_id']); $progress->setProgress($status['post_counter']); $progress->setMessage(round($status['rows_per_second'], 2) . ' rows/s'); @@ -124,15 +149,35 @@ class delete extends command $io->newLine(2); } + catch(index_empty_exception $e) + { + $this->save_state([]); + $io->error($this->language->lang('CLI_SEARCHINDEX_NO_CREATED', $name)); + return command::FAILURE; + } catch (\Exception $e) { $io->error($this->language->lang('CLI_SEARCHINDEX_DELETE_FAILURE', $name)); return command::FAILURE; } + $search->tidy(); + + $this->save_state([]); + $this->log->add('admin', ANONYMOUS, '', 'LOG_SEARCH_INDEX_REMOVED', false, array($name)); $io->success($this->language->lang('CLI_SEARCHINDEX_DELETE_SUCCESS', $name)); return command::SUCCESS; } + + /** + * @param array $state + */ + private function save_state(array $state = []): void + { + ksort($state); + + $this->config->set('search_indexing_state', implode(',', $state), true); + } } diff --git a/phpBB/phpbb/console/command/searchindex/list_all.php b/phpBB/phpbb/console/command/searchindex/list_all.php index 3db054f389..cfa26e7893 100644 --- a/phpBB/phpbb/console/command/searchindex/list_all.php +++ b/phpBB/phpbb/console/command/searchindex/list_all.php @@ -53,7 +53,7 @@ class list_all extends command /** * Sets the command name and description * - * @return null + * @return void */ protected function configure() { @@ -80,9 +80,14 @@ class list_all extends command $search_backends = []; foreach ($this->search_backend_collection as $search_backend) { - $name = get_class($search_backend); - $active = ($name == $this->config['search_type']) ? '(' . $this->language->lang('ACTIVE') . ') ' : ''; + $name = $search_backend->get_type(); + $active = ($name === $this->config['search_type']) ? '(' . $this->language->lang('ACTIVE') . ') ' : ''; $search_backends[] = '' . $name . ' ' . $active . $search_backend->get_name(); + + if ($name === $this->config['search_type'] && !$search_backend->index_created()) + { + $io->error($this->language->lang('CLI_SEARCHINDEX_ACTIVE_NOT_INDEXED')); + } } $io->listing($search_backends); diff --git a/phpBB/phpbb/post/post_helper.php b/phpBB/phpbb/post/post_helper.php new file mode 100644 index 0000000000..f039771e10 --- /dev/null +++ b/phpBB/phpbb/post/post_helper.php @@ -0,0 +1,44 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbb\post; + + +use phpbb\db\driver\driver_interface; + +class post_helper +{ + /** + * @var driver_interface + */ + protected $db; + + public function __construct(driver_interface $db) + { + $this->db = $db; + } + + /** + * Get last post id + */ + public function get_max_post_id(): int + { + $sql = 'SELECT MAX(post_id) as max_post_id + FROM '. POSTS_TABLE; + $result = $this->db->sql_query($sql); + $max_post_id = (int) $this->db->sql_fetchfield('max_post_id'); + $this->db->sql_freeresult($result); + + return $max_post_id; + } +} diff --git a/phpBB/phpbb/search/backend/base.php b/phpBB/phpbb/search/backend/base.php index 3a0049947d..e1b0f142a2 100644 --- a/phpBB/phpbb/search/backend/base.php +++ b/phpBB/phpbb/search/backend/base.php @@ -16,6 +16,8 @@ namespace phpbb\search\backend; use phpbb\cache\service; use phpbb\config\config; use phpbb\db\driver\driver_interface; +use phpbb\search\exception\index_created_exception; +use phpbb\search\exception\index_empty_exception; use phpbb\user; /** @@ -323,6 +325,11 @@ abstract class base implements search_backend_interface */ public function create_index(int &$post_counter = 0): ?array { + if ($this->index_created()) + { + throw new index_created_exception(); + } + $max_post_id = $this->get_max_post_id(); $forums_indexing_enabled = $this->forum_ids_with_indexing_enabled(); @@ -385,6 +392,11 @@ abstract class base implements search_backend_interface */ public function delete_index(int &$post_counter = null): ?array { + if (!$this->index_created()) + { + throw new index_empty_exception(); + } + $max_post_id = $this->get_max_post_id(); $starttime = microtime(true); diff --git a/phpBB/phpbb/search/backend/fulltext_mysql.php b/phpBB/phpbb/search/backend/fulltext_mysql.php index 3e47cb4668..febaee3495 100644 --- a/phpBB/phpbb/search/backend/fulltext_mysql.php +++ b/phpBB/phpbb/search/backend/fulltext_mysql.php @@ -17,6 +17,8 @@ use phpbb\config\config; use phpbb\db\driver\driver_interface; use phpbb\event\dispatcher_interface; use phpbb\language\language; +use phpbb\search\exception\index_created_exception; +use phpbb\search\exception\index_empty_exception; use phpbb\user; use RuntimeException; @@ -913,6 +915,11 @@ class fulltext_mysql extends base implements search_backend_interface */ public function create_index(int &$post_counter = 0): ?array { + if ($this->index_created()) + { + throw new index_created_exception(); + } + // Make sure we can actually use MySQL with fulltext indexes if ($error = $this->init()) { @@ -985,6 +992,11 @@ class fulltext_mysql extends base implements search_backend_interface */ public function delete_index(int &$post_counter = null): ?array { + if (!$this->index_created()) + { + throw new index_empty_exception(); + } + // Make sure we can actually use MySQL with fulltext indexes if ($error = $this->init()) { diff --git a/phpBB/phpbb/search/backend/fulltext_native.php b/phpBB/phpbb/search/backend/fulltext_native.php index a5c9bb8202..f4d141dd59 100644 --- a/phpBB/phpbb/search/backend/fulltext_native.php +++ b/phpBB/phpbb/search/backend/fulltext_native.php @@ -17,6 +17,7 @@ use phpbb\config\config; use phpbb\db\driver\driver_interface; use phpbb\event\dispatcher_interface; use phpbb\language\language; +use phpbb\search\exception\index_empty_exception; use phpbb\user; /** @@ -1598,6 +1599,11 @@ class fulltext_native extends base implements search_backend_interface */ public function delete_index(int &$post_counter = null): ?array { + if (!$this->index_created()) + { + throw new index_empty_exception(); + } + $sql_queries = []; switch ($this->db->get_sql_layer()) diff --git a/phpBB/phpbb/search/backend/fulltext_postgres.php b/phpBB/phpbb/search/backend/fulltext_postgres.php index bd2c24224c..077eee4f30 100644 --- a/phpBB/phpbb/search/backend/fulltext_postgres.php +++ b/phpBB/phpbb/search/backend/fulltext_postgres.php @@ -17,6 +17,8 @@ use phpbb\config\config; use phpbb\db\driver\driver_interface; use phpbb\event\dispatcher_interface; use phpbb\language\language; +use phpbb\search\exception\index_created_exception; +use phpbb\search\exception\index_empty_exception; use phpbb\user; use RuntimeException; @@ -868,6 +870,11 @@ class fulltext_postgres extends base implements search_backend_interface */ public function create_index(int &$post_counter = 0): ?array { + if ($this->index_created()) + { + throw new index_created_exception(); + } + // Make sure we can actually use PostgreSQL with fulltext indexes if ($error = $this->init()) { @@ -927,6 +934,11 @@ class fulltext_postgres extends base implements search_backend_interface */ public function delete_index(int &$post_counter = null): ?array { + if (!$this->index_created()) + { + throw new index_empty_exception(); + } + // Make sure we can actually use PostgreSQL with fulltext indexes if ($error = $this->init()) { diff --git a/phpBB/phpbb/search/backend/fulltext_sphinx.php b/phpBB/phpbb/search/backend/fulltext_sphinx.php index 709690b4f2..74ff23bbf8 100644 --- a/phpBB/phpbb/search/backend/fulltext_sphinx.php +++ b/phpBB/phpbb/search/backend/fulltext_sphinx.php @@ -20,6 +20,7 @@ use phpbb\db\tools\tools_interface; use phpbb\event\dispatcher_interface; use phpbb\language\language; use phpbb\log\log; +use phpbb\search\exception\index_empty_exception; use phpbb\user; /** @@ -633,23 +634,28 @@ class fulltext_sphinx implements search_backend_interface { if (!$this->index_created()) { - $table_data = array( - 'COLUMNS' => array( - 'counter_id' => array('UINT', 0), - 'max_doc_id' => array('UINT', 0), - ), - 'PRIMARY_KEY' => 'counter_id', - ); - $this->db_tools->sql_create_table(SPHINX_TABLE, $table_data); - - $data = array( - 'counter_id' => '1', - 'max_doc_id' => '0', - ); - $sql = 'INSERT INTO ' . SPHINX_TABLE . ' ' . $this->db->sql_build_array('INSERT', $data); - $this->db->sql_query($sql); + throw new index_empty_exception(); } + $table_data = array( + 'COLUMNS' => array( + 'counter_id' => array('UINT', 0), + 'max_doc_id' => array('UINT', 0), + ), + 'PRIMARY_KEY' => 'counter_id', + ); + $this->db_tools->sql_create_table(SPHINX_TABLE, $table_data); + + $sql = 'TRUNCATE TABLE ' . SPHINX_TABLE; + $this->db->sql_query($sql); + + $data = array( + 'counter_id' => '1', + 'max_doc_id' => '0', + ); + $sql = 'INSERT INTO ' . SPHINX_TABLE . ' ' . $this->db->sql_build_array('INSERT', $data); + $this->db->sql_query($sql); + return null; } @@ -658,11 +664,13 @@ class fulltext_sphinx implements search_backend_interface */ public function delete_index(int &$post_counter = null): ?array { - if ($this->index_created()) + if (!$this->index_created()) { - $this->db_tools->sql_table_drop(SPHINX_TABLE); + throw new index_empty_exception(); } + $this->db_tools->sql_table_drop(SPHINX_TABLE); + return null; } diff --git a/phpBB/phpbb/search/exception/index_created_exception.php b/phpBB/phpbb/search/exception/index_created_exception.php new file mode 100644 index 0000000000..9be91bfbb8 --- /dev/null +++ b/phpBB/phpbb/search/exception/index_created_exception.php @@ -0,0 +1,20 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbb\search\exception; + + +class index_created_exception extends search_exception +{ + +} diff --git a/phpBB/phpbb/search/exception/index_empty_exception.php b/phpBB/phpbb/search/exception/index_empty_exception.php new file mode 100644 index 0000000000..a6a55698de --- /dev/null +++ b/phpBB/phpbb/search/exception/index_empty_exception.php @@ -0,0 +1,20 @@ + + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbb\search\exception; + + +class index_empty_exception extends search_exception +{ + +}