[ticket/12683] Add progress bar to create_index

PHPBB3-12683

phpBB/includes/acp/acp_main.php

@@ -652,17 +652,10 @@ class acp_main
 				$search_backend_factory = $phpbb_container->get('search.backend_factory');
 				$search = $search_backend_factory->get_active();
 			}
-			catch (RuntimeException $e)
+			catch (\phpbb\search\exception\no_search_backend_found_exception $e)
-			{
-				if (strpos($e->getMessage(), 'No service found') === 0)
 			{
 				trigger_error('NO_SUCH_SEARCH_MODULE');
 			}
-				else
-				{
-					throw $e;
-				}
-			}
 
 			if (!$search->index_created())
 			{

phpBB/includes/functions.php

@@ -206,6 +206,7 @@ function get_formatted_filesize($value, $string_only = true, $allowed_units = fa
 */
 function still_on_time($extra_time = 15)
 {
+	// TODO: Check the bug with this, it should be possible to restart the start time
 	static $max_execution_time, $start_time;
 
 	$current_time = microtime(true);

phpBB/includes/functions_admin.php

@@ -1092,17 +1092,10 @@ function delete_posts($where_type, $where_ids, $auto_sync = true, $posted_sync =
 		$search_backend_factory = $phpbb_container->get('search.backend_factory');
 		$search = $search_backend_factory->get_active();
 	}
-	catch (RuntimeException $e)
+	catch (\phpbb\search\exception\no_search_backend_found_exception $e)
-	{
-		if (strpos($e->getMessage(), 'No service found') === 0)
 	{
 		trigger_error('NO_SUCH_SEARCH_MODULE');
 	}
-		else
-		{
-			throw $e;
-		}
-	}
 
 	$search->index_remove($post_ids, $poster_ids, $forum_ids);
 

phpBB/includes/functions_posting.php

@@ -2356,17 +2356,10 @@ function submit_post($mode, $subject, $username, $topic_type, &$poll_ary, &$data
 			$search_backend_factory = $phpbb_container->get('search.backend_factory');
 			$search = $search_backend_factory->get_active();
 		}
-		catch (RuntimeException $e)
+		catch (\phpbb\search\exception\no_search_backend_found_exception $e)
-		{
-			if (strpos($e->getMessage(), 'No service found') === 0)
 		{
 			trigger_error('NO_SUCH_SEARCH_MODULE');
 		}
-			else
-			{
-				throw $e;
-			}
-		}
 
 		$search->index($mode, (int) $data_ary['post_id'], $data_ary['message'], $subject, $poster_id, (int) $data_ary['forum_id']);
 	}

phpBB/includes/mcp/mcp_main.php

@@ -1397,17 +1397,10 @@ function mcp_fork_topic($topic_ids)
 					$search_backend_factory = $phpbb_container->get('search.backend_factory');
 					$search = $search_backend_factory->get_active();
 				}
-				catch (RuntimeException $e)
+				catch (\phpbb\search\exception\no_search_backend_found_exception $e)
-				{
-					if (strpos($e->getMessage(), 'No service found') === 0)
 				{
 					trigger_error('NO_SUCH_SEARCH_MODULE');
 				}
-					else
-					{
-						throw $e;
-					}
-				}
 				$search_mode = 'post';
 			}
 			else if (!isset($search) && !$topic_row['enable_indexing'])

phpBB/includes/mcp/mcp_post.php

@@ -637,17 +637,10 @@ function change_poster(&$post_info, $userdata)
 		$search_backend_factory = $phpbb_container->get('search.backend_factory');
 		$search = $search_backend_factory->get_active();
 	}
-	catch (RuntimeException $e)
+	catch (\phpbb\search\exception\no_search_backend_found_exception $e)
-	{
-		if (strpos($e->getMessage(), 'No service found') === 0)
 	{
 		trigger_error('NO_SUCH_SEARCH_MODULE');
 	}
-		else
-		{
-			throw $e;
-		}
-	}
 
 	$search->index_remove([], [$post_info['user_id'], $userdata['user_id']], []);
 

phpBB/includes/mcp/mcp_topic.php

@@ -680,17 +680,10 @@ function split_topic($action, $topic_id, $to_forum_id, $subject)
 				$search_backend_factory = $phpbb_container->get('search.backend_factory');
 				$search = $search_backend_factory->get_active();
 			}
-			catch (RuntimeException $e)
+			catch (\phpbb\search\exception\no_search_backend_found_exception $e)
-			{
-				if (strpos($e->getMessage(), 'No service found') === 0)
 			{
 				trigger_error('NO_SUCH_SEARCH_MODULE');
 			}
-				else
-				{
-					throw $e;
-				}
-			}
 
 			$search->index('edit', (int) $first_post_data['post_id'], $first_post_data['post_text'], $subject, (int) $first_post_data['poster_id'], (int) $first_post_data['forum_id']);
 		}

phpBB/language/en/cli.php

@@ -147,6 +147,7 @@ $lang = array_merge($lang, array(
 	'CLI_REPARSER_REPARSE_SUCCESS'			=> 'Reparsing ended with success',
 
 	'CLI_SEARCHINDEX_SEARCH_BACKEND_NAME'	=> 'Backend class',
+	'CLI_SEARCHINDEX_BACKEND_NOT_FOUND'		=> 'Search module not found',
 	'CLI_SEARCHINDEX_CREATE_SUCCESS'		=> 'Search index created successfully',
 	'CLI_SEARCHINDEX_CREATE_FAILURE'		=> 'Error creating search index',
 	'CLI_SEARCHINDEX_DELETE_SUCCESS'		=> 'Search index deleted successfully',

phpBB/phpbb/console/command/searchindex/create.php

@@ -17,6 +17,7 @@ use phpbb\config\config;
 use phpbb\console\command\command;
 use phpbb\language\language;
 use phpbb\log\log;
+use phpbb\search\exception\no_search_backend_found_exception;
 use phpbb\search\search_backend_factory;
 use phpbb\user;
 use Symfony\Component\Console\Input\InputArgument;
@@ -70,7 +71,7 @@ class create extends command
 			->addArgument(
 				'search-backend',
 				InputArgument::REQUIRED,
-				$this->user->lang('CLI_SEARCHINDEX_SEARCH_BACKEND_NAME')
+				$this->language->lang('CLI_SEARCHINDEX_SEARCH_BACKEND_NAME')
 			)
 		;
 	}
@@ -90,31 +91,47 @@ class create extends command
 		$io = new SymfonyStyle($input, $output);
 
 		$search_backend = $input->getArgument('search-backend');
+
+		try
+		{
 			$search = $this->search_backend_factory->get($search_backend);
 			$name = $search->get_name();
+		}
+		catch (no_search_backend_found_exception $e)
+		{
+			$io->error($this->language->lang('CLI_SEARCHINDEX_BACKEND_NOT_FOUND', $search_backend));
+			return command::FAILURE;
+		}
 
 		try
 		{
 			$counter = 0;
+
+			$progress = $this->create_progress_bar(1, $io, $output, true);
+			$progress->start();
+
 			while (($status = $search->create_index($counter)) !== null)
 			{
-				$this->config->set('search_indexing_state', implode(',', $this->state), true);
+				$progress->setMaxSteps($status['max_post_id']);
+				$progress->setProgress($status['post_counter']);
+				$progress->setMessage(round($status['rows_per_second'], 2) . ' rows/s');
 
-				$io->success($counter);
+				$progress->advance();
-				$io->success(print_r($status, 1));
 			}
 
-			$search->tidy();
+			$progress->finish();
+
+			$io->newLine(2);
 		}
 		catch (\Exception $e)
 		{
-			$io->error($this->user->lang('CLI_SEARCHINDEX_CREATE_FAILURE', $name));
+			$io->error($this->language->lang('CLI_SEARCHINDEX_CREATE_FAILURE', $name));
-			return 1;
+			return command::FAILURE;
 		}
 
 		$this->log->add('admin', ANONYMOUS, '', 'LOG_SEARCH_INDEX_CREATED', false, array($name));
-		$io->success($this->user->lang('CLI_SEARCHINDEX_CREATE_SUCCESS', $name));
+		$io->success($this->language->lang('CLI_SEARCHINDEX_CREATE_SUCCESS', $name));
 
-		return 0;
+		return command::SUCCESS;
 	}
 }

phpBB/phpbb/console/command/searchindex/delete.php

@@ -17,6 +17,7 @@ use phpbb\config\config;
 use phpbb\console\command\command;
 use phpbb\language\language;
 use phpbb\log\log;
+use phpbb\search\exception\no_search_backend_found_exception;
 use phpbb\search\search_backend_factory;
 use phpbb\user;
 use Symfony\Component\Console\Input\InputArgument;
@@ -70,7 +71,7 @@ class delete extends command
 			->addArgument(
 				'search-backend',
 				InputArgument::REQUIRED,
-				$this->user->lang('CLI_SEARCHINDEX_SEARCH_BACKEND_NAME')
+				$this->language->lang('CLI_SEARCHINDEX_SEARCH_BACKEND_NAME')
 			)
 		;
 	}
@@ -90,40 +91,32 @@ class delete extends command
 		$io = new SymfonyStyle($input, $output);
 
 		$search_backend = $input->getArgument('search-backend');
-		$search = $this->search_backend_factory->get($search_backend);
-		$name = $search->get_name();
 
 		try
 		{
-			$this->state = explode(',', $this->config['search_indexing_state']);
+			$search = $this->search_backend_factory->get($search_backend);
-			$this->max_post_id = $this->get_max_post_id();
+			$name = $search->get_name();
+		}
+		catch (no_search_backend_found_exception $e)
+		{
+			$io->error($this->language->lang('CLI_SEARCHINDEX_BACKEND_NOT_FOUND', $search_backend));
+			return command::FAILURE;
+		}
 
+		try
+		{
 			$search->delete_index($this, '');
 			$search->tidy();
 		}
 		catch (\Exception $e)
 		{
-			$io->error($this->user->lang('CLI_SEARCHINDEX_DELETE_FAILURE', $name));
+			$io->error($this->language->lang('CLI_SEARCHINDEX_DELETE_FAILURE', $name));
-			return 1;
+			return command::FAILURE;
 		}
 
 		$this->log->add('admin', ANONYMOUS, '', 'LOG_SEARCH_INDEX_REMOVED', false, array($name));
-		$io->success($this->user->lang('CLI_SEARCHINDEX_DELETE_SUCCESS', $name));
+		$io->success($this->language->lang('CLI_SEARCHINDEX_DELETE_SUCCESS', $name));
 
-		return 0;
+		return command::SUCCESS;
-	}
-
-	function save_state($state = false)
-	{
-		global $config;
-
-		if ($state)
-		{
-			$this->state = $state;
-		}
-
-		ksort($this->state);
-
-		$config->set('search_indexing_state', implode(',', $this->state), true);
 	}
 }

phpBB/phpbb/search/backend/base.php

@@ -1,15 +1,15 @@
 <?php
 /**
-*
+ *
-* This file is part of the phpBB Forum Software package.
+ * This file is part of the phpBB Forum Software package.
-*
+ *
-* @copyright (c) phpBB Limited <https://www.phpbb.com>
+ * @copyright (c) phpBB Limited <https://www.phpbb.com>
-* @license GNU General Public License, version 2 (GPL-2.0)
+ * @license GNU General Public License, version 2 (GPL-2.0)
-*
+ *
-* For full copyright and license information, please see
+ * For full copyright and license information, please see
-* the docs/CREDITS.txt file.
+ * the docs/CREDITS.txt file.
-*
+ *
-*/
+ */
 
 namespace phpbb\search\backend;
 
@@ -19,9 +19,9 @@ use phpbb\db\driver\driver_interface;
 use phpbb\user;
 
 /**
-* optional base class for search plugins providing simple caching based on ACM
+ * optional base class for search plugins providing simple caching based on ACM
-* and functions to retrieve ignore_words and synonyms
+ * and functions to retrieve ignore_words and synonyms
-*/
+ */
 abstract class base implements search_backend_interface
 {
 	public const SEARCH_RESULT_NOT_IN_CACHE = 0;
@@ -348,7 +348,16 @@ abstract class base implements search_backend_interface
 				$row_count++;
 				$post_counter = $row['post_id'];
 			}
+
+			// With cli process only one batch each time to be able to track progress
+			if (PHP_SAPI === 'cli')
+			{
+				break;
 			}
+		}
+
+		// TODO: With cli if the previous bucle have stoped because of lack of time, launch an exception, because is an error
+		// cli commands should be executed in one step
 
 		// pretend the number of posts was as big as the number of ids we indexed so far
 		// just an estimation as it includes deleted posts
@@ -399,6 +408,12 @@ abstract class base implements search_backend_interface
 				$this->index_remove($ids, $posters, $forum_ids);
 				$post_counter = $ids[count($ids) - 1];
 			}
+
+			// With cli process only one batch each time to be able to track progress
+			if (PHP_SAPI === 'cli')
+			{
+				break;
+			}
 		}
 
 		if ($post_counter < $max_post_id)

phpBB/phpbb/search/exception/no_search_backend_found_exception.php

@@ -0,0 +1,19 @@
+<?php
+/**
+ *
+ * This file is part of the phpBB Forum Software package.
+ *
+ * @copyright (c) phpBB Limited <https://www.phpbb.com>
+ * @license GNU General Public License, version 2 (GPL-2.0)
+ *
+ * For full copyright and license information, please see
+ * the docs/CREDITS.txt file.
+ *
+ */
+
+namespace phpbb\search\exception;
+
+class no_search_backend_found_exception extends search_exception
+{
+
+}

phpBB/phpbb/search/exception/search_exception.php

@@ -0,0 +1,21 @@
+<?php
+/**
+ *
+ * This file is part of the phpBB Forum Software package.
+ *
+ * @copyright (c) phpBB Limited <https://www.phpbb.com>
+ * @license GNU General Public License, version 2 (GPL-2.0)
+ *
+ * For full copyright and license information, please see
+ * the docs/CREDITS.txt file.
+ *
+ */
+
+namespace phpbb\search\exception;
+
+use phpbb\exception\runtime_exception;
+
+class search_exception extends runtime_exception
+{
+	// TODO: Launch this exception from search instead of RuntimeException
+}

phpBB/phpbb/search/search_backend_factory.php

@@ -16,6 +16,8 @@ namespace phpbb\search;
 use phpbb\config\config;
 use phpbb\di\service_collection;
 use phpbb\search\backend\search_backend_interface;
+use phpbb\search\exception\no_search_backend_found_exception;
+use RuntimeException;
 
 class search_backend_factory
 {
@@ -46,16 +48,36 @@ class search_backend_factory
 	 *
 	 * @param string $class
 	 *
+	 * @throws no_search_backend_found_exception
+	 *
 	 * @return search_backend_interface
 	 */
 	public function get(string $class): search_backend_interface
 	{
-		return $this->search_backends->get_by_class($class);
+		try
+		{
+			$search = $this->search_backends->get_by_class($class);
+		}
+		catch (RuntimeException $e)
+		{
+			if (strpos($e->getMessage(), 'No service found') === 0)
+			{
+				throw new no_search_backend_found_exception();
+			}
+			else
+			{
+				throw $e;
+			}
+		}
+
+		return $search;
 	}
 
 	/**
 	 * Obtains active search backend
 	 *
+	 * @throws no_search_backend_found_exception
+	 *
 	 * @return search_backend_interface
 	 */
 	public function get_active(): search_backend_interface

phpBB/search.php

@@ -299,17 +299,10 @@ if ($keywords || $author || $author_id || $search_id || $submit)
 		$search_backend_factory = $phpbb_container->get('search.backend_factory');
 		$search = $search_backend_factory->get_active();
 	}
-	catch (RuntimeException $e)
+	catch (\phpbb\search\exception\no_search_backend_found_exception $e)
-	{
-		if (strpos($e->getMessage(), 'No service found') === 0)
 	{
 		trigger_error('NO_SUCH_SEARCH_MODULE');
 	}
-		else
-		{
-			throw $e;
-		}
-	}
 
 	// let the search module split up the keywords
 	if ($keywords)