diff --git a/phpBB/language/en/cli.php b/phpBB/language/en/cli.php index b2e512fd72..dfee8d147e 100644 --- a/phpBB/language/en/cli.php +++ b/phpBB/language/en/cli.php @@ -75,6 +75,9 @@ $lang = array_merge($lang, array( 'CLI_DESCRIPTION_REPARSER_REPARSE' => 'Reparses stored text with the current text_formatter services.', 'CLI_DESCRIPTION_REPARSER_REPARSE_ARG_1' => 'Type of text to reparse. Leave blank to reparse everything.', 'CLI_DESCRIPTION_REPARSER_REPARSE_OPT_DRY_RUN' => 'Do not save any changes; just print what would happen', + 'CLI_DESCRIPTION_REPARSER_REPARSE_OPT_FILTER_CALLBACK' => 'PHP callback that accepts a single array argument for the record and returns a boolean to indicate whether the record must be reparsed, e.g.: "my\\ext\\reparser::filter"', + 'CLI_DESCRIPTION_REPARSER_REPARSE_OPT_FILTER_TEXT_LIKE' => 'SQL LIKE predicate applied on the text, e.g.: " 'PCRE regexp that matches against the text, e.g.: "/youtube/i"', 'CLI_DESCRIPTION_REPARSER_REPARSE_OPT_RANGE_MIN' => 'Lowest record ID to process', 'CLI_DESCRIPTION_REPARSER_REPARSE_OPT_RANGE_MAX' => 'Highest record ID to process', 'CLI_DESCRIPTION_REPARSER_REPARSE_OPT_RANGE_SIZE' => 'Approximate number of records to process at a time', diff --git a/phpBB/phpbb/console/command/reparser/reparse.php b/phpBB/phpbb/console/command/reparser/reparse.php index 1e66a22a73..ffdceddd9d 100644 --- a/phpBB/phpbb/console/command/reparser/reparse.php +++ b/phpBB/phpbb/console/command/reparser/reparse.php @@ -93,6 +93,24 @@ class reparse extends \phpbb\console\command\command InputOption::VALUE_NONE, $this->user->lang('CLI_DESCRIPTION_REPARSER_REPARSE_OPT_DRY_RUN') ) + ->addOption( + 'filter-callback', + null, + InputOption::VALUE_OPTIONAL, + $this->user->lang('CLI_DESCRIPTION_REPARSER_REPARSE_OPT_FILTER_CALLBACK') + ) + ->addOption( + 'filter-text-like', + null, + InputOption::VALUE_OPTIONAL, + $this->user->lang('CLI_DESCRIPTION_REPARSER_REPARSE_OPT_FILTER_TEXT_LIKE') + ) + ->addOption( + 'filter-text-regexp', + null, + InputOption::VALUE_OPTIONAL, + $this->user->lang('CLI_DESCRIPTION_REPARSER_REPARSE_OPT_FILTER_TEXT_REGEXP') + ) ->addOption( 'resume', null, @@ -161,6 +179,29 @@ class reparse extends \phpbb\console\command\command return symfony_command::SUCCESS; } + /** + * Return the record filter set for this command + * + * @see \phpbb\textreparser\reparser_interface::reparse_range() + * + * @return array + */ + protected function get_filter(): array + { + $filter = []; + $filter_options = ['filter-callback', 'filter-text-like', 'filter-text-regexp']; + foreach ($filter_options as $filter_option) + { + $value = $this->get_option($filter_option); + if ($value !== null) + { + $filter[$filter_option] = $value; + } + } + + return $filter; + } + /** * Get an option value, adjusted for given reparser * @@ -221,6 +262,9 @@ class reparse extends \phpbb\console\command\command $progress->setMessage($this->user->lang('CLI_REPARSER_REPARSE_REPARSING_START', $reparser->get_name())); $progress->start(); + // Initialize the record filter + $filter = $this->get_filter(); + // Start from $max and decrement $current by $size until we reach $min $current = $max; while ($current >= $min) @@ -229,12 +273,22 @@ class reparse extends \phpbb\console\command\command $end = max($min, $current); $progress->setMessage($this->user->lang('CLI_REPARSER_REPARSE_REPARSING', $reparser->get_name(), $start, $end)); - $reparser->reparse_range($start, $end); + + $range = ['range-min' => $start, 'range-max' => $end]; + $reparser->reparse($filter + $range); $current = $start - 1; $progress->setProgress($max + 1 - $start); - $this->reparser_manager->update_resume_data($name, $min, $current, $size, !$this->input->getOption('dry-run')); + $this->reparser_manager->update_resume_data( + $name, + $filter + [ + 'range-min' => $min, + 'range-max' => $current, + 'range-size' => $size + ], + !$this->input->getOption('dry-run') + ); } $progress->finish(); diff --git a/phpBB/phpbb/cron/task/text_reparser/reparser.php b/phpBB/phpbb/cron/task/text_reparser/reparser.php index edfdbb4af6..02e151a94e 100644 --- a/phpBB/phpbb/cron/task/text_reparser/reparser.php +++ b/phpBB/phpbb/cron/task/text_reparser/reparser.php @@ -151,7 +151,11 @@ class reparser extends \phpbb\cron\task\base $reparser->reparse_range($start, $end); - $this->reparser_manager->update_resume_data($this->reparser_name, $min, $start - 1, $size); + $this->resume_data['range-min'] = $min; + $this->resume_data['range-max'] = $start - 1; + $this->resume_data['range-size'] = $size; + + $this->reparser_manager->update_resume_data($this->reparser_name, $this->resume_data); } $this->config->set($this->reparser_name . '_last_cron', time()); diff --git a/phpBB/phpbb/textreparser/base.php b/phpBB/phpbb/textreparser/base.php index 84209a4127..310f04cfa7 100644 --- a/phpBB/phpbb/textreparser/base.php +++ b/phpBB/phpbb/textreparser/base.php @@ -31,13 +31,17 @@ abstract class base implements reparser_interface abstract public function get_max_id(); /** - * Return all records in given range + * Return all records that match given criteria * - * @param integer $min_id Lower bound - * @param integer $max_id Upper bound - * @return array Array of records + * The concrete implementation does not have to handle filter-callback or filter-text-regexp + * which are already handled in reparse() via record_matches_filter() + * + * @see reparser_interface::reparse() + * + * @param array $config Criteria used to select records + * @return array Array of records */ - abstract protected function get_records_by_range($min_id, $max_id); + abstract protected function get_records(array $config): array; /** * Save record @@ -219,12 +223,46 @@ abstract class base implements reparser_interface /** * {@inheritdoc} */ + public function reparse(array $config = []): void + { + foreach ($this->get_records($config) as $record) + { + if ($this->record_matches_filter($record, $config)) + { + $this->reparse_record($record); + } + } + } + + /** + * {@inheritdoc} + * + * @deprecated 4.0.0 + */ public function reparse_range($min_id, $max_id) { - foreach ($this->get_records_by_range($min_id, $max_id) as $record) + $this->reparse(['range-min' => $min_id, 'range-max' => $max_id]); + } + + /** + * Test whether a record matches given filter + * + * @param array $record + * @param array $config + * @return bool + */ + protected function record_matches_filter(array $record, array $config): bool + { + if (isset($config['filter-text-regexp']) && !preg_match($config['filter-text-regexp'], $record['text'])) { - $this->reparse_record($record); + return false; } + if (isset($config['filter-callback']) && !$config['filter-callback']($record)) + { + return false; + } + + return true; } /** diff --git a/phpBB/phpbb/textreparser/manager.php b/phpBB/phpbb/textreparser/manager.php index 9a7663f938..888f205397 100644 --- a/phpBB/phpbb/textreparser/manager.php +++ b/phpBB/phpbb/textreparser/manager.php @@ -70,13 +70,23 @@ class manager /** * Updates the resume data in the database * + * Resume data must contain the following elements: + * - range-min: lowest record ID + * - range-max: current record ID + * - range-size: number of records to process at a time + * + * Resume data may contain the following elements: + * - filter-callback: a callback that accepts a record as argument and returns a boolean + * - filter-text-like: a SQL LIKE predicate applied on the text, if applicable, e.g. 'resume_data === null) @@ -84,11 +94,7 @@ class manager $this->get_resume_data(''); } - $this->resume_data[$name] = array( - 'range-min' => $min, - 'range-max' => $current, - 'range-size' => $size, - ); + $this->resume_data[$name] = $data; if ($update_db) { diff --git a/phpBB/phpbb/textreparser/plugins/contact_admin_info.php b/phpBB/phpbb/textreparser/plugins/contact_admin_info.php index 8910f2256b..df2d939427 100644 --- a/phpBB/phpbb/textreparser/plugins/contact_admin_info.php +++ b/phpBB/phpbb/textreparser/plugins/contact_admin_info.php @@ -41,7 +41,7 @@ class contact_admin_info extends \phpbb\textreparser\base /** * {@inheritdoc} */ - protected function get_records_by_range($min_id, $max_id) + protected function get_records(array $config): array { $values = $this->config_text->get_array(array( 'contact_admin_info', diff --git a/phpBB/phpbb/textreparser/plugins/poll_option.php b/phpBB/phpbb/textreparser/plugins/poll_option.php index a3b17b3703..d9bb9c978b 100644 --- a/phpBB/phpbb/textreparser/plugins/poll_option.php +++ b/phpBB/phpbb/textreparser/plugins/poll_option.php @@ -29,14 +29,19 @@ class poll_option extends \phpbb\textreparser\row_based_plugin /** * {@inheritdoc} */ - protected function get_records_by_range_query($min_id, $max_id) + protected function get_records_sql(array $config): string { $sql = 'SELECT o.topic_id, o.poll_option_id, o.poll_option_text AS text, p.enable_bbcode, p.enable_smilies, p.enable_magic_url, p.bbcode_uid FROM ' . POLL_OPTIONS_TABLE . ' o, ' . TOPICS_TABLE . ' t, ' . POSTS_TABLE . ' p - WHERE o.topic_id BETWEEN ' . $min_id . ' AND ' . $max_id .' - AND t.topic_id = o.topic_id + WHERE t.topic_id = o.topic_id AND p.post_id = t.topic_first_post_id'; + $where = $this->get_where_clauses($config, 'o.topic_id', 'o.poll_option_text'); + if (!empty($where)) + { + $sql .= "\nAND " . implode("\nAND ", $where); + } + return $sql; } diff --git a/phpBB/phpbb/textreparser/plugins/poll_title.php b/phpBB/phpbb/textreparser/plugins/poll_title.php index 5ca8bb063b..147a6fcddc 100644 --- a/phpBB/phpbb/textreparser/plugins/poll_title.php +++ b/phpBB/phpbb/textreparser/plugins/poll_title.php @@ -29,14 +29,19 @@ class poll_title extends \phpbb\textreparser\row_based_plugin /** * {@inheritdoc} */ - protected function get_records_by_range_query($min_id, $max_id) + protected function get_records_sql(array $config): string { $sql = 'SELECT t.topic_id AS id, t.poll_title AS text, p.enable_bbcode, p.enable_smilies, p.enable_magic_url, p.bbcode_uid FROM ' . TOPICS_TABLE . ' t, ' . POSTS_TABLE . ' p - WHERE t.topic_id BETWEEN ' . $min_id . ' AND ' . $max_id .' - AND t.poll_start > 0 + WHERE t.poll_start > 0 AND p.post_id = t.topic_first_post_id'; + $where = $this->get_where_clauses($config, 't.topic_id', 't.poll_title'); + if (!empty($where)) + { + $sql .= "\nAND " . implode("\nAND ", $where); + } + return $sql; } } diff --git a/phpBB/phpbb/textreparser/reparser_interface.php b/phpBB/phpbb/textreparser/reparser_interface.php index 912de10058..8b116cdd6a 100644 --- a/phpBB/phpbb/textreparser/reparser_interface.php +++ b/phpBB/phpbb/textreparser/reparser_interface.php @@ -36,9 +36,28 @@ interface reparser_interface */ public function set_name($name); + /** + * Reparse all records that match given criteria + * + * Available criteria passed as $config: + * - filter-callback: a callback that accepts a record as argument and returns a boolean + * - filter-text-like: a SQL LIKE predicate applied on the text, if applicable, e.g. 'get_records_by_range_query($min_id, $max_id); - $result = $this->db->sql_query($sql); + $sql = $this->get_records_sql($config); + $result = $this->db->sql_query($sql); $records = $this->db->sql_fetchrowset($result); $this->db->sql_freeresult($result); @@ -73,13 +73,12 @@ abstract class row_based_plugin extends base } /** - * Generate the query that retrieves all records for given range + * Generate the query that retrieves records that match given criteria * - * @param integer $min_id Lower bound - * @param integer $max_id Upper bound - * @return string SQL query + * @param array $config Criteria used to select records + * @return string SQL query */ - protected function get_records_by_range_query($min_id, $max_id) + protected function get_records_sql(array $config): string { $columns = $this->get_columns(); $fields = array(); @@ -95,13 +94,43 @@ abstract class row_based_plugin extends base } } - $sql = 'SELECT ' . implode(', ', $fields) . ' - FROM ' . $this->table . ' - WHERE ' . $columns['id'] . ' BETWEEN ' . $min_id . ' AND ' . $max_id; + $sql = 'SELECT ' . implode(', ', $fields) . ' FROM ' . $this->table; + $where = $this->get_where_clauses($config, $columns['id'], $columns['text']); + if (!empty($where)) + { + $sql .= ' WHERE ' . implode("\nAND ", $where); + } return $sql; } + /** + * Generate WHERE clauses for given set of criteria + * + * @param array $config + * @param string $column_id Name for the id column, including its table alias + * @param string $column_text Name for the text column, including its table alias + * @return array Potentially empty list of SQL clauses + */ + protected function get_where_clauses(array $config, string $column_id, string $column_text): array + { + $where = []; + if (isset($config['range-min'])) + { + $where[] = $column_id . ' >= ' . $config['range-min']; + } + if (isset($config['range-max'])) + { + $where[] = $column_id . ' <= ' . $config['range-max']; + } + if (isset($config['filter-text-like'])) + { + $where[] = $column_text . ' ' . $this->db->sql_like_expression(str_replace('%', $this->db->get_any_char(), $config['filter-text-like'])); + } + + return $where; + } + /** * {@inheritdoc} */ diff --git a/tests/text_reparser/base_test.php b/tests/text_reparser/base_test.php index edf39e7f10..ec0396fac4 100644 --- a/tests/text_reparser/base_test.php +++ b/tests/text_reparser/base_test.php @@ -81,4 +81,83 @@ class phpbb_textreparser_base_test extends phpbb_database_test_case $this->get_rows([2]) ); } + + public function test_reparse_filter_like() + { + $this->get_reparser()->reparse([ + 'range-min' => 3, + 'range-max' => 4, + 'filter-text-like' => '%foo123%' + ]); + + $this->assertEquals( + [ + [ + 'id' => '3', + 'text' => '[b]foo123[/b]' + ], + [ + 'id' => '4', + 'text' => '[b]bar456[/b]' + ] + ], + $this->get_rows([3, 4]) + ); + } + + public function test_reparse_filter_regexp() + { + $this->get_reparser()->reparse([ + 'range-min' => 3, + 'range-max' => 4, + 'filter-text-regexp' => '(bar456)' + ]); + + $this->assertEquals( + [ + [ + 'id' => '4', + 'text' => '[b]bar456[/b]' + ], + [ + 'id' => '5', + 'text' => '[b]baz789[/b]' + ] + ], + $this->get_rows([4, 5]) + ); + } + + public function test_reparse_filter_callback() + { + $record = [ + 'id' => '5', + 'enable_bbcode' => '1', + 'enable_smilies' => '1', + 'enable_magic_url' => '1', + 'text' => '[b]baz789[/b]', + 'bbcode_uid' => '' + ]; + + $mock = $this->getMockBuilder('stdClass')->setMethods(['foo'])->getMock(); + $mock->expects($this->once()) + ->method('foo') + ->with($record) + ->will($this->returnValue(false)); + + $this->get_reparser()->reparse([ + 'range-min' => 5, + 'range-max' => 5, + 'filter-callback' => [$mock, 'foo'] + ]); + $this->assertEquals( + [ + [ + 'id' => '5', + 'text' => '[b]baz789[/b]' + ] + ], + $this->get_rows([5, 5]) + ); + } } diff --git a/tests/text_reparser/fixtures/base.xml b/tests/text_reparser/fixtures/base.xml index 532a19a8a9..7464639fab 100644 --- a/tests/text_reparser/fixtures/base.xml +++ b/tests/text_reparser/fixtures/base.xml @@ -23,5 +23,29 @@ [IMG]img.png[/IMG]]]> + + 3 + 1 + 1 + 1 + [b]foo123[/b] + + + + 4 + 1 + 1 + 1 + [b]bar456[/b] + + + + 5 + 1 + 1 + 1 + [b]baz789[/b] + + diff --git a/tests/text_reparser/manager_test.php b/tests/text_reparser/manager_test.php index 98882385a2..746570d233 100644 --- a/tests/text_reparser/manager_test.php +++ b/tests/text_reparser/manager_test.php @@ -76,10 +76,25 @@ class phpbb_text_reparser_manager_test extends phpbb_database_test_case ); $this->config_text->set('reparser_resume', serialize($resume_data)); - $this->reparser_manager->update_resume_data('another_reparser', 5, 20, 10, false); + $this->reparser_manager->update_resume_data( + 'another_reparser', + [ + 'range-min' => 5, + 'range-max' => 20, + 'range-size' => 10, + ], + false + ); $this->assert_array_content_equals($resume_data, unserialize($this->config_text->get('reparser_resume'))); - $this->reparser_manager->update_resume_data('test_reparser', 0, 50, 50); + $this->reparser_manager->update_resume_data( + 'test_reparser', + [ + 'range-min' => 0, + 'range-max' => 50, + 'range-size' => 50, + ] + ); $resume_data = array( 'test_reparser' => array( 'range-min' => 0, diff --git a/tests/text_reparser/plugins/fixtures/poll_options.xml b/tests/text_reparser/plugins/fixtures/poll_options.xml index 48ba024315..8f5cbaa3b5 100644 --- a/tests/text_reparser/plugins/fixtures/poll_options.xml +++ b/tests/text_reparser/plugins/fixtures/poll_options.xml @@ -44,6 +44,16 @@ 13 http://example.org]]> + + 1 + 100 + Matches LIKE foo123 + + + 2 + 100 + Does not match LIKE + 1 123 @@ -124,6 +134,11 @@ 13 Magic URLs + + 100 + 1 + Topic #100 + 123 1 diff --git a/tests/text_reparser/plugins/fixtures/polls.xml b/tests/text_reparser/plugins/fixtures/polls.xml index 5247fb906d..40356918d8 100644 --- a/tests/text_reparser/plugins/fixtures/polls.xml +++ b/tests/text_reparser/plugins/fixtures/polls.xml @@ -99,6 +99,18 @@ 1 + + 100 + 1 + Matches LIKE foo123 + 1 + + + 101 + 1 + Does not match LIKE + 1 + 1000 1 diff --git a/tests/text_reparser/plugins/poll_option_test.php b/tests/text_reparser/plugins/poll_option_test.php index 6c7fd98431..702acb7d6e 100644 --- a/tests/text_reparser/plugins/poll_option_test.php +++ b/tests/text_reparser/plugins/poll_option_test.php @@ -27,10 +27,11 @@ class phpbb_textreparser_poll_option_test extends phpbb_database_test_case return new \phpbb\textreparser\plugins\poll_option($this->db, POLL_OPTIONS_TABLE); } - protected function get_rows() + protected function get_rows(array $ids = null) { $sql = 'SELECT topic_id, poll_option_id, poll_option_text FROM ' . POLL_OPTIONS_TABLE . ' + WHERE ' . $this->db->sql_in_set('topic_id', $ids) . ' ORDER BY topic_id, poll_option_id'; $result = $this->db->sql_query($sql); $rows = $this->db->sql_fetchrowset($result); @@ -59,11 +60,11 @@ class phpbb_textreparser_poll_option_test extends phpbb_database_test_case public function test_dry_run() { - $old_rows = $this->get_rows(); + $old_rows = $this->get_rows([1]); $reparser = $this->get_reparser(); $reparser->disable_save(); $reparser->reparse_range(1, 1); - $new_rows = $this->get_rows(); + $new_rows = $this->get_rows([1]); $this->assertEquals($old_rows, $new_rows); } @@ -124,6 +125,32 @@ class phpbb_textreparser_poll_option_test extends phpbb_database_test_case 'poll_option_text' => 'This row should be [b:abcd1234]ignored[/b:abcd1234]', ), ); - $this->assertEquals($expected, $this->get_rows()); + $this->assertEquals($expected, $this->get_rows([1, 2, 11, 12, 13, 123])); + } + + + public function test_filter_like() + { + $reparser = $this->get_reparser(); + $reparser->reparse([ + 'range-min' => 100, + 'range-max' => 100, + 'filter-text-like' => '%foo123%' + ]); + + $expected = [ + [ + 'topic_id' => 100, + 'poll_option_id' => 1, + 'poll_option_text' => 'Matches LIKE foo123' + ], + [ + 'topic_id' => 100, + 'poll_option_id' => 2, + 'poll_option_text' => 'Does not match LIKE' + ], + ]; + + $this->assertEquals($expected, $this->get_rows([100, 100])); } } diff --git a/tests/text_reparser/plugins/poll_title_test.php b/tests/text_reparser/plugins/poll_title_test.php index 046b6019c8..3d5b2fd924 100644 --- a/tests/text_reparser/plugins/poll_title_test.php +++ b/tests/text_reparser/plugins/poll_title_test.php @@ -23,4 +23,27 @@ class phpbb_textreparser_poll_title_test extends phpbb_textreparser_test_row_bas { return new \phpbb\textreparser\plugins\poll_title($this->db, TOPICS_TABLE); } + + public function test_filter_like() + { + $reparser = $this->get_reparser(); + $reparser->reparse([ + 'range-min' => 100, + 'range-max' => 101, + 'filter-text-like' => '%foo123%' + ]); + + $expected = [ + [ + 'id' => '100', + 'text' => 'Matches LIKE foo123' + ], + [ + 'id' => '101', + 'text' => 'Does not match LIKE' + ] + ]; + + $this->assertEquals($expected, $this->get_rows([100, 101])); + } }