makary/phpbb
1
0
Fork 0
mirror of https://github.com/phpbb/phpbb.git synced 2025-06-07 20:08:53 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #4005 from Elsensee/ticket/14257

Browse source 
[ticket/14257] Reparse after update

* Elsensee/ticket/14257:
  [ticket/14257] Add tests for reparser manager
  [ticket/14257] Fix CLI reparser and set cron interval
  [ticket/14257] Fix CLI error message
  [ticket/14257] Fix phpdoc in CLI command
  [ticket/14257] Add text_reparser manager
  [ticket/14257] Use migrations instead of cron job for some reparsers
  [ticket/14257] Fix lock acquire in CLI command
  [ticket/14257] Fix if condition
  [ticket/14257] Add reparse_lock to CLI command
  [ticket/14257] Add cron tasks for reparsing text
This commit is contained in:
Tristan Darricau 2015-12-05 11:21:40 +01:00
commit b4bbc7056a
10 changed files with 647 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
phpBB/config/default/container/services_console.yml
View file

@ -187,8 +187,9 @@ services:
class: phpbb\console\command\reparser\reparse
arguments:
- @user
- @text_reparser.lock
- @text_reparser_collection
- @config_text
- @text_reparser.manager
tags:
- { name: console.command }

70
phpBB/config/default/container/services_cron.yml
View file

@ -146,3 +146,73 @@ services:
- [set_name, [cron.task.core.tidy_warnings]]
tags:
- { name: cron.task }
cron.task.text_reparser.pm_text:
class: phpbb\cron\task\text_reparser\reparser
arguments:
- @config
- @config_text
- @text_reparser.lock
- @text_reparser.manager
- @text_reparser_collection
calls:
- [set_name, [cron.task.text_reparser.pm_text]]
- [set_reparser, [text_reparser.pm_text]]
tags:
- { name: cron.task }
cron.task.text_reparser.poll_option:
class: phpbb\cron\task\text_reparser\reparser
arguments:
- @config
- @config_text
- @text_reparser.lock
- @text_reparser.manager
- @text_reparser_collection
calls:
- [set_name, [cron.task.text_reparser.poll_option]]
- [set_reparser, [text_reparser.poll_option]]
tags:
- { name: cron.task }
cron.task.text_reparser.poll_title:
class: phpbb\cron\task\text_reparser\reparser
arguments:
- @config
- @config_text
- @text_reparser.lock
- @text_reparser.manager
- @text_reparser_collection
calls:
- [set_name, [cron.task.text_reparser.poll_title]]
- [set_reparser, [text_reparser.poll_title]]
tags:
- { name: cron.task }
cron.task.text_reparser.post_text:
class: phpbb\cron\task\text_reparser\reparser
arguments:
- @config
- @config_text
- @text_reparser.lock
- @text_reparser.manager
- @text_reparser_collection
calls:
- [set_name, [cron.task.text_reparser.post_text]]
- [set_reparser, [text_reparser.post_text]]
tags:
- { name: cron.task }
cron.task.text_reparser.user_signature:
class: phpbb\cron\task\text_reparser\reparser
arguments:
- @config
- @config_text
- @text_reparser.lock
- @text_reparser.manager
- @text_reparser_collection
calls:
- [set_name, [cron.task.text_reparser.user_signature]]
- [set_reparser, [text_reparser.user_signature]]
tags:
- { name: cron.task }

14
phpBB/config/default/container/services_text_reparser.yml
View file

@ -1,4 +1,18 @@
services:
text_reparser.manager:
class: phpbb\textreparser\manager
arguments:
- @config
- @config_text
- @text_reparser_collection
text_reparser.lock:
class: phpbb\lock\db
arguments:
- reparse_lock
- @config
- @dbal.conn
text_reparser_collection:
class: phpbb\di\service_collection
arguments:

1
phpBB/language/en/acp/common.php
View file

@ -292,6 +292,7 @@ $lang = array_merge($lang, array(
'RELEASE_ANNOUNCEMENT' => 'Announcement',
'REMIND' => 'Remind',
'REPARSE_LOCK_ERROR' => 'Reparsing is already in progress by another process.',
'RESYNC' => 'Resynchronise',
'RUNNING_TASK' => 'Running task: %s.',

99
phpBB/phpbb/console/command/reparser/reparse.php
View file

@ -13,6 +13,7 @@
namespace phpbb\console\command\reparser;
use phpbb\exception\runtime_exception;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
@ -21,11 +22,6 @@ use Symfony\Component\Console\Style\SymfonyStyle;
class reparse extends \phpbb\console\command\command
{
/**
* @var \phpbb\config\db_text
*/
protected $config_text;
/**
* @var InputInterface
*/
@ -41,13 +37,23 @@ class reparse extends \phpbb\console\command\command
*/
protected $output;
/**
* @var \phpbb\lock\db
*/
protected $reparse_lock;
/**
* @var \phpbb\textreparser\manager
*/
protected $reparser_manager;
/**
* @var \phpbb\di\service_collection
*/
protected $reparsers;
/**
* @var array Reparser names as keys, and their last $current ID as values
* @var array The reparser's last $current ID as values
*/
protected $resume_data;
@ -55,14 +61,16 @@ class reparse extends \phpbb\console\command\command
* Constructor
*
* @param \phpbb\user $user
* @param \phpbb\lock\db $reparse_lock
* @param \phpbb\textreparser\manager $reparser_manager
* @param \phpbb\di\service_collection $reparsers
* @param \phpbb\config\db_text $config_text
*/
public function __construct(\phpbb\user $user, \phpbb\di\service_collection $reparsers, \phpbb\config\db_text $config_text)
public function __construct(\phpbb\user $user, \phpbb\lock\db $reparse_lock, \phpbb\textreparser\manager $reparser_manager, \phpbb\di\service_collection $reparsers)
{
require_once __DIR__ . '/../../../../includes/functions_content.php';
$this->config_text = $config_text;
$this->reparse_lock = $reparse_lock;
$this->reparser_manager = $reparser_manager;
$this->reparsers = $reparsers;
parent::__construct($user);
}
@ -163,7 +171,11 @@ class reparse extends \phpbb\console\command\command
$this->input = $input;
$this->output = $output;
$this->io = new SymfonyStyle($input, $output);
$this->load_resume_data();
if (!$this->reparse_lock->acquire())
{
throw new runtime_exception('REPARSE_LOCK_ERROR', array(), null, 1);
}
$name = $input->getArgument('reparser-name');
if (isset($name))
@ -185,6 +197,8 @@ class reparse extends \phpbb\console\command\command
$this->io->success($this->user->lang('CLI_REPARSER_REPARSE_SUCCESS'));
$this->reparse_lock->release();
return 0;
}
@ -194,36 +208,18 @@ class reparse extends \phpbb\console\command\command
* Will use the last saved value if --resume is set and the option was not specified
* on the command line
*
* @param string $reparser_name Reparser name
* @param string $option_name Option name
* @return integer
*/
protected function get_option($reparser_name, $option_name)
protected function get_option($option_name)
{
// Return the option from the resume_data if applicable
if ($this->input->getOption('resume') && isset($this->resume_data[$reparser_name][$option_name]) && !$this->input->hasParameterOption('--' . $option_name))
if ($this->input->getOption('resume') && isset($this->resume_data[$option_name]) && !$this->input->hasParameterOption('--' . $option_name))
{
return $this->resume_data[$reparser_name][$option_name];
return $this->resume_data[$option_name];
}
$value = $this->input->getOption($option_name);
// range-max has no default value, it must be computed for each reparser
if ($option_name === 'range-max' && $value === null)
{
$value = $this->reparsers[$reparser_name]->get_max_id();
}
return $value;
}
/**
* Load the resume data from the database
*/
protected function load_resume_data()
{
$resume_data = $this->config_text->get('reparser_resume');
$this->resume_data = (empty($resume_data)) ? array() : unserialize($resume_data);
return $this->input->getOption($option_name);
}
/**
@ -234,6 +230,7 @@ class reparse extends \phpbb\console\command\command
protected function reparse($name)
{
$reparser = $this->reparsers[$name];
$this->resume_data = $this->reparser_manager->get_resume_data($name);
if ($this->input->getOption('dry-run'))
{
$reparser->disable_save();
@ -244,9 +241,15 @@ class reparse extends \phpbb\console\command\command
}
// Start at range-max if specified or at the highest ID otherwise
$max = $this->get_option($name, 'range-max');
$min = $this->get_option($name, 'range-min');
$size = $this->get_option($name, 'range-size');
$max = $this->get_option('range-max');
$min = $this->get_option('range-min');
$size = $this->get_option('range-size');
// range-max has no default value, it must be computed for each reparser
if ($max == null)
{
$max = $reparser->get_max_id();
}
if ($max < $min)
{
@ -272,34 +275,10 @@ class reparse extends \phpbb\console\command\command
$current = $start - 1;
$progress->setProgress($max + 1 - $start);
$this->update_resume_data($name, $current);
$this->reparser_manager->update_resume_data($name, $min, $current, $size, !$this->input->getOption('dry-run'));
}
$progress->finish();
$this->io->newLine(2);
}
/**
* Save the resume data to the database
*/
protected function save_resume_data()
{
$this->config_text->set('reparser_resume', serialize($this->resume_data));
}
/**
* Save the resume data to the database
*
* @param string $name Reparser name
* @param string $current Current ID
*/
protected function update_resume_data($name, $current)
{
$this->resume_data[$name] = array(
'range-min' => $this->get_option($name, 'range-min'),
'range-max' => $current,
'range-size' => $this->get_option($name, 'range-size'),
);
$this->save_resume_data();
}
}

168
phpBB/phpbb/cron/task/text_reparser/reparser.php Normal file
View file

@ -0,0 +1,168 @@
<?php
/**
*
* This file is part of the phpBB Forum Software package.
*
* @copyright (c) phpBB Limited <https://www.phpbb.com>
* @license GNU General Public License, version 2 (GPL-2.0)
*
* For full copyright and license information, please see
* the docs/CREDITS.txt file.
*
*/
namespace phpbb\cron\task\text_reparser;
/**
* Reparse text cron task
*/
class reparser extends \phpbb\cron\task\base
{
const MIN = 1;
const SIZE = 100;
/**
* @var \phpbb\config\config
*/
protected $config;
/**
* @var \phpbb\config\db_text
*/
protected $config_text;
/**
* @var \phpbb\lock\db
*/
protected $reparse_lock;
/**
* @var \phpbb\textreparser\manager
*/
protected $reparser_manager;
/**
* @var string
*/
protected $reparser_name;
/**
* @var \phpbb\di\service_collection
*/
protected $reparsers;
/**
* @var array
*/
protected $resume_data;
/**
* Constructor
*
* @param \phpbb\config\config $config
* @param \phpbb\config\db_text $config_text
* @param \phpbb\lock\db $reparse_lock
* @param \phpbb\textreparser\manager $reparser_manager
* @param \phpbb\di\service_collection $reparsers
*/
public function __construct(\phpbb\config\config $config, \phpbb\config\db_text $config_text, \phpbb\lock\db $reparse_lock, \phpbb\textreparser\manager $reparser_manager, \phpbb\di\service_collection $reparsers)
{
$this->config = $config;
$this->config_text = $config_text;
$this->reparse_lock = $reparse_lock;
$this->reparser_manager = $reparser_manager;
$this->reparsers = $reparsers;
}
/**
* Sets the reparser for this cron task
*
* @param string $reparser
*/
public function set_reparser($reparser)
{
$this->reparser_name = (!isset($this->reparsers[$reparser]) ? 'text_reparser.' : '') . $reparser;
if ($this->resume_data === null)
{
$this->reparser_manager->get_resume_data($this->reparser_name);
}
}
/**
* {@inheritdoc}
*/
public function is_runnable()
{
if ($this->resume_data === null)
{
$this->reparser_manager->get_resume_data($this->reparser_name);
}
if (empty($this->resume_data['range-max']) || $this->resume_data['range-max'] >= $this->resume_data['range-min'])
{
return true;
}
return false;
}
/**
* {@inheritdoc}
*/
public function should_run()
{
if (!empty($this->config['reparse_lock']))
{
$last_run = explode(' ', $this->config['reparse_lock']);
if ($last_run[0] + 3600 >= time())
{
return false;
}
}
if ($this->config[$this->reparser_name . '_cron_interval'])
{
return $this->config[$this->reparser_name . '_last_cron'] < time() - $this->config[$this->reparser_name . '_cron_interval'];
}
return false;
}
/**
* {@inheritdoc}
*/
public function run()
{
if ($this->reparse_lock->acquire())
{
if ($this->resume_data === null)
{
$this->resume_data = $this->reparser_manager->get_resume_data($this->reparser_name);
}
/**
* @var \phpbb\textreparser\reparser_interface $reparser
*/
$reparser = $this->reparsers[$this->reparser_name];
$min = !empty($this->resume_data['range-min']) ? $this->resume_data['range-min'] : self::MIN;
$current = !empty($this->resume_data['range-max']) ? $this->resume_data['range-max'] : $reparser->get_max_id();
$size = !empty($this->resume_data['range-size']) ? $this->resume_data['range-size'] : self::SIZE;
if ($current >= $min)
{
$start = max($min, $current + 1 - $size);
$end = max($min, $current);
$reparser->reparse_range($start, $end);
$this->reparser_manager->update_resume_data($this->reparser_name, $min, $start - 1, $size);
}
$this->config->set($this->reparser_name . '_last_cron', time());
$this->reparse_lock->release();
}
}
}

101
phpBB/phpbb/db/migration/data/v320/text_reparser.php Normal file
View file

@ -0,0 +1,101 @@
<?php
/**
*
* This file is part of the phpBB Forum Software package.
*
* @copyright (c) phpBB Limited <https://www.phpbb.com>
* @license GNU General Public License, version 2 (GPL-2.0)
*
* For full copyright and license information, please see
* the docs/CREDITS.txt file.
*
*/
namespace phpbb\db\migration\data\v320;
class text_reparser extends \phpbb\db\migration\container_aware_migration
{
static public function depends_on()
{
return array('\phpbb\db\migration\data\v310\contact_admin_form');
}
public function effectively_installed()
{
return isset($this->config['reparse_lock']);
}
public function update_data()
{
return array(
array('config.add', array('reparse_lock', 0, true)),
array('config.add', array('text_reparser.pm_text_cron_interval', 10)),
array('config.add', array('text_reparser.pm_text_last_cron', 0)),
array('config.add', array('text_reparser.poll_option_cron_interval', 10)),
array('config.add', array('text_reparser.poll_option_last_cron', 0)),
array('config.add', array('text_reparser.poll_title_cron_interval', 10)),
array('config.add', array('text_reparser.poll_title_last_cron', 0)),
array('config.add', array('text_reparser.post_text_cron_interval', 10)),
array('config.add', array('text_reparser.post_text_last_cron', 0)),
array('config.add', array('text_reparser.user_signature_cron_interval', 10)),
array('config.add', array('text_reparser.user_signature_last_cron', 0)),
array('custom', array(array($this, 'reparse'))),
);
}
public function reparse($resume_data)
{
// Somtimes a cron job is too much
$limit = 100;
$fast_reparsers = array(
'text_reparser.contact_admin_info',
'text_reparser.forum_description',
'text_reparser.forum_rules',
'text_reparser.group_description',
);
if (!is_array($resume_data))
{
$resume_data = array(
'reparser' => 0,
'current' => $this->container->get($fast_reparsers[0])->get_max_id(),
);
}
$fast_reparsers_size = sizeof($fast_reparsers);
$processed_records = 0;
while ($processed_records < $limit && $resume_data['reparser'] < $fast_reparsers_size)
{
$reparser = $this->container->get($fast_reparsers[$resume_data['reparser']]);
// New reparser
if ($resume_data['current'] === 0)
{
$resume_data['current'] = $reparser->get_max_id();
}
$start = max(1, $resume_data['current'] + 1 - ($limit - $processed_records));
$end = max(1, $resume_data['current']);
$reparser->reparse_range($start, $end);
$processed_records = $end - $start + 1;
$resume_data['current'] = $start - 1;
if ($start === 1)
{
// Prevent CLI command from running these reparsers again
$reparser_manager = $this->container->get('text_reparser.manager');
$reparser_manager->update_resume_data($fast_reparsers[$resume_data['reparser']], 1, 0, $limit);
$resume_data['reparser']++;
}
}
if ($resume_data['reparser'] === $fast_reparsers_size)
{
return true;
}
return $resume_data;
}
}

128
phpBB/phpbb/textreparser/manager.php Normal file
View file

@ -0,0 +1,128 @@
<?php
/**
*
* This file is part of the phpBB Forum Software package.
*
* @copyright (c) phpBB Limited <https://www.phpbb.com>
* @license GNU General Public License, version 2 (GPL-2.0)
*
* For full copyright and license information, please see
* the docs/CREDITS.txt file.
*
*/
namespace phpbb\textreparser;
class manager
{
/**
* @var \phpbb\config\config
*/
protected $config;
/**
* @var \phpbb\config\db_text
*/
protected $config_text;
/**
* @var \phpbb\di\service_collection
*/
protected $reparsers;
/**
* @var array
*/
protected $resume_data;
/**
* Constructor
*
* @param \phpbb\config\config $config
* @param \phpbb\config\db_text $config_text
* @param \phpbb\di\service_collection $reparsers
*/
public function __construct(\phpbb\config\config $config, \phpbb\config\db_text $config_text, \phpbb\di\service_collection $reparsers)
{
$this->config = $config;
$this->config_text = $config_text;
$this->reparsers = $reparsers;
}
/**
* Loads resume data from the database
*
* @param string $name Name of the reparser to which the resume data belongs
*
* @return array
*/
public function get_resume_data($name)
{
if ($this->resume_data === null)
{
$resume_data = $this->config_text->get('reparser_resume');
$this->resume_data = !empty($resume_data) ? unserialize($resume_data) : array();
}
return isset($this->resume_data[$name]) ? $this->resume_data[$name] : array();
}
/**
* Updates the resume data in the database
*
* @param string $name Name of the reparser to which the resume data belongs
* @param int $min Lowest record ID
* @param int $current Current record ID
* @param int $size Number of records to process at a time
* @param bool $update_db True if the resume data should be written to the database, false if not. (default: true)
*/
public function update_resume_data($name, $min, $current, $size, $update_db = true)
{
// Prevent overwriting the old, stored array
if ($this->resume_data === null)
{
$this->get_resume_data('');
}
$this->resume_data[$name] = array(
'range-min' => $min,
'range-max' => $current,
'range-size' => $size,
);
if ($update_db)
{
$this->config_text->set('reparser_resume', serialize($this->resume_data));
}
}
/**
* Sets the interval for a text_reparser cron task
*
* @param string $name Name of the reparser to schedule
* @param int $interval Interval in seconds, 0 to disable the cron task
*/
public function schedule($name, $interval)
{
if (isset($this->reparsers[$name]) && isset($this->config[$name . '_cron_interval']))
{
$this->config->set($name . '_cron_interval', $interval);
}
}
/**
* Sets the interval for all text_reparser cron tasks
*
* @param int $interval Interval in seconds, 0 to disable the cron task
*/
public function schedule_all($interval)
{
// This way we don't construct every registered reparser
$reparser_array = array_keys($this->reparsers->getArrayCopy());
foreach ($reparser_array as $reparser)
{
$this->schedule($reparser, $interval);
}
}
}

7
tests/text_reparser/fixtures/config_text.xml Normal file
View file

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8" ?>
<dataset>
<table name="phpbb_config_text">
<column>config_name</column>
<column>config_value</column>
</table>
</dataset>

117
tests/text_reparser/manager_test.php Normal file
View file

@ -0,0 +1,117 @@
<?php
/**
*
* This file is part of the phpBB Forum Software package.
*
* @copyright (c) phpBB Limited <https://www.phpbb.com>
* @license GNU General Public License, version 2 (GPL-2.0)
*
* For full copyright and license information, please see
* the docs/CREDITS.txt file.
*
*/
require_once __DIR__ . '/../mock/container_builder.php';
require_once __DIR__ . '/../test_framework/phpbb_database_test_case.php';
class phpbb_text_reparser_manager_test extends phpbb_database_test_case
{
/** @var \phpbb\config\config */
protected $config;
/** @var \phpbb\config\db_text */
protected $config_text;
/** @var \phpbb\textreparser\manager */
protected $reparser_manager;
public function getDataSet()
{
return $this->createXMLDataSet(dirname(__FILE__) . '/fixtures/config_text.xml');
}
public function setUp()
{
parent::setUp();
$this->config = new \phpbb\config\config(array(
'test_reparser_cron_interval' => 0,
'my_reparser_cron_interval' => 100,
));
$db = $this->new_dbal();
$this->config_text = new \phpbb\config\db_text($db, 'phpbb_config_text');
$service_collection = new \phpbb\di\service_collection(new phpbb_mock_container_builder());
$service_collection->add('test_reparser');
$service_collection->add('another_reparser');
$service_collection->add('my_reparser');
$this->reparser_manager = new \phpbb\textreparser\manager($this->config, $this->config_text, $service_collection);
}
public function test_get_resume_data()
{
$resume_data = array(
'test_reparser' => array(
'range-min' => 0,
'range-max' => 100,
'range-size' => 50,
),
);
$this->config_text->set('reparser_resume', serialize($resume_data));
$this->assert_array_content_equals($resume_data['test_reparser'], $this->reparser_manager->get_resume_data('test_reparser'));
$this->assertEmpty($this->reparser_manager->get_resume_data('another_reparser'));
}
public function test_update_resume_data()
{
$resume_data = array(
'test_reparser' => array(
'range-min' => 0,
'range-max' => 100,
'range-size' => 50,
),
);
$this->config_text->set('reparser_resume', serialize($resume_data));
$this->reparser_manager->update_resume_data('another_reparser', 5, 20, 10, false);
$this->assert_array_content_equals($resume_data, unserialize($this->config_text->get('reparser_resume')));
$this->reparser_manager->update_resume_data('test_reparser', 0, 50, 50);
$resume_data = array(
'test_reparser' => array(
'range-min' => 0,
'range-max' => 50,
'range-size' => 50,
),
'another_reparser' => array(
'range-min' => 5,
'range-max' => 20,
'range-size' => 10,
),
);
$this->assert_array_content_equals($resume_data, unserialize($this->config_text->get('reparser_resume')));
}
public function test_schedule()
{
$this->reparser_manager->schedule('no_reparser', 21);
$this->assertArrayNotHasKey('no_reparser_cron_interval', $this->config);
$this->reparser_manager->schedule('another_reparser', 42);
$this->assertArrayNotHasKey('another_reparser_cron_interval', $this->config);
$this->reparser_manager->schedule('test_reparser', 20);
$this->assertEquals(20, $this->config['test_reparser_cron_interval']);
}
public function test_schedule_all()
{
$this->reparser_manager->schedule_all(180);
$this->assertEquals(180, $this->config['test_reparser_cron_interval']);
$this->assertEquals(180, $this->config['my_reparser_cron_interval']);
$this->assertArrayNotHasKey('another_reparser_cron_interval', $this->config);
}
}