From ee89ede59faab3809cc61b7773a99f3e24376fb8 Mon Sep 17 00:00:00 2001 From: Nathan Date: Wed, 2 Oct 2013 14:23:01 -0500 Subject: [PATCH 1/6] [ticket/11881] Timezone migration can take a long time PHPBB3-11881 --- phpBB/phpbb/db/migration/data/v310/timezone.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index dd0c6a2093..8443f7b492 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -39,16 +39,22 @@ class timezone extends \phpbb\db\migration\migration ); } - public function update_timezones() + public function update_timezones($start) { + $start = (int) $start; + $limit = 1; + $converted_timezones = 0; + // Update user timezones $sql = 'SELECT user_dst, user_timezone FROM ' . $this->table_prefix . 'users GROUP BY user_timezone, user_dst'; - $result = $this->db->sql_query($sql); + $result = $this->db->sql_query_limit($sql, $limit, $start); while ($row = $this->db->sql_fetchrow($result)) { + $converted_timezones++; + $sql = 'UPDATE ' . $this->table_prefix . "users SET user_timezone = '" . $this->db->sql_escape($this->convert_phpbb30_timezone($row['user_timezone'], $row['user_dst'])) . "' WHERE user_timezone = '" . $this->db->sql_escape($row['user_timezone']) . "' @@ -57,6 +63,12 @@ class timezone extends \phpbb\db\migration\migration } $this->db->sql_freeresult($result); + if ($converted_timezones == $limit) + { + // There are still more to convert + return $start + $limit; + } + // Update board default timezone $sql = 'UPDATE ' . $this->table_prefix . "config SET config_value = '" . $this->convert_phpbb30_timezone($this->config['board_timezone'], $this->config['board_dst']) . "' From 9653276ebcd1a5c24302cb8d06f905f90e158e45 Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Fri, 4 Oct 2013 12:10:27 -0500 Subject: [PATCH 2/6] [ticket/11881] Better split the timezone conversion into chunks; add test PHPBB3-11881 --- .../phpbb/db/migration/data/v310/timezone.php | 34 ++++--- tests/dbal/convert_timezones.php | 94 +++++++++++++++++++ tests/dbal/fixtures/convert_timezones.xml | 80 ++++++++++++++++ 3 files changed, 195 insertions(+), 13 deletions(-) create mode 100644 tests/dbal/convert_timezones.php create mode 100644 tests/dbal/fixtures/convert_timezones.xml diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index 8443f7b492..13e2a5ceb6 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -42,28 +42,36 @@ class timezone extends \phpbb\db\migration\migration public function update_timezones($start) { $start = (int) $start; - $limit = 1; - $converted_timezones = 0; + $limit = 5000; + $converted = 0; - // Update user timezones - $sql = 'SELECT user_dst, user_timezone + $update_blocks = array(); + + $sql = 'SELECT user_id, user_timezone, user_dst FROM ' . $this->table_prefix . 'users - GROUP BY user_timezone, user_dst'; + ORDER BY user_id ASC'; $result = $this->db->sql_query_limit($sql, $limit, $start); - while ($row = $this->db->sql_fetchrow($result)) { - $converted_timezones++; + $converted++; - $sql = 'UPDATE ' . $this->table_prefix . "users - SET user_timezone = '" . $this->db->sql_escape($this->convert_phpbb30_timezone($row['user_timezone'], $row['user_dst'])) . "' - WHERE user_timezone = '" . $this->db->sql_escape($row['user_timezone']) . "' - AND user_dst = " . (int) $row['user_dst']; - $this->sql_query($sql); + $update_blocks[$row['user_timezone'] . ':' . $row['user_dst']][] = (int) $row['user_id']; } $this->db->sql_freeresult($result); - if ($converted_timezones == $limit) + // Update blocks of users who share the same timezone/dst + foreach ($update_blocks as $timezone => $user_ids) + { + $timezone = explode(':', $timezone); + $converted_timezone = $this->convert_phpbb30_timezone($timezone[0], $timezone[1]); + + $sql = 'UPDATE ' . $this->table_prefix . "users + SET user_timezone = '" . $this->db->sql_escape($converted_timezone) . "' + WHERE " . $this->db->sql_in_set('user_id', $user_ids); + $this->sql_query($sql); + } + + if ($converted == $limit) { // There are still more to convert return $start + $limit; diff --git a/tests/dbal/convert_timezones.php b/tests/dbal/convert_timezones.php new file mode 100644 index 0000000000..718b28b9ff --- /dev/null +++ b/tests/dbal/convert_timezones.php @@ -0,0 +1,94 @@ +db = $this->new_dbal(); + $db_tools = new \phpbb\db\tools($this->db); + + // user_dst doesn't exist anymore, must re-add it to test this + $db_tools->sql_column_add('phpbb_users', 'user_dst', array('BOOL', 1)); + + return $this->createXMLDataSet(dirname(__FILE__) . '/fixtures/convert_timezones.xml'); + } + + public function revert_schema() + { + return array( + 'drop_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst', + ), + ), + ); + } + + public function update_schema() + { + return array( + 'add_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst' => array('BOOL', 0), + ), + ), + ); + } + + protected function setUp() + { + parent::setUp(); + + global $phpbb_root_path, $phpEx; + + $this->db = $this->new_dbal(); + + $this->migration = new \phpbb\db\migration\data\v310\timezone( + new \phpbb\config\config(array()), + $this->db, + new \phpbb\db\tools($this->db), + $phpbb_root_path, + $phpEx, + 'phpbb_' + ); + } + + protected $expected_results = array( + //user_id => user_timezone + 1 => 'Etc/GMT+12', + 2 => 'Etc/GMT+11', + 3 => 'Etc/GMT-3', + 4 => 'Etc/GMT-4', + 5 => 'America/St_Johns', + 6 => 'Australia/Eucla', + ); + + public function test_convert() + { + $this->migration->update_timezones(0); + + $sql = 'SELECT user_id, user_timezone + FROM phpbb_users + ORDER BY user_id ASC'; + $result = $this->db->sql_query($sql); + while ($row = $this->db->sql_fetchrow($result)) + { + $this->assertEquals($this->expected_results[$row['user_id']], $row['user_timezone']); + } + $this->db->sql_freeresult($result); + + $db_tools = new \phpbb\db\tools($this->db); + + // Remove the user_dst field again + $db_tools->sql_column_remove('phpbb_users', 'user_dst'); + } +} diff --git a/tests/dbal/fixtures/convert_timezones.xml b/tests/dbal/fixtures/convert_timezones.xml new file mode 100644 index 0000000000..ce941d8b74 --- /dev/null +++ b/tests/dbal/fixtures/convert_timezones.xml @@ -0,0 +1,80 @@ + + + + user_id + username + username_clean + user_permissions + user_sig + user_occ + user_interests + user_timezone + user_dst + + 1 + 1 + 1 + + + + + -12 + 0 + + + 2 + 2 + 2 + + + + + -12 + 1 + + + 3 + 3 + 3 + + + + + 3 + 0 + + + 4 + 4 + 4 + + + + + 3 + 1 + + + 5 + 5 + 5 + + + + + -3.5 + 0 + + + 6 + 6 + 6 + + + + + 8.75 + 0 + +
+ From cd7d29b90edcf292464beaf3ab814750703f3f17 Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Fri, 4 Oct 2013 15:04:08 -0500 Subject: [PATCH 3/6] [ticket/11881] Make sure user_timezone isn't converted twice PHPBB3-11881 --- phpBB/phpbb/db/migration/data/v310/timezone.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index 13e2a5ceb6..6106e179cb 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -55,7 +55,12 @@ class timezone extends \phpbb\db\migration\migration { $converted++; - $update_blocks[$row['user_timezone'] . ':' . $row['user_dst']][] = (int) $row['user_id']; + // In case this is somehow run twice on a row. + // Otherwise it would just end up as UTC on the second run + if (is_numeric($row['user_timezone'])) + { + $update_blocks[$row['user_timezone'] . ':' . $row['user_dst']][] = (int) $row['user_id']; + } } $this->db->sql_freeresult($result); From efe130bcfc84577294069641621f93f0d9801d99 Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Fri, 4 Oct 2013 15:04:30 -0500 Subject: [PATCH 4/6] [ticket/11881] Limit to 500 PHPBB3-11881 --- phpBB/phpbb/db/migration/data/v310/timezone.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index 6106e179cb..61f8dc488e 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -42,7 +42,7 @@ class timezone extends \phpbb\db\migration\migration public function update_timezones($start) { $start = (int) $start; - $limit = 5000; + $limit = 500; $converted = 0; $update_blocks = array(); From c6491c9078c4dac365227550c7daa5f8eab01aad Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Mon, 30 Dec 2013 14:27:13 -0600 Subject: [PATCH 5/6] [ticket/11881] Fix test filename PHPBB3-11881 --- tests/dbal/{convert_timezones.php => convert_timezones_test.php} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/dbal/{convert_timezones.php => convert_timezones_test.php} (100%) diff --git a/tests/dbal/convert_timezones.php b/tests/dbal/convert_timezones_test.php similarity index 100% rename from tests/dbal/convert_timezones.php rename to tests/dbal/convert_timezones_test.php From 46c5ab64d4bafb04fe062cca89b2d872567f53c1 Mon Sep 17 00:00:00 2001 From: Nils Adermann Date: Fri, 2 May 2014 14:37:54 +0200 Subject: [PATCH 6/6] [ticket/11881] Move convert_timezones_test to migrator PHPBB3-11881 --- tests/{dbal => migrator}/convert_timezones_test.php | 2 +- tests/{dbal => migrator}/fixtures/convert_timezones.xml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{dbal => migrator}/convert_timezones_test.php (96%) rename tests/{dbal => migrator}/fixtures/convert_timezones.xml (100%) diff --git a/tests/dbal/convert_timezones_test.php b/tests/migrator/convert_timezones_test.php similarity index 96% rename from tests/dbal/convert_timezones_test.php rename to tests/migrator/convert_timezones_test.php index 718b28b9ff..a1eed1dbdf 100644 --- a/tests/dbal/convert_timezones_test.php +++ b/tests/migrator/convert_timezones_test.php @@ -7,7 +7,7 @@ * */ -class phpbb_dbal_convert_timezones_test extends phpbb_database_test_case +class phpbb_migrator_convert_timezones_test extends phpbb_database_test_case { protected $notifications, $db, $container, $user, $config, $auth, $cache; diff --git a/tests/dbal/fixtures/convert_timezones.xml b/tests/migrator/fixtures/convert_timezones.xml similarity index 100% rename from tests/dbal/fixtures/convert_timezones.xml rename to tests/migrator/fixtures/convert_timezones.xml