From f817e20f287a21e2dddfba9721f5e02dce162d29 Mon Sep 17 00:00:00 2001 From: Nils Adermann Date: Fri, 15 Jul 2011 11:57:53 -0400 Subject: [PATCH] [feature/migrations] Basic migrations with schema and data changes The migrator takes care of applying migrations as necessary. RFC: http://area51.phpbb.com/phpBB/viewtopic.php?f=84&t=41337 PHPBB3-9737 --- phpBB/develop/create_schema_files.php | 14 ++ phpBB/includes/constants.php | 1 + phpBB/includes/db/migration.php | 79 ++++++++ phpBB/includes/db/migrator.php | 227 ++++++++++++++++++++++ phpBB/install/schemas/firebird_schema.sql | 12 ++ phpBB/install/schemas/mssql_schema.sql | 17 ++ phpBB/install/schemas/mysql_40_schema.sql | 12 ++ phpBB/install/schemas/mysql_41_schema.sql | 12 ++ phpBB/install/schemas/oracle_schema.sql | 15 ++ phpBB/install/schemas/postgres_schema.sql | 14 ++ phpBB/install/schemas/sqlite_schema.sql | 12 ++ tests/dbal/fixtures/migrator.xml | 27 +++ tests/dbal/migration/dummy.php | 26 +++ tests/dbal/migration/unfulfillable.php | 26 +++ tests/dbal/migrator_test.php | 80 ++++++++ 15 files changed, 574 insertions(+) create mode 100644 phpBB/includes/db/migration.php create mode 100644 phpBB/includes/db/migrator.php create mode 100644 tests/dbal/fixtures/migrator.xml create mode 100644 tests/dbal/migration/dummy.php create mode 100644 tests/dbal/migration/unfulfillable.php create mode 100644 tests/dbal/migrator_test.php diff --git a/phpBB/develop/create_schema_files.php b/phpBB/develop/create_schema_files.php index 3d3e478032..82fbb183c1 100644 --- a/phpBB/develop/create_schema_files.php +++ b/phpBB/develop/create_schema_files.php @@ -1258,6 +1258,20 @@ function get_schema_struct() ), ); + $schema_data['phpbb_migrations'] = array( + 'COLUMNS' => array( + 'migration_name' => array('VCHAR', ''), + 'migration_schema_done' => array('BOOL', 0), + 'migration_data_done' => array('BOOL', 0), + 'migration_data_state' => array('TEXT', ''), + 'migration_start_time' => array('TIMESTAMP', 0), + 'migration_end_time' => array('TIMESTAMP', 0), + ), + 'KEYS' => array( + 'migration_name' => array('UNIQUE', 'migration_name'), + ), + ); + $schema_data['phpbb_moderator_cache'] = array( 'COLUMNS' => array( 'forum_id' => array('UINT', 0), diff --git a/phpBB/includes/constants.php b/phpBB/includes/constants.php index 68af41ab20..68c96a2759 100644 --- a/phpBB/includes/constants.php +++ b/phpBB/includes/constants.php @@ -237,6 +237,7 @@ define('ICONS_TABLE', $table_prefix . 'icons'); define('LANG_TABLE', $table_prefix . 'lang'); define('LOG_TABLE', $table_prefix . 'log'); define('LOGIN_ATTEMPT_TABLE', $table_prefix . 'login_attempts'); +define('MIGRATIONS_TABLE', $table_prefix . 'migrations'); define('MODERATOR_CACHE_TABLE', $table_prefix . 'moderator_cache'); define('MODULES_TABLE', $table_prefix . 'modules'); define('POLL_OPTIONS_TABLE', $table_prefix . 'poll_options'); diff --git a/phpBB/includes/db/migration.php b/phpBB/includes/db/migration.php new file mode 100644 index 0000000000..f96fcb9568 --- /dev/null +++ b/phpBB/includes/db/migration.php @@ -0,0 +1,79 @@ +db = &$db; + $this->db_tools = &$db_tools; + } + + /** + * Defines other migrationsto be applied first (abstract method) + * + * @return array An array of migration class names + */ + function depends_on() + { + return array(); + } + + /** + * Updates the database schema + * + * @return null + */ + function update_schema() + { + } + + /** + * Updates data + * + * @return null + */ + function update_data() + { + } + + /** + * Adds a column to a database table + */ + function db_column_add($table_name, $column_name, $column_data) + { + $this->db_tools->sql_column_add($table_name, $column_name, $column_data); + } +} diff --git a/phpBB/includes/db/migrator.php b/phpBB/includes/db/migrator.php new file mode 100644 index 0000000000..d5d938ca28 --- /dev/null +++ b/phpBB/includes/db/migrator.php @@ -0,0 +1,227 @@ +db = &$db; + $this->db_tools = &$db_tools; + $this->migrations_table = $migrations_table; + $this->migrations = array(); + + $this->load_migration_state(); + } + + /** + * Loads all migrations and their application state from the database. + * + * @return null + */ + function load_migration_state() + { + $sql = "SELECT * + FROM " . $this->migrations_table; + $result = $this->db->sql_query($sql); + + $this->migration_state = array(); + while ($migration = $this->db->sql_fetchrow($result)) + { + $this->migration_state[$migration['migration_name']] = $migration; + } + + $this->db->sql_freeresult($result); + } + + /** + * Sets the list of available migration class names to the given array. + * + * @param array $class_names An array of migration class names + * @return null + */ + function set_migrations($class_names) + { + $this->migrations = $class_names; + } + + /** + * Runs a single update step from the next migration to be applied. + * + * The update step can either be a schema or a (partial) data update. To + * check if update() needs to be called again use the finished() method. + * + * @return null + */ + function update() + { + foreach ($this->migrations as $name) + { + if (!isset($this->migration_state[$name]) || + !$this->migration_state[$name]['migration_schema_done'] || + !$this->migration_state[$name]['migration_data_done']) + { + if (!$this->try_apply($name)) + { + continue; + } + else + { + return; + } + } + } + } + + /** + * Attempts to apply a step of the given migration or one of its dependencies + * + * @param string The class name of the migration + * @return bool Whether any update step was successfully run + */ + function try_apply($name) + { + if (!class_exists($name)) + { + return false; + } + + $migration =& new $name($this->db, $this->db_tools); + $state = (isset($this->migration_state[$name])) ? + $this->migration_state[$name] : + array( + 'migration_schema_done' => false, + 'migration_data_done' => false, + 'migration_data_state' => '', + ); + + $depends = $migration->depends_on(); + + foreach ($depends as $depend) + { + if (!isset($this->migration_state[$depend]) || + !$this->migration_state[$depend]['migration_schema_done'] || + !$this->migration_state[$depend]['migration_data_done']) + { + return $this->try_apply($depend); + } + } + + if (!$state['migration_schema_done']) + { + $migration->update_schema(); + $state['migration_schema_done'] = true; + } + else + { + $migration->update_data(); + $state['migration_data_done'] = true; + } + + $sql = 'UPDATE ' . $this->migrations_table . ' + SET ' . $this->db->sql_build_array('UPDATE', $state) . " + WHERE migration_name = '" . $this->db->sql_escape($name) . "'"; + $this->db->sql_query($sql); + + $this->migration_state[$name] = $state; + + return true; + } + + /** + * Checks if a migration's dependencies can even theoretically be satisfied. + * + * @param string $name The class name of the migration + * @return bool Whether the migration cannot be fulfilled + */ + function unfulfillable($name) + { + if (isset($this->migration_state[$name])) + { + return false; + } + + if (!class_exists($name)) + { + return true; + } + + $migration =& new $name($this->db, $this->db_tools); + $depends = $migration->depends_on(); + + foreach ($depends as $depend) + { + if ($this->unfulfillable($depend)) + { + return true; + } + } + + return false; + } + + /** + * Checks whether all available, fulfillable migrations have been applied. + * + * @return bool Whether the migrations have been applied + */ + function finished() + { + foreach ($this->migrations as $name) + { + if (!isset($this->migration_state[$name])) + { + // skip unfulfillable migrations, but fulfillables mean we + // are not finished yet + if ($this->unfulfillable($name)) + { + continue; + } + return false; + } + + $migration = $this->migration_state[$name]; + + if (!$migration['migration_schema_done'] || !$migration['migration_data_done']) + { + return false; + } + } + + return true; + } +} diff --git a/phpBB/install/schemas/firebird_schema.sql b/phpBB/install/schemas/firebird_schema.sql index 767ce68b4a..43099fc33b 100644 --- a/phpBB/install/schemas/firebird_schema.sql +++ b/phpBB/install/schemas/firebird_schema.sql @@ -573,6 +573,18 @@ CREATE INDEX phpbb_login_attempts_att_for ON phpbb_login_attempts(attempt_forwar CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts(attempt_time);; CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts(user_id);; +# Table: 'phpbb_migrations' +CREATE TABLE phpbb_migrations ( + migration_name VARCHAR(255) CHARACTER SET NONE DEFAULT '' NOT NULL, + migration_schema_done INTEGER DEFAULT 0 NOT NULL, + migration_data_done INTEGER DEFAULT 0 NOT NULL, + migration_data_state BLOB SUB_TYPE TEXT CHARACTER SET NONE DEFAULT '' NOT NULL, + migration_start_time INTEGER DEFAULT 0 NOT NULL, + migration_end_time INTEGER DEFAULT 0 NOT NULL +);; + +CREATE UNIQUE INDEX phpbb_migrations_migration_name ON phpbb_migrations(migration_name);; + # Table: 'phpbb_moderator_cache' CREATE TABLE phpbb_moderator_cache ( forum_id INTEGER DEFAULT 0 NOT NULL, diff --git a/phpBB/install/schemas/mssql_schema.sql b/phpBB/install/schemas/mssql_schema.sql index 84c975942f..1174cdfa3d 100644 --- a/phpBB/install/schemas/mssql_schema.sql +++ b/phpBB/install/schemas/mssql_schema.sql @@ -696,6 +696,23 @@ CREATE INDEX [user_id] ON [phpbb_login_attempts]([user_id]) ON [PRIMARY] GO +/* + Table: 'phpbb_migrations' +*/ +CREATE TABLE [phpbb_migrations] ( + [migration_name] [varchar] (255) DEFAULT ('') NOT NULL , + [migration_schema_done] [int] DEFAULT (0) NOT NULL , + [migration_data_done] [int] DEFAULT (0) NOT NULL , + [migration_data_state] [varchar] (8000) DEFAULT ('') NOT NULL , + [migration_start_time] [int] DEFAULT (0) NOT NULL , + [migration_end_time] [int] DEFAULT (0) NOT NULL +) ON [PRIMARY] +GO + +CREATE UNIQUE INDEX [migration_name] ON [phpbb_migrations]([migration_name]) ON [PRIMARY] +GO + + /* Table: 'phpbb_moderator_cache' */ diff --git a/phpBB/install/schemas/mysql_40_schema.sql b/phpBB/install/schemas/mysql_40_schema.sql index 8aab949103..d645c3384d 100644 --- a/phpBB/install/schemas/mysql_40_schema.sql +++ b/phpBB/install/schemas/mysql_40_schema.sql @@ -397,6 +397,18 @@ CREATE TABLE phpbb_login_attempts ( ); +# Table: 'phpbb_migrations' +CREATE TABLE phpbb_migrations ( + migration_name varbinary(255) DEFAULT '' NOT NULL, + migration_schema_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL, + migration_data_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL, + migration_data_state blob NOT NULL, + migration_start_time int(11) UNSIGNED DEFAULT '0' NOT NULL, + migration_end_time int(11) UNSIGNED DEFAULT '0' NOT NULL, + UNIQUE migration_name (migration_name) +); + + # Table: 'phpbb_moderator_cache' CREATE TABLE phpbb_moderator_cache ( forum_id mediumint(8) UNSIGNED DEFAULT '0' NOT NULL, diff --git a/phpBB/install/schemas/mysql_41_schema.sql b/phpBB/install/schemas/mysql_41_schema.sql index 04aef2844a..d10f2dc55a 100644 --- a/phpBB/install/schemas/mysql_41_schema.sql +++ b/phpBB/install/schemas/mysql_41_schema.sql @@ -397,6 +397,18 @@ CREATE TABLE phpbb_login_attempts ( ) CHARACTER SET `utf8` COLLATE `utf8_bin`; +# Table: 'phpbb_migrations' +CREATE TABLE phpbb_migrations ( + migration_name varchar(255) DEFAULT '' NOT NULL, + migration_schema_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL, + migration_data_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL, + migration_data_state text NOT NULL, + migration_start_time int(11) UNSIGNED DEFAULT '0' NOT NULL, + migration_end_time int(11) UNSIGNED DEFAULT '0' NOT NULL, + UNIQUE migration_name (migration_name) +) CHARACTER SET `utf8` COLLATE `utf8_bin`; + + # Table: 'phpbb_moderator_cache' CREATE TABLE phpbb_moderator_cache ( forum_id mediumint(8) UNSIGNED DEFAULT '0' NOT NULL, diff --git a/phpBB/install/schemas/oracle_schema.sql b/phpBB/install/schemas/oracle_schema.sql index 91f906bc8b..9993caeb7a 100644 --- a/phpBB/install/schemas/oracle_schema.sql +++ b/phpBB/install/schemas/oracle_schema.sql @@ -780,6 +780,21 @@ CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts (attempt_time CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts (user_id) / +/* + Table: 'phpbb_migrations' +*/ +CREATE TABLE phpbb_migrations ( + migration_name varchar2(255) DEFAULT '' , + migration_schema_done number(1) DEFAULT '0' NOT NULL, + migration_data_done number(1) DEFAULT '0' NOT NULL, + migration_data_state clob DEFAULT '' , + migration_start_time number(11) DEFAULT '0' NOT NULL, + migration_end_time number(11) DEFAULT '0' NOT NULL, + CONSTRAINT u_phpbb_migration_name UNIQUE (migration_name) +) +/ + + /* Table: 'phpbb_moderator_cache' */ diff --git a/phpBB/install/schemas/postgres_schema.sql b/phpBB/install/schemas/postgres_schema.sql index 619985e0d6..ef138dac0f 100644 --- a/phpBB/install/schemas/postgres_schema.sql +++ b/phpBB/install/schemas/postgres_schema.sql @@ -557,6 +557,20 @@ CREATE INDEX phpbb_login_attempts_att_for ON phpbb_login_attempts (attempt_forwa CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts (attempt_time); CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts (user_id); +/* + Table: 'phpbb_migrations' +*/ +CREATE TABLE phpbb_migrations ( + migration_name varchar(255) DEFAULT '' NOT NULL, + migration_schema_done INT2 DEFAULT '0' NOT NULL CHECK (migration_schema_done >= 0), + migration_data_done INT2 DEFAULT '0' NOT NULL CHECK (migration_data_done >= 0), + migration_data_state varchar(8000) DEFAULT '' NOT NULL, + migration_start_time INT4 DEFAULT '0' NOT NULL CHECK (migration_start_time >= 0), + migration_end_time INT4 DEFAULT '0' NOT NULL CHECK (migration_end_time >= 0) +); + +CREATE UNIQUE INDEX phpbb_migrations_migration_name ON phpbb_migrations (migration_name); + /* Table: 'phpbb_moderator_cache' */ diff --git a/phpBB/install/schemas/sqlite_schema.sql b/phpBB/install/schemas/sqlite_schema.sql index 1690a7dcab..309297779e 100644 --- a/phpBB/install/schemas/sqlite_schema.sql +++ b/phpBB/install/schemas/sqlite_schema.sql @@ -385,6 +385,18 @@ CREATE INDEX phpbb_login_attempts_att_for ON phpbb_login_attempts (attempt_forwa CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts (attempt_time); CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts (user_id); +# Table: 'phpbb_migrations' +CREATE TABLE phpbb_migrations ( + migration_name varchar(255) NOT NULL DEFAULT '', + migration_schema_done INTEGER UNSIGNED NOT NULL DEFAULT '0', + migration_data_done INTEGER UNSIGNED NOT NULL DEFAULT '0', + migration_data_state text(65535) NOT NULL DEFAULT '', + migration_start_time INTEGER UNSIGNED NOT NULL DEFAULT '0', + migration_end_time INTEGER UNSIGNED NOT NULL DEFAULT '0' +); + +CREATE UNIQUE INDEX phpbb_migrations_migration_name ON phpbb_migrations (migration_name); + # Table: 'phpbb_moderator_cache' CREATE TABLE phpbb_moderator_cache ( forum_id INTEGER UNSIGNED NOT NULL DEFAULT '0', diff --git a/tests/dbal/fixtures/migrator.xml b/tests/dbal/fixtures/migrator.xml new file mode 100644 index 0000000000..1f9079c811 --- /dev/null +++ b/tests/dbal/fixtures/migrator.xml @@ -0,0 +1,27 @@ + + + + migration_name + migration_schema_done + migration_data_done + migration_data_state + migration_start_time + migration_end_time + + installed_migration + 1 + 1 + + 1234 + 5678 + +
+ + config_name + config_value + + foo + bar + +
+ diff --git a/tests/dbal/migration/dummy.php b/tests/dbal/migration/dummy.php new file mode 100644 index 0000000000..b286d44f77 --- /dev/null +++ b/tests/dbal/migration/dummy.php @@ -0,0 +1,26 @@ +db_column_add('phpbb_config', 'extra_column', array('UINT', 0)); + } + + function update_data() + { + $this->db->sql_query('UPDATE phpbb_config SET extra_column = 1'); + } +} diff --git a/tests/dbal/migration/unfulfillable.php b/tests/dbal/migration/unfulfillable.php new file mode 100644 index 0000000000..84136ffe6d --- /dev/null +++ b/tests/dbal/migration/unfulfillable.php @@ -0,0 +1,26 @@ +createXMLDataSet(dirname(__FILE__).'/fixtures/migrator.xml'); + } + + public function setup() + { + parent::setup(); + + $this->db = $this->new_dbal(); + $this->db_tools = new phpbb_db_tools($this->db); + $this->migrator = new phpbb_db_migrator($this->db, $this->db_tools, MIGRATIONS_TABLE); + } + + public function test_update() + { + $this->migrator->set_migrations(array('phpbb_dbal_migration_dummy')); + + // schema + $this->migrator->update(); + $this->assertFalse($this->migrator->finished()); + + // data + $this->migrator->update(); + $this->assertTrue($this->migrator->finished()); + + $this->assertSqlResultEquals( + array(array('extra_column' => '1')), + "SELECT extra_column FROM phpbb_config WHERE config_name = 'foo'", + 'Dummy migration created extra_column with value 1 in all rows.' + ); + + // cleanup + $this->db_tools->sql_column_remove('phpbb_config', 'extra_column'); + } + + public function test_unfulfillable() + { + $this->migrator->set_migrations(array('phpbb_dbal_migration_unfulfillable', 'phpbb_dbal_migration_dummy')); + + while (!$this->migrator->finished()) + { + $this->migrator->update(); + } + + $this->assertTrue($this->migrator->finished()); + + $this->assertSqlResultEquals( + array(array('extra_column' => '1')), + "SELECT extra_column FROM phpbb_config WHERE config_name = 'foo'", + 'Dummy migration was run, even though an unfulfillable migration was found.' + ); + + // cleanup + $this->db_tools->sql_column_remove('phpbb_config', 'extra_column'); + } +}