[feature/migrations] Basic migrations with schema and data changes

The migrator takes care of applying migrations as necessary.

RFC: http://area51.phpbb.com/phpBB/viewtopic.php?f=84&t=41337

PHPBB3-9737

phpBB/develop/create_schema_files.php

@@ -1258,6 +1258,20 @@ function get_schema_struct()
 		),
 	);
 
+	$schema_data['phpbb_migrations'] = array(
+		'COLUMNS'		=> array(
+			'migration_name'		=> array('VCHAR', ''),
+			'migration_schema_done'	=> array('BOOL', 0),
+			'migration_data_done'	=> array('BOOL', 0),
+			'migration_data_state'	=> array('TEXT', ''),
+			'migration_start_time'	=> array('TIMESTAMP', 0),
+			'migration_end_time'	=> array('TIMESTAMP', 0),
+		),
+		'KEYS'			=> array(
+			'migration_name'		=> array('UNIQUE', 'migration_name'),
+		),
+	);
+
 	$schema_data['phpbb_moderator_cache'] = array(
 		'COLUMNS'		=> array(
 			'forum_id'				=> array('UINT', 0),

phpBB/includes/constants.php

@@ -237,6 +237,7 @@ define('ICONS_TABLE',				$table_prefix . 'icons');
 define('LANG_TABLE',				$table_prefix . 'lang');
 define('LOG_TABLE',					$table_prefix . 'log');
 define('LOGIN_ATTEMPT_TABLE',		$table_prefix . 'login_attempts');
+define('MIGRATIONS_TABLE',			$table_prefix . 'migrations');
 define('MODERATOR_CACHE_TABLE',		$table_prefix . 'moderator_cache');
 define('MODULES_TABLE',				$table_prefix . 'modules');
 define('POLL_OPTIONS_TABLE',		$table_prefix . 'poll_options');

phpBB/includes/db/migration.php

@@ -0,0 +1,79 @@
+<?php
+/**
+*
+* @package db
+* @copyright (c) 2011 phpBB Group
+* @license http://opensource.org/licenses/gpl-license.php GNU Public License
+*
+*/
+
+/**
+* @ignore
+*/
+if (!defined('IN_PHPBB'))
+{
+	exit;
+}
+
+/**
+* Abstract base class for database migrations
+*
+* Each migration consists of a set of schema and data changes to be implemented
+* in a subclass. This class provides various utility methods to simplify editing
+* a phpBB.
+*
+* @package db
+*/
+class phpbb_db_migration
+{
+	var $db;
+	var $db_tools;
+
+	/**
+	* Migration constructor
+	*
+	* @param dbal			$db			Connected database abstraction instance
+	* @param phpbb_db_tools	$db_tools	Instance of db schema manipulation tools
+	*/
+	function phpbb_db_migration(&$db, &$db_tools)
+	{
+		$this->db = &$db;
+		$this->db_tools = &$db_tools;
+	}
+
+	/**
+	* Defines other migrationsto be applied first (abstract method)
+	*
+	* @return array An array of migration class names
+	*/
+	function depends_on()
+	{
+		return array();
+	}
+
+	/**
+	* Updates the database schema
+	*
+	* @return null
+	*/
+	function update_schema()
+	{
+	}
+
+	/**
+	* Updates data
+	*
+	* @return null
+	*/
+	function update_data()
+	{
+	}
+
+	/**
+	* Adds a column to a database table
+	*/
+	function db_column_add($table_name, $column_name, $column_data)
+	{
+		$this->db_tools->sql_column_add($table_name, $column_name, $column_data);
+	}
+}

phpBB/includes/db/migrator.php

@@ -0,0 +1,227 @@
+<?php
+/**
+*
+* @package db
+* @copyright (c) 2011 phpBB Group
+* @license http://opensource.org/licenses/gpl-license.php GNU Public License
+*
+*/
+
+/**
+* @ignore
+*/
+if (!defined('IN_PHPBB'))
+{
+	exit;
+}
+
+/**
+* The migrator is responsible for applying new migrations in the correct order.
+*
+* @package db
+*/
+class phpbb_db_migrator
+{
+	var $db;
+	var $db_tools;
+
+	var $migrations_table;
+	var $migration_state;
+
+	var $migrations;
+
+	/**
+	* Constructor of the database migrator
+	*
+	* @param dbal			$db			Connected database abstraction instance
+	* @param phpbb_db_tools	$db_tools	Instance of db schema manipulation tools
+	* @param string			$migrations_table The name of the db table storing
+	*									information on applied migrations
+	*/
+	function phpbb_db_migrator(&$db, &$db_tools, $migrations_table)
+	{
+		$this->db = &$db;
+		$this->db_tools = &$db_tools;
+		$this->migrations_table = $migrations_table;
+		$this->migrations = array();
+
+		$this->load_migration_state();
+	}
+
+	/**
+	* Loads all migrations and their application state from the database.
+	*
+	* @return null
+	*/
+	function load_migration_state()
+	{
+		$sql = "SELECT *
+			FROM " . $this->migrations_table;
+		$result = $this->db->sql_query($sql);
+
+		$this->migration_state = array();
+		while ($migration = $this->db->sql_fetchrow($result))
+		{
+			$this->migration_state[$migration['migration_name']] = $migration;
+		}
+
+		$this->db->sql_freeresult($result);
+	}
+
+	/**
+	* Sets the list of available migration class names to the given array.
+	*
+	* @param array $class_names An array of migration class names
+	* @return null
+	*/
+	function set_migrations($class_names)
+	{
+		$this->migrations = $class_names;
+	}
+
+	/**
+	* Runs a single update step from the next migration to be applied.
+	*
+	* The update step can either be a schema or a (partial) data update. To
+	* check if update() needs to be called again use the finished() method.
+	*
+	* @return null
+	*/
+	function update()
+	{
+		foreach ($this->migrations as $name)
+		{
+			if (!isset($this->migration_state[$name]) ||
+				!$this->migration_state[$name]['migration_schema_done'] ||
+				!$this->migration_state[$name]['migration_data_done'])
+			{
+				if (!$this->try_apply($name))
+				{
+					continue;
+				}
+				else
+				{
+					return;
+				}
+			}
+		}
+	}
+
+	/**
+	* Attempts to apply a step of the given migration or one of its dependencies
+	*
+	* @param	string	The class name of the migration
+	* @return	bool	Whether any update step was successfully run
+	*/
+	function try_apply($name)
+	{
+		if (!class_exists($name))
+		{
+			return false;
+		}
+
+		$migration =& new $name($this->db, $this->db_tools);
+		$state = (isset($this->migration_state[$name])) ?
+			$this->migration_state[$name] :
+			array(
+				'migration_schema_done' => false,
+				'migration_data_done' => false,
+				'migration_data_state' => '',
+			);
+
+		$depends = $migration->depends_on();
+
+		foreach ($depends as $depend)
+		{
+			if (!isset($this->migration_state[$depend]) ||
+				!$this->migration_state[$depend]['migration_schema_done'] ||
+				!$this->migration_state[$depend]['migration_data_done'])
+			{
+				return $this->try_apply($depend);
+			}
+		}
+
+		if (!$state['migration_schema_done'])
+		{
+			$migration->update_schema();
+			$state['migration_schema_done'] = true;
+		}
+		else
+		{
+			$migration->update_data();
+			$state['migration_data_done'] = true;
+		}
+
+		$sql = 'UPDATE ' . $this->migrations_table . '
+			SET ' . $this->db->sql_build_array('UPDATE', $state) . "
+			WHERE migration_name = '" . $this->db->sql_escape($name) . "'";
+		$this->db->sql_query($sql);
+
+		$this->migration_state[$name] = $state;
+
+		return true;
+	}
+
+	/**
+	* Checks if a migration's dependencies can even theoretically be satisfied.
+	*
+	* @param	string	$name	The class name of the migration
+	* @return	bool			Whether the migration cannot be fulfilled
+	*/
+	function unfulfillable($name)
+	{
+		if (isset($this->migration_state[$name]))
+		{
+			return false;
+		}
+
+		if (!class_exists($name))
+		{
+			return true;
+		}
+
+		$migration =& new $name($this->db, $this->db_tools);
+		$depends = $migration->depends_on();
+
+		foreach ($depends as $depend)
+		{
+			if ($this->unfulfillable($depend))
+			{
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	* Checks whether all available, fulfillable migrations have been applied.
+	*
+	* @return bool Whether the migrations have been applied
+	*/
+	function finished()
+	{
+		foreach ($this->migrations as $name)
+		{
+			if (!isset($this->migration_state[$name]))
+			{
+				// skip unfulfillable migrations, but fulfillables mean we
+				// are not finished yet
+				if ($this->unfulfillable($name))
+				{
+					continue;
+				}
+				return false;
+			}
+
+			$migration = $this->migration_state[$name];
+
+			if (!$migration['migration_schema_done'] || !$migration['migration_data_done'])
+			{
+				return false;
+			}
+		}
+
+		return true;
+	}
+}

phpBB/install/schemas/firebird_schema.sql

@@ -573,6 +573,18 @@ CREATE INDEX phpbb_login_attempts_att_for ON phpbb_login_attempts(attempt_forwar
 CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts(attempt_time);;
 CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts(user_id);;
 
+# Table: 'phpbb_migrations'
+CREATE TABLE phpbb_migrations (
+	migration_name VARCHAR(255) CHARACTER SET NONE DEFAULT '' NOT NULL,
+	migration_schema_done INTEGER DEFAULT 0 NOT NULL,
+	migration_data_done INTEGER DEFAULT 0 NOT NULL,
+	migration_data_state BLOB SUB_TYPE TEXT CHARACTER SET NONE DEFAULT '' NOT NULL,
+	migration_start_time INTEGER DEFAULT 0 NOT NULL,
+	migration_end_time INTEGER DEFAULT 0 NOT NULL
+);;
+
+CREATE UNIQUE INDEX phpbb_migrations_migration_name ON phpbb_migrations(migration_name);;
+
 # Table: 'phpbb_moderator_cache'
 CREATE TABLE phpbb_moderator_cache (
 	forum_id INTEGER DEFAULT 0 NOT NULL,

phpBB/install/schemas/mssql_schema.sql

@@ -696,6 +696,23 @@ CREATE  INDEX [user_id] ON [phpbb_login_attempts]([user_id]) ON [PRIMARY]
 GO
 
 
+/*
+	Table: 'phpbb_migrations'
+*/
+CREATE TABLE [phpbb_migrations] (
+	[migration_name] [varchar] (255) DEFAULT ('') NOT NULL ,
+	[migration_schema_done] [int] DEFAULT (0) NOT NULL ,
+	[migration_data_done] [int] DEFAULT (0) NOT NULL ,
+	[migration_data_state] [varchar] (8000) DEFAULT ('') NOT NULL ,
+	[migration_start_time] [int] DEFAULT (0) NOT NULL ,
+	[migration_end_time] [int] DEFAULT (0) NOT NULL 
+) ON [PRIMARY]
+GO
+
+CREATE  UNIQUE  INDEX [migration_name] ON [phpbb_migrations]([migration_name]) ON [PRIMARY]
+GO
+
+
 /*
 	Table: 'phpbb_moderator_cache'
 */

phpBB/install/schemas/mysql_40_schema.sql

@@ -397,6 +397,18 @@ CREATE TABLE phpbb_login_attempts (
 );
 
 
+# Table: 'phpbb_migrations'
+CREATE TABLE phpbb_migrations (
+	migration_name varbinary(255) DEFAULT '' NOT NULL,
+	migration_schema_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL,
+	migration_data_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL,
+	migration_data_state blob NOT NULL,
+	migration_start_time int(11) UNSIGNED DEFAULT '0' NOT NULL,
+	migration_end_time int(11) UNSIGNED DEFAULT '0' NOT NULL,
+	UNIQUE migration_name (migration_name)
+);
+
+
 # Table: 'phpbb_moderator_cache'
 CREATE TABLE phpbb_moderator_cache (
 	forum_id mediumint(8) UNSIGNED DEFAULT '0' NOT NULL,

phpBB/install/schemas/mysql_41_schema.sql

@@ -397,6 +397,18 @@ CREATE TABLE phpbb_login_attempts (
 ) CHARACTER SET `utf8` COLLATE `utf8_bin`;
 
 
+# Table: 'phpbb_migrations'
+CREATE TABLE phpbb_migrations (
+	migration_name varchar(255) DEFAULT '' NOT NULL,
+	migration_schema_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL,
+	migration_data_done tinyint(1) UNSIGNED DEFAULT '0' NOT NULL,
+	migration_data_state text NOT NULL,
+	migration_start_time int(11) UNSIGNED DEFAULT '0' NOT NULL,
+	migration_end_time int(11) UNSIGNED DEFAULT '0' NOT NULL,
+	UNIQUE migration_name (migration_name)
+) CHARACTER SET `utf8` COLLATE `utf8_bin`;
+
+
 # Table: 'phpbb_moderator_cache'
 CREATE TABLE phpbb_moderator_cache (
 	forum_id mediumint(8) UNSIGNED DEFAULT '0' NOT NULL,

phpBB/install/schemas/oracle_schema.sql

@@ -780,6 +780,21 @@ CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts (attempt_time
 CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts (user_id)
 /
 
+/*
+	Table: 'phpbb_migrations'
+*/
+CREATE TABLE phpbb_migrations (
+	migration_name varchar2(255) DEFAULT '' ,
+	migration_schema_done number(1) DEFAULT '0' NOT NULL,
+	migration_data_done number(1) DEFAULT '0' NOT NULL,
+	migration_data_state clob DEFAULT '' ,
+	migration_start_time number(11) DEFAULT '0' NOT NULL,
+	migration_end_time number(11) DEFAULT '0' NOT NULL,
+	CONSTRAINT u_phpbb_migration_name UNIQUE (migration_name)
+)
+/
+
+
 /*
 	Table: 'phpbb_moderator_cache'
 */

phpBB/install/schemas/postgres_schema.sql

@@ -557,6 +557,20 @@ CREATE INDEX phpbb_login_attempts_att_for ON phpbb_login_attempts (attempt_forwa
 CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts (attempt_time);
 CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts (user_id);
 
+/*
+	Table: 'phpbb_migrations'
+*/
+CREATE TABLE phpbb_migrations (
+	migration_name varchar(255) DEFAULT '' NOT NULL,
+	migration_schema_done INT2 DEFAULT '0' NOT NULL CHECK (migration_schema_done >= 0),
+	migration_data_done INT2 DEFAULT '0' NOT NULL CHECK (migration_data_done >= 0),
+	migration_data_state varchar(8000) DEFAULT '' NOT NULL,
+	migration_start_time INT4 DEFAULT '0' NOT NULL CHECK (migration_start_time >= 0),
+	migration_end_time INT4 DEFAULT '0' NOT NULL CHECK (migration_end_time >= 0)
+);
+
+CREATE UNIQUE INDEX phpbb_migrations_migration_name ON phpbb_migrations (migration_name);
+
 /*
 	Table: 'phpbb_moderator_cache'
 */

phpBB/install/schemas/sqlite_schema.sql

@@ -385,6 +385,18 @@ CREATE INDEX phpbb_login_attempts_att_for ON phpbb_login_attempts (attempt_forwa
 CREATE INDEX phpbb_login_attempts_att_time ON phpbb_login_attempts (attempt_time);
 CREATE INDEX phpbb_login_attempts_user_id ON phpbb_login_attempts (user_id);
 
+# Table: 'phpbb_migrations'
+CREATE TABLE phpbb_migrations (
+	migration_name varchar(255) NOT NULL DEFAULT '',
+	migration_schema_done INTEGER UNSIGNED NOT NULL DEFAULT '0',
+	migration_data_done INTEGER UNSIGNED NOT NULL DEFAULT '0',
+	migration_data_state text(65535) NOT NULL DEFAULT '',
+	migration_start_time INTEGER UNSIGNED NOT NULL DEFAULT '0',
+	migration_end_time INTEGER UNSIGNED NOT NULL DEFAULT '0'
+);
+
+CREATE UNIQUE INDEX phpbb_migrations_migration_name ON phpbb_migrations (migration_name);
+
 # Table: 'phpbb_moderator_cache'
 CREATE TABLE phpbb_moderator_cache (
 	forum_id INTEGER UNSIGNED NOT NULL DEFAULT '0',

tests/dbal/fixtures/migrator.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<dataset>
+	<table name="phpbb_migrations">
+		<column>migration_name</column>
+		<column>migration_schema_done</column>
+		<column>migration_data_done</column>
+		<column>migration_data_state</column>
+		<column>migration_start_time</column>
+		<column>migration_end_time</column>
+		<row>
+			<value>installed_migration</value>
+			<value>1</value>
+			<value>1</value>
+			<value></value>
+			<value>1234</value>
+			<value>5678</value>
+		</row>
+	</table>
+	<table name="phpbb_config">
+		<column>config_name</column>
+		<column>config_value</column>
+		<row>
+			<value>foo</value>
+			<value>bar</value>
+		</row>
+	</table>
+</dataset>

tests/dbal/migration/dummy.php

@@ -0,0 +1,26 @@
+<?php
+/**
+*
+* @package testing
+* @copyright (c) 2011 phpBB Group
+* @license http://opensource.org/licenses/gpl-license.php GNU Public License
+*
+*/
+
+class phpbb_dbal_migration_dummy extends phpbb_db_migration
+{
+	function depends_on()
+	{
+		return array('installed_migration');
+	}
+
+	function update_schema()
+	{
+		$this->db_column_add('phpbb_config', 'extra_column', array('UINT', 0));
+	}
+
+	function update_data()
+	{
+		$this->db->sql_query('UPDATE phpbb_config SET extra_column = 1');
+	}
+}

tests/dbal/migration/unfulfillable.php

@@ -0,0 +1,26 @@
+<?php
+/**
+*
+* @package testing
+* @copyright (c) 2011 phpBB Group
+* @license http://opensource.org/licenses/gpl-license.php GNU Public License
+*
+*/
+
+class phpbb_dbal_migration_unfulfillable extends phpbb_db_migration
+{
+	function depends_on()
+	{
+		return array('installed_migration', 'phpbb_dbal_migration_dummy', 'non_existant_migration');
+	}
+
+	function update_schema()
+	{
+		trigger_error('Schema update of migration with unfulfillable dependency was run!');
+	}
+
+	function update_data()
+	{
+		trigger_error('Data update of migration with unfulfillable dependency was run!');
+	}
+}

tests/dbal/migrator_test.php

@@ -0,0 +1,80 @@
+<?php
+/**
+*
+* @package testing
+* @copyright (c) 2011 phpBB Group
+* @license http://opensource.org/licenses/gpl-license.php GNU Public License
+*
+*/
+
+require_once dirname(__FILE__) . '/../../phpBB/includes/functions.php';
+require_once dirname(__FILE__) . '/../../phpBB/includes/db/migrator.php';
+require_once dirname(__FILE__) . '/../../phpBB/includes/db/migration.php';
+require_once dirname(__FILE__) . '/../../phpBB/includes/db/db_tools.php';
+
+require_once dirname(__FILE__) . '/migration/dummy.php';
+require_once dirname(__FILE__) . '/migration/unfulfillable.php';
+
+class phpbb_dbal_migrator_test extends phpbb_database_test_case
+{
+	protected $db;
+	protected $db_tools;
+	protected $migrator;
+
+	public function getDataSet()
+	{
+		return $this->createXMLDataSet(dirname(__FILE__).'/fixtures/migrator.xml');
+	}
+
+	public function setup()
+	{
+		parent::setup();
+
+		$this->db = $this->new_dbal();
+		$this->db_tools = new phpbb_db_tools($this->db);
+		$this->migrator = new phpbb_db_migrator($this->db, $this->db_tools, MIGRATIONS_TABLE);
+	}
+
+	public function test_update()
+	{
+		$this->migrator->set_migrations(array('phpbb_dbal_migration_dummy'));
+
+		// schema
+		$this->migrator->update();
+		$this->assertFalse($this->migrator->finished());
+
+		// data
+		$this->migrator->update();
+		$this->assertTrue($this->migrator->finished());
+
+		$this->assertSqlResultEquals(
+			array(array('extra_column' => '1')),
+			"SELECT extra_column FROM phpbb_config WHERE config_name = 'foo'",
+			'Dummy migration created extra_column with value 1 in all rows.'
+		);
+
+		// cleanup
+		$this->db_tools->sql_column_remove('phpbb_config', 'extra_column');
+	}
+
+	public function test_unfulfillable()
+	{
+		$this->migrator->set_migrations(array('phpbb_dbal_migration_unfulfillable', 'phpbb_dbal_migration_dummy'));
+
+		while (!$this->migrator->finished())
+		{
+			$this->migrator->update();
+		}
+
+		$this->assertTrue($this->migrator->finished());
+
+		$this->assertSqlResultEquals(
+			array(array('extra_column' => '1')),
+			"SELECT extra_column FROM phpbb_config WHERE config_name = 'foo'",
+			'Dummy migration was run, even though an unfulfillable migration was found.'
+		);
+
+		// cleanup
+		$this->db_tools->sql_column_remove('phpbb_config', 'extra_column');
+	}
+}