diff --git a/.travis.yml b/.travis.yml index edc5488..a8c971c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,26 @@ language: php -sudo: false + +sudo: required + +services: + - docker php: - 5.6 - 7.0 - 7.1 - before_script: - - composer install --dev + - composer docker:setup + - composer install script: - - mkdir -p build/logs - - php vendor/bin/phpunit --coverage-clover build/logs/clover.xml + - mkdir -p build/logs + - composer test -- --coverage-clover build/logs/clover.xml after_script: - php vendor/bin/coveralls -v + +cache: + directories: + - vendor diff --git a/README.md b/README.md index 188d5ac..49fa5f4 100644 --- a/README.md +++ b/README.md @@ -17,18 +17,18 @@ What is it? DbSync is a tool for efficiently comparing and synchronising two or more remote MySQL database tables. -In order to do this without comparing every byte of data, the tool preforms a checksum (MD5, SHA1, CRC32) -over a range of rows on both the source and destination tables, and compares only the hash. If a block is found to have -an inconsistency in a block, the tool performs a checksum on each half of the block, recursively (down to a minimum +In order to do this without comparing every byte of data, the tool preforms a checksum (MD5, SHA1, CRC32) +over a range of rows on both the source and destination tables, and compares only the hash. If a block is found to have +an inconsistency in a block, the tool performs a checksum on each half of the block, recursively (down to a minimum block transfer size), until it finds the inconsistency. Notes About Deletion -------------------- -DbSync will only delete rows from the destination that no longer exist on the source when the `--delete` option is specified. +DbSync will only delete rows from the destination that no longer exist on the source when the `--delete` option is specified. Use this option with extreme caution. Always perform a dry run first. -If you use DbSync to synchronise a table which has row deletions on the source without using the `--delete` option, +If you use DbSync to synchronise a table which has row deletions on the source without using the `--delete` option, DbSync will find inconsistencies in any block with a deleted row on every run but will not be able to remove the rows from the target. @@ -132,7 +132,7 @@ db-sync --user root --password mypass 127.0.0.1 111.222.3.44 web.customers -i up Sync every column from the table `web.customers` but only use the `updated_at` fields when calculating the hash: - > Inconsistencies in other fields will not be detected. In the event of a hash inconsistency in fields which are + > Inconsistencies in other fields will not be detected. In the event of a hash inconsistency in fields which are included, the excluded fields will still be copied to the target host. ~~~~ @@ -143,7 +143,7 @@ db-sync --user root --password mypass 127.0.0.1 111.222.3.44 web.customers -C up Sync every column from the table `web.customers` and use all fields except for the `notes` or `info` fields when calculating the hash: - > Inconsistencies in excluded fields will not be detected. In the event of a hash inconsistency in fields which are included, + > Inconsistencies in excluded fields will not be detected. In the event of a hash inconsistency in fields which are included, the excluded fields will still be copied to the target host. > This is especially useful for tables with long text fields that don't change after initial insert, or which are associated @@ -200,7 +200,7 @@ $sync->delete(true); $sourceTable = new Table($sourceConnection, $sourceDb, $sourceTable); $targetTable = new Table($targetConnection, $targetDb, $targetTable); -// if you only want specific columns +// if you only want specific columns $columnConfig = new ColumnConfiguration($syncColumns, $ignoreColumns); // optionally apply a where clause @@ -210,6 +210,19 @@ $targetTable->setWhereClause(new WhereClause("column_name > ?", ['value'])); $sync->sync($sourceTable, $targetTable, $columnConfig); ~~~ +Testing +------- + +[Docker](https://www.docker.com/) is used to create a one-off mysql database, so you need to [install docker](https://docs.docker.com/engine/installation/) before you can run the tests. + +When docker is installed, you need to setup [the mysql image](https://hub.docker.com/r/mysql/mysql-server/) and run the tests. +Fortunately, this is very easy to do. + +1. `composer docker:setup` - setup the mysql image. +2. `composer test` - run the tests in phpunit. + +The mysql image will keep running in docker, until you remove it via `composer docker:clean`. You can re-run the tests anytime via `composer test`. Do not run `composer docker:setup` again, unless you have removed the mysql image. + Roadmap ------- diff --git a/composer.json b/composer.json index d414217..4a78eaf 100644 --- a/composer.json +++ b/composer.json @@ -12,8 +12,15 @@ "bin/sync" ], "scripts" : { - "test" : "php vendor/bin/phpunit", - "package" : "composer install --no-dev && ./phar-composer.phar build" + "pretest": "@docker:start", + "test" : "phpunit", + "package" : "composer install --no-dev && ./phar-composer.phar build", + "docker:setup": ["@docker:create", "@docker:start"], + "docker:clean": "@docker:remove", + "docker:create": "docker run -p 0.0.0.0:33556:3306 -v $(pwd)/tests/integration/sql/:/docker-entrypoint-initdb.d/ --name db-sync-test-mysql -e MYSQL_ROOT_PASSWORD=1234 -e MYSQL_ROOT_HOST=% -e MYSQL_DATABASE=dbsync_int_test -d mysql/mysql-server:5.7", + "docker:remove": ["@docker:stop", "docker rm db-sync-test-mysql"], + "docker:start": "docker start db-sync-test-mysql", + "docker:stop": "docker stop db-sync-test-mysql" }, "require": { "psr/log": "^1.0", diff --git a/tests/integration/FullSyncTest.php b/tests/integration/FullSyncTest.php index 7c4091a..c2a7908 100644 --- a/tests/integration/FullSyncTest.php +++ b/tests/integration/FullSyncTest.php @@ -38,14 +38,13 @@ public function testItRunsCommand() $db = self::DATABASE; - $host = $this->config['host']; + $host = isset($this->config['port']) ? $this->config['host'] . ':' . $this->config['port'] : $this->config['host']; $user = $this->config['username']; $password = $this->config['password']; $password and $password = "-p $password"; $command = __DIR__ . "/../../bin/sync $host $host $db.dbsynctest1 --target.table=$db.dbsynctest2 -u $user $password -e"; - exec($command, $output, $code); $this->assertEquals(0, $code); @@ -199,26 +198,6 @@ private function createTestDatabases($populateBoth = false, $deleteFromSource = $deleteFromSource and $this->connection->query("DELETE FROM $dbName.dbsynctest1 WHERE customerNumber % 12 = 0"); } - private function createTestTable($table) - { - $this->connection->query("CREATE TABLE $table ( - `customerNumber` int(11) NOT NULL, - `customerName` varchar(50) NOT NULL, - `contactLastName` varchar(50) DEFAULT NULL, - `contactFirstName` varchar(50) DEFAULT NULL, - `return` varchar(50) DEFAULT NULL, - `addressLine1` varchar(50) DEFAULT NULL, - `addressLine2` varchar(50) DEFAULT NULL, - `city` varchar(50) DEFAULT NULL, - `state` varchar(50) DEFAULT NULL, - `postalCode` varchar(15) DEFAULT NULL, - `country` varchar(50) DEFAULT NULL, - `salesRepEmployeeNumber` int(11) DEFAULT NULL, - `creditLimit` double DEFAULT NULL, - PRIMARY KEY (`customerNumber`,`customerName`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1;"); - } - private function populateTestTable($table) { diff --git a/tests/integration/TestAbstract.php b/tests/integration/TestAbstract.php index b5938fb..6e2b5f7 100644 --- a/tests/integration/TestAbstract.php +++ b/tests/integration/TestAbstract.php @@ -14,8 +14,6 @@ abstract class TestAbstract extends PHPUnit_Framework_TestCase public function setUp() { $this->setUpConnection(); - - $this->setUpTables(); } private function setUpConnection() @@ -24,47 +22,30 @@ private function setUpConnection() foreach($configs as $config) { + list($host, $port) = $this->parseHostPort($config['host']); + $config['host'] = $host; + $config['port'] = $port; + try{ $this->connection = (new \Database\Connectors\ConnectionFactory())->make($config); $this->config = $config; return; }catch (\PDOException $e) { - + throw $e; } } throw new \InvalidArgumentException("No valid database configs"); } - private function setUpTables() + private function parseHostPort($host) { - $dbName = self::DATABASE; - $this->connection->query("CREATE DATABASE IF NOT EXISTS $dbName"); - $this->connection->query("DROP TABLE IF EXISTS $dbName.dbsynctest1"); - $this->connection->query("DROP TABLE IF EXISTS $dbName.dbsynctest2"); + $parts = explode(':', $host, 2); - $this->createTestTable($dbName . ".dbsynctest1"); - $this->createTestTable($dbName . ".dbsynctest2"); - } + isset($parts[1]) or $parts[1] = 3306; - private function createTestTable($table) - { - $this->connection->query("CREATE TABLE $table ( - `customerNumber` int(11) NOT NULL, - `customerName` varchar(50) NOT NULL, - `contactLastName` varchar(50) DEFAULT NULL, - `contactFirstName` varchar(50) DEFAULT NULL, - `return` varchar(50) DEFAULT NULL, - `addressLine1` varchar(50) DEFAULT NULL, - `addressLine2` varchar(50) DEFAULT NULL, - `city` varchar(50) DEFAULT NULL, - `state` varchar(50) DEFAULT NULL, - `postalCode` varchar(15) DEFAULT NULL, - `country` varchar(50) DEFAULT NULL, - `salesRepEmployeeNumber` int(11) DEFAULT NULL, - `creditLimit` double DEFAULT NULL, - PRIMARY KEY (`customerNumber`,`customerName`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1;"); + return $parts; } -} \ No newline at end of file + +} diff --git a/tests/integration/config.php b/tests/integration/config.php index 904152b..be5212e 100644 --- a/tests/integration/config.php +++ b/tests/integration/config.php @@ -2,18 +2,18 @@ return [ [ - 'host' => 'localhost', + 'host' => '0.0.0.0:33556', 'driver' => 'mysql', 'username' => 'root', - 'password' => '', + 'password' => '1234', 'charset' => 'utf8', 'collation' => 'utf8_unicode_ci', ], [ - 'host' => 'localhost', + 'host' => '0.0.0.0:33556', 'driver' => 'mysql', 'username' => 'root', - 'password' => 'password', + 'password' => '1234', 'charset' => 'utf8', 'collation' => 'utf8_unicode_ci', ] diff --git a/tests/integration/sql/dbsync_int_test.sql b/tests/integration/sql/dbsync_int_test.sql new file mode 100644 index 0000000..64afd58 --- /dev/null +++ b/tests/integration/sql/dbsync_int_test.sql @@ -0,0 +1,95 @@ +-- MySQL dump 10.13 Distrib 5.7.18, for Linux (x86_64) +-- +-- Host: 172.18.0.2 Database: dbsync_int_test +-- ------------------------------------------------------ +-- Server version 5.7.18 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `dbsynctest1` +-- + +DROP TABLE IF EXISTS `dbsynctest1`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `dbsynctest1` ( + `customerNumber` int(11) NOT NULL, + `customerName` varchar(50) NOT NULL, + `contactLastName` varchar(50) DEFAULT NULL, + `contactFirstName` varchar(50) DEFAULT NULL, + `return` varchar(50) DEFAULT NULL, + `addressLine1` varchar(50) DEFAULT NULL, + `addressLine2` varchar(50) DEFAULT NULL, + `city` varchar(50) DEFAULT NULL, + `state` varchar(50) DEFAULT NULL, + `postalCode` varchar(15) DEFAULT NULL, + `country` varchar(50) DEFAULT NULL, + `salesRepEmployeeNumber` int(11) DEFAULT NULL, + `creditLimit` double DEFAULT NULL, + PRIMARY KEY (`customerNumber`,`customerName`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `dbsynctest1` +-- + +LOCK TABLES `dbsynctest1` WRITE; +/*!40000 ALTER TABLE `dbsynctest1` DISABLE KEYS */; +/*!40000 ALTER TABLE `dbsynctest1` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `dbsynctest2` +-- + +DROP TABLE IF EXISTS `dbsynctest2`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `dbsynctest2` ( + `customerNumber` int(11) NOT NULL, + `customerName` varchar(50) NOT NULL, + `contactLastName` varchar(50) DEFAULT NULL, + `contactFirstName` varchar(50) DEFAULT NULL, + `return` varchar(50) DEFAULT NULL, + `addressLine1` varchar(50) DEFAULT NULL, + `addressLine2` varchar(50) DEFAULT NULL, + `city` varchar(50) DEFAULT NULL, + `state` varchar(50) DEFAULT NULL, + `postalCode` varchar(15) DEFAULT NULL, + `country` varchar(50) DEFAULT NULL, + `salesRepEmployeeNumber` int(11) DEFAULT NULL, + `creditLimit` double DEFAULT NULL, + PRIMARY KEY (`customerNumber`,`customerName`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `dbsynctest2` +-- + +LOCK TABLES `dbsynctest2` WRITE; +/*!40000 ALTER TABLE `dbsynctest2` DISABLE KEYS */; +/*!40000 ALTER TABLE `dbsynctest2` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2017-05-08 20:56:09