diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9504a93 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +* text=auto + +/tests export-ignore +/utils export-ignore +/.coveralls.yml export-ignore +/.gitattributes export-ignore +/.gitignore export-ignore +/.travis.yml export-ignore +/docker-compose.yml export-ignore +/phpunit.xml export-ignore +/Dockerfile export-ignore diff --git a/.gitignore b/.gitignore index f2a272f..ef2da60 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /.idea /coverage /utils/ccat/ccat +composer.lock diff --git a/.travis.yml b/.travis.yml index 75e6e16..446f7d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,5 @@ language: php -php: - - '7.1' - sudo: required services: @@ -13,20 +10,18 @@ compiler: - g++ before_install: - - composer require satooshi/php-coveralls + - docker-compose run --rm composer require satooshi/php-coveralls before_script: - - composer self-update - - composer install --prefer-source --dev - - docker run -p 9000:9000 -p 8123:8123 -d --name some-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server + - docker-compose run --rm composer install --prefer-source --dev - cd utils/ccat - make && make install - cd ../../ -script: phpunit --coverage-clover ./tests/logs/clover.xml +script: docker-compose run --rm phpunit --coverage-clover ./tests/logs/clover.xml after_script: - - php vendor/bin/php-coveralls -v + - docker-compose run cli vendor/bin/php-coveralls -v notifications: on_success: never diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..de4a5c0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM php:7.1-cli + +RUN pecl install xdebug-2.6.0 \ + && docker-php-ext-enable xdebug + +RUN php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" \ + && php composer-setup.php --install-dir=/usr/bin --filename=composer \ + && php -r "unlink('composer-setup.php');" + +WORKDIR "/app" diff --git a/README.md b/README.md index 7259030..2856246 100644 --- a/README.md +++ b/README.md @@ -289,10 +289,32 @@ $client->writeOne('DROP TABLE table'); ## Testing +Install dependencies (one time) + +``` bash +$ composer install +``` + +Run tests + ``` bash $ composer test ``` +## Testing with Docker + +Install dependencies (one time) + +``` bash +$ docker-compose run --rm composer install +``` + +Run tests + +``` bash +$ docker-compose run --rm phpunit +``` + ## Roadmap * Add ability to save query result in local file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..dd38870 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +version: "3.6" + +services: + clickhouse: + image: "yandex/clickhouse-server:19.11.3.11" + composer: + build: + context: . + dockerfile: Dockerfile + entrypoint: + - "composer" + volumes: + - "./:/app" + - ${COMPOSER_HOME:-$HOME/.composer}:/tmp + phpunit: + build: + context: . + dockerfile: Dockerfile + depends_on: + - clickhouse + volumes: + - "./:/app" + entrypoint: + - "./vendor/bin/phpunit" + environment: + "CH_HOST": "clickhouse" + "CH_PORT": "8123" + cli: + build: + context: . + dockerfile: Dockerfile + volumes: + - "./:/app" \ No newline at end of file diff --git a/phpunit.xml b/phpunit.xml index 2cfb02b..c77b9f6 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -23,8 +23,8 @@ - - + + diff --git a/tests/ClientTest.php b/tests/ClientTest.php index 0eb8e28..2847929 100644 --- a/tests/ClientTest.php +++ b/tests/ClientTest.php @@ -12,6 +12,7 @@ use Tinderbox\Clickhouse\Query\Mapper\NamedMapper; use Tinderbox\Clickhouse\Query\QueryStatistic; use Tinderbox\Clickhouse\Query\Result; +use Tinderbox\Clickhouse\Support\ServerTrait; /** * @covers \Tinderbox\Clickhouse\Client @@ -20,52 +21,54 @@ */ class ClientTest extends TestCase { + use ServerTrait; + public function testGetters() { - $server = new Server('127.0.0.1'); + $server = $this->getServer(); $serverProvider = new ServerProvider(); $serverProvider->addServer($server); - + $client = new Client($serverProvider); - + $this->assertEquals( $serverProvider->getRandomServer(), $client->getServer(), 'Correctly passes server provider' ); } - + public function testMappers() { - $server = new Server('127.0.0.1'); + $server = $this->getServer(); $serverProvider = new ServerProvider(); $serverProvider->addServer($server); - + $client = new Client($serverProvider, new NamedMapper()); - + $result = $client->readOne('select number from numbers(:min, :max)', ['min' => 0, 'max' => 10]); - + $this->assertEquals(10, count($result->rows), 'Correctly changes mapper'); } - + public function testTransports() { - $server = new Server('127.0.0.1'); + $server = $this->getServer(); $serverProvider = new ServerProvider(); $serverProvider->addServer($server); - + $transport = $this->createMock(TransportInterface::class); $transport->method('read')->willReturn([ - new Result(new Query($server, ''), [0,1], new QueryStatistic(0,0,0,0)) + new Result(new Query($server, ''), [0, 1], new QueryStatistic(0, 0, 0, 0)), ]); - + $client = new Client($serverProvider, null, $transport); - + $result = $client->readOne('test query'); - + $this->assertEquals(2, count($result->rows), 'Correctly changes transport'); } - + public function testClusters() { $cluster = new Cluster( @@ -75,7 +78,7 @@ public function testClusters() new Server('127.0.0.3'), ] ); - + $cluster2 = new Cluster( 'test2', [ new Server('127.0.0.4'), @@ -83,89 +86,89 @@ public function testClusters() new Server('127.0.0.6'), ] ); - + $serverProvider = new ServerProvider(); $serverProvider->addCluster($cluster)->addCluster($cluster2); - + $client = new Client($serverProvider); - + $server = $client->onCluster('test')->getServer(); /* will return random server from cluster */ $this->assertContains( $server, $cluster->getServers(), 'Correctly returns random server from specified cluster' ); - + $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); - + $client->using('127.0.0.3'); $server = $client->getServer(); - + $this->assertEquals( '127.0.0.3', $server->getHost(), 'Correctly returns specified server from specified cluster' ); - + $server = $client->onCluster('test2')->getServer(); /* will return random server from cluster */ $this->assertContains( $server, $cluster2->getServers(), 'Correctly returns random server from specified cluster' ); - + $client->usingRandomServer(); $server = $client->getServer(); - + while ($server === $client->getServer()) { /* Randomize while get non used server */ } - + $this->assertTrue(true, 'Correctly randomizes cluster servers on each call'); - + $this->expectException(ClusterException::class); $this->expectExceptionMessage('Server with hostname [127.0.0.0] is not found in cluster'); - + $client->onCluster('test')->using('127.0.0.0')->getServer(); } - + public function testServers() { $server1 = new Server('127.0.0.1'); $server2 = new Server('127.0.0.2'); $server3 = new Server('127.0.0.3'); - + $serverProvider = new ServerProvider(); $serverProvider->addServer($server1)->addServer($server2)->addServer($server3); - + $client = new Client($serverProvider); - + $server = $client->getServer(); $this->assertTrue( in_array($server->getHost(), ['127.0.0.1', '127.0.0.2', '127.0.0.3'], true), 'Correctly returns random server' ); - + $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); - + $server = $client->using('127.0.0.3')->getServer(); $this->assertEquals('127.0.0.3', $server->getHost(), 'Correctly returns specified server'); - + $client->usingRandomServer(); $server = $client->getServer(); - + while ($server === $client->getServer()) { /* Randomize while get non used server */ } - + $this->assertTrue(true, 'Correctly randomizes cluster servers on each call'); - + $this->expectException(ServerProviderException::class); $this->expectExceptionMessage('Can not find server with hostname [127.0.0.0]'); - + $client->using('127.0.0.0')->getServer(); } - + public function testClusterAndServersTogether() { $cluster = new Cluster( @@ -175,116 +178,116 @@ public function testClusterAndServersTogether() new Server('127.0.0.3'), ] ); - + $server1 = new Server('127.0.0.4'); $server2 = new Server('127.0.0.5'); $server3 = new Server('127.0.0.6'); - + $serverProvider = new ServerProvider(); $serverProvider->addCluster($cluster)->addServer($server1)->addServer($server2)->addServer($server3); - + $client = new Client($serverProvider); - + $server = $client->getServer(); $this->assertTrue( in_array($server->getHost(), ['127.0.0.4', '127.0.0.5', '127.0.0.6'], true), 'Correctly returns random server not in cluster' ); - + $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); - + $client->onCluster('test'); - + $server = $client->onCluster('test')->getServer(); /* will return random server from cluster */ $this->assertContains( $server, $cluster->getServers(), 'Correctly returns random server from specified cluster' ); - + $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); - + $server = $client->onCluster(null)->getServer(); $this->assertTrue( in_array($server->getHost(), ['127.0.0.4', '127.0.0.5', '127.0.0.6'], true), 'Correctly returns random server after disabling cluster mode' ); } - + protected function getClient(): Client { $serverProvider = new ServerProvider(); - $serverProvider->addServer(new Server('127.0.0.1', '8123', 'default', 'default', '')); - + $serverProvider->addServer($this->getServer()); + return new Client($serverProvider); } - + public function testReadOne() { $client = $this->getClient(); - + $result = $client->readOne('select * from numbers(?, ?) order by number desc', [0, 10]); - + $this->assertEquals(10, count($result->rows), 'Correctly executes query using mapper'); } - + public function testRead() { $client = $this->getClient(); - + $result = $client->read( [ [ - 'query' => 'select * from numbers(?, ?) order by number desc', + 'query' => 'select * from numbers(?, ?) order by number desc', 'bindings' => [0, 10], ], new Query($client->getServer(), 'select * from numbers(0, 20) order by number desc'), new Query($client->getServer(), 'select * from numbers(0, 20) where number in tab order by number desc', [ - new TempTable('tab', new FileFromString('1'.PHP_EOL.'2'.PHP_EOL), ['number' => 'UInt64'], Format::TSV) + new TempTable('tab', new FileFromString('1'.PHP_EOL.'2'.PHP_EOL), ['number' => 'UInt64'], Format::TSV), ]), ] ); - + $this->assertEquals(10, count($result[0]->rows), 'Correctly converts query from array to query instance'); $this->assertEquals(20, count($result[1]->rows), 'Correctly executes queries'); $this->assertEquals(2, count($result[2]->rows), 'Correctly executes query with file'); } - + public function testWrite() { $client = $this->getClient(); - + $client->write([ new Query($client->getServer(), 'drop table if exists default.tdchc_test_table'), new Query($client->getServer(), 'create table default.tdchc_test_table (number UInt64) engine = Memory'), ], 1); - + $client->writeOne('insert into default.tdchc_test_table (number) FORMAT TSV', [], [ - new FileFromString('1'.PHP_EOL.'2'.PHP_EOL) + new FileFromString('1'.PHP_EOL.'2'.PHP_EOL), ]); - + $result = $client->readOne('select * from default.tdchc_test_table'); - + $this->assertEquals(2, count($result->rows), 'Correctly writes data'); } - + public function testWriteFiles() { $client = $this->getClient(); - + $client->write([ new Query($client->getServer(), 'drop table if exists default.tdchc_test_table'), new Query($client->getServer(), 'create table default.tdchc_test_table (number UInt64) engine = Memory'), ], 1); - + $client->writeFiles('default.tdchc_test_table', ['number'], [ new FileFromString('1'.PHP_EOL.'2'.PHP_EOL), new FileFromString('3'.PHP_EOL.'4'.PHP_EOL), new FileFromString('5'.PHP_EOL.'6'.PHP_EOL), ], Format::TSV); - + $result = $client->readOne('select * from default.tdchc_test_table'); - + $this->assertEquals(6, count($result->rows), 'Correctly writes data'); } } diff --git a/tests/HttpTransportTest.php b/tests/HttpTransportTest.php index e10d3c5..80b0070 100644 --- a/tests/HttpTransportTest.php +++ b/tests/HttpTransportTest.php @@ -12,6 +12,7 @@ use Tinderbox\Clickhouse\Common\MergedFiles; use Tinderbox\Clickhouse\Common\TempTable; use Tinderbox\Clickhouse\Exceptions\TransportException; +use Tinderbox\Clickhouse\Support\ServerTrait; use Tinderbox\Clickhouse\Transport\HttpTransport; /** @@ -19,7 +20,9 @@ */ class HttpTransportTest extends TestCase { - protected function getMockedTransport(array $responses) : HttpTransport + use ServerTrait; + + protected function getMockedTransport(array $responses): HttpTransport { $mock = new MockHandler($responses); @@ -30,17 +33,12 @@ protected function getMockedTransport(array $responses) : HttpTransport ])); } - protected function getQuery() : Query + protected function getQuery(): Query { return new Query($this->getServer(), 'select * from table'); } - protected function getServer() : Server - { - return new Server(CLICKHOUSE_SERVER_HOST, CLICKHOUSE_SERVER_PORT, 'default', 'default'); - } - - protected function getTransport() : HttpTransport + protected function getTransport(): HttpTransport { return new HttpTransport(); } @@ -49,18 +47,18 @@ public function testRead() { $transport = $this->getMockedTransport([ new Response(200, [], json_encode([ - 'data' => [ + 'data' => [ [ '1' => 1, ], ], - 'statistics' => [ + 'statistics' => [ 'rows_read' => 1, 'bytes_read' => 1, 'elapsed' => 0.100, ], - 'rows_before_limit_at_least' => 1024 - ])) + 'rows_before_limit_at_least' => 1024, + ])), ]); $result = $transport->read([$this->getQuery()]); @@ -78,7 +76,7 @@ public function testRead() ], [ 'rows_read' => $result[0]->statistic->rows, 'bytes_read' => $result[0]->statistic->bytes, - 'elapsed' => $result[0]->statistic->time + 'elapsed' => $result[0]->statistic->time, ], 'Returns correct statistic from server'); $this->assertEquals(1024, $result[0]->statistic->rowsBeforeLimitAtLeast, 'Returns correct rows_before_limit_at_least'); @@ -88,32 +86,32 @@ public function testReadMultipleRequests() { $transport = $this->getMockedTransport([ new Response(200, [], json_encode([ - 'data' => [ + 'data' => [ [ '1' => 1, ], ], - 'statistics' => [ + 'statistics' => [ 'rows_read' => 1, 'bytes_read' => 1, 'elapsed' => 0.100, ], - 'rows_before_limit_at_least' => 1024 + 'rows_before_limit_at_least' => 1024, ])), new Response(200, [], json_encode([ - 'data' => [ + 'data' => [ [ '1' => 2, ], ], - 'statistics' => [ + 'statistics' => [ 'rows_read' => 2, 'bytes_read' => 2, 'elapsed' => 0.100, ], - 'rows_before_limit_at_least' => 1025 - ])) + 'rows_before_limit_at_least' => 1025, + ])), ]); $result = $transport->read([$this->getQuery(), $this->getQuery()]); @@ -131,7 +129,7 @@ public function testReadMultipleRequests() ], [ 'rows_read' => $result[0]->statistic->rows, 'bytes_read' => $result[0]->statistic->bytes, - 'elapsed' => $result[0]->statistic->time + 'elapsed' => $result[0]->statistic->time, ], 'Returns correct statistic from server'); $this->assertEquals(1024, $result[0]->statistic->rowsBeforeLimitAtLeast, 'Returns correct rows_before_limit_at_least'); @@ -149,7 +147,7 @@ public function testReadMultipleRequests() ], [ 'rows_read' => $result[1]->statistic->rows, 'bytes_read' => $result[1]->statistic->bytes, - 'elapsed' => $result[1]->statistic->time + 'elapsed' => $result[1]->statistic->time, ], 'Returns correct statistic from server'); $this->assertEquals(1025, $result[1]->statistic->rowsBeforeLimitAtLeast, 'Returns correct rows_before_limit_at_least'); @@ -168,7 +166,7 @@ public function testReadWithTablesUnstructured() $query = new Query($this->getServer(), 'select * from numbers(0, 10) where number in temp', [$table]); $result = $transport->read([$query]); - $this->assertEquals([1,2], array_column($result[0]->rows, 'number')); + $this->assertEquals([1, 2], array_column($result[0]->rows, 'number')); unlink($tableSource); } @@ -186,7 +184,7 @@ public function testReadWithTablesStructured() $query = new Query($this->getServer(), 'select number from temp', [$table]); $result = $transport->read([$query]); - $this->assertEquals([1,2], array_column($result[0]->rows, 'number'), 'Returns correct result when uses temp tables for read queries'); + $this->assertEquals([1, 2], array_column($result[0]->rows, 'number'), 'Returns correct result when uses temp tables for read queries'); unlink($tableSource); } @@ -231,12 +229,12 @@ public function testWrite() $this->assertEquals([ [ 'number' => 1, - 'string' => 'some' + 'string' => 'some', ], [ 'number' => 2, - 'string' => 'string' - ] + 'string' => 'string', + ], ], $result[0]->rows, 'Returns correct result from recently created table and filled with temp files'); $handle = $table->open(); @@ -290,31 +288,31 @@ public function testWriteMultipleFilesPerOneQuery() $this->assertEquals([ [ 'number' => 1, - 'string' => 'some' + 'string' => 'some', ], [ 'number' => 2, - 'string' => 'string' + 'string' => 'string', ], [ 'number' => 3, - 'string' => 'string' + 'string' => 'string', ], [ 'number' => 4, - 'string' => 'some' - ] + 'string' => 'some', + ], ], $result[0]->rows, 'Returns correct result from recently created table and filled with temp files'); $this->assertEquals([ [ 'number' => 3, - 'string' => 'string' + 'string' => 'string', ], [ 'number' => 4, - 'string' => 'some' - ] + 'string' => 'some', + ], ], $result[1]->rows, 'Returns correct result from recently created table and filled with temp files'); $transport->write([ @@ -339,7 +337,7 @@ public function testConnectionError() $this->expectExceptionMessage('Can\'t connect to the server [KHGIUYhakljsfnk:8123]'); $transport->read([ - new Query(new Server('KHGIUYhakljsfnk'), '') + new Query(new Server('KHGIUYhakljsfnk'), ''), ]); } @@ -353,7 +351,7 @@ public function testUnknownReason() $this->expectExceptionMessage('Unknown exception'); $transport->read([ - new Query($this->getServer(), '') + new Query($this->getServer(), ''), ]); } @@ -368,7 +366,7 @@ public function testHttpTransportMalformedResponse() $this->expectExceptionMessage($e->getMessage()); $transport->read([ - new Query($this->getServer(), '') + new Query($this->getServer(), ''), ]); } @@ -379,11 +377,10 @@ public function testConnectionWithPassword() $this->expectException(TransportException::class); $this->expectExceptionMessageRegExp('/Wrong password for user default/'); - $transport->read([ - new Query(new Server('127.0.0.1', 8123, 'default','default', 'pass'), 'select 1', [ - new TempTable('name', new FileFromString('aaa'), ['string' => 'String'], Format::TSV) - ]) - ]); + $server = $this->getServer('default', 'default', 'pass'); + $file = new TempTable('name', new FileFromString('aaa'), ['string' => 'String'], Format::TSV); + + $transport->read([new Query($server, 'select 1', [$file])]); } public function testConnectionWithPasswordOnWrite() @@ -393,10 +390,10 @@ public function testConnectionWithPasswordOnWrite() $this->expectException(TransportException::class); $this->expectExceptionMessageRegExp('/Wrong password for user default/'); + $server = $this->getServer('default', 'default', 'pass'); + $transport->write([ - new Query(new Server('127.0.0.1', 8123, 'default','default', 'pass'), 'insert into table 1', [ - new FileFromString('aaa') - ]) + new Query($server, 'insert into table ', [new FileFromString('aaa')]), ]); } diff --git a/tests/Support/ServerTrait.php b/tests/Support/ServerTrait.php new file mode 100644 index 0000000..dbbfa77 --- /dev/null +++ b/tests/Support/ServerTrait.php @@ -0,0 +1,13 @@ +