Skip to content

Commit c54caf0

Browse files
IvanChepurnyiIvanChepurnyi
committed
wip
1 parent d2aa12a commit c54caf0

20 files changed

+314
-26
lines changed

‎bin/mysql2jsonl

100644100755
+15-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,18 @@
1-
#!env php
1+
#!/usr/bin/env php
22
<?php
33

4+
use EcomDev\MySQL2JSONL\Command\ExportCommand;
5+
use Symfony\Component\Console\Application;
6+
47
require_once __DIR__ . '/../vendor/autoload.php';
8+
9+
$application = new Application(
10+
basename(__FILE__),
11+
'1.0.0'
12+
);
13+
14+
$application->addCommands([
15+
new ExportCommand()
16+
]);
17+
18+
$application->run();

‎composer.json

+4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"amphp/file": "~3.2",
88
"amphp/mysql": "~3.0",
99
"amphp/amp": "~3.0",
10+
"amphp/pipeline": "~1.2",
1011
"revolt/event-loop": "~1.0",
1112
"symfony/console": "~7.2",
1213
"justinrainbow/json-schema": "^6.0"
@@ -29,6 +30,9 @@
2930
}
3031
},
3132
"autoload-dev": {
33+
"files": [
34+
"tests/fixtures.php"
35+
],
3236
"psr-4": {
3337
"EcomDev\\MySQL2JSONL\\": "tests/"
3438
}

‎schema.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
"maxConnections": {
1111
"type": "integer",
1212
"description": "Maximum number of open connection by tool",
13-
"default": 50
13+
"default": 10
1414
},
1515
"idleTimeout": {
1616
"type": "integer",
@@ -26,6 +26,7 @@
2626
"description": "List of tables to exclude from the database dump"
2727
}
2828
},
29+
"required": ["connection"],
2930
"additionalProperties": false,
3031
"definitions": {
3132
"matchExpression": {

‎src/Command/ExportCommand.php

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
<?php
2+
3+
namespace EcomDev\MySQL2JSONL\Command;
4+
5+
use Amp\Pipeline\Queue;
6+
use EcomDev\MySQL2JSONL\Configuration;
7+
use EcomDev\MySQL2JSONL\ConfigurationException;
8+
use EcomDev\MySQL2JSONL\ExportTableFactory;
9+
use EcomDev\MySQL2JSONL\Progress\ExportProgressNotifier;
10+
use Revolt\EventLoop;
11+
use Symfony\Component\Console\Attribute\AsCommand;
12+
use Symfony\Component\Console\Command\Command;
13+
use Symfony\Component\Console\Helper\FormatterHelper;
14+
use Symfony\Component\Console\Input\InputArgument;
15+
use Symfony\Component\Console\Input\InputInterface;
16+
use Symfony\Component\Console\Output\OutputInterface;
17+
use Symfony\Component\Console\Input\InputOption;
18+
use function Amp\async;
19+
use function Amp\delay;
20+
use function Amp\Future\awaitAll;
21+
22+
#[AsCommand(name: 'export', description: 'Export data from MySQL to a directory with JSONL files')]
23+
class ExportCommand extends Command
24+
{
25+
public function configure()
26+
{
27+
$this->addOption(
28+
'config',
29+
'c', InputOption::VALUE_REQUIRED,
30+
'Configuration file',
31+
'config.json'
32+
);
33+
34+
$this->addArgument(
35+
'directory',
36+
InputArgument::OPTIONAL,
37+
'Directory to export data to',
38+
'./data-dump'
39+
);
40+
}
41+
42+
public function execute(InputInterface $input, OutputInterface $output): int
43+
{
44+
$file = $input->getOption('config');
45+
if (!file_exists($file)) {
46+
/* @var FormatterHelper $formatter*/
47+
$formatter = $this->getHelper('formatter');
48+
$output->write($formatter->formatSection('Error', sprintf(
49+
'Configuration file %s does not exist',
50+
$file
51+
), 'error'));
52+
return 1;
53+
}
54+
try {
55+
$config = Configuration::fromJSON(file_get_contents($file));
56+
} catch (ConfigurationException $error) {
57+
$error->output($output);
58+
return 1;
59+
}
60+
61+
$notifiers = new ExportProgressNotifier($output);
62+
$futures = [];
63+
$connectionPool = $config->createConnectionPool();
64+
$factory = new ExportTableFactory($connectionPool, $notifiers);
65+
foreach ($factory->tablesToExport($config) as $table) {
66+
$queue = new Queue(100);
67+
68+
$futures[] = async(function () use ($factory, $table, $queue) {
69+
$factory->createExport($table)->run($queue);
70+
});
71+
72+
$futures[] = async(function () use ($queue, $output) {
73+
foreach ($queue->iterate() as $item) {
74+
}
75+
});
76+
77+
if (count($futures) >= 100) {
78+
awaitAll($futures);
79+
$futures = [];
80+
}
81+
}
82+
83+
EventLoop::run();
84+
return 0;
85+
}
86+
}

‎src/Condition/AnyTableCondition.php

+1-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
{
1616
public function __construct(private array $conditions)
1717
{
18-
1918
}
2019

2120
public function isSatisfiedBy(string $tableName, int $rows): bool
@@ -33,4 +32,4 @@ public function withCondition(TableCondition $condition): TableCondition
3332
$conditions[] = $condition;
3433
return new self($conditions);
3534
}
36-
}
35+
}

‎src/Condition/StaticTableCondition.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,4 @@ public function isSatisfiedBy(string $tableName, int $rows): bool
3434
{
3535
return $this->result;
3636
}
37-
}
37+
}

‎src/Condition/TableNameCondition.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,4 +62,4 @@ public function isSatisfiedBy(string $tableName, int $rows): bool
6262
default => false
6363
};
6464
}
65-
}
65+
}

‎src/Condition/TableRowsCondition.php

+2-4
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,7 @@ class TableRowsCondition implements TableCondition
2121
public function __construct(
2222
private readonly int $rowsCount,
2323
private readonly string $type
24-
)
25-
{
26-
24+
) {
2725
}
2826

2927
public static function minRows(int $minRows): self
@@ -44,4 +42,4 @@ public function isSatisfiedBy(string $tableName, int $rows): bool
4442
default => false,
4543
};
4644
}
47-
}
45+
}

‎src/Configuration.php

+12-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
namespace EcomDev\MySQL2JSONL;
1111

1212
use Amp\Mysql\MysqlConfig;
13+
use Amp\Mysql\MysqlConnectionPool;
1314
use EcomDev\MySQL2JSONL\Condition\AndTableCondition;
1415
use EcomDev\MySQL2JSONL\Condition\StaticTableCondition;
1516
use EcomDev\MySQL2JSONL\Condition\TableNameCondition;
@@ -45,6 +46,7 @@ public static function fromJSON(string $json): Configuration
4546
$json,
4647
flags: JSON_THROW_ON_ERROR
4748
);
49+
4850
$result = $validator->validate($data, $schema, Constraint::CHECK_MODE_APPLY_DEFAULTS);
4951

5052
if ($result !== Validator::ERROR_NONE) {
@@ -102,7 +104,7 @@ function ($errors, $error) {
102104
);
103105
}
104106

105-
public static function fromMySQLConfig(MysqlConfig $config, int $maxConnections = 50, int $idleTimeout = 60): self
107+
public static function fromMySQLConfig(MysqlConfig $config, int $maxConnections = 10, int $idleTimeout = 60): self
106108
{
107109
return new self(
108110
$config,
@@ -155,4 +157,13 @@ private static function buildCondition(string|object $condition): TableCondition
155157
};
156158
}
157159
}
160+
161+
public function createConnectionPool(): MysqlConnectionPool
162+
{
163+
return new MysqlConnectionPool(
164+
$this->connection,
165+
$this->maxConnections,
166+
$this->idleTimeout
167+
);
168+
}
158169
}

‎src/ConfigurationException.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,4 @@ public function output(ConsoleOutputInterface $output): void
4747
$helper->formatSection($this->getMessage(), PHP_EOL . implode(PHP_EOL, $messages), 'error')
4848
);
4949
}
50-
}
50+
}

‎src/ExportTable.php

+30-2
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,43 @@
99

1010
namespace EcomDev\MySQL2JSONL;
1111

12+
use Amp\Future;
1213
use Amp\Mysql\MysqlConnectionPool;
14+
use Amp\Pipeline\Queue;
15+
use function Amp\async;
16+
use function Amp\delay;
1317

14-
class ExportTable
18+
final class ExportTable
1519
{
1620
public function __construct(
1721
private readonly string $tableName,
1822
private readonly MysqlConnectionPool $connectionPool,
1923
private readonly ProgressNotifier $progressNotifier,
2024
) {
25+
}
26+
27+
public function run(Queue $queue): void
28+
{
29+
$total = $this->connectionPool
30+
->execute(sprintf('SELECT COUNT(*) as `total` FROM `%s`', $this->tableName))
31+
->fetchRow();
32+
33+
$this->progressNotifier->start($this->tableName, $total['total']);
34+
35+
$result = $this->connectionPool->execute(sprintf('SELECT * FROM `%s`', $this->tableName));
36+
$header = [];
37+
foreach ($result->getColumnDefinitions() as $column) {
38+
$header[] = $column->getName();
39+
}
40+
$queue->push($header);
41+
42+
$index = 0;
43+
foreach ($result as $row) {
44+
$queue->push(array_values($row));
45+
$this->progressNotifier->update($this->tableName, ++$index);
46+
}
2147

48+
$queue->complete();
49+
$this->progressNotifier->finish($this->tableName);
2250
}
23-
}
51+
}

‎src/ExportTableFactory.php

+20-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,25 @@ public function __construct(
2222

2323
public function createExport(string $tableName): ExportTable
2424
{
25-
return new ExportTable($tableName, $this->connectionPool, $por);
25+
return new ExportTable($tableName, $this->connectionPool, $this->progressNotifier);
26+
}
27+
28+
public function tablesToExport(Configuration $configuration): array
29+
{
30+
$result = $this->connectionPool->execute(
31+
'SELECT TABLE_NAME, TABLE_ROWS FROM information_schema.tables'
32+
. ' WHERE TABLE_SCHEMA = SCHEMA() AND TABLE_TYPE = ?',
33+
['BASE TABLE']
34+
);
35+
36+
$tables = [];
37+
foreach ($result as $row) {
38+
if (!$configuration->includeCondition->isSatisfiedBy($row['TABLE_NAME'], $row['TABLE_ROWS'])
39+
|| $configuration->excludeCondition->isSatisfiedBy($row['TABLE_NAME'], $row['TABLE_ROWS'])) {
40+
continue;
41+
}
42+
$tables[] = $row['TABLE_NAME'];
43+
}
44+
return $tables;
2645
}
2746
}

‎src/ProgressNotifier.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ public function start(string $name, int $total): void;
1616
public function update(string $name, int $current): void;
1717

1818
public function finish(string $name): void;
19-
}
19+
}

‎src/Sql/InsertOnDuplicate.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,4 +44,4 @@ public function generate(string $tableName, array $columns, int $rowCount, $onUp
4444

4545
return $sql;
4646
}
47-
}
47+
}

‎src/TableCondition.php

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,4 @@ interface TableCondition
1414
public function isSatisfiedBy(string $tableName, int $rows): bool;
1515

1616
public function withCondition(TableCondition $condition): TableCondition;
17-
}
17+
}

0 commit comments

Comments
 (0)