Skip to content

Commit 8a49239

Browse files
IvanChepurnyiIvanChepurnyi
authored
Merge pull request #2 from EcomDev/upcoming-release
feat: mysql2json import/export tool new tool that allows quickly export and import data without modifying database structure feat(config): created JSONSchema for configuration files it allows to show nice errors messages for each incorrect value and defines defaults for configuration feat(config): added `includeTables` and `excludeTables` rules it is possible to define various table matching rules for the tool feat(config): added `concurrency` settings it is possible to control number of processes to use for parallel export/import feat(config): added `batchSize` setting it is possible to specify in configuration how many rows to import per batch feat(config): added `importMode` setting of `truncate` and `update` it is possible to specify if import replaces table completely or update current rows in it feat(command): added `mysql2jsonl export` command it exports data into JSONL file into desired directory with possibility to override concurrency feat(command): added `mysql2jsonl import` command it imports data form directory with JSONL files with possibility to override both concurrency and batch size feat(build): added Phar version to release pipeline it is possible to download now phar file with the tool, without installing it via composer
2 parents 26ef87e + 6863545 commit 8a49239

File tree

85 files changed

+4568
-35
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

85 files changed

+4568
-35
lines changed

‎.github/workflows/php-package.yml‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
name: PHP Package
2+
env:
3+
PHAR_TOOL_VERSION: 1.4.0
4+
PHAR_TOOL_REPOSITORY: clue/phar-composer
25
on:
36
push:
47
pull_request:
@@ -26,19 +29,24 @@ jobs:
2629
php-version:
2730
- 8.2
2831
- 8.3
29-
io-driver:
30-
- eio
31-
- uv
32+
- 8.4
3233
steps:
3334
- uses: actions/checkout@v4
3435
- name: Setup PHP
3536
uses: shivammathur/setup-php@v2
3637
with:
3738
php-version: ${{ matrix.php-version }}
38-
extensions: ${{ matrix.io-driver }}
3939
tools: composer:v2
4040
coverage: xdebug3
4141
- run: composer install
4242
shell: bash
4343
- run: composer test
4444
shell: bash
45+
- name: Download build package
46+
run: gh release download v${{ env.PHAR_TOOL_VERSION }} -R=${{ env.PHAR_TOOL_REPOSITORY }}
47+
env:
48+
GH_TOKEN: ${{ github.token }}
49+
- name: Build package
50+
run: |
51+
chmod +x ./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar
52+
./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar build ./ mysql2json

‎.github/workflows/release-please.yml‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ on:
22
push:
33
branches:
44
- main
5+
env:
6+
PHAR_TOOL_VERSION: 1.4.0
7+
PHAR_TOOL_REPOSITORY: clue/phar-composer
58

69
permissions:
710
contents: write
@@ -15,7 +18,36 @@ jobs:
1518
release-please:
1619
needs: verify-release
1720
runs-on: ubuntu-24.04
21+
outputs:
22+
releases_created: ${{ steps.release.outputs.release_created }}
23+
tag: ${{ steps.release.outputs.tag_name }}
1824
steps:
1925
- uses: googleapis/release-please-action@v4
26+
id: release
2027
with:
21-
release-type: php
28+
release-type: php
29+
upload_phar:
30+
needs: release-please
31+
runs-on: ubuntu-24.04
32+
if: ${{ needs.release-please.outputs.releases_created == true }}
33+
steps:
34+
- uses: actions/checkout@v4
35+
with:
36+
ref: ${{ needs.release-please.outputs.tag }}
37+
- name: Setup PHP
38+
uses: shivammathur/setup-php@v2
39+
with:
40+
php-version: 8.2
41+
tools: composer:v2
42+
- name: Download build package
43+
run: gh release download v${{ env.PHAR_TOOL_VERSION }} -R=${{ env.PHAR_TOOL_REPOSITORY }}
44+
- name: Build package
45+
run: |
46+
chmod +x ./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar
47+
./phar-composer-${{ env.PHAR_TOOL_VERSION }}.phar build ./ mysql2json
48+
env:
49+
GH_TOKEN: ${{ github.token }}
50+
- name: Upload package
51+
run: gh release upload ${{ needs.release-please.outputs.tag }} mysql2json
52+
env:
53+
GH_TOKEN: ${{ github.token }}

‎bin/mysql2jsonl‎

100644100755
Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
1-
#!env php
1+
#!/usr/bin/env php
22
<?php
33

4+
use EcomDev\MySQL2JSONL\Command\ExportCommand;
5+
use EcomDev\MySQL2JSONL\Command\ImportCommand;
6+
use Symfony\Component\Console\Application;
7+
48
require_once __DIR__ . '/../vendor/autoload.php';
9+
10+
$application = new Application(
11+
basename(__FILE__),
12+
'1.0.0'
13+
);
14+
15+
$application->addCommands([
16+
new ExportCommand(),
17+
new ImportCommand(),
18+
]);
19+
20+
$application->run();

‎composer.json‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,18 @@
44
"type": "library",
55
"require": {
66
"php": "~8.2",
7-
"amphp/file": "~3.2",
8-
"amphp/mysql": "~3.0",
97
"amphp/amp": "~3.0",
8+
"amphp/parallel": "~v2.3.1",
109
"revolt/event-loop": "~1.0",
11-
"symfony/console": "~7.2"
10+
"symfony/console": "~7.2",
11+
"justinrainbow/json-schema": "^6.0",
12+
"ext-pdo": "*"
1213
},
1314
"require-dev": {
1415
"squizlabs/php_codesniffer": "^3.0",
1516
"phpunit/phpunit": "^11.5",
1617
"brianium/paratest": "^7.7",
17-
"ecomdev/testcontainers-magento-data":"~1.1"
18+
"ecomdev/testcontainers-magento-data":"~1.2"
1819
},
1920
"license": [
2021
"MIT"
@@ -24,16 +25,20 @@
2425
],
2526
"autoload": {
2627
"psr-4": {
27-
"EcomDev\\MySQL2JSONL\\": "src"
28+
"EcomDev\\MySQL2JSONL\\": "src/"
2829
}
2930
},
3031
"autoload-dev": {
32+
"files": [
33+
"tests/fixtures.php"
34+
],
3135
"psr-4": {
32-
"EcomDev\\MySQL2JSONL\\": "tests"
36+
"EcomDev\\MySQL2JSONL\\": "tests/"
3337
}
3438
},
3539
"scripts": {
3640
"test": "XDEBUG_MODE=coverage paratest --coverage-text",
41+
"test:single-threaded": "XDEBUG_MODE=coverage phpunit --coverage-text",
3742
"format:check": "phpcs",
3843
"format:write": "phpcbf"
3944
},

‎phpunit.xml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
<?xml version="1.0" encoding="UTF-8"?>
2-
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/11.5/phpunit.xsd" bootstrap="vendor/autoload.php" executionOrder="depends,defects" beStrictAboutOutputDuringTests="true" colors="true" cacheDirectory=".phpunit.cache">
2+
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/11.5/phpunit.xsd" bootstrap="vendor/autoload.php" executionOrder="depends,defects"
3+
beStrictAboutOutputDuringTests="true"
4+
colors="true"
5+
cacheDirectory=".phpunit.cache"
6+
displayDetailsOnTestsThatTriggerWarnings="true"
7+
displayDetailsOnTestsThatTriggerNotices="true"
8+
displayDetailsOnTestsThatTriggerDeprecations="true"
9+
>
310
<testsuites>
411
<testsuite name="default">
512
<directory>tests</directory>

‎schema.json‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
{
2+
"$schema": "https://json-schema.org/draft-04/schema#",
3+
"title": "MySQL2JSONL configuration file",
4+
"type": "object",
5+
"properties": {
6+
"config": {
7+
"$ref": "#/definitions/configuration",
8+
"description": "Configuration settings"
9+
}
10+
},
11+
"required": ["config"],
12+
"definitions": {
13+
"configuration": {
14+
"properties": {
15+
"connection": {
16+
"$ref": "#/definitions/connection",
17+
"description": "MySQL connections settings"
18+
},
19+
"concurrency": {
20+
"type": "integer",
21+
"description": "Maximum concurrency used by tool",
22+
"default": 4
23+
},
24+
"batchSize": {
25+
"type": "integer",
26+
"description": "Batch size for importing data into database",
27+
"default": 1000
28+
},
29+
"includeTables": {
30+
"$ref": "#/definitions/tableConditions",
31+
"description": "List of tables to include from the database dump"
32+
},
33+
"excludeTables": {
34+
"$ref": "#/definitions/tableConditions",
35+
"description": "List of tables to exclude from the database dump"
36+
},
37+
"importMode": {
38+
"$ref": "#/definitions/importMode"
39+
}
40+
},
41+
"required": ["connection"],
42+
"additionalProperties": false
43+
},
44+
"importMode": {
45+
"enum": ["truncate", "update"],
46+
"default": "truncate"
47+
},
48+
"matchExpression": {
49+
"type": "object",
50+
"description": "Match condition for table name",
51+
"properties": {
52+
"regexp": {
53+
"type": "string",
54+
"description": "Valid regular expression for a table name, # sign is prohibited as used as delimiter",
55+
"pattern": "^[^#]+$"
56+
},
57+
"startsWith": {
58+
"type": "string",
59+
"description": "Table name starts with"
60+
},
61+
"endsWith": {
62+
"type": "string",
63+
"description": "Table name ends with"
64+
},
65+
"contains": {
66+
"type": "string",
67+
"description": "Table name contains"
68+
}
69+
},
70+
"minProperties": 1,
71+
"maxProperties": 1
72+
},
73+
"rowCount": {
74+
"type": "object",
75+
"properties": {
76+
"min": {
77+
"type": "integer",
78+
"description": "Minimum number of rows for table to be exported"
79+
},
80+
"max": {
81+
"type": "integer",
82+
"description": "Maximum number of rows for table to be exported"
83+
}
84+
},
85+
"minProperties": 1,
86+
"maxProperties": 1
87+
} ,
88+
"tableName": {
89+
"type": "string",
90+
"pattern": "^[0-9a-zA-Z$_\u0080-\uFFFF]+$"
91+
},
92+
"andCondition": {
93+
"type": "array",
94+
"description": "Table matching conditions",
95+
"items": {
96+
"oneOf": [
97+
{ "$ref": "#/definitions/matchExpression" },
98+
{ "$ref": "#/definitions/rowCount" },
99+
{ "$ref": "#/definitions/tableName" }
100+
]
101+
}
102+
},
103+
"tableConditions": {
104+
"type": "array",
105+
"description": "Table matching conditions",
106+
"default": [],
107+
"items": {
108+
"anyOf": [
109+
{ "$ref": "#/definitions/matchExpression" },
110+
{ "$ref": "#/definitions/rowCount" },
111+
{ "$ref": "#/definitions/tableName" },
112+
{
113+
"type": "object",
114+
"properties": {
115+
"and": {"$ref": "#/definitions/andCondition"}
116+
}
117+
}
118+
]
119+
}
120+
},
121+
"connection": {
122+
"type": "object",
123+
"properties": {
124+
"host": {
125+
"type": "string",
126+
"description": "Host for MySQL host"
127+
},
128+
"unixSocket": {
129+
"type": "string",
130+
"description": "Unix socket path for MySQL connection"
131+
},
132+
"port": {
133+
"type": "integer",
134+
"description": "Port for MySQL connection",
135+
"default": 3306
136+
},
137+
"user": {
138+
"type": "string",
139+
"description": "Username for MySQL connection",
140+
"default": "root"
141+
},
142+
"password": {
143+
"type": "string",
144+
"description": "Password for MySQL connection",
145+
"default": ""
146+
},
147+
"database": {
148+
"type": "string",
149+
"description": "Database for MySQL connection"
150+
},
151+
"charset": {
152+
"type": "string",
153+
"description": "Charset for MySQL connection",
154+
"default": "utf8mb4"
155+
},
156+
"key": {
157+
"type": "string",
158+
"description": "Path to public key to use for sha256_password auth method in MySQL connection"
159+
}
160+
},
161+
"allOf": [
162+
{
163+
"required": [
164+
"database"
165+
]
166+
},
167+
{
168+
"oneOf": [
169+
{
170+
"required": [
171+
"host"
172+
],
173+
"errorMessage": "For connection host or unixSocket must be specified"
174+
},
175+
{
176+
"required": [
177+
"unixSocket"
178+
],
179+
"errorMessage": "For connection host or unixSocket must be specified"
180+
}
181+
]
182+
}
183+
]
184+
}
185+
}
186+
}

0 commit comments

Comments
 (0)