Skip to content

Commit 2b61eb7

Browse files
IvanChepurnyiIvanChepurnyi
committed
feat: added commands for import and export
1 parent c54caf0 commit 2b61eb7

File tree

80 files changed

+3155
-710
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+3155
-710
lines changed

‎bin/mysql2jsonl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
<?php
33

44
use EcomDev\MySQL2JSONL\Command\ExportCommand;
5+
use EcomDev\MySQL2JSONL\Command\ImportCommand;
56
use Symfony\Component\Console\Application;
67

78
require_once __DIR__ . '/../vendor/autoload.php';
@@ -12,7 +13,8 @@ $application = new Application(
1213
);
1314

1415
$application->addCommands([
15-
new ExportCommand()
16+
new ExportCommand(),
17+
new ImportCommand(),
1618
]);
1719

1820
$application->run();

‎composer.json

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,12 @@
44
"type": "library",
55
"require": {
66
"php": "~8.2",
7-
"amphp/file": "~3.2",
8-
"amphp/mysql": "~3.0",
97
"amphp/amp": "~3.0",
10-
"amphp/pipeline": "~1.2",
8+
"amphp/parallel": "~v2.3.1",
119
"revolt/event-loop": "~1.0",
1210
"symfony/console": "~7.2",
13-
"justinrainbow/json-schema": "^6.0"
11+
"justinrainbow/json-schema": "^6.0",
12+
"ext-pdo": "*"
1413
},
1514
"require-dev": {
1615
"squizlabs/php_codesniffer": "^3.0",

‎phpunit.xml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
<?xml version="1.0" encoding="UTF-8"?>
2-
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/11.5/phpunit.xsd" bootstrap="vendor/autoload.php" executionOrder="depends,defects" beStrictAboutOutputDuringTests="true" colors="true" cacheDirectory=".phpunit.cache">
2+
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/11.5/phpunit.xsd" bootstrap="vendor/autoload.php" executionOrder="depends,defects"
3+
beStrictAboutOutputDuringTests="true"
4+
colors="true"
5+
cacheDirectory=".phpunit.cache"
6+
displayDetailsOnTestsThatTriggerWarnings="true"
7+
displayDetailsOnTestsThatTriggerNotices="true"
8+
>
39
<testsuites>
410
<testsuite name="default">
511
<directory>tests</directory>

‎schema.json

Lines changed: 66 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,48 @@
33
"title": "MySQL2JSONL configuration file",
44
"type": "object",
55
"properties": {
6-
"connection": {
7-
"$ref": "#/definitions/connection",
8-
"description": "MySQL connections settings"
9-
},
10-
"maxConnections": {
11-
"type": "integer",
12-
"description": "Maximum number of open connection by tool",
13-
"default": 10
14-
},
15-
"idleTimeout": {
16-
"type": "integer",
17-
"description": "Duration of keeping idle connection open",
18-
"default": 60
19-
},
20-
"includeTables": {
21-
"$ref": "#/definitions/tableConditions",
22-
"description": "List of tables to include from the database dump"
23-
},
24-
"excludeTables": {
25-
"$ref": "#/definitions/tableConditions",
26-
"description": "List of tables to exclude from the database dump"
6+
"config": {
7+
"$ref": "#/definitions/configuration",
8+
"description": "Configuration settings"
279
}
2810
},
29-
"required": ["connection"],
30-
"additionalProperties": false,
11+
"required": ["config"],
3112
"definitions": {
13+
"configuration": {
14+
"properties": {
15+
"connection": {
16+
"$ref": "#/definitions/connection",
17+
"description": "MySQL connections settings"
18+
},
19+
"concurrency": {
20+
"type": "integer",
21+
"description": "Maximum concurrency used by tool",
22+
"default": 4
23+
},
24+
"batchSize": {
25+
"type": "integer",
26+
"description": "Batch size for importing data into database",
27+
"default": 1000
28+
},
29+
"includeTables": {
30+
"$ref": "#/definitions/tableConditions",
31+
"description": "List of tables to include from the database dump"
32+
},
33+
"excludeTables": {
34+
"$ref": "#/definitions/tableConditions",
35+
"description": "List of tables to exclude from the database dump"
36+
},
37+
"importMode": {
38+
"$ref": "#/definitions/importMode"
39+
}
40+
},
41+
"required": ["connection"],
42+
"additionalProperties": false
43+
},
44+
"importMode": {
45+
"enum": ["truncate", "update"],
46+
"default": "truncate"
47+
},
3248
"matchExpression": {
3349
"type": "object",
3450
"description": "Match condition for table name",
@@ -109,6 +125,10 @@
109125
"type": "string",
110126
"description": "Host for MySQL host"
111127
},
128+
"unixSocket": {
129+
"type": "string",
130+
"description": "Unix socket path for MySQL connection"
131+
},
112132
"port": {
113133
"type": "integer",
114134
"description": "Port for MySQL connection",
@@ -133,28 +153,33 @@
133153
"description": "Charset for MySQL connection",
134154
"default": "utf8mb4"
135155
},
136-
"collate": {
137-
"type": "string",
138-
"description": "Collation for MySQL connection",
139-
"default": "utf8mb4_unicode_ci"
140-
},
141-
"sqlMode": {
142-
"type": "string",
143-
"description": "SQL_MODE to set for the connection",
144-
"default": ""
145-
},
146-
"useCompression": {
147-
"type": "boolean",
148-
"description": "Use compression for MySQL connection",
149-
"default": false
150-
},
151156
"key": {
152157
"type": "string",
153-
"description": "Private key to use for sha256_password auth method in MySQL connection"
158+
"description": "Path to public key to use for sha256_password auth method in MySQL connection"
154159
}
155160
},
156-
"required": [
157-
"host", "database"
161+
"allOf": [
162+
{
163+
"required": [
164+
"database"
165+
]
166+
},
167+
{
168+
"oneOf": [
169+
{
170+
"required": [
171+
"host"
172+
],
173+
"errorMessage": "For connection host or unixSocket must be specified"
174+
},
175+
{
176+
"required": [
177+
"unixSocket"
178+
],
179+
"errorMessage": "For connection host or unixSocket must be specified"
180+
}
181+
]
182+
}
158183
]
159184
}
160185
}

‎src/Command/ExportCommand.php

Lines changed: 39 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,27 @@
11
<?php
22

3+
/**
4+
* Copyright © EcomDev B.V. All rights reserved.
5+
* See LICENSE for license details.
6+
*/
7+
8+
declare(strict_types=1);
9+
310
namespace EcomDev\MySQL2JSONL\Command;
411

5-
use Amp\Pipeline\Queue;
612
use EcomDev\MySQL2JSONL\Configuration;
713
use EcomDev\MySQL2JSONL\ConfigurationException;
8-
use EcomDev\MySQL2JSONL\ExportTableFactory;
9-
use EcomDev\MySQL2JSONL\Progress\ExportProgressNotifier;
10-
use Revolt\EventLoop;
14+
use EcomDev\MySQL2JSONL\Export\TableListService;
15+
use EcomDev\MySQL2JSONL\Export\WorkerExportServiceFactory;
16+
use EcomDev\MySQL2JSONL\Progress\ConsoleProgressNotifier;
1117
use Symfony\Component\Console\Attribute\AsCommand;
1218
use Symfony\Component\Console\Command\Command;
1319
use Symfony\Component\Console\Helper\FormatterHelper;
20+
use Symfony\Component\Console\Helper\Helper;
1421
use Symfony\Component\Console\Input\InputArgument;
1522
use Symfony\Component\Console\Input\InputInterface;
16-
use Symfony\Component\Console\Output\OutputInterface;
1723
use Symfony\Component\Console\Input\InputOption;
18-
use function Amp\async;
19-
use function Amp\delay;
20-
use function Amp\Future\awaitAll;
24+
use Symfony\Component\Console\Output\OutputInterface;
2125

2226
#[AsCommand(name: 'export', description: 'Export data from MySQL to a directory with JSONL files')]
2327
class ExportCommand extends Command
@@ -26,11 +30,19 @@ public function configure()
2630
{
2731
$this->addOption(
2832
'config',
29-
'c', InputOption::VALUE_REQUIRED,
33+
'c',
34+
InputOption::VALUE_REQUIRED,
3035
'Configuration file',
3136
'config.json'
3237
);
3338

39+
$this->addOption(
40+
'concurrency',
41+
'p',
42+
InputOption::VALUE_REQUIRED,
43+
'Number of concurrent workers, by default taken from config file',
44+
);
45+
3446
$this->addArgument(
3547
'directory',
3648
InputArgument::OPTIONAL,
@@ -53,34 +65,31 @@ public function execute(InputInterface $input, OutputInterface $output): int
5365
}
5466
try {
5567
$config = Configuration::fromJSON(file_get_contents($file));
68+
if ($input->getOption('concurrency')) {
69+
$config = $config->withConcurrency((int)$input->getOption('concurrency'));
70+
}
5671
} catch (ConfigurationException $error) {
5772
$error->output($output);
5873
return 1;
5974
}
6075

61-
$notifiers = new ExportProgressNotifier($output);
62-
$futures = [];
63-
$connectionPool = $config->createConnectionPool();
64-
$factory = new ExportTableFactory($connectionPool, $notifiers);
65-
foreach ($factory->tablesToExport($config) as $table) {
66-
$queue = new Queue(100);
67-
68-
$futures[] = async(function () use ($factory, $table, $queue) {
69-
$factory->createExport($table)->run($queue);
70-
});
76+
$start = microtime(true);
77+
$progressNotifier = new ConsoleProgressNotifier($output, 'Exporting', 'Exported');
78+
$exporter = WorkerExportServiceFactory::fromConfiguration(
79+
$config,
80+
$input->getArgument('directory')
81+
)->create();
82+
$tables = (new TableListService($config))->tablesToExport();
7183

72-
$futures[] = async(function () use ($queue, $output) {
73-
foreach ($queue->iterate() as $item) {
74-
}
75-
});
76-
77-
if (count($futures) >= 100) {
78-
awaitAll($futures);
79-
$futures = [];
80-
}
84+
foreach ($tables as $table) {
85+
$exporter->exportTable($table, $progressNotifier);
8186
}
8287

83-
EventLoop::run();
88+
$exporter->await();
89+
$timeTaken = Helper::formatTime(microtime(true) - $start, 3);
90+
$output->writeln(
91+
"Export finished in {$timeTaken}"
92+
);
8493
return 0;
8594
}
86-
}
95+
}

0 commit comments

Comments
 (0)