265 lines
12 KiB
PHP
265 lines
12 KiB
PHP
|
<?php
|
||
|
|
|
||
|
|
namespace App\Console\Commands;
|
||
|
|
|
||
|
|
use Illuminate\Console\Command;
|
||
|
|
use Illuminate\Database\Connection;
|
||
|
|
use Illuminate\Support\Facades\DB;
|
||
|
|
use Illuminate\Support\Facades\Schema;
|
||
|
|
use RuntimeException;
|
||
|
|
use Throwable;
|
||
|
|
|
||
|
|
class MigrateSqliteToMariaDb extends Command
|
||
|
|
{
|
||
|
|
protected $signature = 'trafficpumper:migrate-sqlite-to-mariadb {--execute} {--chunk=500} {--include-operational} {--replace-migrated-seeds}';
|
||
|
|
|
||
|
|
protected $description = 'Inspect or copy legacy SQLite business data to an already migrated MariaDB database';
|
||
|
|
|
||
|
|
private const OPERATIONAL = ['cache', 'cache_locks', 'sessions', 'jobs', 'job_batches', 'failed_jobs'];
|
||
|
|
|
||
|
|
/** @var array<string, true> */
|
||
|
|
private array $alreadyPresent = [];
|
||
|
|
|
||
|
|
/** @var array<string, list<string>> */
|
||
|
|
private array $replaceSeeds = [];
|
||
|
|
|
||
|
|
public function handle(): int
|
||
|
|
{
|
||
|
|
try {
|
||
|
|
$source = DB::connection('sqlite_legacy');
|
||
|
|
$target = DB::connection('mysql');
|
||
|
|
$this->assertConnections($source, $target);
|
||
|
|
$tables = $this->orderedTables($source);
|
||
|
|
$this->assertTargetSchema($target, $tables);
|
||
|
|
$this->reportPlan($source, $target, $tables);
|
||
|
|
if (! $this->option('execute')) {
|
||
|
|
$this->info('Dry-run completed. No rows were written. Use --execute to transfer.');
|
||
|
|
|
||
|
|
return self::SUCCESS;
|
||
|
|
}
|
||
|
|
$this->assertEmptyTarget($target, $tables);
|
||
|
|
$this->copy($source, $target, $tables);
|
||
|
|
$this->verify($source, $target, $tables);
|
||
|
|
$this->info('Transfer and integrity verification completed successfully.');
|
||
|
|
|
||
|
|
return self::SUCCESS;
|
||
|
|
} catch (Throwable $exception) {
|
||
|
|
$this->error($exception->getMessage());
|
||
|
|
|
||
|
|
return self::FAILURE;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private function assertConnections(Connection $source, Connection $target): void
|
||
|
|
{
|
||
|
|
if ($source->getDriverName() !== 'sqlite') {
|
||
|
|
throw new RuntimeException('sqlite_legacy must use SQLite.');
|
||
|
|
}
|
||
|
|
if (! in_array($target->getDriverName(), ['mysql', 'mariadb'], true)) {
|
||
|
|
throw new RuntimeException('The mysql connection must point to MariaDB/MySQL.');
|
||
|
|
}
|
||
|
|
$path = (string) config('database.connections.sqlite_legacy.database');
|
||
|
|
if (! is_file($path) || ! is_readable($path)) {
|
||
|
|
throw new RuntimeException('The legacy SQLite database is missing or unreadable.');
|
||
|
|
}
|
||
|
|
$server = $target->selectOne('select version() version, @@character_set_database charset, @@collation_database collation');
|
||
|
|
if ($server->charset !== 'utf8mb4' || $server->collation !== 'utf8mb4_unicode_ci') {
|
||
|
|
throw new RuntimeException('Target database must use utf8mb4 and utf8mb4_unicode_ci.');
|
||
|
|
}
|
||
|
|
$this->info("Target server: {$server->version}; {$server->charset}/{$server->collation}");
|
||
|
|
}
|
||
|
|
|
||
|
|
/** @return list<string> */
|
||
|
|
private function orderedTables(Connection $source): array
|
||
|
|
{
|
||
|
|
$excluded = ['migrations', ...($this->option('include-operational') ? [] : self::OPERATIONAL)];
|
||
|
|
$tables = collect(Schema::connection($source->getName())->getTables())
|
||
|
|
->map(fn (array $table): string => (string) ($table['name'] ?? ''))
|
||
|
|
->filter(fn (string $table): bool => $table !== '' && ! in_array($table, $excluded, true))->values()->all();
|
||
|
|
$pending = array_fill_keys($tables, true);
|
||
|
|
$ordered = [];
|
||
|
|
while ($pending !== []) {
|
||
|
|
$progress = false;
|
||
|
|
foreach (array_keys($pending) as $table) {
|
||
|
|
$dependencies = collect(Schema::connection($source->getName())->getForeignKeys($table))
|
||
|
|
->map(fn (array $key): string => (string) ($key['foreign_table'] ?? ''))
|
||
|
|
->filter(fn (string $dependency): bool => isset($pending[$dependency]) && $dependency !== $table);
|
||
|
|
if ($dependencies->isNotEmpty()) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
$ordered[] = $table;
|
||
|
|
unset($pending[$table]);
|
||
|
|
$progress = true;
|
||
|
|
}
|
||
|
|
if (! $progress) {
|
||
|
|
throw new RuntimeException('Unresolved foreign-key dependencies: '.implode(', ', array_keys($pending)));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return $ordered;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** @param list<string> $tables */
|
||
|
|
private function assertTargetSchema(Connection $target, array $tables): void
|
||
|
|
{
|
||
|
|
$missing = collect($tables)->reject(fn (string $table): bool => Schema::connection($target->getName())->hasTable($table));
|
||
|
|
if ($missing->isNotEmpty()) {
|
||
|
|
throw new RuntimeException('Run target migrations first. Missing tables: '.$missing->implode(', '));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** @param list<string> $tables */
|
||
|
|
private function reportPlan(Connection $source, Connection $target, array $tables): void
|
||
|
|
{
|
||
|
|
$this->table(['Table', 'SQLite', 'MariaDB'], collect($tables)->map(fn (string $table): array => [$table, $source->table($table)->count(), $target->table($table)->count()])->all());
|
||
|
|
if (! $this->option('include-operational')) {
|
||
|
|
$this->warn('Operational tables are excluded; database sessions will require users to sign in again.');
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** @param list<string> $tables */
|
||
|
|
private function assertEmptyTarget(Connection $target, array $tables): void
|
||
|
|
{
|
||
|
|
$source = DB::connection('sqlite_legacy');
|
||
|
|
foreach ($tables as $table) {
|
||
|
|
if (! $target->table($table)->exists()) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if ($source->table($table)->count() === $target->table($table)->count()
|
||
|
|
&& hash_equals($this->tableChecksum($source, $table), $this->tableChecksum($target, $table))) {
|
||
|
|
$this->alreadyPresent[$table] = true;
|
||
|
|
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
$primary = collect(Schema::connection($source->getName())->getIndexes($table))->first(fn (array $index): bool => (bool) ($index['primary'] ?? false));
|
||
|
|
$primaryColumns = (array) ($primary['columns'] ?? []);
|
||
|
|
if ($this->option('replace-migrated-seeds') && $primaryColumns !== []
|
||
|
|
&& $source->table($table)->count() === $target->table($table)->count()
|
||
|
|
&& hash_equals($this->keyChecksum($source, $table, $primaryColumns), $this->keyChecksum($target, $table, $primaryColumns))) {
|
||
|
|
$this->replaceSeeds[$table] = $primaryColumns;
|
||
|
|
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
throw new RuntimeException("Target table {$table} is non-empty and does not exactly match the source.");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** @param list<string> $tables */
|
||
|
|
private function copy(Connection $source, Connection $target, array $tables): void
|
||
|
|
{
|
||
|
|
$chunk = max(1, min(5000, (int) $this->option('chunk')));
|
||
|
|
foreach ($tables as $table) {
|
||
|
|
if (isset($this->alreadyPresent[$table])) {
|
||
|
|
$this->line("Kept identical migrated seed data in {$table}");
|
||
|
|
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
$columns = collect(Schema::connection($source->getName())->getColumns($table))->pluck('name')->all();
|
||
|
|
$primary = collect(Schema::connection($source->getName())->getIndexes($table))->first(fn (array $index): bool => (bool) ($index['primary'] ?? false));
|
||
|
|
$order = ((array) ($primary['columns'] ?? []))[0] ?? $columns[0] ?? null;
|
||
|
|
if ($order === null) {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
$target->transaction(function () use ($source, $target, $table, $order, $chunk): void {
|
||
|
|
$source->table($table)->orderBy($order)->chunk($chunk, function ($rows) use ($target, $table): void {
|
||
|
|
$data = $rows->map(fn (object $row): array => (array) $row)->all();
|
||
|
|
if ($data !== []) {
|
||
|
|
if (isset($this->replaceSeeds[$table])) {
|
||
|
|
$target->table($table)->upsert($data, $this->replaceSeeds[$table]);
|
||
|
|
} else {
|
||
|
|
$target->table($table)->insert($data);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
$this->line("Copied {$table}");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/** @param list<string> $tables */
|
||
|
|
private function verify(Connection $source, Connection $target, array $tables): void
|
||
|
|
{
|
||
|
|
foreach ($tables as $table) {
|
||
|
|
if ($source->table($table)->count() !== $target->table($table)->count()) {
|
||
|
|
throw new RuntimeException("Row-count mismatch for {$table}.");
|
||
|
|
}
|
||
|
|
$columns = collect(Schema::connection($source->getName())->getColumns($table))->pluck('name');
|
||
|
|
foreach ($columns->filter(fn (string $column): bool => preg_match('/(^|_)(password|token|ulid|uuid)(_hash)?$/', $column) === 1) as $column) {
|
||
|
|
if (! hash_equals($this->checksum($source, $table, $column), $this->checksum($target, $table, $column))) {
|
||
|
|
throw new RuntimeException("Sensitive checksum mismatch for {$table}.{$column}.");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
foreach ($tables as $table) {
|
||
|
|
foreach (Schema::connection($target->getName())->getForeignKeys($table) as $key) {
|
||
|
|
$local = (array) ($key['columns'] ?? []);
|
||
|
|
$foreign = (array) ($key['foreign_columns'] ?? []);
|
||
|
|
$parent = (string) ($key['foreign_table'] ?? '');
|
||
|
|
if (count($local) !== 1 || count($foreign) !== 1 || $parent === '') {
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
$orphans = $target->table("{$table} as child")
|
||
|
|
->leftJoin("{$parent} as parent", "child.{$local[0]}", '=', "parent.{$foreign[0]}")
|
||
|
|
->whereNotNull("child.{$local[0]}")->whereNull("parent.{$foreign[0]}")->count();
|
||
|
|
if ($orphans > 0) {
|
||
|
|
throw new RuntimeException("Foreign-key orphans found in {$table}.{$local[0]}.");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private function checksum(Connection $connection, string $table, string $column): string
|
||
|
|
{
|
||
|
|
$hash = hash_init('sha256');
|
||
|
|
$connection->table($table)->orderBy($column)->select($column)->chunk(1000, function ($rows) use ($hash, $column): void {
|
||
|
|
foreach ($rows as $row) {
|
||
|
|
$value = (string) $row->{$column};
|
||
|
|
hash_update($hash, strlen($value).':'.$value."\n");
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
return hash_final($hash);
|
||
|
|
}
|
||
|
|
|
||
|
|
private function tableChecksum(Connection $connection, string $table): string
|
||
|
|
{
|
||
|
|
$columns = collect(Schema::connection($connection->getName())->getColumns($table))->pluck('name')->all();
|
||
|
|
$primary = collect(Schema::connection($connection->getName())->getIndexes($table))->first(fn (array $index): bool => (bool) ($index['primary'] ?? false));
|
||
|
|
$order = (array) ($primary['columns'] ?? $columns);
|
||
|
|
$hash = hash_init('sha256');
|
||
|
|
$query = $connection->table($table)->select($columns);
|
||
|
|
foreach ($order as $column) {
|
||
|
|
$query->orderBy($column);
|
||
|
|
}
|
||
|
|
$query->chunk(1000, function ($rows) use ($hash, $columns): void {
|
||
|
|
foreach ($rows as $row) {
|
||
|
|
foreach ($columns as $column) {
|
||
|
|
$value = $row->{$column};
|
||
|
|
hash_update($hash, $value === null ? '-1:' : strlen((string) $value).':'.(string) $value);
|
||
|
|
}
|
||
|
|
hash_update($hash, "\n");
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
return hash_final($hash);
|
||
|
|
}
|
||
|
|
|
||
|
|
/** @param list<string> $columns */
|
||
|
|
private function keyChecksum(Connection $connection, string $table, array $columns): string
|
||
|
|
{
|
||
|
|
$hash = hash_init('sha256');
|
||
|
|
$query = $connection->table($table)->select($columns);
|
||
|
|
foreach ($columns as $column) {
|
||
|
|
$query->orderBy($column);
|
||
|
|
}
|
||
|
|
foreach ($query->get() as $row) {
|
||
|
|
foreach ($columns as $column) {
|
||
|
|
hash_update($hash, strlen((string) $row->{$column}).':'.(string) $row->{$column});
|
||
|
|
}
|
||
|
|
hash_update($hash, "\n");
|
||
|
|
}
|
||
|
|
|
||
|
|
return hash_final($hash);
|
||
|
|
}
|
||
|
|
}
|