From e0600d6870400c4c09171341ef81dc623f523947 Mon Sep 17 00:00:00 2001 From: Zuko Date: Mon, 4 Nov 2024 12:39:20 +0700 Subject: [PATCH 1/6] Remove "merging configs from`google.php`" --- src/LaravelSyncroSheetProvider.php | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/LaravelSyncroSheetProvider.php b/src/LaravelSyncroSheetProvider.php index 03a0461..a92f7c0 100644 --- a/src/LaravelSyncroSheetProvider.php +++ b/src/LaravelSyncroSheetProvider.php @@ -17,12 +17,12 @@ namespace Zuko\SyncroSheet; +use Illuminate\Foundation\AliasLoader; use Illuminate\Support\ServiceProvider; -use Zuko\SyncroSheet\Services\SyncManager; -use Zuko\SyncroSheet\Services\StateManager; use Zuko\SyncroSheet\Services\BatchProcessor; use Zuko\SyncroSheet\Services\GoogleClient; -use Illuminate\Foundation\AliasLoader; +use Zuko\SyncroSheet\Services\StateManager; +use Zuko\SyncroSheet\Services\SyncManager; class LaravelSyncroSheetProvider extends ServiceProvider { @@ -47,14 +47,6 @@ public function register() __DIR__.'/../config/syncro-sheet.php', 'syncro-sheet' ); - // Ensure google config is available - if (!$this->app->configurationIsCached()) { - $this->mergeConfigFrom( - __DIR__.'/../vendor/revolution/laravel-google-sheets/config/google.php', - 'google' - ); - } - $this->app->singleton(SyncManager::class); $this->app->singleton(StateManager::class); $this->app->singleton(BatchProcessor::class); @@ -64,4 +56,4 @@ public function register() $loader = AliasLoader::getInstance(); $loader->alias('SyncroSheet', \Zuko\SyncroSheet\Facades\SyncroSheet::class); } -} +} From f8023fa1e8be2d97ea75fbfa5288c40d23101e36 Mon Sep 17 00:00:00 2001 From: Zuko Date: Mon, 4 Nov 2024 15:47:13 +0700 Subject: [PATCH 2/6] Update auto generate header --- README.md | 7 ++ composer.json | 83 +++++++++--------- ...ble.php => 0_create_sync_states_table.php} | 2 +- ...le.php => 1_create_sync_entries_table.php} | 2 +- src/Contracts/SheetSyncable.php | 17 ++++ src/LaravelSyncroSheetProvider.php | 30 ++++--- src/Services/BatchProcessor.php | 51 ++++++----- src/Services/GoogleClient.php | 84 ++++++++++++++++--- 8 files changed, 194 insertions(+), 82 deletions(-) rename database/migrations/{create_sync_states_table.php => 0_create_sync_states_table.php} (95%) rename database/migrations/{create_sync_entries_table.php => 1_create_sync_entries_table.php} (95%) diff --git a/README.md b/README.md index 3ad45a4..692d0f0 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,13 @@ GOOGLE_SHEETS_CLIENT_ID=your-client-id GOOGLE_SHEETS_CLIENT_SECRET=your-client-secret GOOGLE_SHEETS_REDIRECT_URI=your-redirect-uri ``` + +Or using service account: + +```env +GOOGLE_DEVELOPER_KEY=your-service-account-key +GOOGLE_SERVICE_ENABLED=true +``` As a wrapped around `revolution/laravel-google-sheets`. these env vars is taken from `config/google.php` If you already set these authorization values. You can leave env untouched. diff --git a/composer.json b/composer.json index 615d3f0..57e3927 100644 --- a/composer.json +++ b/composer.json @@ -1,40 +1,47 @@ { - "name": "zuko/syncro-sheet", - "description": "Laravel Google Sheets synchronization package", - "type": "library", - "license": "MIT", - "authors": [ - { - "name": "Zuko", - "email": "tansautn@gmail.com" - } - ], - "require": { - "php": "^8.1", - "laravel/framework": "^10.0", - "revolution/laravel-google-sheets": "^6.0" - }, - "require-dev": { - "phpunit/phpunit": "^10.0", - "orchestra/testbench": "^8.0" - }, - "autoload": { - "psr-4": { - "Zuko\\SyncroSheet\\": "src/" - } - }, - "autoload-dev": { - "psr-4": { - "Zuko\\SyncroSheet\\Tests\\": "tests/" - } - }, - "extra": { - "laravel": { - "providers": [ - "Zuko\\SyncroSheet\\LaravelSyncroSheetProvider" - ] - } - }, - "minimum-stability": "dev", - "prefer-stable": true + "name" : "zuko/syncro-sheet", + "replace" : { + "zuko/laravel-sheet-sync" : "self.version", + "zuko/eloquent-sheet-sync" : "self.version", + "zuko/synchro-sheet" : "self.version" + }, + "description" : "Laravel Eloquent and Google Sheets synchronization package", + "type" : "library", + "license" : "MIT", + "authors" : [ + { + "name" : "Zuko", + "email" : "tansautn@gmail.com", + "role" : "Developer", + "homepage": "https://github.com/tansautn" + } + ], + "require" : { + "php" : "^8.1", + "illuminate/database" : ">=10.0", + "revolution/laravel-google-sheets" : ">=6.0" + }, + "require-dev" : { + "roave/security-advisories" : "dev-latest", + "laravel/pint" : ">=1.17.0" + }, + "autoload" : { + "psr-4" : { + "Zuko\\SyncroSheet\\" : "src/" + } + }, + "autoload-dev" : { + "psr-4" : { + "Zuko\\SyncroSheet\\Tests\\" : "tests/" + } + }, + "extra" : { + "laravel" : { + "providers" : [ + "Zuko\\SyncroSheet\\LaravelSyncroSheetProvider" + ] + } + }, + "minimum-stability" : "dev", + "prefer-stable" : true } diff --git a/database/migrations/create_sync_states_table.php b/database/migrations/0_create_sync_states_table.php similarity index 95% rename from database/migrations/create_sync_states_table.php rename to database/migrations/0_create_sync_states_table.php index 68a4614..0b1b0d9 100644 --- a/database/migrations/create_sync_states_table.php +++ b/database/migrations/0_create_sync_states_table.php @@ -23,7 +23,7 @@ { public function up(): void { - Schema::create('sync_states', function (Blueprint $table) { + Schema::create('sync_states', static function (Blueprint $table) { $table->id(); $table->string('model_class'); $table->enum('sync_type', ['full', 'partial']); diff --git a/database/migrations/create_sync_entries_table.php b/database/migrations/1_create_sync_entries_table.php similarity index 95% rename from database/migrations/create_sync_entries_table.php rename to database/migrations/1_create_sync_entries_table.php index 781f1a7..95275af 100644 --- a/database/migrations/create_sync_entries_table.php +++ b/database/migrations/1_create_sync_entries_table.php @@ -23,7 +23,7 @@ { public function up(): void { - Schema::create('sync_entries', function (Blueprint $table) { + Schema::create('sync_entries', static function (Blueprint $table) { $table->id(); $table->foreignId('sync_state_id')->constrained()->cascadeOnDelete(); $table->string('model_class'); diff --git a/src/Contracts/SheetSyncable.php b/src/Contracts/SheetSyncable.php index 2b41b05..933a5a9 100644 --- a/src/Contracts/SheetSyncable.php +++ b/src/Contracts/SheetSyncable.php @@ -38,4 +38,21 @@ public function toSheetRow(): array; * Get the batch size for processing */ public function getBatchSize(): ?int; + + /** + * Get the fields used to identify unique records + * If not implemented, system will use auto-detection + */ + public function getSheetIdentifierFields(): ?array; + + /** + * Get the sheet row ID for this model instance + * Used for bi-directional sync + */ + public function getSheetRowId(): ?string; + + /** + * Set the sheet row ID for this model instance + */ + public function setSheetRowId(string $rowId): void; } diff --git a/src/LaravelSyncroSheetProvider.php b/src/LaravelSyncroSheetProvider.php index a92f7c0..5d94713 100644 --- a/src/LaravelSyncroSheetProvider.php +++ b/src/LaravelSyncroSheetProvider.php @@ -17,6 +17,7 @@ namespace Zuko\SyncroSheet; + use Illuminate\Foundation\AliasLoader; use Illuminate\Support\ServiceProvider; use Zuko\SyncroSheet\Services\BatchProcessor; @@ -29,29 +30,36 @@ class LaravelSyncroSheetProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__.'/../config/syncro-sheet.php' => config_path('syncro-sheet.php'), - ], 'config'); - - $this->loadMigrationsFrom(__DIR__.'/../database/migrations'); - + __DIR__ . '/../config/syncro-sheet.php' => config_path('syncro-sheet.php'), + ], 'config'); + $this->loadMigrationsFrom(__DIR__ . '/../database/migrations'); if ($this->app->runningInConsole()) { $this->commands([ - Console\Commands\SheetSyncCommand::class, - ]); + Console\Commands\SheetSyncCommand::class, + ]); } } public function register() { - $this->mergeConfigFrom( - __DIR__.'/../config/syncro-sheet.php', 'syncro-sheet' - ); + $this->mergeConfigFrom(__DIR__ . '/../config/syncro-sheet.php', + 'syncro-sheet'); + // Ensure google config is available + if (!$this->app->configurationIsCached()) { + $revolutionConfig = base_path('vendor/revolution/laravel-google-sheets/config/google.php'); + if (!file_exists($revolutionConfig)) { + $this->mergeConfigFrom(__DIR__ . '/../vendor/revolution/laravel-google-sheets/config/google.php', + 'google'); + } else { + $this->mergeConfigFrom($revolutionConfig, + 'google'); + } + } $this->app->singleton(SyncManager::class); $this->app->singleton(StateManager::class); $this->app->singleton(BatchProcessor::class); $this->app->singleton(GoogleClient::class); - // Register facade $loader = AliasLoader::getInstance(); $loader->alias('SyncroSheet', \Zuko\SyncroSheet\Facades\SyncroSheet::class); diff --git a/src/Services/BatchProcessor.php b/src/Services/BatchProcessor.php index 77c1020..4c8c3cb 100644 --- a/src/Services/BatchProcessor.php +++ b/src/Services/BatchProcessor.php @@ -52,7 +52,7 @@ public function process(string $modelClass, SyncState $syncState): array if (!empty($rows)) { // Transform to associative arrays with headers - $rowsWithHeaders = $rows->map(function($row) use ($headers) { + $rowsWithHeaders = collect($rows)->map(function($row) use ($headers) { return array_combine($headers, $row); })->toArray(); @@ -78,19 +78,24 @@ public function process(string $modelClass, SyncState $syncState): array ]; } + private function getModelAndKey($modelClass): array + { + $model = new $modelClass; + return [$model, $model->getKeyName(),]; + } /** * Process partial sync for specific records */ public function processPartial(string $modelClass, array $recordIds, SyncState $syncState): array { - $model = new $modelClass; + [$model, $keyName] = $this->getModelAndKey($modelClass); $batchSize = $model->getBatchSize() ?? config('syncro-sheet.defaults.batch_size'); $query = $this->buildPartialSyncQuery($modelClass, $recordIds); $totalProcessed = 0; foreach (array_chunk($recordIds, $batchSize) as $batchIds) { - $records = $query->whereIn('id', $batchIds)->get(); + $records = $query->whereIn($keyName, $batchIds)->get(); if ($records->isEmpty()) { continue; @@ -120,26 +125,29 @@ public function processPartial(string $modelClass, array $recordIds, SyncState $ private function buildFullSyncQuery(string $modelClass, SyncState $syncState): Builder { + $model = new $modelClass; + $keyName = $model->getKeyName(); $query = $modelClass::query(); if ($syncState->last_processed_id) { - $query->where('id', '>', $syncState->last_processed_id); + $query->where($keyName, '>', $syncState->last_processed_id); } // Get records that haven't been synced in the last 7 days - $query->whereNotExists(function ($query) use ($modelClass) { + $query->whereNotExists(function ($query) use ($modelClass, $keyName) { $query->from('sync_entries') ->where('model_class', $modelClass) ->where('synced_at', '>', now()->subDays(7)) - ->whereColumn('record_id', 'id'); + ->whereColumn('record_id', $keyName); }); - return $query->orderBy('id'); + return $query->orderBy($keyName); } private function buildPartialSyncQuery(string $modelClass, array $recordIds): Builder { - return $modelClass::query()->whereIn('id', $recordIds); + $model = new $modelClass; + return $modelClass::query()->whereIn($model->getKeyName(), $recordIds); } private function ensureHeaders(SheetSyncable $model): array @@ -150,20 +158,21 @@ private function ensureHeaders(SheetSyncable $model): array $model->getSheetName() ); - // Get expected headers from a sample transformation - $sampleRow = $model->toSheetRow(); - $expectedHeaders = array_keys($sampleRow); - - // If headers don't match or don't exist, set them up - if (empty($currentHeaders) || $currentHeaders !== $expectedHeaders) { - $this->googleClient->setHeaders( - $model->getSheetIdentifier(), - $model->getSheetName(), - $expectedHeaders - ); - return $expectedHeaders; + if (!empty($currentHeaders)) { + return $currentHeaders; } - return $currentHeaders; + // Get headers in order of preference + $expectedHeaders = method_exists($model, 'defaultSheetHeaders') + ? $model->defaultSheetHeaders() + : array_keys($model->toSheetRow()); + + $this->googleClient->setHeaders( + $model->getSheetIdentifier(), + $model->getSheetName(), + $expectedHeaders + ); + + return $expectedHeaders; } } diff --git a/src/Services/GoogleClient.php b/src/Services/GoogleClient.php index 94e051e..466266c 100644 --- a/src/Services/GoogleClient.php +++ b/src/Services/GoogleClient.php @@ -92,11 +92,29 @@ private function setupServiceAccount(GoogleAPIClient $client): void if (is_array($serviceAccountFile)) { $client->setAuthConfig($serviceAccountFile); - } elseif (file_exists($serviceAccountFile)) { - $client->setAuthConfig($serviceAccountFile); - } else { - throw new GoogleSheetsException('Service account configuration file not found'); + return; + } + + // Search for the file in multiple locations + $searchPaths = [ + base_path($serviceAccountFile), + resource_path($serviceAccountFile), + resource_path('credentials' . DIRECTORY_SEPARATOR . $serviceAccountFile), + storage_path($serviceAccountFile), + storage_path('credentials' . DIRECTORY_SEPARATOR . $serviceAccountFile) + ]; + + foreach ($searchPaths as $path) { + if (file_exists($path) && is_readable($path)) { + $client->setAuthConfig($path); + return; + } } + + throw new GoogleSheetsException( + 'Service account configuration file not found in any of the following locations: ' . + implode(', ' . PHP_EOL, $searchPaths) + ); } /** @@ -120,19 +138,27 @@ private function setupOAuth(GoogleAPIClient $client): void /** * Write a batch of rows to Google Sheets */ - public function writeBatch(string $spreadsheetId, string $sheetName, array $rows): void + public function writeBatch(string $spreadsheetId, string $sheetName, array $rows, $model = null): void { $this->checkRateLimit(); try { - $this->getClient() - ->spreadsheet($spreadsheetId) - ->sheet($sheetName) - ->append($rows); + $client = $this->getClient()->spreadsheet($spreadsheetId)->sheet($sheetName); + + // Check if sheet is empty and needs headers + $existingData = $client->all(); + if (empty($existingData)) { + // Generate and write headers + $headers = $this->generateHeaders($model, $rows); + $client->update([$headers]); + } + + // Write data + $client->append($rows); $this->updateRateLimit(); - $this->logger->info("Written {count($rows)} rows to sheet {$sheetName}"); + $this->logger->info("Written " . count($rows) . " rows to sheet {$sheetName}"); } catch (\Exception $e) { $this->logger->error("Failed to write to Google Sheets: {$e->getMessage()}"); throw new GoogleSheetsException("Failed to write to Google Sheets: {$e->getMessage()}", 0, $e); @@ -249,6 +275,44 @@ public function appendWithHeaders(string $spreadsheetId, string $sheetName, arra } } + /** + * Generate headers from a SheetSyncable model or data + */ + private function generateHeaders($model = null, array $data = []): array + { + // Try to get headers from model's defaultSheetHeaders method + if ($model && method_exists($model, 'defaultSheetHeaders')) { + return $model->defaultSheetHeaders(); + } + + // Try to get headers from first row's keys if associative + if (!empty($data)) { + $firstRow = reset($data); + if (is_array($firstRow) && !$this->isSequentialArray($firstRow)) { + return array_keys($firstRow); + } + } + + // Fallback to Excel notation (A, B, C, ...) + $columnCount = empty($data) ? 26 : count(reset($data)); // Default to 26 columns if no data + return array_map(function($num) { + $letter = ''; + while ($num >= 0) { + $letter = chr(($num % 26) + 65) . $letter; + $num = floor($num / 26) - 1; + } + return $letter; + }, range(0, $columnCount - 1)); + } + + /** + * Check if array is sequential (numeric keys) or associative + */ + private function isSequentialArray(array $arr): bool + { + return array_keys($arr) === range(0, count($arr) - 1); + } + private function checkRateLimit(): void { $maxRequests = config('syncro-sheet.sheets.rate_limit.max_requests'); From 6cf54f7788afac544ab2c25fdb9041f9fdb28dfc Mon Sep 17 00:00:00 2001 From: Zuko Date: Tue, 5 Nov 2024 13:28:13 +0700 Subject: [PATCH 3/6] Add replace mode for full sync --- config/syncro-sheet.php | 7 +- .../migrations/2_create_sync_mode_column.php | 40 ++ src/Console/Commands/SheetSyncCommand.php | 13 +- src/Contracts/SheetSyncable.php | 22 -- src/Models/SyncState.php | 1 + src/Services/BatchProcessor.php | 50 ++- src/Services/FuzzyRecordIdentifier.php | 353 ++++++++++++++++++ src/Services/SheetRowMapper.php | 192 ++++++++++ src/Services/StateManager.php | 3 +- src/Services/SyncLogger.php | 2 +- src/Services/SyncManager.php | 50 ++- 11 files changed, 670 insertions(+), 63 deletions(-) create mode 100644 database/migrations/2_create_sync_mode_column.php create mode 100644 src/Services/FuzzyRecordIdentifier.php create mode 100644 src/Services/SheetRowMapper.php diff --git a/config/syncro-sheet.php b/config/syncro-sheet.php index 8f81717..2791360 100644 --- a/config/syncro-sheet.php +++ b/config/syncro-sheet.php @@ -22,7 +22,8 @@ |-------------------------------------------------------------------------- */ 'defaults' => [ - 'batch_size' => 1000, + 'batch_size' => 100, + 'sync_mode' => 'append', 'timeout' => 600, 'retries' => 3, ], @@ -33,8 +34,8 @@ |-------------------------------------------------------------------------- */ 'logging' => [ - 'channel' => 'sheet-sync', - 'level' => 'info', + 'channel' => env('SHEET_SYNC_LOG_CHANNEL', 'sheet-sync'), + 'level' => env('SHEET_SYNC_LOG_LEVEL', 'info'), 'separate_files' => true, ], diff --git a/database/migrations/2_create_sync_mode_column.php b/database/migrations/2_create_sync_mode_column.php new file mode 100644 index 0000000..6bbb9b7 --- /dev/null +++ b/database/migrations/2_create_sync_mode_column.php @@ -0,0 +1,40 @@ +enum('sync_mode', \Zuko\SyncroSheet\Services\SyncManager::AVAILABLE_SYNC_MODES) + ->default(\Zuko\SyncroSheet\Services\SyncManager::AVAILABLE_SYNC_MODES[0]) + ->after('sync_type') + ->index(); + }); + } + + public function down() + { + Schema::table('sync_states', static function (Blueprint $table) { + $table->dropColumn('sync_mode'); + }); + } +}; diff --git a/src/Console/Commands/SheetSyncCommand.php b/src/Console/Commands/SheetSyncCommand.php index 4573bd4..078e14a 100644 --- a/src/Console/Commands/SheetSyncCommand.php +++ b/src/Console/Commands/SheetSyncCommand.php @@ -24,7 +24,9 @@ class SheetSyncCommand extends Command { protected $signature = 'sheet:sync {model : The model class to sync} - {--ids=* : Specific record IDs for partial sync}'; + {--ids=* : Specific record IDs for partial sync} + {--M|mode= : Sync mode (append/replace)} + {--F|force : using replace mode}'; protected $description = 'Sync model data with Google Sheets'; @@ -38,15 +40,18 @@ public function handle(SyncManager $syncManager): int } $ids = $this->option('ids'); - + $mode = $this->option('mode'); + if(!$mode && $this->option('force')){ + $mode = 'replace'; + } try { if (empty($ids)) { $this->info("Starting full sync for {$modelClass}"); - $syncState = $syncManager->fullSync($modelClass); + $syncState = $syncManager->fullSync($modelClass, ['sync_mode' => $mode]); } else { $ids = is_array($ids) ? $ids : explode(',', $ids[0]); $this->info("Starting partial sync for {$modelClass} with IDs: " . implode(', ', $ids)); - $syncState = $syncManager->partialSync($modelClass, $ids); + $syncState = $syncManager->partialSync($modelClass, $ids, ['sync_mode' => $mode]); } $this->info("Sync completed! Processed {$syncState->total_processed} records."); diff --git a/src/Contracts/SheetSyncable.php b/src/Contracts/SheetSyncable.php index 933a5a9..c892fa0 100644 --- a/src/Contracts/SheetSyncable.php +++ b/src/Contracts/SheetSyncable.php @@ -33,26 +33,4 @@ public function getSheetName(): string; * Transform the model instance to a sheet row */ public function toSheetRow(): array; - - /** - * Get the batch size for processing - */ - public function getBatchSize(): ?int; - - /** - * Get the fields used to identify unique records - * If not implemented, system will use auto-detection - */ - public function getSheetIdentifierFields(): ?array; - - /** - * Get the sheet row ID for this model instance - * Used for bi-directional sync - */ - public function getSheetRowId(): ?string; - - /** - * Set the sheet row ID for this model instance - */ - public function setSheetRowId(string $rowId): void; } diff --git a/src/Models/SyncState.php b/src/Models/SyncState.php index 8ffb9ea..a8c3418 100644 --- a/src/Models/SyncState.php +++ b/src/Models/SyncState.php @@ -25,6 +25,7 @@ class SyncState extends Model protected $fillable = [ 'model_class', 'sync_type', + 'sync_mode', 'status', 'started_at', 'completed_at', diff --git a/src/Services/BatchProcessor.php b/src/Services/BatchProcessor.php index 4c8c3cb..2fe7027 100644 --- a/src/Services/BatchProcessor.php +++ b/src/Services/BatchProcessor.php @@ -38,7 +38,17 @@ public function __construct( public function process(string $modelClass, SyncState $syncState): array { $model = new $modelClass; - $batchSize = $model->getBatchSize() ?? config('syncro-sheet.defaults.batch_size'); + $batchSize = method_exists($model, 'getBatchSize') + ? $model->getBatchSize() + : config('syncro-sheet.defaults.batch_size'); + + // Handle replace mode by clearing sheet first + if ($syncState->sync_mode === 'replace') { + $this->googleClient->clearSheet( + $model->getSheetIdentifier(), + $model->getSheetName() + ); + } // First, ensure headers are set up $headers = $this->ensureHeaders($model); @@ -55,7 +65,6 @@ public function process(string $modelClass, SyncState $syncState): array $rowsWithHeaders = collect($rows)->map(function($row) use ($headers) { return array_combine($headers, $row); })->toArray(); - $this->googleClient->appendWithHeaders( $model->getSheetIdentifier(), $model->getSheetName(), @@ -63,11 +72,11 @@ public function process(string $modelClass, SyncState $syncState): array ); } - $processedIds = $records->pluck('id')->toArray(); + $processedIds = $records->pluck($model->getKeyName())->toArray(); $this->stateManager->recordBatchSync($syncState, $processedIds); $totalProcessed += count($records); - $lastProcessedId = $records->last()->id; + $lastProcessedId = $records->last()->{$model->getKeyName()}; $this->logger->info("Processed batch of {$records->count()} records for {$syncState->model_class}"); }); @@ -78,7 +87,7 @@ public function process(string $modelClass, SyncState $syncState): array ]; } - private function getModelAndKey($modelClass): array + private function getModelAndKeyName($modelClass): array { $model = new $modelClass; return [$model, $model->getKeyName(),]; @@ -88,17 +97,18 @@ private function getModelAndKey($modelClass): array */ public function processPartial(string $modelClass, array $recordIds, SyncState $syncState): array { - [$model, $keyName] = $this->getModelAndKey($modelClass); - $batchSize = $model->getBatchSize() ?? config('syncro-sheet.defaults.batch_size'); + $model = new $modelClass; + $batchSize = method_exists($model, 'getBatchSize') + ? $model->getBatchSize() + : config('syncro-sheet.defaults.batch_size'); $query = $this->buildPartialSyncQuery($modelClass, $recordIds); $totalProcessed = 0; + $lastProcessedId = null; - foreach (array_chunk($recordIds, $batchSize) as $batchIds) { - $records = $query->whereIn($keyName, $batchIds)->get(); - + $query->lazy()->chunk($batchSize)->each(function ($records) use ($model, $syncState, &$totalProcessed, &$lastProcessedId) { if ($records->isEmpty()) { - continue; + return true; } $rows = $this->transformer->transformBatch($records); @@ -111,15 +121,20 @@ public function processPartial(string $modelClass, array $recordIds, SyncState $ ); } - $this->stateManager->recordBatchSync($syncState, $batchIds); + $processedIds = $records->pluck($model->getKeyName())->toArray(); + $this->stateManager->recordBatchSync($syncState, $processedIds); + $totalProcessed += count($records); + $lastProcessedId = $records->last()->{$model->getKeyName()}; $this->logger->info("Processed partial batch of {$records->count()} records for {$syncState->model_class}"); - } + + return true; + }); return [ 'total_processed' => $totalProcessed, - 'last_processed_id' => max($recordIds) + 'last_processed_id' => $lastProcessedId ]; } @@ -146,8 +161,11 @@ private function buildFullSyncQuery(string $modelClass, SyncState $syncState): B private function buildPartialSyncQuery(string $modelClass, array $recordIds): Builder { - $model = new $modelClass; - return $modelClass::query()->whereIn($model->getKeyName(), $recordIds); + [$model, $keyName] = $this->getModelAndKeyName($modelClass); + + return $modelClass::query() + ->whereIn($keyName, $recordIds) + ->orderBy($keyName); } private function ensureHeaders(SheetSyncable $model): array diff --git a/src/Services/FuzzyRecordIdentifier.php b/src/Services/FuzzyRecordIdentifier.php new file mode 100644 index 0000000..b7de28b --- /dev/null +++ b/src/Services/FuzzyRecordIdentifier.php @@ -0,0 +1,353 @@ +fieldScores = []; + $this->schemaInfo = []; + + // Get sample data + $sampleData = $model->toSheetRow(); + + // Get schema information if available + $this->analyzeSchema($model); + + // Score each field + foreach ($sampleData as $field => $value) { + $this->scoreField($field, $value); + } + + // Sort fields by score and filter those above threshold + arsort($this->fieldScores); + + $identifyingFields = array_filter( + $this->fieldScores, + fn($score) => $score >= self::SCORE_THRESHOLD + ); + + // Take best fields within our limits + $identifyingFields = array_slice( + array_keys($identifyingFields), + 0, + self::MAX_IDENTIFYING_FIELDS + ); + + // Ensure we have minimum required fields + if (count($identifyingFields) < self::MIN_IDENTIFYING_FIELDS) { + $identifyingFields = array_merge( + $identifyingFields, + array_slice( + array_keys($this->fieldScores), + count($identifyingFields), + self::MIN_IDENTIFYING_FIELDS - count($identifyingFields) + ) + ); + } + + // Ensure created_at is always included if the model has timestamps + if ($model->timestamps && !in_array('created_at', $identifyingFields)) { + $identifyingFields[] = 'created_at'; + } + + return $identifyingFields; + } + + /** + * Analyze database schema for the model + */ + private function analyzeSchema(Model $model): void + { + try { + $table = $model->getTable(); + + // Get indexes + $indexes = Schema::getConnection() + ->getDoctrineSchemaManager() + ->listTableIndexes($table); + + foreach ($indexes as $index) { + $columns = $index->getColumns(); + $score = $this->getIndexScore($index); + + foreach ($columns as $column) { + if (!isset($this->schemaInfo[$column])) { + $this->schemaInfo[$column] = 0; + } + $this->schemaInfo[$column] = max($this->schemaInfo[$column], $score); + } + } + + // Get foreign keys + $foreignKeys = Schema::getConnection() + ->getDoctrineSchemaManager() + ->listTableForeignKeys($table); + + foreach ($foreignKeys as $foreignKey) { + $columns = $foreignKey->getLocalColumns(); + foreach ($columns as $column) { + $this->schemaInfo[$column] = max( + $this->schemaInfo[$column] ?? 0, + 0.4 // Base score for foreign keys + ); + } + } + + // Get nullable information + $columns = Schema::getConnection() + ->getDoctrineSchemaManager() + ->listTableColumns($table); + + foreach ($columns as $column) { + if (!$column->getNotnull()) { + $this->schemaInfo[$column->getName()] = + ($this->schemaInfo[$column->getName()] ?? 0) * 0.8; // Penalty for nullable + } + } + + } catch (\Exception $e) { + // Silent fail - schema analysis is optional + $this->schemaInfo = []; + } + } + + /** + * Score an index based on its characteristics + */ + private function getIndexScore(\Doctrine\DBAL\Schema\Index $index): float + { + if ($index->isPrimary()) { + return 1.0; + } + + if ($index->isUnique()) { + return 0.9; + } + + return 0.3; // Regular index + } + + /** + * Score a field based on its name, value, and schema information + */ + private function scoreField(string $field, $value): void + { + $score = 0; + + // Schema-based scoring + if (isset($this->schemaInfo[$field])) { + $score += $this->schemaInfo[$field]; + } + + // Name-based scoring + if ($this->isLikelyIdentifier($field)) { + $score += 0.3; + } + + // Value-based scoring + $valueScore = $this->getValueScore($value); + $score += $valueScore; + + // Normalize final score + $this->fieldScores[$field] = min(1.0, $score); + } + + /** + * Score a value based on its characteristics + */ + private function getValueScore($value): float + { + if ($value === null) { + return 0; + } + + // Timestamp/Date handling + if ($value instanceof \DateTime || $value instanceof \Carbon\Carbon) { + if ($this->isLikelyCreationDate($value)) { + return 0.8; + } + return 0.2; + } + + // String handling + if (is_string($value)) { + $length = Str::length($value); + + // Avoid JSON + if ($this->looksLikeJson($value)) { + return 0; + } + + // Avoid emoji/special characters + if ($this->containsEmoji($value)) { + return 0; + } + + // Score based on length + if ($length >= self::MIN_STRING_LENGTH && $length <= self::MAX_STRING_LENGTH) { + return 0.6; + } + + return 0.2; + } + + // Number handling + if (is_numeric($value)) { + // Avoid small integers that might be status codes + if (is_int($value) && $value >= 0 && $value <= 12) { + return 0.1; + } + + // Prefer numbers with 0-3 decimal places + if (is_float($value)) { + $decimals = strlen(substr(strrchr((string)$value, "."), 1)); + if ($decimals > 0 && $decimals <= 3) { + return 0.5; + } + } + + return 0.4; + } + + // Array/Object handling + if (is_array($value) || is_object($value)) { + if ($this->isSimpleArrayOrObject($value)) { + return 0.3; + } + return 0; + } + + // Boolean values are poor identifiers + if (is_bool($value)) { + return 0; + } + + return 0.1; // Default score for unknown types + } + + /** + * Check if field name suggests it's an identifier + */ + private function isLikelyIdentifier(string $field): bool + { + $identifierPatterns = [ + '/^id$/i', + '/_id$/i', + '/^uuid$/i', + '/^email$/i', + '/^username$/i', + '/^slug$/i', + '/^code$/i', + '/^sku$/i', + '/^reference$/i', + ]; + + foreach ($identifierPatterns as $pattern) { + if (preg_match($pattern, $field)) { + return true; + } + } + + return false; + } + + /** + * Check if date field likely represents creation time + */ + private function isLikelyCreationDate($value): bool + { + $creationPatterns = [ + 'created_at', + 'creation_date', + 'registered_at', + 'joined_at', + 'published_at' + ]; + + foreach ($creationPatterns as $pattern) { + if (Str::contains($value, $pattern)) { + return true; + } + } + + return false; + } + + /** + * Check if string looks like JSON + */ + private function looksLikeJson(string $value): bool + { + if (!in_array($value[0] ?? '', ['{', '['])) { + return false; + } + + json_decode($value); + return json_last_error() === JSON_ERROR_NONE; + } + + /** + * Check if string contains emoji + */ + private function containsEmoji(string $value): bool + { + $emojiPattern = '/[\x{1F300}-\x{1F64F}]|[\x{1F680}-\x{1F6FF}]|[\x{2600}-\x{26FF}]|[\x{2700}-\x{27BF}]|[\x{1F900}-\x{1F9FF}]|[\x{1F1E0}-\x{1F1FF}]/u'; + return preg_match($emojiPattern, $value) === 1; + } + + /** + * Check if array/object is simple enough to use + */ + private function isSimpleArrayOrObject($value): bool + { + $array = (array)$value; + + // Too many elements + if (count($array) > 3) { + return false; + } + + // Check each value is simple + foreach ($array as $item) { + if (is_array($item) || is_object($item)) { + return false; + } + } + + return true; + } +} diff --git a/src/Services/SheetRowMapper.php b/src/Services/SheetRowMapper.php new file mode 100644 index 0000000..4b38ca8 --- /dev/null +++ b/src/Services/SheetRowMapper.php @@ -0,0 +1,192 @@ +fuzzyIdentifier = $fuzzyIdentifier ?? new FuzzyRecordIdentifier(); + } + /** + * Generate a unique hash for a model instance + */ + public function generateRecordHash(Model $model): string + { + // If model has a primary key, use it as part of the hash + $identifier = []; + + if ($model->getKey()) { + $identifier[] = $model->getKey(); + } + + // Get unique identifying attributes (configured in model) + if (method_exists($model, 'getSheetUniqueAttributes')) { + $uniqueAttributes = $model->getSheetUniqueAttributes(); + foreach ($uniqueAttributes as $attribute) { + $identifier[] = $model->getAttribute($attribute); + } + } + + // If no identifying attributes, use fuzzy identification + if (empty($identifier)) { + $identifyingFields = $this->fuzzyIdentifier->analyzeModel($model); + foreach ($identifyingFields as $field) { + $identifier[] = $model->getAttribute($field); + } + } + + // Generate hash using all identifying data + return hash('xxh3', serialize($identifier)); + } + + /** + * Map model instances to sheet rows + */ + public function mapRecordsToSheet(Collection $records, array $sheetData): array + { + $headerRow = array_shift($sheetData); + if (!$headerRow) { + return $this->prepareNewSheetData($records); + } + + // Find hash column index + $hashColumnIndex = array_search(self::HASH_COLUMN, $headerRow); + if ($hashColumnIndex === false) { + // Add hash column if it doesn't exist + $headerRow[] = self::HASH_COLUMN; + $hashColumnIndex = count($headerRow) - 1; + + // Add hash column to existing rows + foreach ($sheetData as &$row) { + $row[$hashColumnIndex] = ''; + } + } + + // Build hash index for existing records + $existingHashes = []; + foreach ($sheetData as $rowIndex => $row) { + if (isset($row[$hashColumnIndex]) && $row[$hashColumnIndex] !== '') { + $existingHashes[$row[$hashColumnIndex]] = $rowIndex + 1; // +1 for header row + } + } + + // Process records and prepare updates + $updates = []; + $newRows = []; + + foreach ($records as $record) { + $hash = $this->generateRecordHash($record); + $rowData = $this->prepareRowData($record, $headerRow); + $rowData[$hashColumnIndex] = $hash; + + if (isset($existingHashes[$hash])) { + // Update existing row + $updates[$existingHashes[$hash]] = $rowData; + } else { + // Add new row + $newRows[] = $rowData; + } + } + + return [ + 'header' => $headerRow, + 'updates' => $updates, + 'new_rows' => $newRows + ]; + } + + /** + * Prepare data for a new sheet + */ + private function prepareNewSheetData(Collection $records): array + { + if ($records->isEmpty()) { + return [ + 'header' => [], + 'updates' => [], + 'new_rows' => [] + ]; + } + + // Get header from first record + $firstRecord = $records->first(); + $header = $this->getHeadersFromModel($firstRecord); + $header[] = self::HASH_COLUMN; + + // Prepare rows + $rows = []; + foreach ($records as $record) { + $rowData = $this->prepareRowData($record, $header); + $rowData[] = $this->generateRecordHash($record); + $rows[] = $rowData; + } + + return [ + 'header' => $header, + 'updates' => [], + 'new_rows' => $rows + ]; + } + + /** + * Get headers from model + */ + private function getHeadersFromModel(Model $model): array + { + if (method_exists($model, 'defaultSheetHeaders')) { + return $model->defaultSheetHeaders(); + } + + // Fallback to getting headers from toSheetRow + if (method_exists($model, 'toSheetRow')) { + return array_keys($model->toSheetRow()); + } + + throw new SyncException('Model must implement either defaultSheetHeaders() or toSheetRow()'); + } + + /** + * Prepare row data ensuring it matches headers + */ + private function prepareRowData(Model $model, array $headers): array + { + $rowData = method_exists($model, 'toSheetRow') + ? $model->toSheetRow() + : $model->toArray(); + + // Ensure data matches headers (excluding hash column) + $data = []; + foreach ($headers as $header) { + if ($header === self::HASH_COLUMN) { + continue; + } + $data[] = $rowData[$header] ?? null; + } + + return $data; + } +} diff --git a/src/Services/StateManager.php b/src/Services/StateManager.php index bf46889..18e4cc4 100644 --- a/src/Services/StateManager.php +++ b/src/Services/StateManager.php @@ -30,11 +30,12 @@ public function __construct( /** * Initialize a new sync state */ - public function initializeSync(string $modelClass, string $syncType): SyncState + public function initializeSync(string $modelClass, string $syncType, string $syncMode): SyncState { return SyncState::create([ 'model_class' => $modelClass, 'sync_type' => $syncType, + 'sync_mode' => $syncMode, 'status' => 'running', 'started_at' => now(), ]); diff --git a/src/Services/SyncLogger.php b/src/Services/SyncLogger.php index 8c2d8d7..e0a2e4a 100644 --- a/src/Services/SyncLogger.php +++ b/src/Services/SyncLogger.php @@ -53,7 +53,7 @@ private function configureLogger(): Logger $logger->pushHandler(new RotatingFileHandler( storage_path('logs/sheet-sync.log'), 30, - Logger::INFO + config('syncro-sheet.logging.level', 'debug') )); } else { $logger->pushHandler(Log::channel( diff --git a/src/Services/SyncManager.php b/src/Services/SyncManager.php index a35a673..d257648 100644 --- a/src/Services/SyncManager.php +++ b/src/Services/SyncManager.php @@ -24,32 +24,31 @@ class SyncManager { + const AVAILABLE_SYNC_MODES = ['append', 'replace']; public function __construct( private readonly BatchProcessor $batchProcessor, private readonly StateManager $stateManager, - private readonly SyncLogger $logger + private readonly SyncLogger $logger, + private readonly NotificationManager $notificationManager, + private readonly ErrorHandler $errorHandler ) {} /** * Start a full sync for the given model class */ - public function fullSync(string $modelClass): SyncState + public function fullSync(string $modelClass, array $options = []): SyncState { - $this->validateModel($modelClass); - - $this->logger->info("Starting full sync for {$modelClass}"); - - $syncState = $this->stateManager->initializeSync($modelClass, 'full'); + $syncMode = $this->determineSyncMode($modelClass, $options); + $syncState = $this->stateManager->initializeSync($modelClass, 'full', $syncMode); + $this->notificationManager->notifyStart($syncState); try { - $result = $this->batchProcessor->process($modelClass, $syncState); + $result = $this->batchProcessor->process($modelClass, $syncState, $syncMode); $this->stateManager->completeSync($syncState, $result); - - $this->logger->info("Completed full sync for {$modelClass}"); - - return $syncState->fresh(); + $this->notificationManager->notifyCompletion($syncState); + return $syncState; } catch (\Exception $e) { - $this->handleSyncError($syncState, $e); + $this->errorHandler->handleError($syncState, $e); throw $e; } } @@ -57,13 +56,13 @@ public function fullSync(string $modelClass): SyncState /** * Start a partial sync for specific model instances */ - public function partialSync(string $modelClass, array $recordIds): SyncState + public function partialSync(string $modelClass, array $recordIds, array $options = []): SyncState { $this->validateModel($modelClass); $this->logger->info("Starting partial sync for {$modelClass}"); - - $syncState = $this->stateManager->initializeSync($modelClass, 'partial'); + $syncMode = $this->determineSyncMode($modelClass, $options); + $syncState = $this->stateManager->initializeSync($modelClass, 'partial', $syncMode); try { $result = $this->batchProcessor->processPartial($modelClass, $recordIds, $syncState); @@ -96,4 +95,23 @@ private function handleSyncError(SyncState $syncState, \Exception $e): void $this->logger->error("Sync failed for {$syncState->model_class}: {$e->getMessage()}"); $this->stateManager->failSync($syncState, $e->getMessage()); } + + private function determineSyncMode(string $modelClass, array $options = []): string + { + // 1. Command line option (passed through options) + if (!empty($options['sync_mode'])) { + return $options['sync_mode']; + } + + // 2. Runtime configuration through model instance + $model = new $modelClass; + if (method_exists($model, 'getPreferredSyncMode')) { + $modelMode = $model->getPreferredSyncMode(); + if ($modelMode) { + return $modelMode; + } + } + // 3. Default from config + return config('syncro-sheet.defaults.sync_mode', 'append'); + } } From f50d554d7dab015ef78fbf526e025cf91a9b90ef Mon Sep 17 00:00:00 2001 From: Zuko Date: Tue, 5 Nov 2024 13:33:29 +0700 Subject: [PATCH 4/6] Respect $syncMode from params --- src/Services/BatchProcessor.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Services/BatchProcessor.php b/src/Services/BatchProcessor.php index 2fe7027..fa520e5 100644 --- a/src/Services/BatchProcessor.php +++ b/src/Services/BatchProcessor.php @@ -35,15 +35,15 @@ public function __construct( /** * Process full sync in batches */ - public function process(string $modelClass, SyncState $syncState): array + public function process(string $modelClass, SyncState $syncState, string $syncMode = null): array { $model = new $modelClass; $batchSize = method_exists($model, 'getBatchSize') ? $model->getBatchSize() : config('syncro-sheet.defaults.batch_size'); - + $syncMode = $syncMode ?? $syncState->sync_mode; // Handle replace mode by clearing sheet first - if ($syncState->sync_mode === 'replace') { + if ($syncMode === 'replace') { $this->googleClient->clearSheet( $model->getSheetIdentifier(), $model->getSheetName() From d69122863b9549965a54609b711fc57b8d7b2d56 Mon Sep 17 00:00:00 2001 From: Zuko Date: Tue, 5 Nov 2024 16:16:47 +0700 Subject: [PATCH 5/6] Update docs --- README.md | 4 + docs/COMPONENTS_OVERVIEW.md | 193 ++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 docs/COMPONENTS_OVERVIEW.md diff --git a/README.md b/README.md index 692d0f0..19fddcb 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +# WIP + +This package is not avail on packagist yet. If you interesting, take a local clone or zip download then try it out. + # SyncroSheet Laravel package for efficient synchronization between your models and Google Sheets with advanced state tracking and error handling. diff --git a/docs/COMPONENTS_OVERVIEW.md b/docs/COMPONENTS_OVERVIEW.md new file mode 100644 index 0000000..ce2d55c --- /dev/null +++ b/docs/COMPONENTS_OVERVIEW.md @@ -0,0 +1,193 @@ +# Component Overview + +This document describes the components used in this package. A minimal diagram also provides to illustrate the flow and mind-set at package built-stage. +Namespace: `Zuko\SyncroSheet` +Composer package name: `zuko/syncro-sheet` +Laravel compatible provider: \Zuko\SyncroSheet\LaravelSyncroSheetProvider + +# Component List & Summary + +### 1. Core Components + +#### `SyncManager` (Main Orchestrator) +- Primary entry point for sync operations +- Coordinates between all other components +- Handles high-level error management +- Ensures proper sync state tracking + +#### `SheetSyncable` (Interface) +- Contract for models that can be synced +- Defines required configuration methods +- Provides data transformation rules +```php +interface SheetSyncable +{ + public function getSheetIdentifier(): string; // Google Sheet ID + public function getSheetName(): string; // Sheet name + public function toSheetRow(): array; // Data transformation + public function getBatchSize(): int; // Optional, defaults to 1000 +} +``` + +### 2. Data Processing Components + +#### `BatchProcessor` +- Manages chunked data processing +- Uses Laravel's lazy loading for memory efficiency +- Reports progress to state manager +- Handles chunk-level errors + +#### `DataTransformer` +- Converts model data to sheet rows +- Handles data type conversions +- Manages date/time formatting +- Optional custom transformers support + +### 3. State Management Components + +#### `StateManager` +- Tracks sync progress and status +- Manages sync history +- Provides resume capability +- Stores performance metrics +- more sophisticated state tracking system that can handle both full and partial syncs while maintaining the sync history of individual records + +#### `SyncState` (Model) +```php +// Used for tracking sync state of one sync call +class SyncState extends Model +{ + protected $fillable = [ + 'model_class', // Model class name + 'sync_type', // 'full' or 'partial' + 'sync_mode', // 'append' or 'replace' + 'status', // 'running', 'completed', 'failed' + 'started_at', + 'completed_at', + 'total_processed', + 'last_processed_id', + ]; +} + +// Used for tracking individual records +class SyncEntry extends Model +{ + protected $fillable = [ + 'model_class', + 'record_id', + 'synced_at', + 'sync_state_id', // Reference to parent SyncState + 'sync_type', // 'full' or 'partial' + 'status' // 'success' or 'failed' + ]; + + protected $casts = [ + 'synced_at' => 'datetime' + ]; +}``` + +### 4. Google Sheets Integration + +#### `SheetClient` (Wrapper around revolution/laravel-google-sheets) +- Manages Google Sheets connections +- Handles authentication +- Provides sheet operations interface +- Manages API rate limits + +### 5. Logging & Monitoring + +#### `SyncLogger` +- Dedicated logging channel +- Structured log format +- Performance logging +- Error tracking + +#### `Events` +```php +class Events +{ + const SYNC_STARTED = 'sheet-sync.started'; + const CHUNK_PROCESSED = 'sheet-sync.chunk-processed'; + const SYNC_COMPLETED = 'sheet-sync.completed'; + const SYNC_FAILED = 'sheet-sync.failed'; +} +``` + +### 6. Configuration + +#### Package Config (`syncro-sheet.php`) +```php +return [ + 'defaults' => [ + 'batch_size' => 1000, + 'timeout' => 600, + 'retries' => 3 + ], + + 'logging' => [ + 'channel' => 'sheet-sync', + 'level' => 'info', + 'separate_files' => true + ], + + 'sheets' => [ + 'cache_ttl' => 3600, + 'rate_limit' => [ + 'max_requests' => 100, + 'per_seconds' => 60 + ] + ] +]; +``` + +# Component Interactions + + +```mermaid +graph TB + A[SyncManager] --> B[BatchProcessor] + A --> C[StateManager] + A --> D[SheetClient] + + B --> E[DataTransformer] + B --> F[SyncLogger] + + D --> G[Google Sheets API] + + C --> H[Database] + F --> I[Log Files] + + J[Model] --> A + J --> E +``` + + +# Flow: + + +```mermaid +sequenceDiagram + participant Command as SyncCommand + participant Manager as SyncManager + participant Processor as BatchProcessor + participant State as StateManager + participant Transformer as DataTransformer + participant Syncer as GoogleSheetsClient + participant Client as GoogleSheetsClient + + Command->>Manager: sync(ModelClass) + Manager->>State: startSync() + Manager->>Processor: process(ModelClass) + + loop For each chunk + Processor->>Transformer: transform(chunk) + Transformer->>Syncer: sync(transformedData) + Syncer->>Client: write(sheetId, data) + Client-->>Syncer: response + Syncer-->>Processor: result + Processor->>State: updateProgress() + end + + Manager->>State: completeSync() + Manager-->>Command: SyncResult +``` From 406f7ef857dc1644ea8d50d5e01cdc4d86d0eb7e Mon Sep 17 00:00:00 2001 From: Zuko Date: Tue, 5 Nov 2024 16:20:45 +0700 Subject: [PATCH 6/6] Running PINT --- config/syncro-sheet.php | 2 +- .../migrations/0_create_sync_states_table.php | 2 +- .../1_create_sync_entries_table.php | 2 +- .../migrations/2_create_sync_mode_column.php | 8 +-- src/Console/Commands/SheetSyncCommand.php | 14 +++-- src/Events/SyncEvent.php | 5 +- src/Facades/SyncroSheet.php | 4 +- src/Jobs/PartialSyncJob.php | 5 +- src/LaravelSyncroSheetProvider.php | 25 ++++---- src/Models/SyncEntry.php | 2 +- src/Models/SyncState.php | 2 +- src/Notifications/BaseNotification.php | 8 ++- .../SyncCompletedNotification.php | 2 +- src/Notifications/SyncFailedNotification.php | 2 +- src/Notifications/SyncRetryNotification.php | 8 +-- src/Services/BatchProcessor.php | 55 ++++++++--------- src/Services/DataTransformer.php | 4 +- src/Services/ErrorHandler.php | 7 ++- src/Services/FuzzyRecordIdentifier.php | 38 +++++++----- src/Services/GoogleClient.php | 60 ++++++++++--------- src/Services/NotificationManager.php | 12 ++-- src/Services/SheetRowMapper.php | 13 ++-- src/Services/StateManager.php | 8 +-- src/Services/SyncLogger.php | 4 +- src/Services/SyncManager.php | 20 ++++--- src/Services/TokenManager.php | 7 ++- 26 files changed, 175 insertions(+), 144 deletions(-) diff --git a/config/syncro-sheet.php b/config/syncro-sheet.php index 2791360..f9ec9cc 100644 --- a/config/syncro-sheet.php +++ b/config/syncro-sheet.php @@ -65,4 +65,4 @@ 'sync_completed' => false, ], ], -]; +]; diff --git a/database/migrations/0_create_sync_states_table.php b/database/migrations/0_create_sync_states_table.php index 0b1b0d9..a71364f 100644 --- a/database/migrations/0_create_sync_states_table.php +++ b/database/migrations/0_create_sync_states_table.php @@ -43,4 +43,4 @@ public function down(): void { Schema::dropIfExists('sync_states'); } -}; +}; diff --git a/database/migrations/1_create_sync_entries_table.php b/database/migrations/1_create_sync_entries_table.php index 95275af..fc2a552 100644 --- a/database/migrations/1_create_sync_entries_table.php +++ b/database/migrations/1_create_sync_entries_table.php @@ -43,4 +43,4 @@ public function down(): void { Schema::dropIfExists('sync_entries'); } -}; +}; diff --git a/database/migrations/2_create_sync_mode_column.php b/database/migrations/2_create_sync_mode_column.php index 6bbb9b7..299d277 100644 --- a/database/migrations/2_create_sync_mode_column.php +++ b/database/migrations/2_create_sync_mode_column.php @@ -25,9 +25,9 @@ public function up() { Schema::table('sync_states', static function (Blueprint $table) { $table->enum('sync_mode', \Zuko\SyncroSheet\Services\SyncManager::AVAILABLE_SYNC_MODES) - ->default(\Zuko\SyncroSheet\Services\SyncManager::AVAILABLE_SYNC_MODES[0]) - ->after('sync_type') - ->index(); + ->default(\Zuko\SyncroSheet\Services\SyncManager::AVAILABLE_SYNC_MODES[0]) + ->after('sync_type') + ->index(); }); } @@ -37,4 +37,4 @@ public function down() $table->dropColumn('sync_mode'); }); } -}; +}; diff --git a/src/Console/Commands/SheetSyncCommand.php b/src/Console/Commands/SheetSyncCommand.php index 078e14a..50e20e6 100644 --- a/src/Console/Commands/SheetSyncCommand.php +++ b/src/Console/Commands/SheetSyncCommand.php @@ -33,15 +33,15 @@ class SheetSyncCommand extends Command public function handle(SyncManager $syncManager): int { $modelClass = $this->argument('model'); - + // Add namespace if not provided - if (!str_contains($modelClass, '\\')) { - $modelClass = 'App\\Models\\' . $modelClass; + if (! str_contains($modelClass, '\\')) { + $modelClass = 'App\\Models\\'.$modelClass; } $ids = $this->option('ids'); $mode = $this->option('mode'); - if(!$mode && $this->option('force')){ + if (! $mode && $this->option('force')) { $mode = 'replace'; } try { @@ -50,15 +50,17 @@ public function handle(SyncManager $syncManager): int $syncState = $syncManager->fullSync($modelClass, ['sync_mode' => $mode]); } else { $ids = is_array($ids) ? $ids : explode(',', $ids[0]); - $this->info("Starting partial sync for {$modelClass} with IDs: " . implode(', ', $ids)); + $this->info("Starting partial sync for {$modelClass} with IDs: ".implode(', ', $ids)); $syncState = $syncManager->partialSync($modelClass, $ids, ['sync_mode' => $mode]); } $this->info("Sync completed! Processed {$syncState->total_processed} records."); + return self::SUCCESS; } catch (\Exception $e) { $this->error("Sync failed: {$e->getMessage()}"); + return self::FAILURE; } } -} +} diff --git a/src/Events/SyncEvent.php b/src/Events/SyncEvent.php index b457ee2..557ddcd 100644 --- a/src/Events/SyncEvent.php +++ b/src/Events/SyncEvent.php @@ -20,7 +20,10 @@ class SyncEvent { const SYNC_STARTED = 'sync.started'; + const CHUNK_PROCESSED = 'sync.chunk_processed'; + const SYNC_COMPLETED = 'sync.completed'; + const SYNC_FAILED = 'sync.failed'; -} +} diff --git a/src/Facades/SyncroSheet.php b/src/Facades/SyncroSheet.php index b287f6d..a650c43 100644 --- a/src/Facades/SyncroSheet.php +++ b/src/Facades/SyncroSheet.php @@ -24,7 +24,7 @@ * @method static \Zuko\SyncroSheet\Models\SyncState fullSync(string $modelClass) * @method static \Zuko\SyncroSheet\Models\SyncState partialSync(string $modelClass, array $recordIds) * @method static \Zuko\SyncroSheet\Models\SyncState|null getLastSync(string $modelClass) - * + * * @see \Zuko\SyncroSheet\Services\SyncManager */ class SyncroSheet extends Facade @@ -33,4 +33,4 @@ protected static function getFacadeAccessor() { return SyncManager::class; } -} +} diff --git a/src/Jobs/PartialSyncJob.php b/src/Jobs/PartialSyncJob.php index 4a92934..2762e23 100644 --- a/src/Jobs/PartialSyncJob.php +++ b/src/Jobs/PartialSyncJob.php @@ -31,7 +31,8 @@ class PartialSyncJob implements ShouldQueue public function __construct( private readonly string $modelClass, private readonly array $recordIds - ) {} + ) { + } public function handle(SyncManager $syncManager): void { @@ -61,4 +62,4 @@ public function maxExceptions(): int { return 3; } -} +} diff --git a/src/LaravelSyncroSheetProvider.php b/src/LaravelSyncroSheetProvider.php index 5d94713..1ea0f0f 100644 --- a/src/LaravelSyncroSheetProvider.php +++ b/src/LaravelSyncroSheetProvider.php @@ -17,7 +17,6 @@ namespace Zuko\SyncroSheet; - use Illuminate\Foundation\AliasLoader; use Illuminate\Support\ServiceProvider; use Zuko\SyncroSheet\Services\BatchProcessor; @@ -30,29 +29,29 @@ class LaravelSyncroSheetProvider extends ServiceProvider public function boot() { $this->publishes([ - __DIR__ . '/../config/syncro-sheet.php' => config_path('syncro-sheet.php'), - ], 'config'); - $this->loadMigrationsFrom(__DIR__ . '/../database/migrations'); + __DIR__.'/../config/syncro-sheet.php' => config_path('syncro-sheet.php'), + ], 'config'); + $this->loadMigrationsFrom(__DIR__.'/../database/migrations'); if ($this->app->runningInConsole()) { $this->commands([ - Console\Commands\SheetSyncCommand::class, - ]); + Console\Commands\SheetSyncCommand::class, + ]); } } public function register() { - $this->mergeConfigFrom(__DIR__ . '/../config/syncro-sheet.php', - 'syncro-sheet'); + $this->mergeConfigFrom(__DIR__.'/../config/syncro-sheet.php', + 'syncro-sheet'); // Ensure google config is available - if (!$this->app->configurationIsCached()) { + if (! $this->app->configurationIsCached()) { $revolutionConfig = base_path('vendor/revolution/laravel-google-sheets/config/google.php'); - if (!file_exists($revolutionConfig)) { - $this->mergeConfigFrom(__DIR__ . '/../vendor/revolution/laravel-google-sheets/config/google.php', - 'google'); + if (! file_exists($revolutionConfig)) { + $this->mergeConfigFrom(__DIR__.'/../vendor/revolution/laravel-google-sheets/config/google.php', + 'google'); } else { $this->mergeConfigFrom($revolutionConfig, - 'google'); + 'google'); } } diff --git a/src/Models/SyncEntry.php b/src/Models/SyncEntry.php index e6a7df8..810dbfd 100644 --- a/src/Models/SyncEntry.php +++ b/src/Models/SyncEntry.php @@ -40,4 +40,4 @@ public function syncState(): BelongsTo { return $this->belongsTo(SyncState::class); } -} +} diff --git a/src/Models/SyncState.php b/src/Models/SyncState.php index a8c3418..9c59733 100644 --- a/src/Models/SyncState.php +++ b/src/Models/SyncState.php @@ -43,4 +43,4 @@ public function entries(): HasMany { return $this->hasMany(SyncEntry::class); } -} +} diff --git a/src/Notifications/BaseNotification.php b/src/Notifications/BaseNotification.php index 4b72c6c..bd45556 100644 --- a/src/Notifications/BaseNotification.php +++ b/src/Notifications/BaseNotification.php @@ -17,16 +17,17 @@ namespace Zuko\SyncroSheet\Notifications; -use Illuminate\Notifications\Notification; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Notifications\Messages\SlackMessage; +use Illuminate\Notifications\Notification; use Zuko\SyncroSheet\Models\SyncState; abstract class BaseNotification extends Notification { public function __construct( protected SyncState $syncState - ) {} + ) { + } public function via($notifiable): array { @@ -39,5 +40,6 @@ protected function getModelName(): string } abstract protected function getMailMessage(): MailMessage; + abstract protected function getSlackMessage(): SlackMessage; -} +} diff --git a/src/Notifications/SyncCompletedNotification.php b/src/Notifications/SyncCompletedNotification.php index a2e72c0..d214c6c 100644 --- a/src/Notifications/SyncCompletedNotification.php +++ b/src/Notifications/SyncCompletedNotification.php @@ -44,4 +44,4 @@ protected function getSlackMessage(): SlackMessage ]); }); } -} +} diff --git a/src/Notifications/SyncFailedNotification.php b/src/Notifications/SyncFailedNotification.php index 0f2659b..216810c 100644 --- a/src/Notifications/SyncFailedNotification.php +++ b/src/Notifications/SyncFailedNotification.php @@ -53,4 +53,4 @@ protected function getSlackMessage(): SlackMessage ]); }); } -} +} diff --git a/src/Notifications/SyncRetryNotification.php b/src/Notifications/SyncRetryNotification.php index 83656cd..8f60ea0 100644 --- a/src/Notifications/SyncRetryNotification.php +++ b/src/Notifications/SyncRetryNotification.php @@ -35,8 +35,8 @@ protected function getMailMessage(): MailMessage return (new MailMessage) ->subject("Sheet Sync Retry: {$this->getModelName()}") ->line("Attempting retry #{$this->retryCount} for {$this->getModelName()} sync.") - ->line("Failed records: " . count($this->failedRecords)) - ->line("Next retry scheduled in: " . $this->getRetryDelay() . " minutes"); + ->line('Failed records: '.count($this->failedRecords)) + ->line('Next retry scheduled in: '.$this->getRetryDelay().' minutes'); } protected function getSlackMessage(): SlackMessage @@ -49,7 +49,7 @@ protected function getSlackMessage(): SlackMessage ->fields([ 'Retry Attempt' => $this->retryCount, 'Failed Records' => count($this->failedRecords), - 'Next Retry In' => $this->getRetryDelay() . ' minutes', + 'Next Retry In' => $this->getRetryDelay().' minutes', ]); }); } @@ -58,4 +58,4 @@ private function getRetryDelay(): int { return pow(2, $this->retryCount - 1); } -} +} diff --git a/src/Services/BatchProcessor.php b/src/Services/BatchProcessor.php index fa520e5..4786d80 100644 --- a/src/Services/BatchProcessor.php +++ b/src/Services/BatchProcessor.php @@ -18,10 +18,8 @@ namespace Zuko\SyncroSheet\Services; use Illuminate\Database\Eloquent\Builder; -use Illuminate\Support\Collection; -use Zuko\SyncroSheet\Models\SyncState; use Zuko\SyncroSheet\Contracts\SheetSyncable; -use Zuko\SyncroSheet\Exceptions\SyncException; +use Zuko\SyncroSheet\Models\SyncState; class BatchProcessor { @@ -30,16 +28,17 @@ public function __construct( private readonly DataTransformer $transformer, private readonly GoogleClient $googleClient, private readonly SyncLogger $logger - ) {} + ) { + } /** * Process full sync in batches */ - public function process(string $modelClass, SyncState $syncState, string $syncMode = null): array + public function process(string $modelClass, SyncState $syncState, ?string $syncMode = null): array { $model = new $modelClass; - $batchSize = method_exists($model, 'getBatchSize') - ? $model->getBatchSize() + $batchSize = method_exists($model, 'getBatchSize') + ? $model->getBatchSize() : config('syncro-sheet.defaults.batch_size'); $syncMode = $syncMode ?? $syncState->sync_mode; // Handle replace mode by clearing sheet first @@ -49,20 +48,20 @@ public function process(string $modelClass, SyncState $syncState, string $syncMo $model->getSheetName() ); } - + // First, ensure headers are set up $headers = $this->ensureHeaders($model); - + $query = $this->buildFullSyncQuery($modelClass, $syncState); $totalProcessed = 0; $lastProcessedId = null; $query->chunk($batchSize, function ($records) use ($model, $headers, $syncState, &$totalProcessed, &$lastProcessedId) { $rows = $this->transformer->transformBatch($records); - - if (!empty($rows)) { + + if (! empty($rows)) { // Transform to associative arrays with headers - $rowsWithHeaders = collect($rows)->map(function($row) use ($headers) { + $rowsWithHeaders = collect($rows)->map(function ($row) use ($headers) { return array_combine($headers, $row); })->toArray(); $this->googleClient->appendWithHeaders( @@ -74,34 +73,36 @@ public function process(string $modelClass, SyncState $syncState, string $syncMo $processedIds = $records->pluck($model->getKeyName())->toArray(); $this->stateManager->recordBatchSync($syncState, $processedIds); - + $totalProcessed += count($records); $lastProcessedId = $records->last()->{$model->getKeyName()}; - + $this->logger->info("Processed batch of {$records->count()} records for {$syncState->model_class}"); }); return [ 'total_processed' => $totalProcessed, - 'last_processed_id' => $lastProcessedId + 'last_processed_id' => $lastProcessedId, ]; } private function getModelAndKeyName($modelClass): array { $model = new $modelClass; - return [$model, $model->getKeyName(),]; + + return [$model, $model->getKeyName()]; } + /** * Process partial sync for specific records */ public function processPartial(string $modelClass, array $recordIds, SyncState $syncState): array { $model = new $modelClass; - $batchSize = method_exists($model, 'getBatchSize') - ? $model->getBatchSize() + $batchSize = method_exists($model, 'getBatchSize') + ? $model->getBatchSize() : config('syncro-sheet.defaults.batch_size'); - + $query = $this->buildPartialSyncQuery($modelClass, $recordIds); $totalProcessed = 0; $lastProcessedId = null; @@ -112,8 +113,8 @@ public function processPartial(string $modelClass, array $recordIds, SyncState $ } $rows = $this->transformer->transformBatch($records); - - if (!empty($rows)) { + + if (! empty($rows)) { $this->googleClient->writeBatch( $records->first()->getSheetIdentifier(), $records->first()->getSheetName(), @@ -123,10 +124,10 @@ public function processPartial(string $modelClass, array $recordIds, SyncState $ $processedIds = $records->pluck($model->getKeyName())->toArray(); $this->stateManager->recordBatchSync($syncState, $processedIds); - + $totalProcessed += count($records); $lastProcessedId = $records->last()->{$model->getKeyName()}; - + $this->logger->info("Processed partial batch of {$records->count()} records for {$syncState->model_class}"); return true; @@ -134,7 +135,7 @@ public function processPartial(string $modelClass, array $recordIds, SyncState $ return [ 'total_processed' => $totalProcessed, - 'last_processed_id' => $lastProcessedId + 'last_processed_id' => $lastProcessedId, ]; } @@ -162,7 +163,7 @@ private function buildFullSyncQuery(string $modelClass, SyncState $syncState): B private function buildPartialSyncQuery(string $modelClass, array $recordIds): Builder { [$model, $keyName] = $this->getModelAndKeyName($modelClass); - + return $modelClass::query() ->whereIn($keyName, $recordIds) ->orderBy($keyName); @@ -176,7 +177,7 @@ private function ensureHeaders(SheetSyncable $model): array $model->getSheetName() ); - if (!empty($currentHeaders)) { + if (! empty($currentHeaders)) { return $currentHeaders; } @@ -193,4 +194,4 @@ private function ensureHeaders(SheetSyncable $model): array return $expectedHeaders; } -} +} diff --git a/src/Services/DataTransformer.php b/src/Services/DataTransformer.php index cc1faac..ad3a660 100644 --- a/src/Services/DataTransformer.php +++ b/src/Services/DataTransformer.php @@ -43,7 +43,7 @@ public function transformBatch(Collection $records): array private function transformRecord(SheetSyncable $record): array { $row = $record->toSheetRow(); - + return array_map(function ($value) { return $this->formatValue($value); }, $row); @@ -72,4 +72,4 @@ private function formatValue($value): string return (string) $value; } -} +} diff --git a/src/Services/ErrorHandler.php b/src/Services/ErrorHandler.php index 90a0be9..6911dae 100644 --- a/src/Services/ErrorHandler.php +++ b/src/Services/ErrorHandler.php @@ -17,8 +17,8 @@ namespace Zuko\SyncroSheet\Services; -use Zuko\SyncroSheet\Models\SyncState; use Illuminate\Support\Facades\DB; +use Zuko\SyncroSheet\Models\SyncState; class ErrorHandler { @@ -26,7 +26,8 @@ public function __construct( private readonly NotificationManager $notificationManager, private readonly SyncLogger $logger, private readonly int $maxRetries = 3 - ) {} + ) { + } /** * Handle sync error @@ -87,4 +88,4 @@ private function schedulePartialSync(string $modelClass, array $recordIds, int $ new \Zuko\SyncroSheet\Jobs\PartialSyncJob($modelClass, $recordIds) ); } -} +} diff --git a/src/Services/FuzzyRecordIdentifier.php b/src/Services/FuzzyRecordIdentifier.php index b7de28b..0c91a91 100644 --- a/src/Services/FuzzyRecordIdentifier.php +++ b/src/Services/FuzzyRecordIdentifier.php @@ -19,18 +19,22 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Support\Facades\Schema; -use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; class FuzzyRecordIdentifier { private const MIN_STRING_LENGTH = 10; + private const MAX_STRING_LENGTH = 150; + private const MIN_IDENTIFYING_FIELDS = 2; + private const MAX_IDENTIFYING_FIELDS = 5; + private const SCORE_THRESHOLD = 0.6; private array $fieldScores = []; + private array $schemaInfo = []; /** @@ -57,7 +61,7 @@ public function analyzeModel(Model $model): array $identifyingFields = array_filter( $this->fieldScores, - fn($score) => $score >= self::SCORE_THRESHOLD + fn ($score) => $score >= self::SCORE_THRESHOLD ); // Take best fields within our limits @@ -80,7 +84,7 @@ public function analyzeModel(Model $model): array } // Ensure created_at is always included if the model has timestamps - if ($model->timestamps && !in_array('created_at', $identifyingFields)) { + if ($model->timestamps && ! in_array('created_at', $identifyingFields)) { $identifyingFields[] = 'created_at'; } @@ -97,15 +101,15 @@ private function analyzeSchema(Model $model): void // Get indexes $indexes = Schema::getConnection() - ->getDoctrineSchemaManager() - ->listTableIndexes($table); + ->getDoctrineSchemaManager() + ->listTableIndexes($table); foreach ($indexes as $index) { $columns = $index->getColumns(); $score = $this->getIndexScore($index); foreach ($columns as $column) { - if (!isset($this->schemaInfo[$column])) { + if (! isset($this->schemaInfo[$column])) { $this->schemaInfo[$column] = 0; } $this->schemaInfo[$column] = max($this->schemaInfo[$column], $score); @@ -114,8 +118,8 @@ private function analyzeSchema(Model $model): void // Get foreign keys $foreignKeys = Schema::getConnection() - ->getDoctrineSchemaManager() - ->listTableForeignKeys($table); + ->getDoctrineSchemaManager() + ->listTableForeignKeys($table); foreach ($foreignKeys as $foreignKey) { $columns = $foreignKey->getLocalColumns(); @@ -129,11 +133,11 @@ private function analyzeSchema(Model $model): void // Get nullable information $columns = Schema::getConnection() - ->getDoctrineSchemaManager() - ->listTableColumns($table); + ->getDoctrineSchemaManager() + ->listTableColumns($table); foreach ($columns as $column) { - if (!$column->getNotnull()) { + if (! $column->getNotnull()) { $this->schemaInfo[$column->getName()] = ($this->schemaInfo[$column->getName()] ?? 0) * 0.8; // Penalty for nullable } @@ -200,6 +204,7 @@ private function getValueScore($value): float if ($this->isLikelyCreationDate($value)) { return 0.8; } + return 0.2; } @@ -234,7 +239,7 @@ private function getValueScore($value): float // Prefer numbers with 0-3 decimal places if (is_float($value)) { - $decimals = strlen(substr(strrchr((string)$value, "."), 1)); + $decimals = strlen(substr(strrchr((string) $value, '.'), 1)); if ($decimals > 0 && $decimals <= 3) { return 0.5; } @@ -248,6 +253,7 @@ private function getValueScore($value): float if ($this->isSimpleArrayOrObject($value)) { return 0.3; } + return 0; } @@ -295,7 +301,7 @@ private function isLikelyCreationDate($value): bool 'creation_date', 'registered_at', 'joined_at', - 'published_at' + 'published_at', ]; foreach ($creationPatterns as $pattern) { @@ -312,11 +318,12 @@ private function isLikelyCreationDate($value): bool */ private function looksLikeJson(string $value): bool { - if (!in_array($value[0] ?? '', ['{', '['])) { + if (! in_array($value[0] ?? '', ['{', '['])) { return false; } json_decode($value); + return json_last_error() === JSON_ERROR_NONE; } @@ -326,6 +333,7 @@ private function looksLikeJson(string $value): bool private function containsEmoji(string $value): bool { $emojiPattern = '/[\x{1F300}-\x{1F64F}]|[\x{1F680}-\x{1F6FF}]|[\x{2600}-\x{26FF}]|[\x{2700}-\x{27BF}]|[\x{1F900}-\x{1F9FF}]|[\x{1F1E0}-\x{1F1FF}]/u'; + return preg_match($emojiPattern, $value) === 1; } @@ -334,7 +342,7 @@ private function containsEmoji(string $value): bool */ private function isSimpleArrayOrObject($value): bool { - $array = (array)$value; + $array = (array) $value; // Too many elements if (count($array) > 3) { diff --git a/src/Services/GoogleClient.php b/src/Services/GoogleClient.php index 466266c..5388710 100644 --- a/src/Services/GoogleClient.php +++ b/src/Services/GoogleClient.php @@ -17,20 +17,22 @@ namespace Zuko\SyncroSheet\Services; -use Revolution\Google\Sheets\Facades\Sheets; use Google\Client as GoogleAPIClient; use Google\Service\Sheets as GoogleSheets; +use Revolution\Google\Sheets\Facades\Sheets; use Zuko\SyncroSheet\Exceptions\GoogleSheetsException; class GoogleClient { private array $rateLimits = []; + private $sheetsClient = null; public function __construct( private readonly TokenManager $tokenManager, private readonly SyncLogger $logger - ) {} + ) { + } /** * Get or initialize the Sheets client @@ -48,17 +50,17 @@ private function getClient() } else { // Non-Laravel environment - manual setup using config $client = new GoogleAPIClient($this->getGoogleConfig()); - + // Set required scopes for sheets $client->setScopes([GoogleSheets::DRIVE, GoogleSheets::SPREADSHEETS]); - + // Handle authentication based on config if (config('google.service.enable')) { $this->setupServiceAccount($client); } else { $this->setupOAuth($client); } - + $service = new GoogleSheets($client); $this->sheetsClient = Sheets::setService($service); } @@ -72,10 +74,10 @@ private function getClient() private function getGoogleConfig(): array { $config = config('google.config', []); - + // Add basic configuration $config['application_name'] = config('google.application_name'); - + if (config('google.developer_key')) { $config['developer_key'] = config('google.developer_key'); } @@ -89,9 +91,10 @@ private function getGoogleConfig(): array private function setupServiceAccount(GoogleAPIClient $client): void { $serviceAccountFile = config('google.service.file'); - + if (is_array($serviceAccountFile)) { $client->setAuthConfig($serviceAccountFile); + return; } @@ -99,21 +102,22 @@ private function setupServiceAccount(GoogleAPIClient $client): void $searchPaths = [ base_path($serviceAccountFile), resource_path($serviceAccountFile), - resource_path('credentials' . DIRECTORY_SEPARATOR . $serviceAccountFile), + resource_path('credentials'.DIRECTORY_SEPARATOR.$serviceAccountFile), storage_path($serviceAccountFile), - storage_path('credentials' . DIRECTORY_SEPARATOR . $serviceAccountFile) + storage_path('credentials'.DIRECTORY_SEPARATOR.$serviceAccountFile), ]; foreach ($searchPaths as $path) { if (file_exists($path) && is_readable($path)) { $client->setAuthConfig($path); + return; } } throw new GoogleSheetsException( - 'Service account configuration file not found in any of the following locations: ' . - implode(', ' . PHP_EOL, $searchPaths) + 'Service account configuration file not found in any of the following locations: '. + implode(', '.PHP_EOL, $searchPaths) ); } @@ -127,7 +131,7 @@ private function setupOAuth(GoogleAPIClient $client): void $client->setRedirectUri(config('google.redirect_uri')); $client->setAccessType(config('google.access_type', 'online')); $client->setApprovalPrompt(config('google.approval_prompt', 'auto')); - + // Set the token if available $token = $this->tokenManager->getToken(); if ($token) { @@ -144,7 +148,7 @@ public function writeBatch(string $spreadsheetId, string $sheetName, array $rows try { $client = $this->getClient()->spreadsheet($spreadsheetId)->sheet($sheetName); - + // Check if sheet is empty and needs headers $existingData = $client->all(); if (empty($existingData)) { @@ -157,8 +161,8 @@ public function writeBatch(string $spreadsheetId, string $sheetName, array $rows $client->append($rows); $this->updateRateLimit(); - - $this->logger->info("Written " . count($rows) . " rows to sheet {$sheetName}"); + + $this->logger->info('Written '.count($rows)." rows to sheet {$sheetName}"); } catch (\Exception $e) { $this->logger->error("Failed to write to Google Sheets: {$e->getMessage()}"); throw new GoogleSheetsException("Failed to write to Google Sheets: {$e->getMessage()}", 0, $e); @@ -179,7 +183,7 @@ public function clearSheet(string $spreadsheetId, string $sheetName): void ->clear(); $this->updateRateLimit(); - + $this->logger->info("Cleared sheet {$sheetName}"); } catch (\Exception $e) { $this->logger->error("Failed to clear sheet: {$e->getMessage()}"); @@ -201,7 +205,7 @@ public function readSheet(string $spreadsheetId, string $sheetName): array ->all(); $this->updateRateLimit(); - + return $values; } catch (\Exception $e) { $this->logger->error("Failed to read from Google Sheets: {$e->getMessage()}"); @@ -223,7 +227,7 @@ public function getHeaders(string $spreadsheetId, string $sheetName): array ->all(); $this->updateRateLimit(); - + return $rows[0] ?? []; } catch (\Exception $e) { $this->logger->error("Failed to get headers: {$e->getMessage()}"); @@ -245,7 +249,7 @@ public function setHeaders(string $spreadsheetId, string $sheetName, array $head ->update([$headers]); // Update first row with headers $this->updateRateLimit(); - + $this->logger->info("Set headers in sheet {$sheetName}"); } catch (\Exception $e) { $this->logger->error("Failed to set headers: {$e->getMessage()}"); @@ -267,8 +271,8 @@ public function appendWithHeaders(string $spreadsheetId, string $sheetName, arra ->append($rows); // The client will handle header matching $this->updateRateLimit(); - - $this->logger->info("Written " . count($rows) . " rows to sheet {$sheetName}"); + + $this->logger->info('Written '.count($rows)." rows to sheet {$sheetName}"); } catch (\Exception $e) { $this->logger->error("Failed to write to Google Sheets: {$e->getMessage()}"); throw new GoogleSheetsException("Failed to write to Google Sheets: {$e->getMessage()}", 0, $e); @@ -286,21 +290,23 @@ private function generateHeaders($model = null, array $data = []): array } // Try to get headers from first row's keys if associative - if (!empty($data)) { + if (! empty($data)) { $firstRow = reset($data); - if (is_array($firstRow) && !$this->isSequentialArray($firstRow)) { + if (is_array($firstRow) && ! $this->isSequentialArray($firstRow)) { return array_keys($firstRow); } } // Fallback to Excel notation (A, B, C, ...) $columnCount = empty($data) ? 26 : count(reset($data)); // Default to 26 columns if no data - return array_map(function($num) { + + return array_map(function ($num) { $letter = ''; while ($num >= 0) { - $letter = chr(($num % 26) + 65) . $letter; + $letter = chr(($num % 26) + 65).$letter; $num = floor($num / 26) - 1; } + return $letter; }, range(0, $columnCount - 1)); } @@ -335,4 +341,4 @@ private function updateRateLimit(): void { $this->rateLimits[] = time(); } -} +} diff --git a/src/Services/NotificationManager.php b/src/Services/NotificationManager.php index b261a29..bbe5d72 100644 --- a/src/Services/NotificationManager.php +++ b/src/Services/NotificationManager.php @@ -19,11 +19,11 @@ use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Notification; -use Zuko\SyncroSheet\Models\SyncState; use Zuko\SyncroSheet\Events\SyncEvent; +use Zuko\SyncroSheet\Models\SyncState; +use Zuko\SyncroSheet\Notifications\SyncCompletedNotification; use Zuko\SyncroSheet\Notifications\SyncFailedNotification; use Zuko\SyncroSheet\Notifications\SyncRetryNotification; -use Zuko\SyncroSheet\Notifications\SyncCompletedNotification; class NotificationManager { @@ -42,7 +42,7 @@ public function notifyChunkProcessed(SyncState $syncState, int $processed): void { Event::dispatch(SyncEvent::CHUNK_PROCESSED, [ 'sync_state' => $syncState, - 'processed' => $processed + 'processed' => $processed, ]); } @@ -79,7 +79,7 @@ public function notifyFinalFailure(SyncState $syncState, \Exception $exception): { Event::dispatch(SyncEvent::SYNC_FAILED, [ 'sync_state' => $syncState, - 'error' => $exception->getMessage() + 'error' => $exception->getMessage(), ]); if (config('syncro-sheet.notifications.notify_on.error')) { @@ -90,7 +90,7 @@ public function notifyFinalFailure(SyncState $syncState, \Exception $exception): private function sendNotification($notification): void { $channels = config('syncro-sheet.notifications.channels', ['mail']); - + foreach ($channels as $channel) { if ($channel === 'mail') { Notification::route('mail', config('syncro-sheet.notifications.mail_to')) @@ -101,4 +101,4 @@ private function sendNotification($notification): void } } } -} +} diff --git a/src/Services/SheetRowMapper.php b/src/Services/SheetRowMapper.php index 4b38ca8..7e446c2 100644 --- a/src/Services/SheetRowMapper.php +++ b/src/Services/SheetRowMapper.php @@ -24,13 +24,16 @@ class SheetRowMapper { private const HASH_COLUMN = '_record_hash'; + private const ROW_INDEX_COLUMN = '_row_index'; + private FuzzyRecordIdentifier $fuzzyIdentifier; - public function __construct(FuzzyRecordIdentifier $fuzzyIdentifier = null) + public function __construct(?FuzzyRecordIdentifier $fuzzyIdentifier = null) { $this->fuzzyIdentifier = $fuzzyIdentifier ?? new FuzzyRecordIdentifier(); } + /** * Generate a unique hash for a model instance */ @@ -69,7 +72,7 @@ public function generateRecordHash(Model $model): string public function mapRecordsToSheet(Collection $records, array $sheetData): array { $headerRow = array_shift($sheetData); - if (!$headerRow) { + if (! $headerRow) { return $this->prepareNewSheetData($records); } @@ -115,7 +118,7 @@ public function mapRecordsToSheet(Collection $records, array $sheetData): array return [ 'header' => $headerRow, 'updates' => $updates, - 'new_rows' => $newRows + 'new_rows' => $newRows, ]; } @@ -128,7 +131,7 @@ private function prepareNewSheetData(Collection $records): array return [ 'header' => [], 'updates' => [], - 'new_rows' => [] + 'new_rows' => [], ]; } @@ -148,7 +151,7 @@ private function prepareNewSheetData(Collection $records): array return [ 'header' => $header, 'updates' => [], - 'new_rows' => $rows + 'new_rows' => $rows, ]; } diff --git a/src/Services/StateManager.php b/src/Services/StateManager.php index 18e4cc4..6ea649b 100644 --- a/src/Services/StateManager.php +++ b/src/Services/StateManager.php @@ -17,15 +17,15 @@ namespace Zuko\SyncroSheet\Services; -use Zuko\SyncroSheet\Models\SyncState; -use Zuko\SyncroSheet\Models\SyncEntry; use Illuminate\Support\Facades\DB; +use Zuko\SyncroSheet\Models\SyncState; class StateManager { public function __construct( private readonly SyncLogger $logger - ) {} + ) { + } /** * Initialize a new sync state @@ -106,4 +106,4 @@ public function getLastSuccessfulSync(string $modelClass): ?SyncState ->latest('completed_at') ->first(); } -} +} diff --git a/src/Services/SyncLogger.php b/src/Services/SyncLogger.php index e0a2e4a..1f7a71c 100644 --- a/src/Services/SyncLogger.php +++ b/src/Services/SyncLogger.php @@ -18,8 +18,8 @@ namespace Zuko\SyncroSheet\Services; use Illuminate\Support\Facades\Log; -use Monolog\Logger; use Monolog\Handler\RotatingFileHandler; +use Monolog\Logger; class SyncLogger { @@ -63,4 +63,4 @@ private function configureLogger(): Logger return $logger; } -} +} diff --git a/src/Services/SyncManager.php b/src/Services/SyncManager.php index d257648..db2dfad 100644 --- a/src/Services/SyncManager.php +++ b/src/Services/SyncManager.php @@ -25,13 +25,15 @@ class SyncManager { const AVAILABLE_SYNC_MODES = ['append', 'replace']; + public function __construct( private readonly BatchProcessor $batchProcessor, private readonly StateManager $stateManager, private readonly SyncLogger $logger, private readonly NotificationManager $notificationManager, private readonly ErrorHandler $errorHandler - ) {} + ) { + } /** * Start a full sync for the given model class @@ -46,6 +48,7 @@ public function fullSync(string $modelClass, array $options = []): SyncState $result = $this->batchProcessor->process($modelClass, $syncState, $syncMode); $this->stateManager->completeSync($syncState, $result); $this->notificationManager->notifyCompletion($syncState); + return $syncState; } catch (\Exception $e) { $this->errorHandler->handleError($syncState, $e); @@ -67,9 +70,9 @@ public function partialSync(string $modelClass, array $recordIds, array $options try { $result = $this->batchProcessor->processPartial($modelClass, $recordIds, $syncState); $this->stateManager->completeSync($syncState, $result); - + $this->logger->info("Completed partial sync for {$modelClass}"); - + return $syncState->fresh(); } catch (\Exception $e) { $this->handleSyncError($syncState, $e); @@ -79,13 +82,13 @@ public function partialSync(string $modelClass, array $recordIds, array $options private function validateModel(string $modelClass): void { - if (!class_exists($modelClass)) { + if (! class_exists($modelClass)) { throw new SyncException("Model class {$modelClass} does not exist"); } $model = new $modelClass; - - if (!$model instanceof Model || !$model instanceof SheetSyncable) { + + if (! $model instanceof Model || ! $model instanceof SheetSyncable) { throw new SyncException("Model class {$modelClass} must implement SheetSyncable interface"); } } @@ -99,7 +102,7 @@ private function handleSyncError(SyncState $syncState, \Exception $e): void private function determineSyncMode(string $modelClass, array $options = []): string { // 1. Command line option (passed through options) - if (!empty($options['sync_mode'])) { + if (! empty($options['sync_mode'])) { return $options['sync_mode']; } @@ -111,7 +114,8 @@ private function determineSyncMode(string $modelClass, array $options = []): str return $modelMode; } } + // 3. Default from config return config('syncro-sheet.defaults.sync_mode', 'append'); } -} +} diff --git a/src/Services/TokenManager.php b/src/Services/TokenManager.php index 2b9be97..b7376f0 100644 --- a/src/Services/TokenManager.php +++ b/src/Services/TokenManager.php @@ -23,6 +23,7 @@ class TokenManager { private const CACHE_KEY = 'google_sheets_token'; + private const TOKEN_BUFFER = 300; // 5 minutes buffer before expiration /** @@ -50,8 +51,8 @@ public function storeToken(array $tokenData): void private function shouldRefreshToken(): bool { $tokenData = Cache::get(self::CACHE_KEY); - - if (!$tokenData) { + + if (! $tokenData) { return true; } @@ -66,4 +67,4 @@ private function refreshToken(): void $this->storeToken($client->getAccessToken()); } } -} +}