Merge pull request #1 from ultra-bugs/z-dev

Basic sync

README.md

@@ -1,3 +1,7 @@
+# WIP
+
+This package is not avail on packagist yet. If you interesting, take a local clone or zip download then try it out.
+
 # SyncroSheet
 
 Laravel package for efficient synchronization between your models and Google Sheets with advanced state tracking and error handling.
@@ -46,6 +50,13 @@ GOOGLE_SHEETS_CLIENT_ID=your-client-id
 GOOGLE_SHEETS_CLIENT_SECRET=your-client-secret
 GOOGLE_SHEETS_REDIRECT_URI=your-redirect-uri
 ```
+
+Or using service account:
+
+```env
+GOOGLE_DEVELOPER_KEY=your-service-account-key
+GOOGLE_SERVICE_ENABLED=true
+```
 As a wrapped around `revolution/laravel-google-sheets`. these env vars is taken from `config/google.php`
 
 If you already set these authorization values. You can leave env untouched.

composer.json

@@ -1,40 +1,47 @@
 {
-    "name": "zuko/syncro-sheet",
-    "description": "Laravel Google Sheets synchronization package",
-    "type": "library",
-    "license": "MIT",
-    "authors": [
-        {
-            "name": "Zuko",
-            "email": "[email protected]"
-        }
-    ],
-    "require": {
-        "php": "^8.1",
-        "laravel/framework": "^10.0",
-        "revolution/laravel-google-sheets": "^6.0"
-    },
-    "require-dev": {
-        "phpunit/phpunit": "^10.0",
-        "orchestra/testbench": "^8.0"
-    },
-    "autoload": {
-        "psr-4": {
-            "Zuko\\SyncroSheet\\": "src/"
-        }
-    },
-    "autoload-dev": {
-        "psr-4": {
-            "Zuko\\SyncroSheet\\Tests\\": "tests/"
-        }
-    },
-    "extra": {
-        "laravel": {
-            "providers": [
-                "Zuko\\SyncroSheet\\LaravelSyncroSheetProvider"
-            ]
-        }
-    },
-    "minimum-stability": "dev",
-    "prefer-stable": true
+  "name"              : "zuko/syncro-sheet",
+  "replace"           : {
+    "zuko/laravel-sheet-sync"  : "self.version",
+    "zuko/eloquent-sheet-sync" : "self.version",
+    "zuko/synchro-sheet" : "self.version"
+  },
+  "description"       : "Laravel Eloquent and Google Sheets synchronization package",
+  "type"              : "library",
+  "license"           : "MIT",
+  "authors"           : [
+    {
+      "name"  : "Zuko",
+      "email" : "[email protected]",
+      "role"  : "Developer",
+      "homepage": "https://github.com/tansautn"
+    }
+  ],
+  "require"           : {
+    "php"                              : "^8.1",
+    "illuminate/database"              : ">=10.0",
+    "revolution/laravel-google-sheets" : ">=6.0"
+  },
+  "require-dev"       : {
+    "roave/security-advisories"     : "dev-latest",
+    "laravel/pint" : ">=1.17.0"
+  },
+  "autoload"          : {
+    "psr-4" : {
+      "Zuko\\SyncroSheet\\" : "src/"
+    }
+  },
+  "autoload-dev"      : {
+    "psr-4" : {
+      "Zuko\\SyncroSheet\\Tests\\" : "tests/"
+    }
+  },
+  "extra"             : {
+    "laravel" : {
+      "providers" : [
+        "Zuko\\SyncroSheet\\LaravelSyncroSheetProvider"
+      ]
+    }
+  },
+  "minimum-stability" : "dev",
+  "prefer-stable"     : true
 }

config/syncro-sheet.php

@@ -22,7 +22,8 @@
     |--------------------------------------------------------------------------
     */
     'defaults' => [
-        'batch_size' => 1000,
+        'batch_size' => 100,
+        'sync_mode' => 'append',
         'timeout' => 600,
         'retries' => 3,
     ],
@@ -33,8 +34,8 @@
     |--------------------------------------------------------------------------
     */
     'logging' => [
-        'channel' => 'sheet-sync',
-        'level' => 'info',
+        'channel' => env('SHEET_SYNC_LOG_CHANNEL', 'sheet-sync'),
+        'level' => env('SHEET_SYNC_LOG_LEVEL', 'info'),
         'separate_files' => true,
     ],
 
@@ -64,4 +65,4 @@
             'sync_completed' => false,
         ],
     ],
-]; 
+];

database/migrations/create_sync_states_table.php → database/migrations/0_create_sync_states_table.php

@@ -23,7 +23,7 @@
 {
     public function up(): void
     {
-        Schema::create('sync_states', function (Blueprint $table) {
+        Schema::create('sync_states', static function (Blueprint $table) {
             $table->id();
             $table->string('model_class');
             $table->enum('sync_type', ['full', 'partial']);
@@ -43,4 +43,4 @@ public function down(): void
     {
         Schema::dropIfExists('sync_states');
     }
-}; 
+};

database/migrations/create_sync_entries_table.php → database/migrations/1_create_sync_entries_table.php

@@ -23,7 +23,7 @@
 {
     public function up(): void
     {
-        Schema::create('sync_entries', function (Blueprint $table) {
+        Schema::create('sync_entries', static function (Blueprint $table) {
             $table->id();
             $table->foreignId('sync_state_id')->constrained()->cascadeOnDelete();
             $table->string('model_class');
@@ -43,4 +43,4 @@ public function down(): void
     {
         Schema::dropIfExists('sync_entries');
     }
-}; 
+};

database/migrations/2_create_sync_mode_column.php

@@ -0,0 +1,40 @@
+<?php
+/*
+ *          M""""""""`M            dP
+ *          Mmmmmm   .M            88
+ *          MMMMP  .MMM  dP    dP  88  .dP   .d8888b.
+ *          MMP  .MMMMM  88    88  88888"    88'  `88
+ *          M' .MMMMMMM  88.  .88  88  `8b.  88.  .88
+ *          M         M  `88888P'  dP   `YP  `88888P'
+ *          MMMMMMMMMMM    -*-  Created by Zuko  -*-
+ *
+ *          * * * * * * * * * * * * * * * * * * * * *
+ *          * -    - -   F.R.E.E.M.I.N.D   - -    - *
+ *          * -  Copyright © 2024 (Z) Programing  - *
+ *          *    -  -  All Rights Reserved  -  -    *
+ *          * * * * * * * * * * * * * * * * * * * * *
+ */
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up()
+    {
+        Schema::table('sync_states', static function (Blueprint $table) {
+            $table->enum('sync_mode', \Zuko\SyncroSheet\Services\SyncManager::AVAILABLE_SYNC_MODES)
+                ->default(\Zuko\SyncroSheet\Services\SyncManager::AVAILABLE_SYNC_MODES[0])
+                ->after('sync_type')
+                ->index();
+        });
+    }
+
+    public function down()
+    {
+        Schema::table('sync_states', static function (Blueprint $table) {
+            $table->dropColumn('sync_mode');
+        });
+    }
+};

docs/COMPONENTS_OVERVIEW.md

@@ -0,0 +1,193 @@
+# Component Overview
+
+This document describes the components used in this package. A minimal diagram also provides to illustrate the flow and mind-set at package built-stage.
+Namespace: `Zuko\SyncroSheet`
+Composer package name: `zuko/syncro-sheet`
+Laravel compatible provider: \Zuko\SyncroSheet\LaravelSyncroSheetProvider
+
+# Component List & Summary
+
+### 1. Core Components
+
+#### `SyncManager` (Main Orchestrator)
+- Primary entry point for sync operations
+- Coordinates between all other components
+- Handles high-level error management
+- Ensures proper sync state tracking
+
+#### `SheetSyncable` (Interface)
+- Contract for models that can be synced
+- Defines required configuration methods
+- Provides data transformation rules
+```php
+interface SheetSyncable
+{
+    public function getSheetIdentifier(): string;    // Google Sheet ID
+    public function getSheetName(): string;          // Sheet name
+    public function toSheetRow(): array;             // Data transformation
+    public function getBatchSize(): int;             // Optional, defaults to 1000
+}
+```
+
+### 2. Data Processing Components
+
+#### `BatchProcessor`
+- Manages chunked data processing
+- Uses Laravel's lazy loading for memory efficiency
+- Reports progress to state manager
+- Handles chunk-level errors
+
+#### `DataTransformer`
+- Converts model data to sheet rows
+- Handles data type conversions
+- Manages date/time formatting
+- Optional custom transformers support
+
+### 3. State Management Components
+
+#### `StateManager`
+- Tracks sync progress and status
+- Manages sync history
+- Provides resume capability
+- Stores performance metrics
+- more sophisticated state tracking system that can handle both full and partial syncs while maintaining the sync history of individual records
+
+#### `SyncState` (Model)
+```php
+// Used for tracking sync state of one sync call
+class SyncState extends Model
+{
+    protected $fillable = [
+        'model_class',      // Model class name
+        'sync_type',        // 'full' or 'partial'
+        'sync_mode',        // 'append' or 'replace'
+        'status',           // 'running', 'completed', 'failed'
+        'started_at',
+        'completed_at',
+        'total_processed',
+        'last_processed_id',
+    ];
+}
+
+// Used for tracking individual records
+class SyncEntry extends Model
+{
+    protected $fillable = [
+        'model_class',
+        'record_id',
+        'synced_at',
+        'sync_state_id',    // Reference to parent SyncState
+        'sync_type',        // 'full' or 'partial'
+        'status'            // 'success' or 'failed'
+    ];
+
+    protected $casts = [
+        'synced_at' => 'datetime'
+    ];
+}```
+
+### 4. Google Sheets Integration
+
+#### `SheetClient` (Wrapper around revolution/laravel-google-sheets)
+- Manages Google Sheets connections
+- Handles authentication
+- Provides sheet operations interface
+- Manages API rate limits
+
+### 5. Logging & Monitoring
+
+#### `SyncLogger`
+- Dedicated logging channel
+- Structured log format
+- Performance logging
+- Error tracking
+
+#### `Events`
+```php
+class Events
+{
+    const SYNC_STARTED = 'sheet-sync.started';
+    const CHUNK_PROCESSED = 'sheet-sync.chunk-processed';
+    const SYNC_COMPLETED = 'sheet-sync.completed';
+    const SYNC_FAILED = 'sheet-sync.failed';
+}
+```
+
+### 6. Configuration
+
+#### Package Config (`syncro-sheet.php`)
+```php
+return [
+    'defaults' => [
+        'batch_size' => 1000,
+        'timeout' => 600,
+        'retries' => 3
+    ],
+
+    'logging' => [
+        'channel' => 'sheet-sync',
+        'level' => 'info',
+        'separate_files' => true
+    ],
+
+    'sheets' => [
+        'cache_ttl' => 3600,
+        'rate_limit' => [
+            'max_requests' => 100,
+            'per_seconds' => 60
+        ]
+    ]
+];
+```
+
+# Component Interactions
+
+
+```mermaid
+graph TB
+    A[SyncManager] --> B[BatchProcessor]
+    A --> C[StateManager]
+    A --> D[SheetClient]
+
+    B --> E[DataTransformer]
+    B --> F[SyncLogger]
+
+    D --> G[Google Sheets API]
+
+    C --> H[Database]
+    F --> I[Log Files]
+
+    J[Model] --> A
+    J --> E
+```
+
+
+# Flow:
+
+
+```mermaid
+sequenceDiagram
+    participant Command as SyncCommand
+    participant Manager as SyncManager
+    participant Processor as BatchProcessor
+    participant State as StateManager
+    participant Transformer as DataTransformer
+    participant Syncer as GoogleSheetsClient
+    participant Client as GoogleSheetsClient
+
+    Command->>Manager: sync(ModelClass)
+    Manager->>State: startSync()
+    Manager->>Processor: process(ModelClass)
+
+    loop For each chunk
+        Processor->>Transformer: transform(chunk)
+        Transformer->>Syncer: sync(transformedData)
+        Syncer->>Client: write(sheetId, data)
+        Client-->>Syncer: response
+        Syncer-->>Processor: result
+        Processor->>State: updateProgress()
+    end
+
+    Manager->>State: completeSync()
+    Manager-->>Command: SyncResult
+```