Fetch device JSONs #550
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fetch device JSONs | |
| on: | |
| schedule: | |
| - cron: '0 */12 * * *' # every 12 hours (reduced frequency due to polling workflow) | |
| workflow_dispatch: | |
| repository_dispatch: | |
| types: [ota-update, ota-device-update] | |
| jobs: | |
| fetch: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: true | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - name: Process webhook payload | |
| id: webhook | |
| if: github.event_name == 'repository_dispatch' | |
| run: | | |
| echo "Webhook event type: ${{ github.event.action }}" | |
| echo "Webhook client payload: ${{ toJson(github.event.client_payload) }}" | |
| # Check if this is a relevant update | |
| if [[ "${{ github.event.action }}" == "ota-update" ]] || [[ "${{ github.event.action }}" == "ota-device-update" ]]; then | |
| echo "should_update=true" >> $GITHUB_OUTPUT | |
| echo "trigger_reason=webhook" >> $GITHUB_OUTPUT | |
| # Extract changed files if available in payload | |
| CHANGED_FILES="${{ github.event.client_payload.changed_files }}" | |
| if [[ -n "$CHANGED_FILES" ]]; then | |
| echo "changed_files=$CHANGED_FILES" >> $GITHUB_OUTPUT | |
| # Check if any JSON files were changed | |
| if echo "$CHANGED_FILES" | grep -q "\.json$"; then | |
| echo "json_files_changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "json_files_changed=false" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| echo "changed_files=" >> $GITHUB_OUTPUT | |
| echo "json_files_changed=true" >> $GITHUB_OUTPUT # Assume JSON files changed if not specified | |
| fi | |
| else | |
| echo "should_update=false" >> $GITHUB_OUTPUT | |
| echo "trigger_reason=unrecognized_webhook" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Check if update is needed | |
| id: check_update | |
| if: github.event_name != 'repository_dispatch' || steps.webhook.outputs.should_update == 'true' | |
| run: | | |
| if [[ "${{ github.event_name }}" == "repository_dispatch" ]]; then | |
| if [[ "${{ steps.webhook.outputs.json_files_changed }}" == "false" ]]; then | |
| echo "No JSON files were changed, skipping update" | |
| echo "should_proceed=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "JSON files were changed, proceeding with update" | |
| echo "should_proceed=true" >> $GITHUB_OUTPUT | |
| echo "trigger_reason=${{ steps.webhook.outputs.trigger_reason }}" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| echo "Scheduled or manual trigger, proceeding with update" | |
| echo "should_proceed=true" >> $GITHUB_OUTPUT | |
| echo "trigger_reason=${{ github.event_name }}" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Fetch and aggregate device JSONs | |
| if: steps.check_update.outputs.should_proceed == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TRIGGER_REASON: ${{ steps.check_update.outputs.trigger_reason }} | |
| run: | | |
| # write a JS script using a quoted heredoc to avoid shell quoting/escaping issues | |
| mkdir -p .github/scripts | |
| cat > .github/scripts/fetch-devices.js <<'JS' | |
| const https = require('https'); | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const API_BASE = 'https://api.github.com/repos/AlphaDroid-devices/OTA'; | |
| const API_CONTENTS = `${API_BASE}/contents/`; | |
| const TOKEN = process.env.GITHUB_TOKEN; | |
| const TRIGGER_REASON = process.env.TRIGGER_REASON || 'unknown'; | |
| const opts = { | |
| headers: { | |
| 'User-Agent': 'AlphaDroid-Website-Bot/1.0', | |
| 'Accept': 'application/vnd.github.v3+json', | |
| ...(TOKEN ? { 'Authorization': `token ${TOKEN}` } : {}) | |
| } | |
| }; | |
| function get(url, retries = 3) { | |
| return new Promise((resolve, reject) => { | |
| const attempt = (retryCount) => { | |
| const request = https.get(url, opts, response => { | |
| let body = ''; | |
| response.on('data', chunk => body += chunk); | |
| response.on('end', () => { | |
| if (response.statusCode >= 200 && response.statusCode < 300) { | |
| resolve({ | |
| statusCode: response.statusCode, | |
| headers: response.headers, | |
| body, | |
| url | |
| }); | |
| } else if (response.statusCode === 403 && retryCount > 0) { | |
| // Rate limit, wait and retry | |
| console.warn(`Rate limited, retrying in ${(4 - retryCount) * 2} seconds... (${retryCount} retries left)`); | |
| setTimeout(() => attempt(retryCount - 1), (4 - retryCount) * 2000); | |
| } else { | |
| reject(new Error(`HTTP ${response.statusCode}: ${response.statusMessage} for ${url}`)); | |
| } | |
| }); | |
| }); | |
| request.on('error', error => { | |
| if (retryCount > 0) { | |
| console.warn(`Network error, retrying... (${retryCount} retries left)`, error.message); | |
| setTimeout(() => attempt(retryCount - 1), 1000); | |
| } else { | |
| reject(error); | |
| } | |
| }); | |
| request.setTimeout(30000, () => { | |
| request.destroy(); | |
| if (retryCount > 0) { | |
| console.warn(`Timeout, retrying... (${retryCount} retries left)`); | |
| setTimeout(() => attempt(retryCount - 1), 1000); | |
| } else { | |
| reject(new Error('Request timeout')); | |
| } | |
| }); | |
| }; | |
| attempt(retries); | |
| }); | |
| } | |
| async function fetchDeviceFiles() { | |
| console.log(`Starting device fetch (trigger: ${TRIGGER_REASON})`); | |
| const startTime = Date.now(); | |
| try { | |
| console.log('Fetching repository contents...'); | |
| const listResp = await get(API_CONTENTS); | |
| const items = JSON.parse(listResp.body) | |
| .filter(item => item.type === 'file' && item.name.toLowerCase().endsWith('.json')) | |
| .sort((a, b) => a.name.localeCompare(b.name)); // Sort for consistent ordering | |
| console.log(`Found ${items.length} JSON files to process`); | |
| if (items.length === 0) { | |
| console.warn('No JSON files found in repository'); | |
| return { success: false, error: 'No JSON files found' }; | |
| } | |
| const results = []; | |
| let successCount = 0; | |
| let errorCount = 0; | |
| // Process files in batches to avoid overwhelming the API | |
| const batchSize = 5; | |
| for (let i = 0; i < items.length; i += batchSize) { | |
| const batch = items.slice(i, i + batchSize); | |
| console.log(`Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(items.length / batchSize)} (${batch.length} files)`); | |
| const batchPromises = batch.map(async (item) => { | |
| try { | |
| const response = await get(item.download_url); | |
| const data = JSON.parse(response.body); | |
| // Validate that this looks like a device JSON | |
| if (!data || typeof data !== 'object') { | |
| throw new Error('Invalid JSON structure'); | |
| } | |
| results.push({ | |
| name: item.name, | |
| data, | |
| rawUrl: item.download_url, | |
| lastModified: response.headers['last-modified'] || null, | |
| size: response.headers['content-length'] || null, | |
| sha: item.sha | |
| }); | |
| successCount++; | |
| console.log(`✓ ${item.name}`); | |
| } catch (error) { | |
| errorCount++; | |
| console.error(`✗ ${item.name}: ${error.message}`); | |
| } | |
| }); | |
| await Promise.all(batchPromises); | |
| // Small delay between batches to be respectful to the API | |
| if (i + batchSize < items.length) { | |
| await new Promise(resolve => setTimeout(resolve, 100)); | |
| } | |
| } | |
| if (results.length === 0) { | |
| console.error('No device files were successfully fetched'); | |
| return { success: false, error: 'All fetches failed' }; | |
| } | |
| // Ensure data directory exists | |
| const dataDir = 'data'; | |
| if (!fs.existsSync(dataDir)) { | |
| fs.mkdirSync(dataDir, { recursive: true }); | |
| } | |
| // Write the aggregated data | |
| const outputPath = path.join(dataDir, 'devices.json'); | |
| const outputData = { | |
| metadata: { | |
| fetchedAt: new Date().toISOString(), | |
| trigger: TRIGGER_REASON, | |
| totalFiles: items.length, | |
| successfulFetches: successCount, | |
| failedFetches: errorCount, | |
| fetchDurationMs: Date.now() - startTime | |
| }, | |
| devices: results | |
| }; | |
| // Compare device-level data with existing file before writing to avoid spurious commits | |
| let shouldWrite = true; | |
| try { | |
| if (fs.existsSync(outputPath)) { | |
| const existing = fs.readFileSync(outputPath, 'utf8'); | |
| const existingJson = JSON.parse(existing); | |
| // Check if the file is stale (older than 24 hours) | |
| // We want to force an update at least once a day to keep the timestamp fresh | |
| // for health checks, even if the device data itself hasn't changed. | |
| let isStale = false; | |
| if (existingJson.metadata && existingJson.metadata.fetchedAt) { | |
| const lastFetch = new Date(existingJson.metadata.fetchedAt); | |
| const ageMs = Date.now() - lastFetch.getTime(); | |
| const ageHours = ageMs / (1000 * 60 * 60); | |
| if (ageHours > 24) { | |
| console.log(`Existing data is ${ageHours.toFixed(1)} hours old (older than 24h). Forcing update.`); | |
| isStale = true; | |
| } | |
| } | |
| const normalize = (devices) => ( | |
| devices | |
| .map(d => ({ name: d.name, data: d.data })) | |
| .sort((a, b) => a.name.localeCompare(b.name)) | |
| ); | |
| const existingNormalized = normalize(existingJson.devices || []); | |
| const outputNormalized = normalize(outputData.devices || []); | |
| if (!isStale && JSON.stringify(existingNormalized) === JSON.stringify(outputNormalized)) { | |
| console.log('No device-level changes detected and data is fresh (< 24h); skipping write.'); | |
| shouldWrite = false; | |
| } | |
| } | |
| } catch (err) { | |
| console.warn('Comparison failed:', err.message); | |
| shouldWrite = true; | |
| } | |
| const duration = Date.now() - startTime; | |
| if (shouldWrite) { | |
| fs.writeFileSync(outputPath, JSON.stringify(outputData, null, 2)); | |
| console.log(`\n✓ Successfully wrote ${outputPath}`); | |
| console.log(` Devices: ${results.length}/${items.length}`); | |
| console.log(` Duration: ${duration}ms`); | |
| console.log(` Success rate: ${((successCount / items.length) * 100).toFixed(1)}%`); | |
| return { | |
| success: true, | |
| deviceCount: results.length, | |
| totalFiles: items.length, | |
| successRate: (successCount / items.length) * 100, | |
| duration | |
| }; | |
| } else { | |
| console.log(`Skipped writing ${outputPath} — device data unchanged.`); | |
| return { | |
| success: true, | |
| skipped: true, | |
| reason: 'No device-level changes', | |
| deviceCount: results.length, | |
| totalFiles: items.length, | |
| duration | |
| }; | |
| } | |
| } catch (error) { | |
| console.error('Fatal error during fetch:', error.message); | |
| return { success: false, error: error.message }; | |
| } | |
| } | |
| // Execute the fetch | |
| fetchDeviceFiles() | |
| .then(result => { | |
| if (result.success) { | |
| console.log('Device fetch completed successfully'); | |
| process.exit(0); | |
| } else { | |
| console.error('Device fetch failed:', result.error); | |
| process.exit(1); | |
| } | |
| }) | |
| .catch(error => { | |
| console.error('Unexpected error:', error); | |
| process.exit(1); | |
| }); | |
| JS | |
| echo "Trigger reason: $TRIGGER_REASON" | |
| node .github/scripts/fetch-devices.js | |
| - name: Commit and push changes | |
| if: steps.check_update.outputs.should_proceed == 'true' | |
| id: commit | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add --force data/devices.json || true | |
| if git diff --quiet HEAD -- data/devices.json; then | |
| echo "No changes to commit (data/devices.json unchanged)" | |
| echo "has_changes=false" >> $GITHUB_OUTPUT | |
| else | |
| # Generate a more descriptive commit message | |
| TRIGGER="${{ steps.check_update.outputs.trigger_reason }}" | |
| if [[ "$TRIGGER" == "webhook" ]]; then | |
| COMMIT_MSG="chore: update device data from OTA repository webhook [skip ci]" | |
| elif [[ "$TRIGGER" == "schedule" ]]; then | |
| COMMIT_MSG="chore: scheduled update of device data [skip ci]" | |
| elif [[ "$TRIGGER" == "workflow_dispatch" ]]; then | |
| COMMIT_MSG="chore: manual update of device data [skip ci]" | |
| else | |
| COMMIT_MSG="chore: update aggregated devices.json [skip ci]" | |
| fi | |
| git commit -m "$COMMIT_MSG" | |
| git push | |
| echo "has_changes=true" >> $GITHUB_OUTPUT | |
| echo "Changes committed and pushed successfully" | |
| fi | |
| - name: Create summary | |
| if: always() && steps.check_update.outputs.should_proceed == 'true' | |
| run: | | |
| echo "## Device Data Update Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Trigger:** ${{ steps.check_update.outputs.trigger_reason }}" >> $GITHUB_STEP_SUMMARY | |
| echo "**Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY | |
| echo "**Changes Committed:** ${{ steps.commit.outputs.has_changes }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [[ "${{ job.status }}" == "success" ]]; then | |
| echo "✅ **Success:** Device data updated successfully" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ **Failed:** Device data update failed" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Workflow Details" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Repository:** AlphaDroid-devices/OTA" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Target File:** data/devices.json" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Workflow Run:** [View Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY |