From 654e44237cdb5b8d95616b38ef1282bb32e98d5d Mon Sep 17 00:00:00 2001 From: rcooke-warwick Date: Fri, 17 Nov 2023 13:41:02 +0000 Subject: [PATCH 1/6] use rsync to send image instead of http POST Change-type: patch Signed-off-by: Ryan Cooke --- core/lib/common/worker.js | 91 +++++++++++++-------------------------- 1 file changed, 31 insertions(+), 60 deletions(-) diff --git a/core/lib/common/worker.js b/core/lib/common/worker.js index 636e32da7..d975b5782 100644 --- a/core/lib/common/worker.js +++ b/core/lib/common/worker.js @@ -135,72 +135,43 @@ module.exports = class Worker { * @category helper */ async flash(imagePath) { - let attempt = 0; + this.logger.log(`Gzipping image...`); + await exec(`gzip ${imagePath}`) + + + const TARGET_PATH = '/tmp/os.img.gz'; + // file is not imagePath.gz await retry( async () => { - attempt++; - this.logger.log(`Preparing to flash, attempt ${attempt}...`); - - await new Promise(async (resolve, reject) => { - const req = rp.post({ uri: `${this.url}/dut/flash`, timeout: 0 }); - - req.catch((error) => { - this.logger.log(`client side error: `) - this.logger.log(error.message) - reject(error); - }); - req.finally(() => { - if (lastStatus !== 'done') { - reject(new Error('Unexpected end of TCP connection')); - } - - resolve(); - }); - - let lastStatus; - req.on('data', (data) => { - const computedLine = RegExp('(.+?): (.*)').exec(data.toString()); - - if (computedLine) { - if (computedLine[1] === 'error') { - req.cancel(); - reject(new Error(computedLine[2])); - } - - if (computedLine[1] === 'progress') { - once(() => { - this.logger.log('Flashing'); - }); - // Hide any errors as the lines we get can be half written - const state = JSON.parse(computedLine[2]); - if (state != null && isNumber(state.percentage)) { - this.logger.status({ - message: 'Flashing', - percentage: state.percentage, - }); - } - } - - if (computedLine[1] === 'status') { - lastStatus = computedLine[2]; - } - } - }); - - pipeline( - fs.createReadStream(imagePath), - createGzip({ level: 6 }), - req, - ); - }); - this.logger.log('Flash completed'); + this.logger.log(`Sending image ${imagePath}.gz`); + try{ + // arrives at worker as /tmp/imagePathBaseName.gz + await this.sendFile(`${imagePath}.gz`, TARGET_PATH, 'worker'); + }catch(e){ + console.log(e); + throw new Error(`Rysnc error: ${e.message}`) + } }, { max_tries: 5, interval: 1000 * 5, - throw_original: true, - }, + throw_original: true + } + ) + + this.logger.log(`Trying to flash ${TARGET_PATH}`); + let res = await rp.post( + { + uri: `${this.url}/dut/flash`, + body: { + filename: `${TARGET_PATH}` + }, + timeout: 0, + json: true + } ); + console.log(res) + } @@ -522,7 +493,7 @@ module.exports = class Worker { ); // todo : replace with npm package await exec( - `rsync -av -e "ssh ${this.workerUser}@${this.workerHost} -p ${this.workerPort} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q ${this.sshPrefix}balena exec -i" ${filePath} ${containerId}:${destination}`, + `rsync -av --partial -e "ssh ${this.workerUser}@${this.workerHost} -p ${this.workerPort} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q ${this.sshPrefix}balena exec -i" ${filePath} ${containerId}:${destination}`, ); } else { let ip = await this.ip(target); From 0c407052e89ecb318bbeb73d5da2fe389adb153c Mon Sep 17 00:00:00 2001 From: rcooke-warwick Date: Fri, 17 Nov 2023 13:43:13 +0000 Subject: [PATCH 2/6] add gzip Change-type: patch Signed-off-by: Ryan Cooke --- core/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Dockerfile b/core/Dockerfile index e7714a9fd..27f10ad89 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND noninteractive # https://github.com/balena-io/balena-cli/blob/master/INSTALL-LINUX.md#additional-dependencies # hadolint ignore=DL3008 RUN apt-get update && apt-get install --no-install-recommends -y \ - bind9-dnsutils \ + bind9-dnsutils gzip\ ca-certificates \ docker.io \ git \ From fe39360a537750733a5df173fe07093b74d04d9a Mon Sep 17 00:00:00 2001 From: rcooke-warwick Date: Fri, 24 Nov 2023 13:01:59 +0000 Subject: [PATCH 3/6] write to /data Change-type: patch Signed-off-by: Ryan Cooke --- core/lib/common/worker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/common/worker.js b/core/lib/common/worker.js index d975b5782..f8aba7440 100644 --- a/core/lib/common/worker.js +++ b/core/lib/common/worker.js @@ -139,7 +139,7 @@ module.exports = class Worker { await exec(`gzip ${imagePath}`) - const TARGET_PATH = '/tmp/os.img.gz'; + const TARGET_PATH = '/data/os.img.gz'; // file is not imagePath.gz await retry( async () => { From fab1a776eaee56bc24d3c7f7591788b3fce8c668 Mon Sep 17 00:00:00 2001 From: rcooke-warwick Date: Tue, 12 Dec 2023 12:19:40 +0000 Subject: [PATCH 4/6] take qemu into account Change-type: patch Signed-off-by: Ryan Cooke --- core/lib/common/worker.js | 50 +++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/core/lib/common/worker.js b/core/lib/common/worker.js index f8aba7440..79dc67119 100644 --- a/core/lib/common/worker.js +++ b/core/lib/common/worker.js @@ -135,36 +135,40 @@ module.exports = class Worker { * @category helper */ async flash(imagePath) { - this.logger.log(`Gzipping image...`); - await exec(`gzip ${imagePath}`) + // if using remote worker i.e autokit/testbot, send the image over ssh + if(!this.directConnect){ + this.logger.log(`Gzipping image...`); + await exec(`gzip ${imagePath}`); + imagePath = '/data/os.img.gz' - const TARGET_PATH = '/data/os.img.gz'; - // file is not imagePath.gz - await retry( - async () => { - this.logger.log(`Sending image ${imagePath}.gz`); - try{ - // arrives at worker as /tmp/imagePathBaseName.gz - await this.sendFile(`${imagePath}.gz`, TARGET_PATH, 'worker'); - }catch(e){ - console.log(e); - throw new Error(`Rysnc error: ${e.message}`) + // file is not imagePath.gz + await retry( + async () => { + this.logger.log(`Sending image ${imagePath}`); + try{ + // arrives at worker as /tmp/imagePathBaseName.gz + await this.sendFile(`${imagePath}.gz`, TARGET_PATH, 'worker'); + }catch(e){ + console.log(e); + throw new Error(`Rysnc error: ${e.message}`) + } + }, + { + max_tries: 5, + interval: 1000 * 5, + throw_original: true } - }, - { - max_tries: 5, - interval: 1000 * 5, - throw_original: true - } - ) + ) + } - this.logger.log(`Trying to flash ${TARGET_PATH}`); + // note: for the qemu case, we are assuming a shared volume between core and worker - this means that any image we want to flash must be in this volume - is it in all our cases? + this.logger.log(`Trying to flash ${imagePath}`); let res = await rp.post( { uri: `${this.url}/dut/flash`, body: { - filename: `${TARGET_PATH}` + filename: `${imagePath}` }, timeout: 0, json: true @@ -498,7 +502,7 @@ module.exports = class Worker { } else { let ip = await this.ip(target); await exec( - `rsync -av -e "ssh -p 22222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i ${this.sshKey}" ${filePath} root@${ip}:${destination}`, + `rsync -av -e --partial "ssh -p 22222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i ${this.sshKey}" ${filePath} root@${ip}:${destination}`, ); } } From 59c152ce99fc3ac581d23ff9f4195cb4a6d114d6 Mon Sep 17 00:00:00 2001 From: rcooke-warwick Date: Wed, 13 Dec 2023 15:24:58 +0000 Subject: [PATCH 5/6] adjust image paths Change-type: patch Signed-off-by: Ryan Cooke --- core/lib/common/worker.js | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/core/lib/common/worker.js b/core/lib/common/worker.js index 79dc67119..ed28a5260 100644 --- a/core/lib/common/worker.js +++ b/core/lib/common/worker.js @@ -136,19 +136,24 @@ module.exports = class Worker { */ async flash(imagePath) { + let TARGET_PATH = imagePath; // if using remote worker i.e autokit/testbot, send the image over ssh if(!this.directConnect){ this.logger.log(`Gzipping image...`); await exec(`gzip ${imagePath}`); - imagePath = '/data/os.img.gz' + // adjust image path to take new gz suffix + imagePath = `${imagePath}.gz` + // ensure we always send to the same location on worker to avoid overflowing of images + TARGET_PATH = `/data/os.img.gz` - // file is not imagePath.gz + // Wrap sending of image in a retry - hopefully the --partial arguement in the rsync command means it will resume + // in the case of an error await retry( async () => { this.logger.log(`Sending image ${imagePath}`); try{ - // arrives at worker as /tmp/imagePathBaseName.gz - await this.sendFile(`${imagePath}.gz`, TARGET_PATH, 'worker'); + // arrives at worker as /data/os.img.gz + await this.sendFile(imagePath, TARGET_PATH, 'worker'); }catch(e){ console.log(e); throw new Error(`Rysnc error: ${e.message}`) @@ -163,19 +168,17 @@ module.exports = class Worker { } // note: for the qemu case, we are assuming a shared volume between core and worker - this means that any image we want to flash must be in this volume - is it in all our cases? - this.logger.log(`Trying to flash ${imagePath}`); + this.logger.log(`Trying to flash ${TARGET_PATH}`); let res = await rp.post( { - uri: `${this.url}/dut/flash`, + uri: `${this.url}/dut/flashFromFile`, body: { - filename: `${imagePath}` + filename: TARGET_PATH }, timeout: 0, json: true } - ); - console.log(res) - + ); } @@ -502,7 +505,7 @@ module.exports = class Worker { } else { let ip = await this.ip(target); await exec( - `rsync -av -e --partial "ssh -p 22222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i ${this.sshKey}" ${filePath} root@${ip}:${destination}`, + `rsync -av --partial -e "ssh -p 22222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i ${this.sshKey}" ${filePath} root@${ip}:${destination}`, ); } } From 5fa0a373a9d57ac1d6b66f566602f64e4eaee1cd Mon Sep 17 00:00:00 2001 From: rcooke-warwick Date: Fri, 26 Jul 2024 14:09:14 +0100 Subject: [PATCH 6/6] log worker messages Change-type: patch Signed-off-by: Ryan Cooke --- core/lib/common/worker.js | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/core/lib/common/worker.js b/core/lib/common/worker.js index ed28a5260..a1ef2110f 100644 --- a/core/lib/common/worker.js +++ b/core/lib/common/worker.js @@ -169,16 +169,26 @@ module.exports = class Worker { // note: for the qemu case, we are assuming a shared volume between core and worker - this means that any image we want to flash must be in this volume - is it in all our cases? this.logger.log(`Trying to flash ${TARGET_PATH}`); - let res = await rp.post( - { - uri: `${this.url}/dut/flashFromFile`, - body: { - filename: TARGET_PATH - }, - timeout: 0, - json: true - } - ); + await new Promise(async (resolve, reject) => { + let res = rp.post( + { + uri: `${this.url}/dut/flashFromFile`, + body: { + filename: TARGET_PATH + }, + timeout: 0, + json: true + } + ); + + req.on('data', (data) => { + this.logger.log(data.toString()); + }); + + req.finally(() => { + resolve(); + }); + }); }