diff --git a/.github/actions/provision-cluster/lib/kubeception.js b/.github/actions/provision-cluster/lib/kubeception.js index 4dd1f020..771fa7fb 100644 --- a/.github/actions/provision-cluster/lib/kubeception.js +++ b/.github/actions/provision-cluster/lib/kubeception.js @@ -64,24 +64,34 @@ class Client { kubeceptionProfile = "default" } - return utils.fibonacciRetry(async ()=>{ - const response = await this.client.put(`https://sw.bakerstreet.io/kubeception/api/klusters/${name}?version=${version}&profile=${kubeceptionProfile}&timeoutSecs=${lifespan}`) + return utils.fibonacciRetry(async () => { + const response = await this.client.put( + `https://sw.bakerstreet.io/kubeception/api/klusters/${name}?version=${version}&profile=${kubeceptionProfile}&timeoutSecs=${lifespan}` + ); + if (!response || !response.message) { - throw Error("Unknown error getting response") + throw new utils.Transient("Unknown error getting response"); } - if (response.message.statusCode == 200) { - return await response.readBody() - } else if (response.message.statusCode == 425) { - // The kubeception API uses 425 to signal that cluster creation is "in progress", so we want - // to retry later. - throw new utils.Transient(`status code ${response.message.statusCode}`) - } else { - // Any other status code is likely a permanent error. - let body = await response.readBody() - throw new Error(`Status code ${response.message.statusCode}: ${body}`) + switch (response.message.statusCode) { + case 200: + case 201: + return await response.readBody(); + case 202: + throw new utils.Retry("Request is still pending"); + default: + if (response.message.statusCode >= 400) { + throw new utils.Transient( + `Status code ${response.message.statusCode}` + ); + } else { + let body = await response.readBody(); + throw new Error( + `Status code ${response.message.statusCode}: ${body}` + ); + } } - }) + }); } async deleteKluster(name) { diff --git a/.github/actions/provision-cluster/lib/utils.js b/.github/actions/provision-cluster/lib/utils.js index 14bcfe5a..46e53601 100644 --- a/.github/actions/provision-cluster/lib/utils.js +++ b/.github/actions/provision-cluster/lib/utils.js @@ -54,12 +54,14 @@ function uid() { return crypto.randomBytes(16).toString("hex") } +class Retry extends Error {} + class Transient extends Error {} // Retry the supplied action with a fibonacci backoff until it returns true or timeout seconds have // passed. Use minDelay and maxDelay to tune the delay times. The action should signal retry with -// `throw new Transient(...)`, and return upon success. The result of the final successful -// invocation will be returned. +// `throw new Transient(...)` or `throw new Retry(...)`, and return upon success. +// The result of the final successful invocation will be returned. async function fibonacciRetry(action, timeout=600000, minDelay=1000, maxDelay=30000) { let start = Date.now() let nextDelay = fibonacciDelaySequence(minDelay, maxDelay) @@ -67,7 +69,7 @@ async function fibonacciRetry(action, timeout=600000, minDelay=1000, maxDelay=30 try { return await action() } catch (e) { - if (!(e instanceof Transient)) { + if (!(e instanceof Transient) && !(e instanceof Retry)) { throw e } let delay = nextDelay() @@ -76,13 +78,13 @@ async function fibonacciRetry(action, timeout=600000, minDelay=1000, maxDelay=30 let remaining = timeout - elapsed if (remaining > 0) { let t = Math.min(delay, remaining) - core.info(`Transient error (${e.message}) retrying after ${t/1000}s ...`) + core.info(`Error (${e.message}) retrying after ${t/1000}s ...`) await sleep(t) } else { - throw new Error(`Transient error (${e.message}) failing after ${count} attempts over ${elapsed/1000}s.`) + throw new Error(`Error (${e.message}) failing after ${count} attempts over ${elapsed/1000}s.`) } } } } -module.exports = { getUniqueClusterName, writeFile, sleep, fibonacciDelaySequence, uid, Transient, fibonacciRetry } +module.exports = { getUniqueClusterName, writeFile, sleep, fibonacciDelaySequence, uid, Retry, Transient, fibonacciRetry } diff --git a/.github/actions/provision-cluster/lib/utils.test.js b/.github/actions/provision-cluster/lib/utils.test.js index 83fadeb5..f7408892 100644 --- a/.github/actions/provision-cluster/lib/utils.test.js +++ b/.github/actions/provision-cluster/lib/utils.test.js @@ -75,7 +75,7 @@ test('fibonacciRetry fail all', async () => { returned = true } catch (err) { let elapsed = Date.now() - start - expect(err.message).toContain("Transient error") + expect(err.message).toContain("Error") expect(err.message).toContain("never big enough") expect(err.message).toContain("failing after") expect(err.message).toContain("attempts over")