Refactor Kubeception retry handling

.github/actions/provision-cluster/lib/kubeception.js

@@ -64,24 +64,34 @@ class Client {
       kubeceptionProfile = "default"
     }
 
-    return utils.fibonacciRetry(async ()=>{
-      const response = await this.client.put(`https://sw.bakerstreet.io/kubeception/api/klusters/${name}?version=${version}&profile=${kubeceptionProfile}&timeoutSecs=${lifespan}`)
+    return utils.fibonacciRetry(async () => {
+      const response = await this.client.put(
+        `https://sw.bakerstreet.io/kubeception/api/klusters/${name}?version=${version}&profile=${kubeceptionProfile}&timeoutSecs=${lifespan}`
+      );
+
       if (!response || !response.message) {
-        throw Error("Unknown error getting response")
+        throw new utils.Transient("Unknown error getting response");
       }
 
-      if (response.message.statusCode == 200) {
-        return await response.readBody()
-      } else if (response.message.statusCode == 425) {
-        // The kubeception API uses 425 to signal that cluster creation is "in progress", so we want
-        // to retry later.
-        throw new utils.Transient(`status code ${response.message.statusCode}`)
-      } else {
-        // Any other status code is likely a permanent error.
-        let body = await response.readBody()
-        throw new Error(`Status code ${response.message.statusCode}: ${body}`)
+      switch (response.message.statusCode) {
+        case 200:
+        case 201:
+          return await response.readBody();
+        case 202:
+          throw new utils.Retry("Request is still pending");
+        default:
+          if (response.message.statusCode >= 400) {
+            throw new utils.Transient(
+              `Status code ${response.message.statusCode}`
+            );
+          } else {
+            let body = await response.readBody();
+            throw new Error(
+              `Status code ${response.message.statusCode}: ${body}`
+            );
+          }
       }
-    })
+    });
   }
 
   async deleteKluster(name) {

.github/actions/provision-cluster/lib/utils.js

@@ -54,20 +54,22 @@ function uid() {
   return crypto.randomBytes(16).toString("hex")
 }
 
+class Retry extends Error {}
+
 class Transient extends Error {}
 
 // Retry the supplied action with a fibonacci backoff until it returns true or timeout seconds have
 // passed. Use minDelay and maxDelay to tune the delay times. The action should signal retry with
-// `throw new Transient(...)`, and return upon success. The result of the final successful
-// invocation will be returned.
+// `throw new Transient(...)` or `throw new Retry(...)`, and return upon success.
+// The result of the final successful invocation will be returned.
 async function fibonacciRetry(action, timeout=600000, minDelay=1000, maxDelay=30000) {
   let start = Date.now()
   let nextDelay = fibonacciDelaySequence(minDelay, maxDelay)
   for (let count = 1; true; count++) {
     try {
       return await action()
     } catch (e) {
-      if (!(e instanceof Transient)) {
+      if (!(e instanceof Transient) && !(e instanceof Retry)) {
         throw e
       }
       let delay = nextDelay()
@@ -76,13 +78,13 @@ async function fibonacciRetry(action, timeout=600000, minDelay=1000, maxDelay=30
       let remaining = timeout - elapsed
       if (remaining > 0) {
         let t = Math.min(delay, remaining)
-        core.info(`Transient error (${e.message}) retrying after ${t/1000}s ...`)
+        core.info(`Error (${e.message}) retrying after ${t/1000}s ...`)
         await sleep(t)
       } else {
-        throw new Error(`Transient error (${e.message}) failing after ${count} attempts over ${elapsed/1000}s.`)
+        throw new Error(`Error (${e.message}) failing after ${count} attempts over ${elapsed/1000}s.`)
       }
     }
   }
 }
 
-module.exports = { getUniqueClusterName, writeFile, sleep, fibonacciDelaySequence, uid, Transient, fibonacciRetry }
+module.exports = { getUniqueClusterName, writeFile, sleep, fibonacciDelaySequence, uid, Retry, Transient, fibonacciRetry }

.github/actions/provision-cluster/lib/utils.test.js

@@ -75,7 +75,7 @@ test('fibonacciRetry fail all', async () => {
     returned = true
   } catch (err) {
     let elapsed = Date.now() - start
-    expect(err.message).toContain("Transient error")
+    expect(err.message).toContain("Error")
     expect(err.message).toContain("never big enough")
     expect(err.message).toContain("failing after")
     expect(err.message).toContain("attempts over")