Add request retries for network/connection and 502 errors

Author: stanleyphu

package-lock.json

@@ -37,10 +37,11 @@
     "devDependencies": {
         "@types/chai": "^4.3.9",
         "@types/mocha": "^10.0.3",
-        "@types/node": "^18.11.18",
+        "@types/node": "^18.13.0",
         "chai": "^4.3.10",
         "mocha": "^10.2.0",
         "ts-node": "^10.9.1",
-        "typescript": "^4.3.2"
+        "typescript": "^4.3.2",
+        "undici": "^6.0.1"
     }
 }

package.json

@@ -34,6 +34,12 @@ interface FetchRequestOptions {
     body?: string;
 }
 
+const MAX_RETRY_ATTEMPTS = 3;
+const BACKOFF_MULTIPLIER = 1.5;
+const MINIMUM_SLEEP_TIME = 500;
+
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
+
 export default class ApiClient implements HttpClient {
     private config: HttpClientConfig;
 
@@ -44,51 +50,85 @@ export default class ApiClient implements HttpClient {
     public async get(requestOptions: HttpClientRequestOptions): Promise<any> {
         const [requestUrl, fetchRequestOptions] = this.buildRequestUrlAndOptions("GET", requestOptions);
 
-        /* @ts-ignore */
-        const response = await fetch(requestUrl, fetchRequestOptions);
-        if (!response.ok) {
-            throw this.buildError(await response.json());
-        }
+        const response = await this.fetchWithRetry(requestUrl, fetchRequestOptions);
 
         return this.parseResponse(response);
     }
 
     public async delete(requestOptions: HttpClientRequestOptions): Promise<any> {
         const [requestUrl, fetchRequestOptions] = this.buildRequestUrlAndOptions("DELETE", requestOptions);
 
-        /* @ts-ignore */
-        const response = await fetch(requestUrl, fetchRequestOptions);
-        if (!response.ok) {
-            throw this.buildError(await response.json());
-        }
+        const response = await this.fetchWithRetry(requestUrl, fetchRequestOptions);
 
         return this.parseResponse(response);
     }
 
     public async post(requestOptions: HttpClientRequestOptions): Promise<any> {
         const [requestUrl, fetchRequestOptions] = this.buildRequestUrlAndOptions("POST", requestOptions);
 
-        /* @ts-ignore */
-        const response = await fetch(requestUrl, fetchRequestOptions);
-        if (!response.ok) {
-            throw this.buildError(await response.json());
-        }
+        const response = await this.fetchWithRetry(requestUrl, fetchRequestOptions);
 
         return this.parseResponse(response);
     }
 
     public async put(requestOptions: HttpClientRequestOptions): Promise<any> {
         const [requestUrl, fetchRequestOptions] = this.buildRequestUrlAndOptions("PUT", requestOptions);
 
-        /* @ts-ignore */
-        const response = await fetch(requestUrl, fetchRequestOptions);
-        if (!response.ok) {
-            throw this.buildError(await response.json());
-        }
+        const response = await this.fetchWithRetry(requestUrl, fetchRequestOptions);
 
         return this.parseResponse(response);
     }
 
+    private async fetchWithRetry(requestUrl: string, fetchRequestOptions: FetchRequestOptions): Promise<any> {
+        let response: any = null;
+        let requestError: any = null;
+        let retryAttempts = 1;
+
+        const makeRequest = async (): Promise<any> => {
+            try {
+                response = await fetch(requestUrl, fetchRequestOptions);
+            } catch (e) {
+                requestError = e;
+            }
+
+            if (this.shouldRetryRequest(response, requestError, retryAttempts)) {
+                retryAttempts++;
+                await sleep(this.getSleepTime(retryAttempts));
+                return makeRequest();
+            }
+
+            if (!response.ok) {
+                throw this.buildError(await response.json());
+            }
+
+            return response;
+        }
+
+        return makeRequest();
+    }
+
+    private shouldRetryRequest(response: any, requestError: any, retryAttempt: number): boolean {
+        if (retryAttempt > MAX_RETRY_ATTEMPTS) {
+            return false;
+        }
+
+        if (requestError != null && requestError instanceof TypeError) {
+            return true;
+        }
+
+        if (response?.status == 502) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private getSleepTime(retryAttempt: number): number {
+        let sleepTime = MINIMUM_SLEEP_TIME * Math.pow(BACKOFF_MULTIPLIER, retryAttempt);
+        const jitter = Math.random() + 0.5;
+        return sleepTime * jitter;
+    }
+
     private buildRequestUrlAndOptions(method: FetchRequestOptions["method"], requestOptions?: HttpClientRequestOptions): [string, FetchRequestOptions] {
         let baseUrl = this.config.baseUrl;
         const fetchRequestOptions: FetchRequestOptions = {

src/HttpClient.ts

@@ -0,0 +1,80 @@
+import WarrantClient from "../src/WarrantClient";
+import { ApiError } from "../src/types";
+import { assert } from "chai";
+import { MockAgent, setGlobalDispatcher } from "undici";
+
+describe('WarrantClientTest', function () {
+    before(function () {
+        this.warrant = new WarrantClient({ apiKey: "my_api_key", endpoint: "http://localhost:8000" });
+
+        const agent = new MockAgent();
+        agent.disableNetConnect();
+        this.client = agent.get("http://localhost:8000")
+        setGlobalDispatcher(agent);
+    });
+
+    it('should make request after retries', async function () {
+        this.timeout(10000);
+        this.client
+            .intercept({
+                path: "/v2/objects/user/some-user",
+                method: "GET"
+            })
+            .reply(502);
+
+        this.client
+            .intercept({
+                path: "/v2/objects/user/some-user",
+                method: "GET"
+            })
+            .reply(502);
+
+        this.client
+            .intercept({
+                path: "/v2/objects/user/some-user",
+                method: "GET"
+            })
+            .reply(200, { "objectType": "user", "objectId": "some-user" });
+
+        const fetchedUser = await this.warrant.User.get("some-user");
+
+        assert.strictEqual(fetchedUser.userId, "some-user");
+        assert.strictEqual(fetchedUser.meta, undefined);
+    });
+
+    it('should stop requests after max retries', async function () {
+        this.timeout(10000);
+        this.client
+            .intercept({
+                path: "/v2/objects/user/some-user",
+                method: "GET"
+            })
+            .reply(502, {
+                message: "Bad Gateway"
+            });
+
+        this.client
+            .intercept({
+                path: "/v2/objects/user/some-user",
+                method: "GET"
+            })
+            .reply(502, {
+                message: "Bad Gateway"
+            });
+
+        this.client
+            .intercept({
+                path: "/v2/objects/user/some-user",
+                method: "GET"
+            })
+            .reply(502, {
+                message: "Bad Gateway"
+            });
+
+        try {
+            await this.warrant.User.get("some-user");
+        } catch (e) {
+            assert.instanceOf(e, ApiError);
+        }
+    });
+});