From b542588c11c969b08df993562b02357ed803d50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Varela?= Date: Mon, 10 Jul 2023 10:02:59 +0100 Subject: [PATCH] Add ability to retry Graphql requests (#26) * Retry twice * Retry multiple times * Extract util functions * Fix typo Co-authored-by: Rijk van Zanten * Increase delay between retries using exponential backoff * Regenerate pnpm lockfile to pease CI --------- Co-authored-by: Rijk van Zanten --- gatsby-node.js | 35 +++++++++++++++++++++++++++++++- package.json | 7 ++++--- pnpm-lock.yaml | 54 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/gatsby-node.js b/gatsby-node.js index ce9948d..813728d 100644 --- a/gatsby-node.js +++ b/gatsby-node.js @@ -5,6 +5,7 @@ const chalk = require('chalk'); const { Directus } = require('@directus/sdk'); const { sourceNodes, createSchemaCustomization } = require('gatsby-source-graphql/gatsby-node'); const { createRemoteFileNode } = require('gatsby-source-filesystem'); +const nodeFetch = require('node-fetch').default; /** * Validate plugin options @@ -32,6 +33,7 @@ exports.pluginOptionsSchema = ({ Joi }) => { }), graphql: Joi.object(), concurrency: Joi.number().default(10), + retries: Joi.number().default(5), }); }; @@ -149,10 +151,11 @@ class Plugin { this.refreshInterval = 0; this.authPromise = null; this.concurrency = 10; + this.retries = 5; } async setOptions(options) { - const { url, dev, auth, concurrency } = options; + const { url, dev, auth, concurrency, retries } = options; if (isEmpty(url)) error('"url" must be defined'); @@ -202,6 +205,7 @@ class Plugin { this.options = options; this.concurrency = concurrency; + this.retries = retries; return this.authPromise; } @@ -219,6 +223,31 @@ class Plugin { typeName: this.options?.type?.name || 'DirectusData', fieldName: this.options?.type?.field || 'directus', headers: this.headers.bind(this), + fetch: async (uri, options) => { + function request() { + return nodeFetch(uri, options); + } + + let count = 0; + let response = null; + let error = null; + + while (response === null && count++ < this.retries) { + try { + response = await request(); + } catch (err) { + error = err; + } + + if (count > 0) { + await sleep(Math.pow(2, count) * 1000); + } + } + + if (response === null) throw error; + + return response; + }, }; } @@ -317,4 +346,8 @@ function warning(message) { Log.warning(message); } +function sleep(ms) { + return new Promise((res) => setTimeout(res, ms)); +} + const plugin = new Plugin(); diff --git a/package.json b/package.json index 552059e..d92d41b 100644 --- a/package.json +++ b/package.json @@ -11,12 +11,13 @@ ], "dependencies": { "@directus/sdk": "10.2.0", - "ms": "2.1.3" + "ms": "2.1.3", + "node-fetch": "3.3.1" }, "peerDependencies": { + "eslint": "7||8", "gatsby-source-filesystem": "4||5", - "gatsby-source-graphql": "4||5", - "eslint": "7||8" + "gatsby-source-graphql": "4||5" }, "repository": "directus/gatsby-source-directus", "bugs": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cca0fe7..c4c3265 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ dependencies: ms: specifier: 2.1.3 version: 2.1.3 + node-fetch: + specifier: 3.3.1 + version: 3.3.1 devDependencies: eslint-config-prettier: @@ -3601,7 +3604,7 @@ packages: /axios@0.27.2: resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: - follow-redirects: 1.15.2(debug@3.2.7) + follow-redirects: 1.15.2 form-data: 4.0.0 transitivePeerDependencies: - debug @@ -4633,6 +4636,11 @@ packages: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} dev: false + /data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + dev: false + /dataloader@2.0.0: resolution: {integrity: sha512-YzhyDAwA4TaQIhM5go+vCLmU0UikghC/t9DTQYZR2M/UvZ1MdOhPezSDZcjj9uqQJOMqjLcpWtyW2iNINdlatQ==} dev: false @@ -5731,6 +5739,14 @@ packages: resolution: {integrity: sha512-iAHrIslQb3U68OcMSP0kkNWabp7sSN6d2TBSb2JO3gcLJVDd4owr/hKM4SFJovFOUeeXeItjYgouEDTMWiVAnA==} dev: false + /fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.2.1 + dev: false + /figures@3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -5857,6 +5873,16 @@ packages: resolution: {integrity: sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==} dev: false + /follow-redirects@1.15.2: + resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: false + /follow-redirects@1.15.2(debug@3.2.7): resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==} engines: {node: '>=4.0'} @@ -5925,6 +5951,13 @@ packages: mime-types: 2.1.35 dev: false + /formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: false + /forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -7976,6 +8009,11 @@ packages: resolution: {integrity: sha512-vgbBJTS4m5/KkE16t5Ly0WW9hz46swAstv0hYYwMtbG7AznRhNyfLRe8HZAiWIpcHzoO7HxhLuBQj9rJ/Ho0ZA==} dev: false + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + /node-fetch@2.6.1: resolution: {integrity: sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==} engines: {node: 4.x || >=6.0.0} @@ -7993,6 +8031,15 @@ packages: whatwg-url: 5.0.0 dev: false + /node-fetch@3.3.1: + resolution: {integrity: sha512-cRVc/kyto/7E5shrWca1Wsea4y6tL9iYJE5FBCius3JQfb/4P4I295PfhgbJQBLTx6lATE4z+wK0rPM4VS2uow==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: false + /node-gyp-build-optional-packages@5.0.3: resolution: {integrity: sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==} hasBin: true @@ -10466,6 +10513,11 @@ packages: resolution: {integrity: sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==} dev: false + /web-streams-polyfill@3.2.1: + resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} + engines: {node: '>= 8'} + dev: false + /webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} dev: false