implement retries

server/src/scripts/ingest-option-quotes-from-polygon-to-clickhouse.ts

@@ -3,6 +3,7 @@ import { getApiKey } from "./polygon.js";
 import pAll from "p-all";
 import pQueue from "p-queue";
 import pSeries from "p-series";
+import pRetry from "p-retry";
 
 const optionContractToTicker = ({
   symbol,
@@ -51,26 +52,34 @@ async function getOptionAggregates(
     type,
   });
   // first mark the sync of this particular option contract as "pending":
-  await clickhouse.insert({
-    table: "amg_option_aggregate_sync_statuses",
-    values: [
-      {
-        asOfDate,
-        symbol: underlyingSymbol,
-        expirationDate,
-        strike,
-        type,
-        status: "pending",
-      },
-    ],
-    format: "JSONEachRow",
-  });
+  await pRetry(
+    () =>
+      clickhouse.insert({
+        table: "amg_option_aggregate_sync_statuses",
+        values: [
+          {
+            asOfDate,
+            symbol: underlyingSymbol,
+            expirationDate,
+            strike,
+            type,
+            status: "pending",
+          },
+        ],
+        format: "JSONEachRow",
+      }),
+    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
+  );
 
-  let latestBatchResponse = (await (
-    await fetch(
-      `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`
-    )
-  ).json()) as PolygonResponse;
+  let latestBatchResponse = await pRetry(
+    async () =>
+      (await (
+        await fetch(
+          `https://api.polygon.io/v2/aggs/ticker/${optionContractTicker}/range/1/minute/${asOfDate}/${asOfDate}?adjusted=false&sort=asc&limit=50000&apiKey=${await getApiKey()}`
+        )
+      ).json()) as PolygonResponse,
+    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
+  );
   if (!latestBatchResponse.results) {
     console.log(latestBatchResponse);
     return;
@@ -87,25 +96,33 @@ async function getOptionAggregates(
     low: result.l,
     high: result.h,
   }));
-  await clickhouse.insert({
-    table: "option_aggregates",
-    values: latestBatch,
-    format: "JSONEachRow",
-  });
-  await clickhouse.insert({
-    table: "amg_option_aggregate_sync_statuses",
-    values: [
-      {
-        asOfDate,
-        symbol: underlyingSymbol,
-        expirationDate,
-        strike,
-        type,
-        status: "done",
-      },
-    ],
-    format: "JSONEachRow",
-  });
+  await pRetry(
+    () =>
+      clickhouse.insert({
+        table: "option_aggregates",
+        values: latestBatch,
+        format: "JSONEachRow",
+      }),
+    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
+  );
+  await pRetry(
+    () =>
+      clickhouse.insert({
+        table: "amg_option_aggregate_sync_statuses",
+        values: [
+          {
+            asOfDate,
+            symbol: underlyingSymbol,
+            expirationDate,
+            strike,
+            type,
+            status: "done",
+          },
+        ],
+        format: "JSONEachRow",
+      }),
+    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
+  );
 }
 
 type OptionContract = {
@@ -122,7 +139,9 @@ async function getNextBatchOfUnstartedOptionAggregates(
   if (typeof previousUnstartedOptionContract === "undefined") {
     return;
   }
-  const optionContractsWithoutAggregates = await query<OptionContractDay>(`
+  const optionContractsWithoutAggregates = await pRetry(
+    () =>
+      query<OptionContractDay>(`
     SELECT 
       asOfDate,
       symbol, 
@@ -161,7 +180,9 @@ async function getNextBatchOfUnstartedOptionAggregates(
     AND status = 'not-started'
     ORDER BY asOfDate, symbol, expirationDate, strike, type
     LIMIT ${limit}
-  `);
+  `),
+    { retries: 5, factor: 2, minTimeout: 1000, maxTimeout: 60 * 1000 }
+  );
   return optionContractsWithoutAggregates;
 }
 
@@ -239,7 +260,8 @@ async function revertPendingSyncs() {
                   console.log();
                 }),
           ])
-    )
+    ),
+    { concurrency: 1 }
   );
 }
 
@@ -250,13 +272,13 @@ async function revertPendingSyncs() {
  *
  * This queries Polygon with a concurrency of 6.
  */
-const q = new pQueue({ concurrency: 60 });
+const q = new pQueue({ concurrency: 6 });
 /** Initialized with the lowest possible option contract.
  *  It's passed into `getNextUnstartedSymbolAndAsOfDate()`.
  */
 let nextBatchOfUnstartedOptionContracts: Array<OptionContractDay> = [
   {
-    asOfDate: "2022-03-15",
+    asOfDate: "2022-03-18",
     symbol: "A",
     expirationDate: "2022-02-01",
     strike: 0,
@@ -287,9 +309,9 @@ while (
         })
     )
   );
-  // don't loop again until the queue has less than 50 items; we don't want it to grow in memory without bound:
-  console.log("Waiting till less than 50 in queue");
-  await q.onSizeLessThan(50);
+  // don't loop again until the queue has less than 2 items; we don't want it to grow in memory without bound:
+  console.log("Waiting till less than 2 in queue");
+  await q.onSizeLessThan(2);
 }
 // wait until pending queue operations are done:
 await q.onSizeLessThan(1);