ingest symbols and symbol_sync_statuses

2024-02-18 20:40:09 -05:00
parent e6a5002ec0
commit 113b6e66e5
5 changed files with 102 additions and 2 deletions
@@ -0,0 +1,62 @@
+import { clickhouse, query } from "../clickhouse.js";
+import { getApiKey } from "./polygon.js";
+import pAll from 'p-all';
+
+type PolygonResponse = {next_url?:string, results:Array<{ticker:string}>};
+async function getOptionContracts(underlyingSymbol, asOfDate){
+  let latestBatch = await (await fetch(`https://api.polygon.io/v3/reference/options/contracts?underlying_ticker=${underlyingSymbol}&as_of=${asOfDate}&sort=ticker&limit=1000&apiKey=${await getApiKey()}`)).json() as PolygonResponse;
+  console.log(latestBatch.results.map((r)=>r.ticker));
+  while(latestBatch.hasOwnProperty('next_url')){
+    latestBatch = await (await fetch(`${latestBatch.next_url}&apiKey=${await getApiKey()}`)).json() as PolygonResponse;
+    console.log(latestBatch.results.map((r)=>r.ticker));
+  }
+}
+
+//await getOptionContracts('AAPL','2024-01-30');
+
+/**
+ * For each symbol in `symbols` table, check the latest `asOfDate` 
+ * in `symbol_sync_statuses` for that symbol. Then fill-in the rest 
+ * of the dates until today's date.
+ */
+async function fillSyncStatuses(){
+  const symbols = (await query(`
+    SELECT symbol from symbols
+  `)).map(({symbol})=>symbol);
+  
+  console.log('symbols', symbols);
+  await pAll(symbols.map(
+    (symbol)=>
+      ()=>query<{latestAsOfDate:string}>(`
+        SELECT
+          latestAsOfDate
+        FROM(
+          SELECT last_value(asOfDate) as latestAsOfDate
+          FROM (
+            SELECT * 
+            FROM symbol_sync_statuses 
+            WHERE symbol = '${symbol}' 
+            ORDER BY asOfDate ASC
+          )
+        )
+        WHERE latestAsOfDate > '2022-02-18'
+      `).then((rows)=>
+        clickhouse.command({
+          query: `
+            INSERT INTO symbol_sync_statuses
+            SELECT
+              '${symbol}' as symbol,
+              Date(dateAdd(DAY,number,'${rows[0]?.latestAsOfDate || '2022-02-19'}')) as asOfDate,
+              'not-started' as status
+            FROM system.numbers
+            WHERE number < dateDiff('days',Date('${rows[0]?.latestAsOfDate || '2022-02-19'}'), Date(now()))
+            AND number > 0
+          `
+        }).then(()=>{console.log(`Done ${symbol}`);})
+      )
+    ), 
+    {concurrency: 6}
+  );
+}
+
+await fillSyncStatuses();
@@ -0,0 +1,4 @@
+import pThrottle from 'p-throttle';
+
+const apiKey = 'H95NTsatM1iTWLUwDLxM2J5zhUVYdCEz';
+export const getApiKey = pThrottle({limit: 5, interval: 60000})(()=>apiKey);