improve and extract nextDate() function; improve clickhouse-to-lmdbx sync script

server/src/scripts/clickhouse-to-lmdbx.ts

@@ -3,47 +3,52 @@ import type { AggregateDatabase } from "../interfaces.js";
 // import { stockDatabase as stockDatabaseLmdbx } from "../stockdb.lmdbx.js";
 import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
 import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js";
-
-function nextDate(date: string) {
-  const dateObject = new Date(date);
-  dateObject.setDate(dateObject.getDate() + 1);
-  return dateObject.toISOString().substring(0, 10);
-}
+import { nextDate } from "../lib/util.js";
 
 async function syncAggregates<T>({
-  from,
-  to,
+  fromDatabase,
+  toDatabase,
   key,
   date,
 }: {
-  from: AggregateDatabase<T>;
-  to: AggregateDatabase<T>;
+  fromDatabase: AggregateDatabase<T>;
+  toDatabase: AggregateDatabase<T>;
   key: T;
   date: string;
 }) {
-  const aggregatesFrom = (await from.getAggregates({ key, date })).map(
+  const aggregatesFrom = (await fromDatabase.getAggregates({ key, date })).map(
     (aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }),
   );
-  await to.insertAggregates(aggregatesFrom);
+  await toDatabase.insertAggregates(aggregatesFrom);
 }
 
 const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"];
 async function run() {
-  const startDate = "2022-02-01";
-  const endDate = "2024-07-15";
+  const startDate = process.argv[2];
+  const endDate = process.argv[3];
+
+  if (!startDate || !endDate) {
+    console.error("Usage: node clickhouse-to-lmdbx.js <startDate> <endDate>");
+    console.error("Dates should be in YYYY-MM-DD format");
+    process.exit(1);
+  }
   for (let date = startDate; date <= endDate; date = nextDate(date)) {
     // const symbols = await stockDatabaseClickhouse.getSymbols({ date });
     for (const symbol of symbols) {
       console.log(date, symbol);
-			const keys = await optionContractDatabaseClickhouse.getKeys({key: {symbol}, date});
-			for(const key of keys){
-				await syncAggregates({
-					from: optionContractDatabaseClickhouse,
-					to: optionContractDatabaseLmdbx,
-					key,
-					date,
-				});
-			}
+      const keys = await optionContractDatabaseClickhouse.getKeys({
+        key: { symbol },
+        date,
+      });
+      for (const key of keys) {
+        // console.log(date, symbol, key.expirationDate, key.strike, key.type);
+        await syncAggregates({
+          fromDatabase: optionContractDatabaseClickhouse,
+          toDatabase: optionContractDatabaseLmdbx,
+          key,
+          date,
+        });
+      }
     }
   }
 }