clickhouse.ts

@@ -5,12 +5,10 @@ import { Env } from "@humanwhocodes/env";
 dotenv.config();
 
 const env = new Env();
-const CLICKHOUSE_URL = env.require("CLICKHOUSE_URL");
 const CLICKHOUSE_USERNAME = env.require("CLICKHOUSE_USERNAME");
 const CLICKHOUSE_PASSWORD = env.require("CLICKHOUSE_PASSWORD");
 
 export const client = createClient({
-  url: CLICKHOUSE_URL,
   username: CLICKHOUSE_USERNAME,
   password: CLICKHOUSE_PASSWORD,
   clickhouse_settings: {
@@ -21,7 +19,7 @@ export const client = createClient({
   },
 });
 
-// client.insert({
+client.insert({
-//   table: "option_quotes",
+  table: "option_quotes",
-//   values: [],
+  values: [],
-// });
+});

ingest-stock-quotes.ts

@@ -1,96 +0,0 @@
-import { restClient } from "@polygon.io/client-js";
-import { client } from "./clickhouse.js";
-import PQueue from "p-queue";
-import dotenv from "dotenv";
-import { Env } from "@humanwhocodes/env";
-
-dotenv.config();
-
-const env = new Env();
-const POLYGON_API_KEY = env.require("POLYGON_API_KEY");
-const CONCURRENCY = Number.parseInt(env.get("CONCURRENCY", "6"));
-const STOCK_QUOTES_TABLE = env.get("STOCK_QUOTES_TABLE", "stock_aggregates");
-const START_DATE = env.get("START_DATE", "2020-01-01");
-const END_DATE = env.get("END_DATE", "2022-03-01");
-
-const polygon = restClient(POLYGON_API_KEY, "https://api.polygon.io", {
-  pagination: true,
-});
-
-const queue = new PQueue({ concurrency: CONCURRENCY });
-
-function* dateRange(
-  startDateStr: string,
-  endDateStr: string
-): Generator<string, void, unknown> {
-  // Convert the start and end date strings to Date objects
-  const startDate = new Date(startDateStr);
-  const endDate = new Date(endDateStr);
-
-  // Loop from the start date to the end date
-  for (
-    let currentDate = startDate;
-    currentDate <= endDate;
-    currentDate.setDate(currentDate.getDate() + 1)
-  ) {
-    // Format the current date as YYYY-MM-DD
-    const formattedDate = currentDate.toISOString().split("T")[0];
-
-    // Yield the formatted date
-    yield formattedDate;
-  }
-}
-
-for (const date of dateRange(START_DATE, END_DATE)) {
-  const quotes = await polygon.stocks.aggregates(
-    "SPY",
-    1,
-    "second",
-    date,
-    date,
-    {
-      adjusted: "true",
-      sort: "asc",
-      limit: 50000,
-    },
-    { pagination: true }
-  );
-  if (quotes.status?.toLowerCase() === "ok" && quotes.results) {
-    await client.insert({
-      table: STOCK_QUOTES_TABLE,
-      values: quotes.results.map(({ h: high, l: low, t: timestamp }) =>
-        typeof high === "number" &&
-        typeof low === "number" &&
-        typeof timestamp === "number"
-          ? {
-              symbol: "SPY",
-              ts: new Date(timestamp),
-              price: high + low / 2,
-            }
-          : {
-              symbol: "",
-              ts: 0,
-              price: 0,
-            }
-      ),
-      format: "JSONEachRow",
-    });
-    // console.log(date, contract.ticker, quotes.results?.length);
-  }
-  console.log("finished", date);
-}
-
-// const data = await polygon.options.quotes(
-//   "O:SPY241220P00600000",
-//   {
-//     "timestamp.gte": "2024-12-15",
-//     "timestamp.lte": "2024-12-16",
-//     sort: "timestamp",
-//     order: "asc",
-//     limit: 50000,
-//   },
-//   { pagination: true }
-// );
-
-// console.log(data.status);
-// console.log(data.results?.length);