add clickhouse pushing

clickhouse.ts

@@ -0,0 +1,26 @@
+import { createClient } from "@clickhouse/client";
+import dotenv from "dotenv";
+import { Env } from "@humanwhocodes/env";
+
+dotenv.config();
+
+const env = new Env();
+const CLICKHOUSE_USERNAME = env.require("CLICKHOUSE_USERNAME");
+const CLICKHOUSE_PASSWORD = env.require("CLICKHOUSE_PASSWORD");
+
+export const client = createClient({
+  username: CLICKHOUSE_USERNAME,
+  password: CLICKHOUSE_PASSWORD,
+  clickhouse_settings: {
+    async_insert: 1,
+    output_format_json_quote_32bit_integers: 0,
+    output_format_json_quote_64bit_integers: 0,
+    output_format_json_quote_64bit_floats: 0,
+    output_format_json_quote_32bit_floats: 0,
+  },
+});
+
+client.insert({
+  table: "option_quotes",
+  values: [],
+});

ingest-option-quotes.ts

@@ -1,4 +1,5 @@
 import { restClient } from "@polygon.io/client-js";
+import { client } from "./clickhouse.js";
 import PQueue from "p-queue";
 import dotenv from "dotenv";
 import { Env } from "@humanwhocodes/env";
@@ -59,6 +60,20 @@ for (const date of dateRange("2024-05-02", "2024-05-03")) {
               { pagination: true }
             );
             if (quotes.status?.toLowerCase() === "ok" && quotes.results) {
+              await client.insert({
+                table: "option_quotes2",
+                values: quotes.results.map(
+                  ({ bid_price, ask_price, sip_timestamp }) => ({
+                    option_type: contract.contract_type,
+                    symbol: contract.underlying_ticker,
+                    expiration_date: contract.expiration_date,
+                    strike: contract.strike_price,
+                    ts: sip_timestamp,
+                    bid: bid_price,
+                    ask: ask_price,
+                  })
+                ),
+              });
               console.log(date, contract.ticker, quotes.results?.length);
             }
           }

package.json

@@ -16,6 +16,7 @@
     "typescript": "^5.7.2"
   },
   "dependencies": {
+    "@clickhouse/client": "^1.10.0",
     "@humanwhocodes/env": "^4.0.0",
     "@polygon.io/client-js": "^7.3.2",
     "dotenv": "^16.4.7",

pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      '@clickhouse/client':
+        specifier: ^1.10.0
+        version: 1.10.0
       '@humanwhocodes/env':
         specifier: ^4.0.0
         version: 4.0.0
@@ -33,6 +36,13 @@ importers:
 
 packages:
 
+  '@clickhouse/client-common@1.10.0':
+    resolution: {integrity: sha512-dxXgJEGxwhOclflDH+GI8z7fQMh7KjJ/r0kSGE7yLKJlW0YHgbfxVufOkrEVg2QJMOmqi+YynkO4w1rXhetC0w==}
+
+  '@clickhouse/client@1.10.0':
+    resolution: {integrity: sha512-vyLkrE7iqWw3QPTet7no5Rd3qZFV4ujyo01FbspaTpME04F07AkBkRFJ/haUpf6GbZr/LsGIH5I4jy1vzGoRbQ==}
+    engines: {node: '>=16'}
+
   '@esbuild/aix-ppc64@0.23.1':
     resolution: {integrity: sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==}
     engines: {node: '>=18'}
@@ -343,6 +353,12 @@ packages:
 
 snapshots:
 
+  '@clickhouse/client-common@1.10.0': {}
+
+  '@clickhouse/client@1.10.0':
+    dependencies:
+      '@clickhouse/client-common': 1.10.0
+
   '@esbuild/aix-ppc64@0.23.1':
     optional: true