well-interfaced pluggable databases
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
import { stockDatabase } from "./stockdb.clickhouse.js";
|
||||
import { calendarDatabase } from "./calendardb.clickhouse.js";
|
||||
import type { CalendarKey } from "./calendardb.interfaces.js";
|
||||
import type { Aggregate } from "./interfaces.js";
|
||||
|
||||
function nextDate(date: string) {
|
||||
const dateObject = new Date(date);
|
||||
dateObject.setDate(dateObject.getDate() + 1);
|
||||
return dateObject.toISOString().substring(0, 10);
|
||||
}
|
||||
|
||||
type BacktestInput = {
|
||||
symbol: string;
|
||||
startDate: string;
|
||||
endDate: string;
|
||||
/** Between 0 and 1. The frequency that similar calendars have historically ended (i.e. within the last hour) at a higher price than the current calendar's price. */
|
||||
historicalProbabilityOfSuccess?: number;
|
||||
initialAvailableValue?: number;
|
||||
};
|
||||
export async function backtest({
|
||||
symbol,
|
||||
startDate,
|
||||
endDate,
|
||||
historicalProbabilityOfSuccess = 0.8,
|
||||
initialAvailableValue: initialBuyingPower = 2000,
|
||||
}: BacktestInput) {
|
||||
let buyingPower = initialBuyingPower;
|
||||
const portfolio = new Set<CalendarKey>();
|
||||
// for each day:
|
||||
for (
|
||||
let date = startDate, didBuyCalendar = false;
|
||||
date <= endDate;
|
||||
date = nextDate(date), didBuyCalendar = false
|
||||
) {
|
||||
console.log("Current Date:", date);
|
||||
const calendars = await calendarDatabase.getCalendars({
|
||||
key: { symbol },
|
||||
date,
|
||||
});
|
||||
const stockAggregates = await stockDatabase.getAggregates({
|
||||
key: symbol,
|
||||
date,
|
||||
});
|
||||
const calendarsAggregates = new Map<
|
||||
CalendarKey,
|
||||
Array<Pick<Aggregate<CalendarKey>, "tsStart" | "open" | "close">>
|
||||
>();
|
||||
for (const calendar of calendars) {
|
||||
calendarsAggregates.set(
|
||||
calendar,
|
||||
await calendarDatabase.getAggregates({
|
||||
key: {
|
||||
...calendar,
|
||||
},
|
||||
date,
|
||||
}),
|
||||
);
|
||||
}
|
||||
// for each minute of that day for which we have a stock candlestick:
|
||||
for (const stockAggregate of stockAggregates) {
|
||||
console.log("Current Time:", new Date(stockAggregate.tsStart));
|
||||
// filter-out calendars that are far-from-the-money (10%)
|
||||
const calendarsNearTheMoney = calendars.filter(
|
||||
({ strike }) =>
|
||||
Math.abs((stockAggregate.open - strike) / stockAggregate.open) < 0.1,
|
||||
);
|
||||
// for each relevant calendar on that day:
|
||||
for (const calendar of calendarsNearTheMoney) {
|
||||
const strikePercentageFromTheMoney = Math.abs(
|
||||
(stockAggregate.open - calendar.strike) / stockAggregate.open,
|
||||
);
|
||||
/** In days. */
|
||||
const calendarSpan =
|
||||
(new Date(calendar.backExpirationDate).valueOf() -
|
||||
new Date(calendar.frontExpirationDate).valueOf()) /
|
||||
(1000 * 60 * 60 * 24);
|
||||
const targetCalendarPrice =
|
||||
await calendarDatabase.getTargetPriceByProbability({
|
||||
symbol,
|
||||
calendarSpan,
|
||||
strikePercentageFromTheMoney,
|
||||
historicalProbabilityOfSuccess,
|
||||
});
|
||||
const calendarAggregates = calendarsAggregates.get(calendar);
|
||||
const calendarAggregateAtCurrentTime = calendarAggregates.find(
|
||||
({ tsStart }) => tsStart === stockAggregate.tsStart,
|
||||
);
|
||||
// if there exists a matching calendar candlestick for the current minute:
|
||||
if (calendarAggregateAtCurrentTime) {
|
||||
// if the current candlestick is a good price (i.e. less than the target price):
|
||||
const minCalendarPriceInCandlestick = Math.min(
|
||||
calendarAggregateAtCurrentTime.open,
|
||||
calendarAggregateAtCurrentTime.close,
|
||||
);
|
||||
if (
|
||||
minCalendarPriceInCandlestick < targetCalendarPrice &&
|
||||
minCalendarPriceInCandlestick >
|
||||
0.07 /* sometimes the calendar price is zero or negative, which is of course impossible; some institution got a good deal */
|
||||
) {
|
||||
// if we can afford to buy the calendar:
|
||||
if (buyingPower > minCalendarPriceInCandlestick) {
|
||||
// buy the calendar, and continue to the next day:
|
||||
portfolio.add(calendar);
|
||||
buyingPower = buyingPower - minCalendarPriceInCandlestick * 100;
|
||||
console.log(
|
||||
"Bought",
|
||||
calendar,
|
||||
"for",
|
||||
minCalendarPriceInCandlestick * 100,
|
||||
"...$",
|
||||
buyingPower,
|
||||
"left",
|
||||
);
|
||||
didBuyCalendar = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (didBuyCalendar) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (didBuyCalendar) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// for each calendar in portfolio, if today is the last day, close the position:
|
||||
for (const calendar of portfolio.values()) {
|
||||
if (calendar.frontExpirationDate === date) {
|
||||
const calendarClosingPrice = await calendarDatabase.getClosingPrice({
|
||||
key: {
|
||||
...calendar,
|
||||
},
|
||||
});
|
||||
portfolio.delete(calendar);
|
||||
buyingPower = buyingPower + calendarClosingPrice * 100;
|
||||
console.log(
|
||||
"Sold",
|
||||
calendar,
|
||||
"for",
|
||||
calendarClosingPrice,
|
||||
"...$",
|
||||
buyingPower,
|
||||
"left",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Ending Buying Power:", buyingPower);
|
||||
console.log("Portfolio:", portfolio.values());
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import type { CalendarDatabase, CalendarKey } from "./calendardb.interfaces.js";
|
||||
import type { Aggregate } from "./interfaces.js";
|
||||
import { query } from "./lib/clickhouse.js";
|
||||
|
||||
function makeCalendarDatabase(): CalendarDatabase {
|
||||
const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
|
||||
getKeys: async ({ key: { symbol }, date }) => {
|
||||
const calendarsForSymbolOnDate = await query<
|
||||
Omit<CalendarKey, "symbol">
|
||||
>(`
|
||||
WITH today_option_contracts AS (
|
||||
SELECT expirationDate, strike, type
|
||||
FROM option_contract_existences
|
||||
WHERE symbol = '${symbol}'
|
||||
AND asOfDate = '${date}'
|
||||
)
|
||||
SELECT
|
||||
front_option_contract.type as type,
|
||||
front_option_contract.strike as strike,
|
||||
front_option_contract.expirationDate as frontExpirationDate,
|
||||
back_option_contract.expirationDate as backExpirationDate
|
||||
FROM today_option_contracts AS front_option_contract
|
||||
ASOF INNER JOIN today_option_contracts AS back_option_contract
|
||||
ON front_option_contract.type = back_option_contract.type
|
||||
AND front_option_contract.strike = back_option_contract.strike
|
||||
AND front_option_contract.expirationDate < back_option_contract.expirationDate
|
||||
`);
|
||||
|
||||
return calendarsForSymbolOnDate.map((calendarWithoutSymbol) => ({
|
||||
...calendarWithoutSymbol,
|
||||
symbol,
|
||||
}));
|
||||
},
|
||||
getAggregates: async ({
|
||||
key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
|
||||
date,
|
||||
}) => {
|
||||
return (
|
||||
await query<Omit<Aggregate<CalendarKey>, "key">>(`
|
||||
WITH front_option_contract_candlestick AS (
|
||||
SELECT
|
||||
tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low
|
||||
FROM option_contract_aggregates
|
||||
WHERE symbol = '${symbol}'
|
||||
AND type = '${type}'
|
||||
AND strike = '${strike}'
|
||||
AND expirationDate = '${frontExpirationDate}'
|
||||
AND toDate(tsStart) = '${date}'
|
||||
),
|
||||
back_option_contract_candlestick AS (
|
||||
SELECT
|
||||
tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low
|
||||
FROM option_contract_aggregates
|
||||
WHERE symbol = '${symbol}'
|
||||
AND type = '${type}'
|
||||
AND strike = '${strike}'
|
||||
AND expirationDate = '${backExpirationDate}'
|
||||
AND toDate(tsStart) = '${date}'
|
||||
)
|
||||
SELECT
|
||||
toUnixTimestamp(front_option_contract_candlestick.tsStart) as tsStart,
|
||||
back_option_contract_candlestick.open - front_option_contract_candlestick.open as open,
|
||||
back_option_contract_candlestick.close - front_option_contract_candlestick.close as close
|
||||
FROM front_option_contract_candlestick
|
||||
INNER JOIN back_option_contract_candlestick
|
||||
ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
|
||||
ORDER BY front_option_contract_candlestick.tsStart ASC
|
||||
`)
|
||||
).map((aggregate) => ({
|
||||
...aggregate,
|
||||
tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
|
||||
}));
|
||||
},
|
||||
insertAggregates: async (aggregates) => {
|
||||
// no-op: we insert individual option contracts, not calendars
|
||||
},
|
||||
getClosingPrice: async ({
|
||||
key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
|
||||
}) => {
|
||||
return (
|
||||
await query<{ calendarClosingPrice: number }>(`
|
||||
WITH front_option_contract_candlestick AS (
|
||||
SELECT
|
||||
tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low
|
||||
FROM option_contract_aggregates
|
||||
WHERE symbol = '${symbol}'
|
||||
AND type = '${type}'
|
||||
AND strike = '${strike}'
|
||||
AND expirationDate = '${frontExpirationDate}'
|
||||
AND toDate(tsStart) = '${frontExpirationDate}'
|
||||
),
|
||||
back_option_contract_candlestick AS (
|
||||
SELECT
|
||||
tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low
|
||||
FROM option_contract_aggregates
|
||||
WHERE symbol = '${symbol}'
|
||||
AND type = '${type}'
|
||||
AND strike = '${strike}'
|
||||
AND expirationDate = '${backExpirationDate}'
|
||||
AND toDate(tsStart) = '${frontExpirationDate}'
|
||||
)
|
||||
SELECT
|
||||
min(back_option_contract_candlestick.close - front_option_contract_candlestick.close) as calendarClosingPrice
|
||||
FROM front_option_contract_candlestick
|
||||
INNER JOIN back_option_contract_candlestick
|
||||
ON front_option_contract_candlestick.tsStart = back_option_contract_candlestick.tsStart
|
||||
`)
|
||||
)[0]?.calendarClosingPrice;
|
||||
},
|
||||
getTargetPriceByProbability: async ({
|
||||
symbol,
|
||||
calendarSpan,
|
||||
strikePercentageFromTheMoney,
|
||||
historicalProbabilityOfSuccess,
|
||||
}) => {
|
||||
return 0.24;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...calendarDatabase,
|
||||
getCalendars: calendarDatabase.getKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export const calendarDatabase: CalendarDatabase = makeCalendarDatabase();
|
||||
@@ -0,0 +1,24 @@
|
||||
import type { AggregateDatabase } from "./interfaces.js";
|
||||
|
||||
export type CalendarKey = {
|
||||
symbol: string;
|
||||
type: "call" | "put";
|
||||
strike: number;
|
||||
frontExpirationDate: string;
|
||||
backExpirationDate: string;
|
||||
};
|
||||
|
||||
export type CalendarDatabase = AggregateDatabase<CalendarKey> & {
|
||||
getCalendars: AggregateDatabase<CalendarKey>["getKeys"];
|
||||
getTargetPriceByProbability: ({
|
||||
symbol,
|
||||
calendarSpan,
|
||||
strikePercentageFromTheMoney,
|
||||
historicalProbabilityOfSuccess,
|
||||
}: {
|
||||
symbol: string;
|
||||
calendarSpan: number;
|
||||
strikePercentageFromTheMoney: number;
|
||||
historicalProbabilityOfSuccess: number;
|
||||
}) => Promise<number>;
|
||||
};
|
||||
@@ -0,0 +1,153 @@
|
||||
import type { CalendarDatabase } from "./calendardb.interfaces.js";
|
||||
import { open } from "lmdbx";
|
||||
|
||||
const calendarAggregatesDb = open({
|
||||
path: "/tmp/calendar-aggregates.db",
|
||||
// any options go here, we can turn on compression like this:
|
||||
compression: true,
|
||||
});
|
||||
|
||||
const calendarExistenceDb = open({
|
||||
path: "/tmp/calendar-existence.db",
|
||||
// any options go here, we can turn on compression like this:
|
||||
compression: true,
|
||||
});
|
||||
|
||||
/** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
|
||||
const MAXIMUM_KEY = Buffer.from([0xff]);
|
||||
|
||||
function makeCalendarDatabase(): CalendarDatabase {
|
||||
const calendarDatabase: Omit<CalendarDatabase, "getCalendars"> = {
|
||||
getKeys: async ({ key: { symbol }, date }) => {
|
||||
return calendarExistenceDb
|
||||
.getRange({
|
||||
start: [date, symbol],
|
||||
end: [date, symbol, MAXIMUM_KEY],
|
||||
})
|
||||
.map(({ key }) => ({
|
||||
symbol,
|
||||
frontExpirationDate: key[2],
|
||||
backExpirationDate: key[3],
|
||||
strike: key[4],
|
||||
type: key[5],
|
||||
})).asArray;
|
||||
},
|
||||
getAggregates: async ({
|
||||
key: { symbol, frontExpirationDate, backExpirationDate, strike, type },
|
||||
date,
|
||||
}) => {
|
||||
const startOfDayUnix = new Date(`${date}T00:00:00Z`).valueOf();
|
||||
const endOfDayUnix = startOfDayUnix + 3600 * 24 * 1000;
|
||||
return calendarAggregatesDb
|
||||
.getRange({
|
||||
start: [
|
||||
symbol,
|
||||
frontExpirationDate,
|
||||
backExpirationDate,
|
||||
strike,
|
||||
type,
|
||||
startOfDayUnix,
|
||||
],
|
||||
end: [
|
||||
symbol,
|
||||
frontExpirationDate,
|
||||
backExpirationDate,
|
||||
strike,
|
||||
type,
|
||||
endOfDayUnix,
|
||||
],
|
||||
})
|
||||
.map(({ value }) => ({
|
||||
tsStart: value.tsStart,
|
||||
open: value.open,
|
||||
close: value.close,
|
||||
high: value.high,
|
||||
low: value.low,
|
||||
})).asArray;
|
||||
},
|
||||
insertAggregates: async (aggregates) => {
|
||||
await calendarExistenceDb.batch(() => {
|
||||
for (const aggregate of aggregates) {
|
||||
calendarExistenceDb.put(
|
||||
[
|
||||
new Date(aggregate.tsStart).toISOString().substring(0, 10),
|
||||
aggregate.key.symbol,
|
||||
aggregate.key.frontExpirationDate,
|
||||
aggregate.key.backExpirationDate,
|
||||
aggregate.key.strike,
|
||||
aggregate.key.type,
|
||||
],
|
||||
null,
|
||||
);
|
||||
}
|
||||
});
|
||||
await calendarAggregatesDb.batch(() => {
|
||||
for (const aggregate of aggregates) {
|
||||
calendarAggregatesDb.put(
|
||||
[
|
||||
aggregate.key.symbol,
|
||||
aggregate.key.frontExpirationDate,
|
||||
aggregate.key.backExpirationDate,
|
||||
aggregate.key.strike,
|
||||
aggregate.key.type,
|
||||
aggregate.tsStart,
|
||||
],
|
||||
{
|
||||
open: aggregate.open,
|
||||
close: aggregate.close,
|
||||
high: aggregate.high,
|
||||
low: aggregate.low,
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
getClosingPrice: async ({
|
||||
key: { symbol, strike, type, frontExpirationDate, backExpirationDate },
|
||||
}) => {
|
||||
const startOfLastHourUnix = new Date(
|
||||
`${frontExpirationDate}T00:00:00Z`,
|
||||
).valueOf();
|
||||
const endOfLastHourUnix = startOfLastHourUnix + 3600 * 1000;
|
||||
let minPrice = 0;
|
||||
for (const { value } of calendarAggregatesDb.getRange({
|
||||
start: [
|
||||
symbol,
|
||||
frontExpirationDate,
|
||||
backExpirationDate,
|
||||
strike,
|
||||
type,
|
||||
startOfLastHourUnix,
|
||||
],
|
||||
end: [
|
||||
symbol,
|
||||
frontExpirationDate,
|
||||
backExpirationDate,
|
||||
strike,
|
||||
type,
|
||||
endOfLastHourUnix,
|
||||
],
|
||||
})) {
|
||||
if (value.close < minPrice || minPrice === 0) {
|
||||
minPrice = value.close;
|
||||
}
|
||||
}
|
||||
return minPrice;
|
||||
},
|
||||
getTargetPriceByProbability: async ({
|
||||
symbol,
|
||||
calendarSpan,
|
||||
strikePercentageFromTheMoney,
|
||||
historicalProbabilityOfSuccess,
|
||||
}) => {
|
||||
return 0.24;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...calendarDatabase,
|
||||
getCalendars: calendarDatabase.getKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export const calendarDatabase: CalendarDatabase = makeCalendarDatabase();
|
||||
+212
-212
@@ -3,14 +3,14 @@ import { query } from "./lib/clickhouse.js";
|
||||
import { createHTTPHandler } from "@trpc/server/adapters/standalone";
|
||||
import cors from "cors";
|
||||
import {
|
||||
Object as ObjectT,
|
||||
String as StringT,
|
||||
TSchema,
|
||||
Number as NumberT,
|
||||
Object as ObjectT,
|
||||
String as StringT,
|
||||
type TSchema,
|
||||
Number as NumberT,
|
||||
} from "@sinclair/typebox";
|
||||
import { TypeCompiler } from "@sinclair/typebox/compiler";
|
||||
import { TRPCError } from "@trpc/server";
|
||||
import { createServer } from "http";
|
||||
import { createServer } from "node:http";
|
||||
import { Env } from "@humanwhocodes/env";
|
||||
|
||||
const env = new Env();
|
||||
@@ -24,53 +24,53 @@ const LISTEN_PORT = env.get("LISTEN_PORT", 3005);
|
||||
* @returns A TRPC-compatible validator function
|
||||
*/
|
||||
export function RpcType<T extends TSchema>(schema: T) {
|
||||
const check = TypeCompiler.Compile(schema);
|
||||
return (value: unknown) => {
|
||||
if (check.Check(value)) return value;
|
||||
const { path, message } = check.Errors(value).First()!;
|
||||
throw new TRPCError({
|
||||
message: `${message} for ${path}`,
|
||||
code: "BAD_REQUEST",
|
||||
});
|
||||
};
|
||||
const check = TypeCompiler.Compile(schema);
|
||||
return (value: unknown) => {
|
||||
if (check.Check(value)) return value;
|
||||
const { path, message } = check.Errors(value).First();
|
||||
throw new TRPCError({
|
||||
message: `${message} for ${path}`,
|
||||
code: "BAD_REQUEST",
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
const appRouter = router({
|
||||
getAvailableUnderlyings: publicProcedure.query(async (opts) => {
|
||||
// return (
|
||||
// await query<{ symbol: string }>(`
|
||||
// SELECT DISTINCT(symbol) as symbol FROM option_contract_existences WHERE asOfDate = (SELECT max(asOfDate) FROM option_contract_existences)
|
||||
// `)
|
||||
// ).map(({ symbol }) => symbol);
|
||||
return ["AAPL", "AMD", "GOOGL", "MSFT", "NFLX"];
|
||||
}),
|
||||
getAvailableAsOfDates: publicProcedure
|
||||
.input(RpcType(ObjectT({ underlying: StringT() })))
|
||||
.query(async (opts) => {
|
||||
const underlying = opts.input.underlying;
|
||||
return (
|
||||
await query<{ asOfDate: string }>(`
|
||||
getAvailableUnderlyings: publicProcedure.query(async (opts) => {
|
||||
// return (
|
||||
// await query<{ symbol: string }>(`
|
||||
// SELECT DISTINCT(symbol) as symbol FROM option_contract_existences WHERE asOfDate = (SELECT max(asOfDate) FROM option_contract_existences)
|
||||
// `)
|
||||
// ).map(({ symbol }) => symbol);
|
||||
return ["AAPL", "AMD", "GOOGL", "MSFT", "NFLX"];
|
||||
}),
|
||||
getAvailableAsOfDates: publicProcedure
|
||||
.input(RpcType(ObjectT({ underlying: StringT() })))
|
||||
.query(async (opts) => {
|
||||
const underlying = opts.input.underlying;
|
||||
return (
|
||||
await query<{ asOfDate: string }>(`
|
||||
SELECT
|
||||
DISTINCT(asOfDate) as asOfDate
|
||||
FROM option_contract_existences
|
||||
WHERE symbol = '${underlying}'
|
||||
ORDER BY asOfDate
|
||||
`)
|
||||
).map(({ asOfDate }) => asOfDate);
|
||||
}),
|
||||
getExpirationsForUnderlying: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
asOfDate: StringT(),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, asOfDate } = opts.input;
|
||||
return (
|
||||
await query<{ expirationDate: string }>(`
|
||||
).map(({ asOfDate }) => asOfDate);
|
||||
}),
|
||||
getExpirationsForUnderlying: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
asOfDate: StringT(),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, asOfDate } = opts.input;
|
||||
return (
|
||||
await query<{ expirationDate: string }>(`
|
||||
SELECT
|
||||
DISTINCT(expirationDate) as expirationDate
|
||||
FROM option_contract_existences
|
||||
@@ -78,22 +78,22 @@ const appRouter = router({
|
||||
AND asOfDate = '${asOfDate}'
|
||||
ORDER BY expirationDate
|
||||
`)
|
||||
).map(({ expirationDate }) => expirationDate);
|
||||
}),
|
||||
getStrikesForUnderlying: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
asOfDate: StringT(),
|
||||
expirationDate: StringT(),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, asOfDate, expirationDate } = opts.input;
|
||||
return (
|
||||
await query<{ strike: string }>(`
|
||||
).map(({ expirationDate }) => expirationDate);
|
||||
}),
|
||||
getStrikesForUnderlying: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
asOfDate: StringT(),
|
||||
expirationDate: StringT(),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, asOfDate, expirationDate } = opts.input;
|
||||
return (
|
||||
await query<{ strike: string }>(`
|
||||
SELECT
|
||||
DISTINCT(strike) as strike
|
||||
FROM option_contract_existences
|
||||
@@ -102,20 +102,20 @@ const appRouter = router({
|
||||
AND expirationDate = '${expirationDate}'
|
||||
ORDER BY strike
|
||||
`)
|
||||
).map(({ strike }) => strike);
|
||||
}),
|
||||
getOpensForUnderlying: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying } = opts.input;
|
||||
return await query<{ x: number; y: number }>(
|
||||
`
|
||||
).map(({ strike }) => strike);
|
||||
}),
|
||||
getOpensForUnderlying: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying } = opts.input;
|
||||
return await query<{ x: number; y: number }>(
|
||||
`
|
||||
SELECT
|
||||
toUnixTimestamp(tsStart) as x,
|
||||
open as y
|
||||
@@ -123,23 +123,23 @@ const appRouter = router({
|
||||
WHERE symbol = '${underlying}'
|
||||
ORDER BY tsStart ASC
|
||||
`,
|
||||
"JSONEachRow"
|
||||
);
|
||||
}),
|
||||
getOpensForOptionContract: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
expirationDate: StringT(),
|
||||
strike: NumberT(),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, expirationDate, strike } = opts.input;
|
||||
return await query<{ x: number; y: number }>(
|
||||
`
|
||||
"JSONEachRow",
|
||||
);
|
||||
}),
|
||||
getOpensForOptionContract: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
expirationDate: StringT(),
|
||||
strike: NumberT(),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, expirationDate, strike } = opts.input;
|
||||
return await query<{ x: number; y: number }>(
|
||||
`
|
||||
SELECT
|
||||
toUnixTimestamp(tsStart) as x,
|
||||
open as y
|
||||
@@ -150,32 +150,32 @@ const appRouter = router({
|
||||
AND type = 'call'
|
||||
ORDER BY tsStart ASC
|
||||
`,
|
||||
"JSONEachRow"
|
||||
);
|
||||
}),
|
||||
getHistoricalCalendarPrices: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
daysToFrontExpiration: NumberT(),
|
||||
daysBetweenFrontAndBackExpiration: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMin: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMax: NumberT(),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const {
|
||||
underlying,
|
||||
daysToFrontExpiration,
|
||||
daysBetweenFrontAndBackExpiration,
|
||||
strikePercentageFromUnderlyingPriceRangeMin,
|
||||
strikePercentageFromUnderlyingPriceRangeMax,
|
||||
} = opts.input;
|
||||
return (
|
||||
await query<[number, number]>(
|
||||
`
|
||||
"JSONEachRow",
|
||||
);
|
||||
}),
|
||||
getHistoricalCalendarPrices: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
daysToFrontExpiration: NumberT(),
|
||||
daysBetweenFrontAndBackExpiration: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMin: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMax: NumberT(),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const {
|
||||
underlying,
|
||||
daysToFrontExpiration,
|
||||
daysBetweenFrontAndBackExpiration,
|
||||
strikePercentageFromUnderlyingPriceRangeMin,
|
||||
strikePercentageFromUnderlyingPriceRangeMax,
|
||||
} = opts.input;
|
||||
return (
|
||||
await query<[number, number]>(
|
||||
`
|
||||
SELECT
|
||||
toUnixTimestamp(tsStart) as asOfTs,
|
||||
calendarPrice
|
||||
@@ -186,31 +186,31 @@ const appRouter = router({
|
||||
AND strikePercentageFromUnderlyingPrice <= ${strikePercentageFromUnderlyingPriceRangeMax}
|
||||
AND daysBetweenFrontAndBackExpiration = ${daysBetweenFrontAndBackExpiration}
|
||||
`,
|
||||
"JSONCompactEachRow"
|
||||
)
|
||||
).reduce(
|
||||
(columns, row) => {
|
||||
columns[0].push(row[0]);
|
||||
columns[1].push(row[1]);
|
||||
return columns;
|
||||
},
|
||||
[[], []]
|
||||
);
|
||||
}),
|
||||
getHistoricalStockQuoteChartData: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
lookbackPeriodStart: StringT(),
|
||||
lookbackPeriodEnd: StringT(),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, lookbackPeriodStart, lookbackPeriodEnd } = opts.input;
|
||||
return await query<[number, number]>(
|
||||
`
|
||||
"JSONCompactEachRow",
|
||||
)
|
||||
).reduce(
|
||||
(columns, row) => {
|
||||
columns[0].push(row[0]);
|
||||
columns[1].push(row[1]);
|
||||
return columns;
|
||||
},
|
||||
[[], []],
|
||||
);
|
||||
}),
|
||||
getHistoricalStockQuoteChartData: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
lookbackPeriodStart: StringT(),
|
||||
lookbackPeriodEnd: StringT(),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const { underlying, lookbackPeriodStart, lookbackPeriodEnd } = opts.input;
|
||||
return await query<[number, number]>(
|
||||
`
|
||||
SELECT
|
||||
toUnixTimestamp(tsStart) as x,
|
||||
open as y
|
||||
@@ -220,35 +220,35 @@ const appRouter = router({
|
||||
AND tsStart <= '${lookbackPeriodEnd} 00:00:00'
|
||||
ORDER BY x ASC
|
||||
`,
|
||||
"JSONEachRow"
|
||||
);
|
||||
}),
|
||||
getHistoricalCalendarQuoteChartData: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
daysToFrontExpiration: NumberT(),
|
||||
daysBetweenFrontAndBackExpiration: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMin: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMax: NumberT(),
|
||||
lookbackPeriodStart: StringT(),
|
||||
lookbackPeriodEnd: StringT(),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const {
|
||||
underlying,
|
||||
daysToFrontExpiration,
|
||||
daysBetweenFrontAndBackExpiration,
|
||||
strikePercentageFromUnderlyingPriceRangeMin,
|
||||
strikePercentageFromUnderlyingPriceRangeMax,
|
||||
lookbackPeriodStart,
|
||||
lookbackPeriodEnd,
|
||||
} = opts.input;
|
||||
return await query<[number, number]>(
|
||||
`
|
||||
"JSONEachRow",
|
||||
);
|
||||
}),
|
||||
getHistoricalCalendarQuoteChartData: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
daysToFrontExpiration: NumberT(),
|
||||
daysBetweenFrontAndBackExpiration: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMin: NumberT(),
|
||||
strikePercentageFromUnderlyingPriceRangeMax: NumberT(),
|
||||
lookbackPeriodStart: StringT(),
|
||||
lookbackPeriodEnd: StringT(),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const {
|
||||
underlying,
|
||||
daysToFrontExpiration,
|
||||
daysBetweenFrontAndBackExpiration,
|
||||
strikePercentageFromUnderlyingPriceRangeMin,
|
||||
strikePercentageFromUnderlyingPriceRangeMax,
|
||||
lookbackPeriodStart,
|
||||
lookbackPeriodEnd,
|
||||
} = opts.input;
|
||||
return await query<[number, number]>(
|
||||
`
|
||||
SELECT
|
||||
toUnixTimestamp(tsStart) as x,
|
||||
truncate(calendarPrice, 2) as y
|
||||
@@ -261,33 +261,33 @@ const appRouter = router({
|
||||
AND tsStart >= '${lookbackPeriodStart} 00:00:00'
|
||||
AND tsStart <= '${lookbackPeriodEnd} 00:00:00'
|
||||
`,
|
||||
"JSONEachRow"
|
||||
);
|
||||
}),
|
||||
getHistoricalCalendarExitQuoteChartData: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
daysToFrontExpiration: NumberT(),
|
||||
daysBetweenFrontAndBackExpiration: NumberT(),
|
||||
lookbackPeriodStart: StringT({
|
||||
pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}",
|
||||
}),
|
||||
lookbackPeriodEnd: StringT({ pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}" }),
|
||||
})
|
||||
)
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const {
|
||||
underlying,
|
||||
daysToFrontExpiration,
|
||||
daysBetweenFrontAndBackExpiration,
|
||||
lookbackPeriodStart,
|
||||
lookbackPeriodEnd,
|
||||
} = opts.input;
|
||||
return await query<[number, number, number]>(
|
||||
`
|
||||
"JSONEachRow",
|
||||
);
|
||||
}),
|
||||
getHistoricalCalendarExitQuoteChartData: publicProcedure
|
||||
.input(
|
||||
RpcType(
|
||||
ObjectT({
|
||||
underlying: StringT({ maxLength: 5 }),
|
||||
daysToFrontExpiration: NumberT(),
|
||||
daysBetweenFrontAndBackExpiration: NumberT(),
|
||||
lookbackPeriodStart: StringT({
|
||||
pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}",
|
||||
}),
|
||||
lookbackPeriodEnd: StringT({ pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}" }),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.query(async (opts) => {
|
||||
const {
|
||||
underlying,
|
||||
daysToFrontExpiration,
|
||||
daysBetweenFrontAndBackExpiration,
|
||||
lookbackPeriodStart,
|
||||
lookbackPeriodEnd,
|
||||
} = opts.input;
|
||||
return await query<[number, number, number]>(
|
||||
`
|
||||
SELECT
|
||||
FLOOR(strikePercentageFromUnderlyingPrice, 1) as x,
|
||||
FLOOR(calendarPrice, 1) as y,
|
||||
@@ -303,9 +303,9 @@ const appRouter = router({
|
||||
GROUP BY x, y
|
||||
ORDER BY x ASC, y ASC
|
||||
`,
|
||||
"JSONEachRow"
|
||||
);
|
||||
}),
|
||||
"JSONEachRow",
|
||||
);
|
||||
}),
|
||||
});
|
||||
|
||||
// Export type router type signature,
|
||||
@@ -313,20 +313,20 @@ const appRouter = router({
|
||||
export type AppRouter = typeof appRouter;
|
||||
|
||||
const handler = createHTTPHandler({
|
||||
middleware: cors(),
|
||||
router: appRouter,
|
||||
createContext() {
|
||||
return {};
|
||||
},
|
||||
middleware: cors(),
|
||||
router: appRouter,
|
||||
createContext() {
|
||||
return {};
|
||||
},
|
||||
});
|
||||
|
||||
const server = createServer((req, res) => {
|
||||
if (req.url.startsWith("/healthz")) {
|
||||
res.statusCode = 200;
|
||||
res.end("OK");
|
||||
} else {
|
||||
handler(req, res);
|
||||
}
|
||||
if (req.url.startsWith("/healthz")) {
|
||||
res.statusCode = 200;
|
||||
res.end("OK");
|
||||
} else {
|
||||
handler(req, res);
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(parseInt(LISTEN_PORT));
|
||||
server.listen(Number.parseInt(LISTEN_PORT));
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
export type Candlestick = {
|
||||
open: number;
|
||||
close: number;
|
||||
high: number;
|
||||
low: number;
|
||||
};
|
||||
|
||||
export type Aggregate<T> = {
|
||||
key: T;
|
||||
/** UNIX time in milliseconds */
|
||||
tsStart: number;
|
||||
} & Candlestick;
|
||||
|
||||
export type AggregateDatabase<T> = {
|
||||
getKeys: ({
|
||||
key,
|
||||
date,
|
||||
}: { key?: T | Partial<T>; date?: string }) => Promise<Array<T>>;
|
||||
getAggregates: ({
|
||||
key,
|
||||
date,
|
||||
}: { key: T; date: string }) => Promise<Array<Omit<Aggregate<T>, "key">>>;
|
||||
insertAggregates: (aggregates: Array<Aggregate<T>>) => Promise<void>;
|
||||
getClosingPrice: ({ key }: { key: T }) => Promise<number>;
|
||||
};
|
||||
@@ -0,0 +1,90 @@
|
||||
import type {
|
||||
OptionContractDatabase,
|
||||
OptionContractKey,
|
||||
} from "./optiondb.interfaces.js";
|
||||
import type { Aggregate } from "./interfaces.js";
|
||||
import { clickhouse, query } from "./lib/clickhouse.js";
|
||||
|
||||
function makeOptionContractDatabase(): OptionContractDatabase {
|
||||
const optionContractDatabase: Omit<
|
||||
OptionContractDatabase,
|
||||
"getOptionContracts"
|
||||
> = {
|
||||
getKeys: async ({ key: { symbol }, date }) => {
|
||||
return (
|
||||
await query<Omit<OptionContractKey, "symbol">>(`
|
||||
SELECT expirationDate, strike, type
|
||||
FROM option_contract_existences
|
||||
WHERE symbol = '${symbol}'
|
||||
AND asOfDate = '${date}'
|
||||
`)
|
||||
).map((optionContractWithoutKey) => ({
|
||||
...optionContractWithoutKey,
|
||||
symbol,
|
||||
}));
|
||||
},
|
||||
getAggregates: async ({
|
||||
key: { symbol, expirationDate, strike, type },
|
||||
date,
|
||||
}) => {
|
||||
return (
|
||||
await query<Omit<Aggregate<OptionContractKey>, "key">>(`
|
||||
SELECT
|
||||
toUnixTimestamp(tsStart) as tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low
|
||||
FROM option_contract_aggregates
|
||||
WHERE symbol = '${symbol}'
|
||||
AND type = '${type}'
|
||||
AND strike = '${strike}'
|
||||
AND expirationDate = '${expirationDate}'
|
||||
AND toDate(tsStart) = '${date}'
|
||||
ORDER BY tsStart ASC
|
||||
`)
|
||||
).map((aggregate) => ({
|
||||
...aggregate,
|
||||
tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
|
||||
}));
|
||||
},
|
||||
insertAggregates: async (aggregates) => {
|
||||
// stock existence is taken care of by clickhouse materialized view
|
||||
await clickhouse.insert({
|
||||
table: "option_contract_aggregates",
|
||||
values: aggregates.map(
|
||||
({
|
||||
key: { symbol, expirationDate, strike, type },
|
||||
tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low,
|
||||
}) => ({
|
||||
symbol,
|
||||
expirationDate,
|
||||
strike,
|
||||
type,
|
||||
tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low,
|
||||
}),
|
||||
),
|
||||
});
|
||||
},
|
||||
getClosingPrice: async ({ key }) => {
|
||||
// no-op: not used since stocks don't have a "closing" price, unlike options.
|
||||
return 0;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...optionContractDatabase,
|
||||
getOptionContracts: optionContractDatabase.getKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export const optionContractDatabase: OptionContractDatabase =
|
||||
makeOptionContractDatabase();
|
||||
@@ -0,0 +1,12 @@
|
||||
import type { AggregateDatabase } from "./interfaces.js";
|
||||
|
||||
export type OptionContractKey = {
|
||||
symbol: string;
|
||||
expirationDate: string;
|
||||
strike: number;
|
||||
type: "call" | "put";
|
||||
};
|
||||
|
||||
export type OptionContractDatabase = AggregateDatabase<OptionContractKey> & {
|
||||
getOptionContracts: AggregateDatabase<OptionContractKey>["getKeys"];
|
||||
};
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { OptionContractDatabase } from "./optiondb.interfaces.js";
|
||||
import { open } from "lmdbx";
|
||||
|
||||
const optionContractAggregatesDb = open({
|
||||
path: "/tmp/option-contract-aggregates.db",
|
||||
// any options go here, we can turn on compression like this:
|
||||
compression: true,
|
||||
});
|
||||
|
||||
const optionContractExistenceDb = open({
|
||||
path: "/tmp/option-contract-existence.db",
|
||||
// any options go here, we can turn on compression like this:
|
||||
compression: true,
|
||||
});
|
||||
|
||||
/** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
|
||||
const MAXIMUM_KEY = Buffer.from([0xff]);
|
||||
|
||||
function makeOptionContractDatabase(): OptionContractDatabase {
|
||||
const optionContractDatabase: Omit<
|
||||
OptionContractDatabase,
|
||||
"getOptionContracts"
|
||||
> = {
|
||||
getKeys: async ({ key: { symbol }, date }) => {
|
||||
return optionContractExistenceDb
|
||||
.getRange({
|
||||
start: [date, symbol],
|
||||
end: [date, symbol, MAXIMUM_KEY],
|
||||
})
|
||||
.map(({ key }) => ({
|
||||
symbol,
|
||||
expirationDate: key[2],
|
||||
strike: key[3],
|
||||
type: key[4],
|
||||
})).asArray;
|
||||
},
|
||||
getAggregates: async ({
|
||||
key: { symbol, expirationDate, strike, type },
|
||||
date,
|
||||
}) => {
|
||||
const startOfDayUnix = new Date(`${date}T00:00:00Z`).valueOf();
|
||||
const endOfDayUnix = startOfDayUnix + 3600 * 24 * 1000;
|
||||
return optionContractAggregatesDb
|
||||
.getRange({
|
||||
start: [symbol, expirationDate, strike, type, startOfDayUnix],
|
||||
end: [symbol, expirationDate, strike, type, endOfDayUnix],
|
||||
})
|
||||
.map(({ value }) => ({
|
||||
tsStart: value.tsStart,
|
||||
open: value.open,
|
||||
close: value.close,
|
||||
high: value.high,
|
||||
low: value.low,
|
||||
})).asArray;
|
||||
},
|
||||
insertAggregates: async (aggregates) => {
|
||||
await optionContractExistenceDb.batch(() => {
|
||||
for (const aggregate of aggregates) {
|
||||
optionContractExistenceDb.put(
|
||||
[
|
||||
new Date(aggregate.tsStart).toISOString().substring(0, 10),
|
||||
aggregate.key.symbol,
|
||||
aggregate.key.expirationDate,
|
||||
aggregate.key.strike,
|
||||
aggregate.key.type,
|
||||
],
|
||||
null,
|
||||
);
|
||||
}
|
||||
});
|
||||
await optionContractAggregatesDb.batch(() => {
|
||||
for (const aggregate of aggregates) {
|
||||
optionContractAggregatesDb.put(
|
||||
[
|
||||
aggregate.key.symbol,
|
||||
aggregate.key.expirationDate,
|
||||
aggregate.key.strike,
|
||||
aggregate.key.type,
|
||||
aggregate.tsStart,
|
||||
],
|
||||
{
|
||||
open: aggregate.open,
|
||||
close: aggregate.close,
|
||||
high: aggregate.high,
|
||||
low: aggregate.low,
|
||||
},
|
||||
);
|
||||
}
|
||||
});
|
||||
},
|
||||
getClosingPrice: async ({
|
||||
key: { symbol, strike, type, expirationDate },
|
||||
}) => {
|
||||
const startOfLastHourUnix = new Date(
|
||||
`${expirationDate}T00:00:00Z`,
|
||||
).valueOf();
|
||||
const endOfLastHourUnix = startOfLastHourUnix + 3600 * 1000;
|
||||
let minPrice = 0;
|
||||
for (const { value } of optionContractAggregatesDb.getRange({
|
||||
start: [symbol, expirationDate, strike, type, startOfLastHourUnix],
|
||||
end: [symbol, expirationDate, strike, type, endOfLastHourUnix],
|
||||
})) {
|
||||
if (value.close < minPrice || minPrice === 0) {
|
||||
minPrice = value.close;
|
||||
}
|
||||
}
|
||||
return minPrice;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...optionContractDatabase,
|
||||
getOptionContracts: optionContractDatabase.getKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export const optionContractDatabase: OptionContractDatabase =
|
||||
makeOptionContractDatabase();
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { AggregateDatabase } from "../interfaces.js";
|
||||
// import { stockDatabase as stockDatabaseClickhouse } from "../stockdb.clickhouse.js";
|
||||
// import { stockDatabase as stockDatabaseLmdbx } from "../stockdb.lmdbx.js";
|
||||
import { optionContractDatabase as optionContractDatabaseClickhouse } from "../optiondb.clickhouse.js";
|
||||
import { optionContractDatabase as optionContractDatabaseLmdbx } from "../optiondb.lmdbx.js";
|
||||
|
||||
function nextDate(date: string) {
|
||||
const dateObject = new Date(date);
|
||||
dateObject.setDate(dateObject.getDate() + 1);
|
||||
return dateObject.toISOString().substring(0, 10);
|
||||
}
|
||||
|
||||
async function syncAggregates<T>({
|
||||
from,
|
||||
to,
|
||||
key,
|
||||
date,
|
||||
}: {
|
||||
from: AggregateDatabase<T>;
|
||||
to: AggregateDatabase<T>;
|
||||
key: T;
|
||||
date: string;
|
||||
}) {
|
||||
const aggregatesFrom = (await from.getAggregates({ key, date })).map(
|
||||
(aggregateWithoutKey) => ({ ...aggregateWithoutKey, key }),
|
||||
);
|
||||
await to.insertAggregates(aggregatesFrom);
|
||||
}
|
||||
|
||||
const symbols = ["AMD", "AAPL", "MSFT", "GOOGL", "NFLX", "NVDA"];
|
||||
async function run() {
|
||||
const startDate = "2022-02-01";
|
||||
const endDate = "2024-07-15";
|
||||
for (let date = startDate; date <= endDate; date = nextDate(date)) {
|
||||
// const symbols = await stockDatabaseClickhouse.getSymbols({ date });
|
||||
for (const symbol of symbols) {
|
||||
console.log(date, symbol);
|
||||
const keys = await optionContractDatabaseClickhouse.getKeys({key: {symbol}, date});
|
||||
for(const key of keys){
|
||||
await syncAggregates({
|
||||
from: optionContractDatabaseClickhouse,
|
||||
to: optionContractDatabaseLmdbx,
|
||||
key,
|
||||
date,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await run();
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { StockDatabase, StockKey } from "./stockdb.interfaces.js";
|
||||
import type { Aggregate } from "./interfaces.js";
|
||||
import { clickhouse, query } from "./lib/clickhouse.js";
|
||||
|
||||
function makeStockDatabase(): StockDatabase {
|
||||
const stockDatabase: Omit<StockDatabase, "getSymbols"> = {
|
||||
getKeys: async ({ date }) => {
|
||||
return (
|
||||
await query(`
|
||||
SELECT DISTINCT symbol FROM stock_aggregates WHERE toDate(tsStart) = '${date}'
|
||||
`)
|
||||
).map(({ symbol }) => symbol);
|
||||
},
|
||||
getAggregates: async ({ key: symbol, date }) => {
|
||||
return (
|
||||
await query<Omit<Aggregate<StockKey>, "key">>(`
|
||||
SELECT
|
||||
toUnixTimestamp(tsStart) as tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low
|
||||
FROM stock_aggregates
|
||||
WHERE symbol = '${symbol}'
|
||||
AND toDate(tsStart) = '${date}'
|
||||
ORDER BY tsStart ASC
|
||||
`)
|
||||
).map((aggregate) => ({
|
||||
...aggregate,
|
||||
tsStart: aggregate.tsStart * 1000, // unfortunately, `toUnixTimestamp` only returns second-precision
|
||||
}));
|
||||
},
|
||||
insertAggregates: async (aggregates) => {
|
||||
// stock existence is taken care of by clickhouse materialized view
|
||||
await clickhouse.insert({
|
||||
table: "stock_aggregates",
|
||||
values: aggregates.map(({ key, tsStart, open, close, high, low }) => ({
|
||||
symbol: key,
|
||||
tsStart,
|
||||
open,
|
||||
close,
|
||||
high,
|
||||
low,
|
||||
})),
|
||||
});
|
||||
},
|
||||
getClosingPrice: async ({ key }) => {
|
||||
// no-op: not used since stocks don't have a "closing" price, unlike options.
|
||||
return 0;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...stockDatabase,
|
||||
getSymbols: stockDatabase.getKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export const stockDatabase: StockDatabase = makeStockDatabase();
|
||||
@@ -0,0 +1,7 @@
|
||||
import type { AggregateDatabase } from "./interfaces.js";
|
||||
|
||||
export type StockKey = string;
|
||||
|
||||
export type StockDatabase = AggregateDatabase<StockKey> & {
|
||||
getSymbols: AggregateDatabase<StockKey>["getKeys"];
|
||||
};
|
||||
@@ -0,0 +1,80 @@
|
||||
import type { StockDatabase } from "./stockdb.interfaces.js";
|
||||
import { open } from "lmdbx";
|
||||
|
||||
const stockAggregatesDb = open({
|
||||
path: "/tmp/stock-aggregates.db",
|
||||
// any options go here, we can turn on compression like this:
|
||||
compression: true,
|
||||
});
|
||||
|
||||
const stockExistenceDb = open({
|
||||
path: "/tmp/stock-existence.db",
|
||||
// any options go here, we can turn on compression like this:
|
||||
compression: true,
|
||||
});
|
||||
|
||||
/** Largest possible key according to the `ordered-binary` (used by lmdbx) docs. */
|
||||
const MAXIMUM_KEY = Buffer.from([0xff]);
|
||||
|
||||
function makeStockDatabase(): StockDatabase {
|
||||
const stockDatabase: Omit<StockDatabase, "getSymbols"> = {
|
||||
getKeys: async ({ date }) => {
|
||||
return stockExistenceDb
|
||||
.getRange({
|
||||
start: [date],
|
||||
end: [date, MAXIMUM_KEY],
|
||||
})
|
||||
.map(({ key }) => key[1]).asArray;
|
||||
},
|
||||
getAggregates: async ({ key: symbol, date }) => {
|
||||
const startOfDayUnix = new Date(`${date}T00:00:00Z`).valueOf();
|
||||
const endOfDayUnix = startOfDayUnix + 3600 * 24 * 1000;
|
||||
return stockAggregatesDb
|
||||
.getRange({
|
||||
start: [symbol, startOfDayUnix],
|
||||
end: [symbol, endOfDayUnix],
|
||||
})
|
||||
.map(({ key, value }) => ({
|
||||
tsStart: key[1],
|
||||
open: value.open,
|
||||
close: value.close,
|
||||
high: value.high,
|
||||
low: value.low,
|
||||
})).asArray;
|
||||
},
|
||||
insertAggregates: async (aggregates) => {
|
||||
await stockExistenceDb.batch(() => {
|
||||
for (const aggregate of aggregates) {
|
||||
stockExistenceDb.put(
|
||||
[
|
||||
new Date(aggregate.tsStart).toISOString().substring(0, 10),
|
||||
aggregate.key,
|
||||
],
|
||||
null,
|
||||
);
|
||||
}
|
||||
});
|
||||
await stockAggregatesDb.batch(() => {
|
||||
for (const aggregate of aggregates) {
|
||||
stockAggregatesDb.put([aggregate.key, aggregate.tsStart], {
|
||||
open: aggregate.open,
|
||||
close: aggregate.close,
|
||||
high: aggregate.high,
|
||||
low: aggregate.low,
|
||||
});
|
||||
}
|
||||
});
|
||||
},
|
||||
getClosingPrice: async ({ key }) => {
|
||||
// no-op: not used since stocks don't have a "closing" price, unlike options.
|
||||
return 0;
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...stockDatabase,
|
||||
getSymbols: stockDatabase.getKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export const stockDatabase: StockDatabase = makeStockDatabase();
|
||||
Reference in New Issue
Block a user