abstract application-to-database interface

database/common.ts

@@ -0,0 +1,9 @@
+export interface Entity<T, ID> {
+  construct: (data: T) => T;
+  create: (data: T) => Promise<T>;
+  createMany: (data: T[]) => Promise<T[]>;
+  findAll: () => Promise<T[]>;
+  findById: (id: ID) => Promise<T | undefined>;
+  update: (id: ID, data: Partial<T>) => Promise<void>;
+  delete: (id: ID) => Promise<void>;
+}

database/lowdb.ts

@@ -1,6 +1,8 @@
 import { Low } from "lowdb";
 import { JSONFile } from "lowdb/node";
 import type { CommittedMessage } from "../types";
+import type { Entity } from "./common";
+import { nanoid } from "nanoid";
 
 export type Conversation = {
   id: string;
@@ -51,3 +53,209 @@ export const db = new Low<DB>(new JSONFile("db.json"), {
 await db.read();
 /** Write the database to the file, in case it didn't exist before. */
 await db.write();
+
+const conversations: Entity<Conversation, string> & {
+  fetchMessages: (conversationId: string) => Promise<Array<CommittedMessage>>;
+} = {
+  construct: (conversation: Conversation) => conversation,
+  create: async (conversation: Conversation) => {
+    conversation.id = conversation.id ?? nanoid();
+    await db.data.conversations.push(conversation);
+    await db.write();
+    return conversation;
+  },
+  createMany: async (conversations: Array<Conversation>) => {
+    await db.data.conversations.push(...conversations);
+    await db.write();
+    return conversations;
+  },
+  findAll: async () => {
+    return db.data.conversations;
+  },
+  findById: async (id) => {
+    return db.data.conversations.find((c) => c.id === id);
+  },
+  update: async (id, data: Partial<Conversation>) => {
+    const conversationIndex = db.data.conversations.findIndex(
+      (c) => c.id === id
+    );
+    if (conversationIndex === -1) throw new Error("Conversation not found");
+    db.data.conversations[conversationIndex] = {
+      ...db.data.conversations[conversationIndex],
+      ...data,
+    };
+    await db.write();
+  },
+  delete: async (id) => {
+    db.data.conversations.splice(
+      db.data.conversations.findIndex((c) => c.id === id),
+      1
+    );
+    const deletedMessageIds = db.data.messages
+      .filter((m) => m.conversationId === id)
+      .map((m) => m.id);
+    db.data.messages = db.data.messages.filter((m) => m.conversationId !== id);
+    const deletedFactIds = db.data.facts
+      .filter((fact) => deletedMessageIds.includes(fact.sourceMessageId))
+      .map((fact) => fact.id);
+    db.data.facts = db.data.facts.filter(
+      (fact) => !deletedFactIds.includes(fact.id)
+    );
+    db.data.factTriggers = db.data.factTriggers.filter(
+      (factTrigger) => !deletedFactIds.includes(factTrigger.sourceFactId)
+    );
+    await db.write();
+  },
+  fetchMessages: async (conversationId) => {
+    const rows = await db.data.messages.filter(
+      (m) => m.conversationId === conversationId
+    );
+    return rows as Array<CommittedMessage>;
+  },
+};
+
+const factTriggers: Entity<FactTrigger, string> & {
+  findByFactId: (factId: string) => Promise<Array<FactTrigger>>;
+} = {
+  construct: (factTrigger: FactTrigger) => factTrigger,
+  create: async (factTrigger: FactTrigger) => {
+    factTrigger.id = factTrigger.id ?? nanoid();
+    await db.data.factTriggers.push(factTrigger);
+    await db.write();
+    return factTrigger;
+  },
+  createMany: async (factTriggers: Array<FactTrigger>) => {
+    await db.data.factTriggers.push(...factTriggers);
+    await db.write();
+    return factTriggers;
+  },
+  findAll: async () => {
+    return db.data.factTriggers;
+  },
+  findById: async (id) => {
+    return db.data.factTriggers.find((factTrigger) => factTrigger.id === id);
+  },
+  update: async (id, data: Partial<FactTrigger>) => {
+    const factTriggerIndex = db.data.factTriggers.findIndex(
+      (factTrigger) => factTrigger.id === id
+    );
+    if (factTriggerIndex === -1) throw new Error("Fact trigger not found");
+    db.data.factTriggers[factTriggerIndex] = {
+      ...db.data.factTriggers[factTriggerIndex],
+      ...data,
+    };
+    await db.write();
+  },
+  delete: async (id) => {
+    const deletedFactTriggerIndex = db.data.factTriggers.findIndex(
+      (factTrigger) => factTrigger.id === id
+    );
+    if (deletedFactTriggerIndex === -1)
+      throw new Error("Fact trigger not found");
+    db.data.factTriggers.splice(deletedFactTriggerIndex, 1);
+    await db.write();
+  },
+  findByFactId: async (factId: string) => {
+    return db.data.factTriggers.filter(
+      (factTrigger) => factTrigger.sourceFactId === factId
+    );
+  },
+};
+
+const facts: Entity<Fact, string> & {
+  findByConversationId: (conversationId: string) => Promise<Array<Fact>>;
+} = {
+  construct: (fact: Fact) => fact,
+  create: async (fact: Fact) => {
+    fact.id = fact.id ?? nanoid();
+    await db.data.facts.push(fact);
+    await db.write();
+    return fact;
+  },
+  createMany: async (facts: Array<Fact>) => {
+    await db.data.facts.push(...facts);
+    await db.write();
+    return facts;
+  },
+  findAll: async () => {
+    return db.data.facts;
+  },
+  findById: async (id) => {
+    return db.data.facts.find((fact) => fact.id === id);
+  },
+  update: async (id, data: Partial<Fact>) => {
+    const factIndex = db.data.facts.findIndex((fact) => fact.id === id);
+    if (factIndex === -1) throw new Error("Fact not found");
+    db.data.facts[factIndex] = {
+      ...db.data.facts[factIndex],
+      ...data,
+    };
+    await db.write();
+  },
+  delete: async (id) => {
+    const deletedFactId = db.data.facts.findIndex((fact) => fact.id === id);
+    if (deletedFactId === -1) throw new Error("Fact not found");
+    db.data.facts.splice(deletedFactId, 1);
+    await db.write();
+  },
+  findByConversationId: async (conversationId) => {
+    const conversationMessageIds = db.data.messages
+      .filter((m) => m.conversationId === conversationId)
+      .map((m) => m.id);
+    const rows = await db.data.facts.filter((f) =>
+      conversationMessageIds.includes(f.sourceMessageId)
+    );
+    return rows as Array<Fact>;
+  },
+};
+
+const messages: Entity<CommittedMessage, string> & {
+  findByConversationId: (
+    conversationId: string
+  ) => Promise<Array<CommittedMessage>>;
+} = {
+  construct: (message: CommittedMessage) => message,
+  create: async (message: CommittedMessage) => {
+    message.id = message.id ?? nanoid();
+    await db.data.messages.push(message);
+    await db.write();
+    return message;
+  },
+  createMany: async (messages: Array<CommittedMessage>) => {
+    await db.data.messages.push(...messages);
+    await db.write();
+    return messages;
+  },
+  findAll: async () => {
+    return db.data.messages;
+  },
+  findById: async (id) => {
+    return db.data.messages.find((m) => m.id === id);
+  },
+  update: async (id, data: Partial<CommittedMessage>) => {
+    const messageIndex = db.data.messages.findIndex((m) => m.id === id);
+    if (messageIndex === -1) throw new Error("Message not found");
+    db.data.messages[messageIndex] = {
+      ...db.data.messages[messageIndex],
+      ...data,
+    };
+    await db.write();
+  },
+  delete: async (id) => {
+    db.data.messages.splice(
+      db.data.messages.findIndex((m) => m.id === id),
+      1
+    );
+    await db.write();
+  },
+  findByConversationId: async (conversationId) => {
+    return db.data.messages.filter((m) => m.conversationId === conversationId);
+  },
+};
+
+export const _db = {
+  conversations,
+  factTriggers,
+  facts,
+  messages,
+};

pages/chat/conversations.ts

@@ -3,55 +3,29 @@ import {
   publicProcedure,
   createCallerFactory,
 } from "../../trpc/server";
-import type { CommittedMessage } from "../../types.js";
+import { _db } from "../../database/lowdb";
-import { db } from "../../database/lowdb";
-import { nanoid } from "nanoid";
 
 export const conversations = router({
   fetchAll: publicProcedure.query(async () => {
-    const rows = await db.data.conversations;
+    return await _db.conversations.findAll();
-    return rows;
   }),
   fetchOne: publicProcedure
     .input((x) => x as { id: string })
     .query(async ({ input: { id } }) => {
-      const row = await db.data.conversations.find((c) => c.id === id);
+      return await _db.conversations.findById(id);
-      return row;
     }),
   start: publicProcedure.mutation(async () => {
-    const title = "New Conversation";
     const row = {
-      id: nanoid(),
+      id: "",
-      title,
+      title: "New Conversation",
       userId: "1",
     };
-    await db.data.conversations.push(row);
+    return await _db.conversations.create(row);
-    db.write();
-    return row;
   }),
   deleteOne: publicProcedure
     .input((x) => x as { id: string })
     .mutation(async ({ input: { id } }) => {
-      db.data.conversations.splice(
+      await _db.conversations.delete(id);
-        db.data.conversations.findIndex((c) => c.id === id),
-        1
-      );
-      const deletedMessageIds = db.data.messages
-        .filter((m) => m.conversationId === id)
-        .map((m) => m.id);
-      db.data.messages = db.data.messages.filter(
-        (m) => m.conversationId !== id
-      );
-      const deletedFactIds = db.data.facts
-        .filter((fact) => deletedMessageIds.includes(fact.sourceMessageId))
-        .map((fact) => fact.id);
-      db.data.facts = db.data.facts.filter(
-        (fact) => !deletedFactIds.includes(fact.id)
-      );
-      db.data.factTriggers = db.data.factTriggers.filter(
-        (factTrigger) => !deletedFactIds.includes(factTrigger.sourceFactId)
-      );
-      db.write();
       return { ok: true };
     }),
   updateTitle: publicProcedure
@@ -63,19 +37,13 @@ export const conversations = router({
         }
     )
     .mutation(async ({ input: { id, title } }) => {
-      const conversation = await db.data.conversations.find((c) => c.id === id);
+      await _db.conversations.update(id, { title });
-      if (!conversation) throw new Error("Conversation not found");
-      conversation.title = title;
-      db.write();
       return { ok: true };
     }),
   fetchMessages: publicProcedure
     .input((x) => x as { conversationId: string })
     .query(async ({ input: { conversationId } }) => {
-      const rows = await db.data.messages.filter(
+      return await _db.conversations.fetchMessages(conversationId);
-        (m) => m.conversationId === conversationId
-      );
-      return rows as Array<CommittedMessage>;
     }),
 });
 

pages/chat/fact-triggers.ts

@@ -3,7 +3,7 @@ import {
   publicProcedure,
   createCallerFactory,
 } from "../../trpc/server.js";
-import { db, type Fact } from "../../database/lowdb.js";
+import { _db, type Fact } from "../../database/lowdb.js";
 import type { DraftMessage } from "../../types.js";
 import { openrouter, MODEL_NAME } from "./provider.js";
 import { generateObject, generateText, jsonSchema } from "ai";
@@ -61,9 +61,7 @@ export const factTriggers = router({
   fetchByFactId: publicProcedure
     .input((x) => x as { factId: string })
     .query(async ({ input: { factId } }) => {
-      return db.data.factTriggers.filter(
+      return _db.factTriggers.findByFactId(factId);
-        (factTrigger) => factTrigger.sourceFactId === factId
-      );
     }),
   deleteOne: publicProcedure
     .input(
@@ -73,13 +71,7 @@ export const factTriggers = router({
         }
     )
     .mutation(async ({ input: { factTriggerId } }) => {
-      const deletedFactTriggerIndex = db.data.factTriggers.findIndex(
+      await _db.factTriggers.delete(factTriggerId);
-        (factTrigger) => factTrigger.id === factTriggerId
-      );
-      if (deletedFactTriggerIndex === -1)
-        throw new Error("Fact trigger not found");
-      db.data.factTriggers.splice(deletedFactTriggerIndex, 1);
-      await db.write();
       return { ok: true };
     }),
   update: publicProcedure
@@ -91,12 +83,7 @@ export const factTriggers = router({
         }
     )
     .mutation(async ({ input: { factTriggerId, content } }) => {
-      const factTriggerIndex = db.data.factTriggers.findIndex(
+      _db.factTriggers.update(factTriggerId, { content });
-        (factTrigger) => factTrigger.id === factTriggerId
-      );
-      if (factTriggerIndex === -1) throw new Error("Fact trigger not found");
-      db.data.factTriggers[factTriggerIndex].content = content;
-      await db.write();
       return { ok: true };
     }),
   generateFromFact: publicProcedure

pages/chat/facts.ts

@@ -3,7 +3,7 @@ import {
   publicProcedure,
   createCallerFactory,
 } from "../../trpc/server.js";
-import { db, type Fact } from "../../database/lowdb.js";
+import { _db } from "../../database/lowdb.js";
 import type { DraftMessage } from "../../types.js";
 import { MODEL_NAME, openrouter } from "./provider.js";
 import { generateObject, generateText, jsonSchema } from "ai";
@@ -57,13 +57,7 @@ export const facts = router({
   fetchByConversationId: publicProcedure
     .input((x) => x as { conversationId: string })
     .query(async ({ input: { conversationId } }) => {
-      const conversationMessageIds = db.data.messages
+      return await _db.facts.findByConversationId(conversationId);
-        .filter((m) => m.conversationId === conversationId)
-        .map((m) => m.id);
-      const rows = await db.data.facts.filter((f) =>
-        conversationMessageIds.includes(f.sourceMessageId)
-      );
-      return rows as Array<Fact>;
     }),
   deleteOne: publicProcedure
     .input(
@@ -73,12 +67,7 @@ export const facts = router({
         }
     )
     .mutation(async ({ input: { factId } }) => {
-      const deletedFactId = db.data.facts.findIndex(
+      await _db.facts.delete(factId);
-        (fact) => fact.id === factId
-      );
-      if (deletedFactId === -1) throw new Error("Fact not found");
-      db.data.facts.splice(deletedFactId, 1);
-      db.write();
       return { ok: true };
     }),
   update: publicProcedure
@@ -90,10 +79,7 @@ export const facts = router({
         }
     )
     .mutation(async ({ input: { factId, content } }) => {
-      const factIndex = db.data.facts.findIndex((fact) => fact.id === factId);
+      await _db.facts.update(factId, { content });
-      if (factIndex === -1) throw new Error("Fact not found");
-      db.data.facts[factIndex].content = content;
-      await db.write();
       return { ok: true };
     }),
   extractFromNewMessages: publicProcedure

pages/chat/messages.ts

@@ -3,10 +3,10 @@ import {
   publicProcedure,
   createCallerFactory,
 } from "../../trpc/server";
-import { createCaller as createConversationsCaller } from "./conversations.js";
 import { MODEL_NAME, openrouter } from "./provider.js";
 import { generateObject, generateText, jsonSchema } from "ai";
 import type { DraftMessage } from "../../types.js";
+import { _db } from "../../database/lowdb";
 
 const runningSummarySystemPrompt = ({
   previousRunningSummary,
@@ -52,10 +52,7 @@ export const messages = router({
   fetchByConversationId: publicProcedure
     .input((x) => x as { conversationId: string })
     .query(async ({ input: { conversationId } }) => {
-      const caller = createConversationsCaller({});
+      return await _db.messages.findByConversationId(conversationId);
-      return await caller.fetchMessages({
-        conversationId,
-      });
     }),
   generateRunningSummary: publicProcedure
     .input(

pages/chat/trpc.ts

@@ -14,7 +14,7 @@ import type {
 //   ConsistencyLevelEnum,
 //   type NumberArrayId,
 // } from "@zilliz/milvus2-sdk-node";
-import { db, type FactTrigger, type Fact } from "../../database/lowdb.js";
+import { db, type FactTrigger, type Fact, _db } from "../../database/lowdb.js";
 import { nanoid } from "nanoid";
 import { conversations } from "./conversations.js";
 import { messages } from "./messages.js";
@@ -105,8 +105,7 @@ export const chat = router({
           index: messages.length - 1,
           createdAt: new Date().toISOString(),
         };
-        db.data.messages.push(insertedUserMessage);
+        await _db.messages.create(insertedUserMessage);
-        // do not db.write() until the end
 
         /** Generate a new message from the model, but hold-off on adding it to
          * the database until we produce the associated running-summary, below.
@@ -163,7 +162,7 @@ export const chat = router({
             content: fact,
             createdAt: new Date().toISOString(),
           }));
-        db.data.facts.push(...insertedFactsFromUserMessage);
+        _db.facts.createMany(insertedFactsFromUserMessage);
 
         /** Produce a running summary of the conversation, and save that along
          * with the model's response to the database. The new running summary is
@@ -185,7 +184,7 @@ export const chat = router({
           index: messages.length,
           createdAt: new Date().toISOString(),
         };
-        db.data.messages.push(insertedAssistantMessage);
+        await _db.messages.create(insertedAssistantMessage);
         /** Extract Facts from the model's response, and add them to the database,
          * linking the Facts with the messages they came from. */
         const factsFromAssistantMessageResponse =
@@ -209,7 +208,7 @@ export const chat = router({
             content: factContent,
             createdAt: new Date().toISOString(),
           }));
-        db.data.facts.push(...insertedFactsFromAssistantMessage);
+        _db.facts.createMany(insertedFactsFromAssistantMessage);
 
         const insertedFacts = [
           ...insertedFactsFromUserMessage,
@@ -238,7 +237,7 @@ export const chat = router({
               scopeConversationId: conversationId,
               createdAt: new Date().toISOString(),
             }));
-          db.data.factTriggers.push(...insertedFactTriggers);
+          _db.factTriggers.createMany(insertedFactTriggers);
         }
 
         await db.write();