factored-out fact generation into facts module

pages/chat/facts.ts

@@ -4,6 +4,48 @@ import {
   createCallerFactory,
 } from "../../trpc/server.js";
 import { db, type Fact } from "../../database/lowdb.js";
+import type { DraftMessage } from "../../types.js";
+import { openrouter } from "./provider.js";
+import { generateObject, generateText, jsonSchema } from "ai";
+
+const factsFromNewMessagesSystemPrompt = ({
+  previousRunningSummary,
+  messagesSincePreviousRunningSummary,
+}: {
+  previousRunningSummary: string;
+  messagesSincePreviousRunningSummary: Array<DraftMessage>;
+}) => `You are an expert at extracting facts from conversations.
+
+An AI assistant is in the middle of a conversation whose data is given below. The data consists of a summary of a conversation, and optionally some messages exchanged since that summary was produced. The user will provide you with *new* messages.
+
+Your task is to extract *new* facts that can be gleaned from the *new* messages that the user sends.
+
+* You should not extract any facts that are already in the summary.
+* The user should be referred to as "the user" in the fact text.
+* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous.
+* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context.
+
+<running_summary>
+  ${previousRunningSummary}
+</running_summary>
+
+${messagesSincePreviousRunningSummary.map(
+  (message) =>
+    `<${message.role}_message>${message.content}</${message.role}_message>`,
+)}
+`;
+
+const factsFromNewMessagesUserPrompt = ({
+  newMessages,
+}: {
+  newMessages: Array<DraftMessage>;
+}) =>
+  `${newMessages.map(
+    (message) =>
+      `<${message.role}_message>${message.content}</${message.role}_message>`,
+  )}
+
+Extract new facts from these messages.`;
 
 export const facts = router({
   fetchByConversationId: publicProcedure
@@ -33,6 +75,60 @@ export const facts = router({
       db.write();
       return { ok: true };
     }),
+  extractFromNewMessages: publicProcedure
+    .input(
+      (x) =>
+        x as {
+          previousRunningSummary: string;
+          /** will *not* have facts extracted */
+          messagesSincePreviousRunningSummary: Array<DraftMessage>;
+          /** *will* have facts extracted */
+          newMessages: Array<DraftMessage>;
+        },
+    )
+    .query(
+      async ({
+        input: {
+          previousRunningSummary,
+          messagesSincePreviousRunningSummary,
+          newMessages,
+        },
+      }) => {
+        const factsFromUserMessageResponse = await generateObject<{
+          facts: Array<string>;
+        }>({
+          model: openrouter("mistralai/mistral-nemo"),
+          messages: [
+            {
+              role: "system" as const,
+              content: factsFromNewMessagesSystemPrompt({
+                previousRunningSummary,
+                messagesSincePreviousRunningSummary,
+              }),
+            },
+            {
+              role: "user" as const,
+              content: factsFromNewMessagesUserPrompt({
+                newMessages,
+              }),
+            },
+          ],
+          schema: jsonSchema({
+            type: "object",
+            properties: {
+              facts: {
+                type: "array",
+                items: {
+                  type: "string",
+                },
+              },
+            },
+          }),
+          temperature: 0.4,
+        });
+        return factsFromUserMessageResponse;
+      },
+    ),
 });
 
 export const createCaller = createCallerFactory(facts);

pages/chat/provider.ts

@@ -0,0 +1,5 @@
+import { createOpenRouter } from "@openrouter/ai-sdk-provider";
+import { env } from "../../server/env.js";
+export const openrouter = createOpenRouter({
+  apiKey: env.OPENROUTER_API_KEY,
+});

pages/chat/trpc.ts

@@ -3,15 +3,12 @@ import {
   publicProcedure,
   createCallerFactory,
 } from "../../trpc/server.js";
-import { createOpenRouter } from "@openrouter/ai-sdk-provider";
 import { generateObject, generateText, jsonSchema } from "ai";
-import type { Message as UIMessage } from "ai";
 import type {
   OtherParameters,
   CommittedMessage,
   DraftMessage,
 } from "../../types.js";
-import { env } from "../../server/env.js";
 // import { client } from "../../database/milvus";
 // import {
 //   ConsistencyLevelEnum,
@@ -21,7 +18,10 @@ import { db, type FactTrigger, type Fact } from "../../database/lowdb.js";
 import { nanoid } from "nanoid";
 import { conversations } from "./conversations.js";
 import { messages } from "./messages.js";
-import { facts } from "./facts.js";
+import { facts, createCaller as createCallerFacts } from "./facts.js";
+import { openrouter } from "./provider.js";
+
+const factsCaller = createCallerFacts({});
 
 const mainSystemPrompt = ({
   systemPrompt,
@@ -33,74 +33,6 @@ This is a summary of the conversation so far, from your point-of-view (so "I" an
 ${previousRunningSummary}
 </running_summary>
 `;
-const factsFromUserMessageSystemPrompt = ({
-  previousRunningSummary,
-}: {
-  previousRunningSummary: string;
-}) => `You are an expert at extracting facts from conversations.
-
-You will be given a summary of a conversation, and the messages exchanged since that summary was produced.
-
-Your task is to extract *new* facts that can be gleaned from the messages exchanged since the summary was produced.
-
-* You should not extract any facts that are already in the summary.
-* The user should be referred to as "the user" in the fact text.
-* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous.
-* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context.
-
-<running_summary>
-  ${previousRunningSummary}
-</running_summary>
-`;
-
-const factsFromUserMessageUserPrompt = ({
-  messagesSincePreviousRunningSummary,
-}: {
-  messagesSincePreviousRunningSummary: Array<DraftMessage>;
-}) =>
-  `${messagesSincePreviousRunningSummary.map(
-    (message) =>
-      `<${message.role}_message>${message.content}</${message.role}_message>`,
-  )}
-
-Extract new facts from these messages.`;
-
-const factsFromAssistantMessageSystemPrompt = ({
-  previousRunningSummary,
-}: {
-  previousRunningSummary: string;
-}) => `You are an expert at extracting facts from conversations.
-
-You will be given a summary of a conversation, and the messages exchanged since that summary was produced. 
-
-Your task is to extract *new* facts that can be gleaned from the *final assistant response*.
-
-* You should not extract any facts that are already in the summary or in the ensuing conversation; you should only extract new facts from the final assistant response.
-* The user should be referred to as "the user" in the fact text.
-* The user's pronouns should be either he or she, NOT "they" or "them", because this summary will be read by an AI assistant to give it context; and excessive use of "they" or "them" will make what they refer to unclear or ambiguous.
-* The assistant should be referred to as "I" or "me", because these facts will be read by an AI assistant to give it context.
-
-<running_summary>
-  ${previousRunningSummary}
-</running_summary>
-`;
-
-const factsFromAssistantMessageUserPrompt = ({
-  messagesSincePreviousRunningSummary,
-  mainResponseContent,
-}: {
-  messagesSincePreviousRunningSummary: Array<DraftMessage>;
-  mainResponseContent: string;
-}) =>
-  `${messagesSincePreviousRunningSummary.map(
-    (message) =>
-      `<${message.role}_message>${message.content}</${message.role}_message>`,
-  )}
-<assistant_response>
-${mainResponseContent}
-</assistant_response>
-
-Extract facts from the assistant's response.`;
 
 const factTriggersSystemPrompt = ({
   previousRunningSummary,
@@ -185,10 +117,6 @@ ${mainResponseContent}
 
 Generate a new running summary of the conversation.`;
 
-const openrouter = createOpenRouter({
-  apiKey: env.OPENROUTER_API_KEY,
-});
-
 export const chat = router({
   conversations,
   messages,
@@ -269,39 +197,12 @@ export const chat = router({
          * injection, because we're sending the user's message unadulterated to
          * the model; there's no reason to inject the same Facts that the model is
          * already using to generate its response.) */
-        const factsFromUserMessageResponse = await generateObject<{
+        const factsFromUserMessageResponse =
-          facts: Array<string>;
+          await factsCaller.extractFromNewMessages({
-        }>({
+            previousRunningSummary,
-          model: openrouter("mistralai/mistral-nemo"),
+            messagesSincePreviousRunningSummary: [],
-          messages: [
+            newMessages: messagesSincePreviousRunningSummary,
-            {
+          });
-              role: "system" as const,
-              content: factsFromUserMessageSystemPrompt({
-                previousRunningSummary,
-              }),
-            },
-            {
-              role: "user" as const,
-              content: factsFromUserMessageUserPrompt({
-                messagesSincePreviousRunningSummary,
-              }),
-            },
-          ],
-          schema: jsonSchema({
-            type: "object",
-            properties: {
-              facts: {
-                type: "array",
-                items: {
-                  type: "string",
-                },
-              },
-            },
-          }),
-          maxSteps: 3,
-          tools: undefined,
-          ...parameters,
-        });
         const insertedFactsFromUserMessage: Array<Fact> =
           factsFromUserMessageResponse.object.facts.map((fact) => ({
             id: nanoid(),
@@ -349,45 +250,18 @@ export const chat = router({
         db.data.messages.push(insertedAssistantMessage);
         /** Extract Facts from the model's response, and add them to the database,
          * linking the Facts with the messages they came from. */
-        const factsFromAssistantMessageResponse = await generateObject<{
+        const factsFromAssistantMessageResponse =
-          facts: Array<string>;
+          await factsCaller.extractFromNewMessages({
-        }>({
+            previousRunningSummary,
-          model: openrouter("mistralai/mistral-nemo"),
+            messagesSincePreviousRunningSummary,
-          messages: [
+            newMessages: [
-            {
+              {
-              role: "system" as const,
+                role: "assistant" as const,
-              content: factsFromAssistantMessageSystemPrompt({
+                content: mainResponse.text,
-                previousRunningSummary,
-              }),
-            },
-            /** Yes, the next message is a `user` message, because models are
-             * trained to respond to `user` messages. So we wrap the assistant
-             * response in XML tags to show that it's not the user speaking,
-             * rather it's input for the model to process. The user is only
-             * saying "Extract facts..." */
-            {
-              role: "user" as const,
-              content: factsFromAssistantMessageUserPrompt({
-                messagesSincePreviousRunningSummary,
-                mainResponseContent: mainResponse.text,
-              }),
-            },
-          ],
-          schema: jsonSchema({
-            type: "object",
-            properties: {
-              facts: {
-                type: "array",
-                items: {
-                  type: "string",
-                },
               },
-            },
+            ],
-          }),
+          });
-          maxSteps: 3,
+
-          tools: undefined,
-          ...parameters,
-        });
         const insertedFactsFromAssistantMessage: Array<Fact> =
           factsFromAssistantMessageResponse.object.facts.map((factContent) => ({
             id: nanoid(),