From d3ab5f67bdd8fbf147c08f7c666f6609c35b20a3 Mon Sep 17 00:00:00 2001
From: Ovidijus Parsiunas <oparsiunas@googlemail.com>
Date: Mon, 10 Jun 2024 22:27:04 +0100
Subject: [PATCH] fix for OpenAI Assistants v2 message annotations

---
 .../openAI/utils/openAIAssistantUtils.ts      | 47 ++++++++++---------
 1 file changed, 25 insertions(+), 22 deletions(-)

diff --git a/component/src/services/openAI/utils/openAIAssistantUtils.ts b/component/src/services/openAI/utils/openAIAssistantUtils.ts
index 7b5486574..5d5dc33f9 100644
--- a/component/src/services/openAI/utils/openAIAssistantUtils.ts
+++ b/component/src/services/openAI/utils/openAIAssistantUtils.ts
@@ -1,5 +1,4 @@
 import {OpenAIAssistantData, OpenAIAssistantContent, OpenAIAssistantMessagesResult} from '../../../types/openAIResult';
-import {FileMessageUtils} from '../../../views/chat/messages/fileMessageUtils';
 import {MessageFileType, MessageFile} from '../../../types/messageFile';
 import {Messages} from '../../../views/chat/messages/messages';
 import {RequestUtils} from '../../../utils/HTTP/requestUtils';
@@ -81,14 +80,37 @@ export class OpenAIAssistantUtils {
     return parts[parts.length - 1];
   }
 
+  // prettier-ignore
+  private static async getFilesAndNewText(io: ServiceIO, fileDetails: FileDetails,
+      role?: string, content?: OpenAIAssistantContent) {
+    let files: MessageFile[] | undefined;
+    if (fileDetails.length > 0) {
+      files = await OpenAIAssistantUtils.getFiles(io, fileDetails);
+      if (content?.text?.value) {
+        files.forEach((file, index) => {
+          if (!file.src) return;
+          const path = fileDetails[index].path;
+          if (content?.text?.value && path) {
+            content.text.value = content.text.value.replace(path, file.src);
+          }
+        });
+      }
+    }
+    // not displaying a separate file if annotated
+    return content?.text?.value ? {text: content.text.value, role} : {files, role};
+  }
+
+  // Noticed an issue where text contains a sandbox hyperlink to a csv, but no annotation provided
+  // To reproduce use the following text:
+  // give example data for a csv and create a suitable bar chart for it with a link
+  // Don't think it can be fixed and it is something on OpenAI side of things
   // prettier-ignore
   private static getFileDetails(lastMessage: OpenAIAssistantData, content?: OpenAIAssistantContent) {
     const fileDetails: FileDetails = [];
     if (content?.text?.value) {
       lastMessage.content.forEach((content) => {
         content.text?.annotations?.forEach((annotation) => {
-          if (annotation.text && annotation.text.startsWith('sandbox:')
-              && !FileMessageUtils.isImageFile({src: annotation.text}) && annotation.file_path?.file_id) {
+          if (annotation.text && annotation.text.startsWith('sandbox:') && annotation.file_path?.file_id) {
             fileDetails.push({
               path: annotation.text,
               fileId: annotation.file_path.file_id,
@@ -106,25 +128,6 @@ export class OpenAIAssistantUtils {
     return fileDetails;
   }
 
-  // prettier-ignore
-  private static async getFilesAndNewText(io: ServiceIO, fileDetails: FileDetails,
-      role?: string, content?: OpenAIAssistantContent) {
-    let files: MessageFile[] | undefined;
-    if (fileDetails.length > 0) {
-      files = await OpenAIAssistantUtils.getFiles(io, fileDetails);
-      if (content?.text?.value) {
-        files.forEach((file, index) => {
-          if (!file.src) return;
-          const path = fileDetails[index].path;
-          if (content?.text?.value && path) {
-            content.text.value = content.text.value.replace(path, file.src);
-          }
-        });
-      }
-    }
-    return {files, text: content?.text?.value, role};
-  }
-
   public static async getFilesAndText(io: ServiceIO, message: OpenAIAssistantData, content?: OpenAIAssistantContent) {
     const fileDetails = OpenAIAssistantUtils.getFileDetails(message, content);
     // gets files and replaces hyperlinks with base64 file encodings