immediate display transcribed text added: frontend changes (#179)

* immediate display transcribed text added * error handling
ohcnetwork · Feb 20, 2024 · f440c22 · f440c22
1 parent bb11822
commit f440c22
Show file tree

Hide file tree

Showing 3 changed files with 70 additions and 16 deletions.
diff --git a/src/app/(main)/project/[project_id]/chat/[chat_id]/page.tsx b/src/app/(main)/project/[project_id]/chat/[chat_id]/page.tsx
@@ -25,6 +25,7 @@ export default function Chat(
   const [chatMessage, setChatMessage] = useState<string>("");
   const [storage] = useAtom(storageAtom);
   const [isTyping, setIsTyping] = useState<boolean>(false);
+  const [apiError, setApiError] = useState<undefined | string>(undefined);
 
   const chatQuery = useQuery({
     queryKey: ["chat", chat_id],
@@ -120,8 +121,23 @@ export default function Chat(
 
   const handleAudio = async (blobUrl: string) => {
     setIsTyping(true);
-    const fd = await getFormData(storage, blobUrl);
-    converseMutation.mutate({ formdata: fd });
+    const sttFormData = await getFormData(storage, blobUrl)
+    try{
+      const {transcript, stats} = await API.chat.speechToText(
+        project_id,
+        chat_id,
+        sttFormData,
+      )
+      setNewChat(transcript);
+
+      const fd = await getFormData(storage, undefined, transcript);
+      fd.append("transcript_start_time", stats.transcript_start_time.toString());
+      fd.append("transcript_end_time", stats.transcript_end_time.toString());
+      converseMutation.mutate({ formdata: fd });
+    }
+    catch(e: any){
+      setApiError(e.message);
+    }
   };
 
   const messagesContainerRef = useRef<HTMLDivElement>(null);
@@ -186,7 +202,7 @@ export default function Chat(
           onChange={(e) => setNewChat(e.target.value)}
           onSubmit={handleSubmit}
           onAudio={handleAudio}
-          errors={[(converseMutation.error as any)?.error?.error]}
+          errors={[(converseMutation.error as any)?.error?.error, apiError]}
           loading={converseMutation.isPending || isTyping}
           projectId={project_id}
           forceLanguage={chat_language}

diff --git a/src/app/(main)/project/[project_id]/page.tsx b/src/app/(main)/project/[project_id]/page.tsx
@@ -20,6 +20,7 @@ export default function Chat(params: { params: { project_id: string } }) {
   const [chatMessage, setChatMessage] = useState<string>("");
   const [isTyping, setIsTyping] = useState<boolean>(false);
   const [chatID, setChatID] = useState<string>("");
+  const [apiError, setApiError] = useState<undefined | string>(undefined);
 
   const openai_key =
     !storage?.user?.allow_key || storage?.override_api_key
@@ -55,20 +56,13 @@ export default function Chat(params: { params: { project_id: string } }) {
 
   const newChatMutation = useMutation(
     {
-      mutationFn: (params: { formdata: FormData }) =>
+      mutationFn: () =>
         API.chat.create(
           project_id,
           chat !== "" ? chat.slice(0, 50) : "new chat",
           storage.openai_api_key,
         ),
       retry: false,
-      onSuccess: async (data, vars) => {
-        setChatID(data.external_id);
-        await converseMutation.mutateAsync({
-          external_id: data.external_id,
-          formdata: vars.formdata,
-        });
-      },
     },
   );
 
@@ -113,14 +107,41 @@ export default function Chat(params: { params: { project_id: string } }) {
   const handleSubmit = async (e: React.FormEvent<HTMLFormElement>) => {
     setIsTyping(true);
     e.preventDefault();
-    const fd = await getFormData(undefined, chat);
-    newChatMutation.mutate({ formdata: fd });
+
+    try{
+      const {external_id} = await newChatMutation.mutateAsync();
+      setChatID(external_id);
+
+      const fd = await getFormData(undefined, chat);
+      converseMutation.mutate({ external_id, formdata: fd });
+    }catch(e: any){
+      setApiError(e.message);
+    }
+
   };
 
   const handleAudio = async (blobUrl: string) => {
     setIsTyping(true);
-    const fd = await getFormData(blobUrl);
-    newChatMutation.mutate({ formdata: fd });
+    try{
+      const {external_id} =  await newChatMutation.mutateAsync();
+      setChatID(external_id);
+
+      const sttFormData = await getFormData(blobUrl);
+      const {transcript, stats} = await API.chat.speechToText(
+        project_id,
+        external_id,
+        sttFormData,
+      )
+      setChat(transcript);
+
+      const fd = await getFormData(undefined, transcript);
+      fd.append("transcript_start_time", stats.transcript_start_time.toString());
+      fd.append("transcript_end_time", stats.transcript_end_time.toString());
+      converseMutation.mutate({ external_id, formdata: fd });
+    }
+    catch(e: any){
+      setApiError(e.message);
+    }
   };
 
   return (
@@ -144,7 +165,14 @@ export default function Chat(params: { params: { project_id: string } }) {
                           setChat(prompt);
                           setIsTyping(true);
                           const fd = await getFormData(undefined, prompt);
-                          newChatMutation.mutate({ formdata: fd });
+                          const {external_id} = await newChatMutation.mutateAsync();
+                          if(external_id === "") return;
+                          setChatID(external_id);
+
+                          converseMutation.mutate({
+                            external_id,
+                            formdata: fd,
+                          });
                         }}
                         disabled={newChatMutation.isPending}
                         className="bg-white hover:shadow-lg hover:bg-gray-100 hover:text-indigo-500 text-left border border-gray-200 rounded-lg p-4 transition disabled:opacity-50 disabled:hover:text-gray-400"
@@ -203,6 +231,7 @@ export default function Chat(params: { params: { project_id: string } }) {
           errors={[
             (newChatMutation.error as any)?.error?.error,
             (newChatMutation.error as any)?.error?.non_field_errors,
+            apiError,
           ]}
           loading={
             newChatMutation.isPending || converseMutation.isPending || isTyping

diff --git a/src/utils/api.ts b/src/utils/api.ts
@@ -295,6 +295,15 @@ export const API = {
       request(`projects/${project_id}/chats/${id}`, "PATCH", fields),
     delete: (project_id: string, id: string) =>
       request(`projects/${project_id}/chats/${id}`, "DELETE"),
+    speechToText: (project_id: string, chat_id: string, formdata: FormData) =>
+    request(
+      `projects/${project_id}/chats/${chat_id}/speech_to_text`,
+      "POST",
+      formdata,
+      {
+        formdata: true,
+      },
+    ),
     converse: (
       project_id: string,
       chat_id: string,