From a586f6f50b84c83a0ffe781874bbb722ce63f98f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Sat, 16 Nov 2024 11:48:29 +0700 Subject: [PATCH 1/5] Support memory context for proactive notification apps --- backend/utils/llm.py | 7 +++- backend/utils/plugins.py | 84 +++++++++++++++++++++++++++++++++------- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/backend/utils/llm.py b/backend/utils/llm.py index ff8c4ab88..e44c758ce 100644 --- a/backend/utils/llm.py +++ b/backend/utils/llm.py @@ -861,10 +861,10 @@ def provide_advice_message(uid: str, segments: List[TranscriptSegment], context: # ************************************************** -# ************* MENTOR PLUGIN ************** +# ************* PROACTIVE NOTIFICATION PLUGIN ************** # ************************************************** -def get_metoring_message(uid: str, plugin_prompt: str, params: [str]) -> str: +def get_proactive_message(uid: str, plugin_prompt: str, params: [str], context: str) -> str: user_name, facts_str = get_prompt_facts(uid) prompt = plugin_prompt @@ -875,6 +875,9 @@ def get_metoring_message(uid: str, plugin_prompt: str, params: [str]) -> str: if param == "user_facts": prompt = prompt.replace("{{user_facts}}", facts_str) continue + if param == "user_context": + prompt = prompt.replace("{{user_context}}", context if context else "") + continue prompt = prompt.replace(' ', '').strip() return llm_mini.invoke(prompt).content diff --git a/backend/utils/plugins.py b/backend/utils/plugins.py index 8a2227aa8..2263aeff9 100644 --- a/backend/utils/plugins.py +++ b/backend/utils/plugins.py @@ -14,7 +14,12 @@ from models.plugin import Plugin, UsageHistoryType from utils.notifications import send_notification from utils.other.endpoints import timeit -from utils.llm import get_metoring_message +from utils.llm import ( + generate_embedding, + get_proactive_message +) +from database.vector_db import query_vectors_by_metadata +import database.memories as memories_db def get_github_docs_content(repo="BasedHardware/omi", path="docs/docs"): @@ -219,6 +224,65 @@ async def trigger_realtime_integrations(uid: str, segments: list[dict]): _trigger_realtime_integrations(uid, token, segments) +# proactive notification +def _retrieve_contextual_memories(uid: str, user_context): + vector = ( + generate_embedding(user_context.get('question', '')) + if user_context.get('question') + else [0] * 3072 + ) + print("query_vectors vector:", vector[:5]) + + date_filters = {} # not support yet + filters = user_context.get('filters', {}) + memories_id = query_vectors_by_metadata( + uid, + vector, + dates_filter=[date_filters.get("start"), date_filters.get("end")], + people=filters.get("people", []), + topics=filters.get("topics", []), + entities=filters.get("entities", []), + dates=filters.get("dates", []), + ) + return memories_db.get_memories_by_id(uid, memories_id) + + +def _process_proactive_notification(uid: str, token: str, plugin: Plugin, data): + if not plugin.has_capability("proactive_notification") or not data: + print(f"Plugins {plugin.id} is not proactive_notification or data invalid", uid) + return + + max_prompt_char_limit = 8000 + min_message_char_limit = 5 + + prompt = data.get('prompt', '') + if len(prompt) > max_prompt_char_limit: + send_plugin_notification(token, plugin.name, plugin.id, f"Prompt too long: {len(prompt)}/{max_prompt_char_limit} characters. Please shorten.") + print(f"Plugin {plugin.id}, prompt too long, length: {len(prompt)}/{max_prompt_char_limit}", uid) + return None + + filter_scopes = plugin.fitler_proactive_notification_scopes(data.get('params', [])) + + # context + context = None + if 'user_context' in filter_scopes: + memories = _retrieve_contextual_memories(uid, data.get('user_context', {})) + if len(memories) > 0: + context = Memory.memories_to_string(memories, True) + + print(f'_process_proactive_notification context {context[100] if context else "empty"}') + + # retrive message + message = get_proactive_message(uid, prompt, filter_scopes, context) + if not message or len(message) < min_message_char_limit: + print(f"Plugins {plugin.id}, message too short", uid) + return None + + # send notification + send_plugin_notification(token, plugin.name, plugin.id, message) + return message + + def _trigger_realtime_integrations(uid: str, token: str, segments: List[dict]) -> dict: plugins: List[Plugin] = get_plugins_data_from_db(uid, include_reviews=False) filtered_plugins = [ @@ -259,20 +323,12 @@ def _single(plugin: Plugin): results[plugin.id] = message # proactive_notification + noti = response_data.get('notification', None) + print('Plugin', plugin.id, 'response notification:', noti) if plugin.has_capability("proactive_notification"): - noti = response_data.get('notification', None) - print('Plugin', plugin.id, 'response notification:', noti) - if noti: - prompt = noti.get('prompt', '') - if len(prompt) > 0 and len(prompt) <= 8000: - params = noti.get('params', []) - message = get_metoring_message(uid, prompt, plugin.fitler_proactive_notification_scopes(params)) - if message and len(message) > 5: - send_plugin_notification(token, plugin.name, plugin.id, message) - results[plugin.id] = message - elif len(prompt) > 8000: - send_plugin_notification(token, plugin.name, plugin.id, f"Prompt too long: {len(prompt)}/8000 characters. Please shorten.") - print(f"Plugin {plugin.id} prompt too long, length: {len(prompt)}/8000") + message = _process_proactive_notification(uid, token, plugin, noti) + if message: + results[plugin.id] = message except Exception as e: print(f"Plugin integration error: {e}") From e0df4aa8c970a55cc6d85108ec5ba45150ae24f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Sat, 16 Nov 2024 11:56:36 +0700 Subject: [PATCH 2/5] Simplify context, add example to Mentor01 app --- backend/utils/plugins.py | 2 +- plugins/example/basic/mentor.py | 21 +++++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/backend/utils/plugins.py b/backend/utils/plugins.py index 2263aeff9..ba7858b61 100644 --- a/backend/utils/plugins.py +++ b/backend/utils/plugins.py @@ -266,7 +266,7 @@ def _process_proactive_notification(uid: str, token: str, plugin: Plugin, data): # context context = None if 'user_context' in filter_scopes: - memories = _retrieve_contextual_memories(uid, data.get('user_context', {})) + memories = _retrieve_contextual_memories(uid, data.get('context', {})) if len(memories) > 0: context = Memory.memories_to_string(memories, True) diff --git a/plugins/example/basic/mentor.py b/plugins/example/basic/mentor.py index c11fa4223..9e8f9e110 100644 --- a/plugins/example/basic/mentor.py +++ b/plugins/example/basic/mentor.py @@ -43,6 +43,7 @@ def normalize(text): user_name = "{{user_name}}" user_facts = "{{user_facts}}" + user_context = "{{user_context}}" prompt = f""" You are an experienced mentor, that helps people achieve their goals during the meeting. @@ -67,15 +68,27 @@ def normalize(text): Output your response in plain text, without markdown. If you cannot find the topic or problem of the meeting, respond 'Nah 🤷 ~'. + + Conversation: ``` ${transcript} ``` + + Context: + ``` + {user_context} + ``` """.replace(' ', '').strip() - # 3. Respond with the format {notification: {prompt, params}} - return {'session_id': data.session_id, - 'notification': {'prompt': prompt, - 'params': ['user_name', 'user_facts']}} + # 3. Respond with the format {notification: {prompt, params, context}} + # - context: {question, filters: {people, topics, entities, dates}} | None + return { + 'session_id': data.session_id, + 'notification': { + 'prompt': prompt, + 'params': ['user_name', 'user_facts', 'user_context'], + } + } @ router.get('/setup/mentor', tags=['mentor']) def is_setup_completed(uid: str): From fc6642b374e0b43669e94e948962dbc85769837f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Sat, 16 Nov 2024 11:59:48 +0700 Subject: [PATCH 3/5] Remove dates supports --- plugins/example/basic/mentor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/example/basic/mentor.py b/plugins/example/basic/mentor.py index 9e8f9e110..11fc91fb8 100644 --- a/plugins/example/basic/mentor.py +++ b/plugins/example/basic/mentor.py @@ -81,7 +81,7 @@ def normalize(text): """.replace(' ', '').strip() # 3. Respond with the format {notification: {prompt, params, context}} - # - context: {question, filters: {people, topics, entities, dates}} | None + # - context: {question, filters: {people, topics, entities}} | None return { 'session_id': data.session_id, 'notification': { From 8fac7081e3deb8898aef779c364924c460eb72b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Sat, 16 Nov 2024 12:10:16 +0700 Subject: [PATCH 4/5] Add the context example model to Mentor01 app --- plugins/example/basic/mentor.py | 6 +++--- plugins/example/models.py | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/plugins/example/basic/mentor.py b/plugins/example/basic/mentor.py index 11fc91fb8..36b601c3c 100644 --- a/plugins/example/basic/mentor.py +++ b/plugins/example/basic/mentor.py @@ -2,7 +2,7 @@ from fastapi import APIRouter -from models import TranscriptSegment, MentorEndpointResponse, RealtimePluginRequest +from models import TranscriptSegment, ProactiveNotificationEndpointResponse, RealtimePluginRequest from db import get_upsert_segment_to_transcript_plugin router = APIRouter() @@ -10,10 +10,10 @@ scan_segment_session = {} # ******************************************************* -# ************ Basic Mentor Plugin ************ +# ************ Basic Proactive Notification Plugin ************ # ******************************************************* -@router.post('/mentor', tags=['mentor', 'basic', 'realtime'], response_model=MentorEndpointResponse) +@router.post('/mentor', tags=['mentor', 'basic', 'realtime', 'proactive_notification'], response_model=ProactiveNotificationEndpointResponse) def mentoring(data: RealtimePluginRequest): def normalize(text): return re.sub(r' +', ' ',re.sub(r'[,?.!]', ' ', text)).lower().strip() diff --git a/plugins/example/models.py b/plugins/example/models.py index aba0e6f21..abf5f64bf 100644 --- a/plugins/example/models.py +++ b/plugins/example/models.py @@ -168,10 +168,20 @@ class RealtimePluginRequest(BaseModel): segments: List[TranscriptSegment] -class MentorResponse(BaseModel): +class ProactiveNotificationContextFitlersResponse(BaseModel): + people: List[str] = Field(description="A list of people. ", default=[]) + entities: List[str] = Field(description="A list of entity. ", default=[]) + topics: List[str] = Field(description="A list of topic. ", default=[]) + +class ProactiveNotificationContextResponse(BaseModel): + question: str = Field(description="A question to query the embeded vector database.", default='') + filters: ProactiveNotificationContextFitlersResponse = Field(description="Filter options to query the embeded vector database. ", default=None) + +class ProactiveNotificationResponse(BaseModel): prompt: str = Field(description="A prompt or a template with the parameters such as {{user_name}} {{user_facts}}.", default='') params: List[str] = Field(description="A list of string that match with proactive notification scopes. ", default=[]) + context: ProactiveNotificationContextResponse = Field(description="An object to guide the system in retrieving the users context", default=None) -class MentorEndpointResponse(BaseModel): +class ProactiveNotificationEndpointResponse(BaseModel): message: str = Field(description="A short message to be sent as notification to the user, if needed.", default='') - notification: MentorResponse = Field(description="An object to guide the system in generating the proactive notification", default=None) + notification: ProactiveNotificationResponse = Field(description="An object to guide the system in generating the proactive notification", default=None) From 91fad0a913c417396b3cf79daecd9baa55607021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Sat, 16 Nov 2024 15:27:59 +0700 Subject: [PATCH 5/5] Fix metor01's context param --- backend/utils/llm.py | 1 + backend/utils/plugins.py | 2 +- plugins/example/basic/mentor.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/utils/llm.py b/backend/utils/llm.py index e44c758ce..352412889 100644 --- a/backend/utils/llm.py +++ b/backend/utils/llm.py @@ -879,5 +879,6 @@ def get_proactive_message(uid: str, plugin_prompt: str, params: [str], context: prompt = prompt.replace("{{user_context}}", context if context else "") continue prompt = prompt.replace(' ', '').strip() + #print(prompt) return llm_mini.invoke(prompt).content diff --git a/backend/utils/plugins.py b/backend/utils/plugins.py index ba7858b61..5ca8c7b3b 100644 --- a/backend/utils/plugins.py +++ b/backend/utils/plugins.py @@ -270,7 +270,7 @@ def _process_proactive_notification(uid: str, token: str, plugin: Plugin, data): if len(memories) > 0: context = Memory.memories_to_string(memories, True) - print(f'_process_proactive_notification context {context[100] if context else "empty"}') + print(f'_process_proactive_notification context {context[:100] if context else "empty"}') # retrive message message = get_proactive_message(uid, prompt, filter_scopes, context) diff --git a/plugins/example/basic/mentor.py b/plugins/example/basic/mentor.py index 36b601c3c..0ebd1509d 100644 --- a/plugins/example/basic/mentor.py +++ b/plugins/example/basic/mentor.py @@ -13,7 +13,7 @@ # ************ Basic Proactive Notification Plugin ************ # ******************************************************* -@router.post('/mentor', tags=['mentor', 'basic', 'realtime', 'proactive_notification'], response_model=ProactiveNotificationEndpointResponse) +@router.post('/mentor', tags=['mentor', 'basic', 'realtime', 'proactive_notification'], response_model=ProactiveNotificationEndpointResponse, response_model_exclude_none=True) def mentoring(data: RealtimePluginRequest): def normalize(text): return re.sub(r' +', ' ',re.sub(r'[,?.!]', ' ', text)).lower().strip()