diff --git a/bots/admin.py b/bots/admin.py index 82da0aab2..3163abca7 100644 --- a/bots/admin.py +++ b/bots/admin.py @@ -71,6 +71,21 @@ "slack_create_personal_channels", ] web_fields = ["web_allowed_origins", "web_config_extras"] +twilio_fields = [ + "twilio_account_sid", + "twilio_auth_token", + "twilio_phone_number", + "twilio_phone_number_sid", + "twilio_default_to_gooey_asr", + "twilio_default_to_gooey_tts", + "twilio_voice", + "twilio_asr_language", + "twilio_initial_text", + "twilio_initial_audio_url", + "twilio_use_missed_call", + "twilio_waiting_audio_url", + "twilio_waiting_text", +] class BotIntegrationAdminForm(forms.ModelForm): @@ -133,12 +148,14 @@ class BotIntegrationAdmin(admin.ModelAdmin): "slack_channel_name", "slack_channel_hook_url", "slack_access_token", + "twilio_phone_number", ] list_display = [ "name", "get_display_name", "platform", "wa_phone_number", + "twilio_phone_number", "created_at", "updated_at", "billing_account_uid", @@ -193,6 +210,7 @@ class BotIntegrationAdmin(admin.ModelAdmin): *wa_fields, *slack_fields, *web_fields, + *twilio_fields, ] }, ), @@ -489,6 +507,7 @@ class ConversationAdmin(admin.ModelAdmin): "slack_user_name", "slack_channel_id", "slack_channel_name", + "twilio_phone_number", ] + [f"bot_integration__{field}" for field in BotIntegrationAdmin.search_fields] actions = [export_to_csv, export_to_excel] diff --git a/bots/migrations/0077_botintegration_twilio_account_sid_and_more.py b/bots/migrations/0077_botintegration_twilio_account_sid_and_more.py new file mode 100644 index 000000000..0f1550e92 --- /dev/null +++ b/bots/migrations/0077_botintegration_twilio_account_sid_and_more.py @@ -0,0 +1,89 @@ +# Generated by Django 4.2.7 on 2024-07-11 01:30 + +from django.db import migrations, models +import phonenumber_field.modelfields + + +class Migration(migrations.Migration): + + dependencies = [ + ('bots', '0076_alter_workflowmetadata_default_image_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='botintegration', + name='twilio_account_sid', + field=models.TextField(blank=True, default='', help_text='Twilio account sid as found on twilio.com/console (mandatory)'), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_asr_language', + field=models.TextField(default='en-US', help_text='The language to use for Twilio ASR (https://www.twilio.com/docs/voice/twiml/gather#languagetags)'), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_auth_token', + field=models.TextField(blank=True, default='', help_text='Twilio auth token as found on twilio.com/console (mandatory)'), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_default_to_gooey_asr', + field=models.BooleanField(default=False, help_text="If true, the bot will use Gooey ASR for speech recognition instead of Twilio's when available on the attached run"), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_default_to_gooey_tts', + field=models.BooleanField(default=False, help_text="If true, the bot will use Gooey TTS for text to speech instead of Twilio's when available on the attached run"), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_initial_audio_url', + field=models.TextField(blank=True, default='', help_text='The initial audio url to play to the user when a call is started'), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_initial_text', + field=models.TextField(blank=True, default='', help_text='The initial text to send to the user when a call is started'), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_phone_number', + field=phonenumber_field.modelfields.PhoneNumberField(blank=True, default='', help_text='Twilio unformatted phone number as found on twilio.com/console/phone-numbers/incoming (mandatory)', max_length=128, region=None), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_phone_number_sid', + field=models.TextField(blank=True, default='', help_text='Twilio phone number sid as found on twilio.com/console/phone-numbers/incoming (mandatory)'), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_use_missed_call', + field=models.BooleanField(default=False, help_text="If true, the bot will reject incoming calls and call back the user instead so they don't get charged for the call"), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_voice', + field=models.TextField(default='woman', help_text="The voice to use for Twilio TTS ('man', 'woman', or Amazon Polly/Google Voices: https://www.twilio.com/docs/voice/twiml/say/text-speech#available-voices-and-languages)"), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_waiting_audio_url', + field=models.TextField(blank=True, default='', help_text='The audio url to play to the user while waiting for a response if using voice'), + ), + migrations.AddField( + model_name='botintegration', + name='twilio_waiting_text', + field=models.TextField(blank=True, default='', help_text='The text to send to the user while waiting for a response if using sms'), + ), + migrations.AddField( + model_name='conversation', + name='twilio_phone_number', + field=phonenumber_field.modelfields.PhoneNumberField(blank=True, default='', help_text="User's Twilio phone number (mandatory)", max_length=128, region=None), + ), + migrations.AlterField( + model_name='botintegration', + name='platform', + field=models.IntegerField(choices=[(1, 'Facebook Messenger'), (2, 'Instagram'), (3, 'WhatsApp'), (4, 'Slack'), (5, 'Web'), (6, 'Twilio')], help_text='The platform that the bot is integrated with'), + ), + ] diff --git a/bots/models.py b/bots/models.py index 6ff6674ff..8b8ad343a 100644 --- a/bots/models.py +++ b/bots/models.py @@ -59,11 +59,14 @@ class Platform(models.IntegerChoices): WHATSAPP = (3, "WhatsApp") SLACK = (4, "Slack") WEB = (5, "Web") + TWILIO = (6, "Twilio") def get_icon(self): match self: case Platform.WEB: return f'' + case Platform.TWILIO: + return f'' case _: return f'' @@ -622,6 +625,67 @@ class BotIntegration(models.Model): help_text="Extra configuration for the bot's web integration", ) + twilio_account_sid = models.TextField( + blank=True, + default="", + help_text="Twilio account sid as found on twilio.com/console (mandatory)", + ) + twilio_auth_token = models.TextField( + blank=True, + default="", + help_text="Twilio auth token as found on twilio.com/console (mandatory)", + ) + twilio_phone_number = PhoneNumberField( + blank=True, + default="", + help_text="Twilio unformatted phone number as found on twilio.com/console/phone-numbers/incoming (mandatory)", + ) + twilio_phone_number_sid = models.TextField( + blank=True, + default="", + help_text="Twilio phone number sid as found on twilio.com/console/phone-numbers/incoming (mandatory)", + ) + twilio_default_to_gooey_asr = models.BooleanField( + default=False, + help_text="If true, the bot will use Gooey ASR for speech recognition instead of Twilio's when available on the attached run", + ) + twilio_default_to_gooey_tts = models.BooleanField( + default=False, + help_text="If true, the bot will use Gooey TTS for text to speech instead of Twilio's when available on the attached run", + ) + twilio_voice = models.TextField( + default="woman", + help_text="The voice to use for Twilio TTS ('man', 'woman', or Amazon Polly/Google Voices: https://www.twilio.com/docs/voice/twiml/say/text-speech#available-voices-and-languages)", + ) + twilio_initial_text = models.TextField( + default="", + blank=True, + help_text="The initial text to send to the user when a call is started", + ) + twilio_initial_audio_url = models.TextField( + default="", + blank=True, + help_text="The initial audio url to play to the user when a call is started", + ) + twilio_use_missed_call = models.BooleanField( + default=False, + help_text="If true, the bot will reject incoming calls and call back the user instead so they don't get charged for the call", + ) + twilio_waiting_audio_url = models.TextField( + default="", + blank=True, + help_text="The audio url to play to the user while waiting for a response if using voice", + ) + twilio_waiting_text = models.TextField( + default="", + blank=True, + help_text="The text to send to the user while waiting for a response if using sms", + ) + twilio_asr_language = models.TextField( + default="en-US", + help_text="The language to use for Twilio ASR (https://www.twilio.com/docs/voice/twiml/gather#languagetags)", + ) + streaming_enabled = models.BooleanField( default=False, help_text="If set, the bot will stream messages to the frontend (Slack & Web only)", @@ -667,6 +731,7 @@ def get_display_name(self): or " | #".join( filter(None, [self.slack_team_name, self.slack_channel_name]) ) + or (self.twilio_phone_number and self.twilio_phone_number.as_international) or self.name or ( self.platform == Platform.WEB @@ -699,6 +764,21 @@ def get_web_widget_config(self, target="#gooey-embed") -> dict: ) return config + def translate(self, text: str) -> str: + from daras_ai_v2.asr import run_google_translate, should_translate_lang + + if text and should_translate_lang(self.user_language): + active_run = self.get_active_saved_run() + return run_google_translate( + [text], + self.user_language, + glossary_url=( + active_run.state.get("output_glossary") if active_run else None + ), + )[0] + else: + return text + class BotIntegrationAnalysisRun(models.Model): bot_integration = models.ForeignKey( @@ -769,7 +849,12 @@ class ConversationQuerySet(models.QuerySet): def get_unique_users(self) -> "ConversationQuerySet": """Get unique conversations""" return self.distinct( - "fb_page_id", "ig_account_id", "wa_phone_number", "slack_user_id" + "fb_page_id", + "ig_account_id", + "wa_phone_number", + "slack_user_id", + "twilio_phone_number", + "web_user_id", ) def to_df(self, tz=pytz.timezone(settings.TIME_ZONE)) -> "pd.DataFrame": @@ -962,6 +1047,12 @@ class Conversation(models.Model): help_text="Whether this is a personal slack channel between the bot and the user", ) + twilio_phone_number = PhoneNumberField( + blank=True, + default="", + help_text="User's Twilio phone number (mandatory)", + ) + web_user_id = models.CharField( max_length=512, blank=True, @@ -1007,6 +1098,7 @@ def get_display_name(self): or self.fb_page_id or self.slack_user_id or self.web_user_id + or (self.twilio_phone_number and self.twilio_phone_number.as_international) ) get_display_name.short_description = "User" diff --git a/bots/tasks.py b/bots/tasks.py index d49abd6bc..85d55d34b 100644 --- a/bots/tasks.py +++ b/bots/tasks.py @@ -23,6 +23,7 @@ create_personal_channel, SlackBot, ) +from daras_ai_v2.twilio_bot import create_voice_call, send_sms_message from daras_ai_v2.vector_search import references_as_prompt from gooeysite.bg_db_conn import get_celery_result_db_safe from recipes.VideoBots import ReplyButton, messages_as_prompt @@ -153,6 +154,7 @@ def send_broadcast_msgs_chunked( buttons: list[ReplyButton] = None, convo_qs: QuerySet[Conversation], bi: BotIntegration, + medium: str, ): convo_ids = list(convo_qs.values_list("id", flat=True)) for i in range(0, len(convo_ids), 100): @@ -164,6 +166,7 @@ def send_broadcast_msgs_chunked( documents=documents, bi_id=bi.id, convo_ids=convo_ids[i : i + 100], + medium=medium, ) @@ -177,6 +180,7 @@ def send_broadcast_msg( documents: list[str] = None, bi_id: int, convo_ids: list[int], + medium: str, ): bi = BotIntegration.objects.get(id=bi_id) convos = Conversation.objects.filter(id__in=convo_ids) @@ -205,6 +209,11 @@ def send_broadcast_msg( username=bi.name, token=bi.slack_access_token, )[0] + case Platform.TWILIO: + if medium == "Voice Call": + create_voice_call(convo, text, audio) + else: + send_sms_message(convo, text, media_url=audio) case _: raise NotImplementedError( f"Platform {bi.platform} doesn't support broadcasts yet" diff --git a/daras_ai_v2/bot_integration_widgets.py b/daras_ai_v2/bot_integration_widgets.py index 92937ef3b..ab89f82a2 100644 --- a/daras_ai_v2/bot_integration_widgets.py +++ b/daras_ai_v2/bot_integration_widgets.py @@ -26,24 +26,33 @@ def general_integration_settings(bi: BotIntegration, current_user: AppUser): st.session_state[f"_bi_show_feedback_buttons_{bi.id}"] = ( BotIntegration._meta.get_field("show_feedback_buttons").default ) + st.session_state[f"_bi_user_language_{bi.id}"] = BotIntegration._meta.get_field( + "user_language" + ).default st.session_state["analysis_urls"] = [] st.session_state.pop("--list-view:analysis_urls", None) - bi.streaming_enabled = st.checkbox( - "**πŸ“‘ Streaming Enabled**", - value=bi.streaming_enabled, - key=f"_bi_streaming_enabled_{bi.id}", - ) - st.caption("Responses will be streamed to the user in real-time if enabled.") - bi.show_feedback_buttons = st.checkbox( - "**πŸ‘πŸΎ πŸ‘ŽπŸ½ Show Feedback Buttons**", - value=bi.show_feedback_buttons, - key=f"_bi_show_feedback_buttons_{bi.id}", - ) - st.caption( - "Users can rate and provide feedback on every copilot response if enabled." - ) + if bi.platform != Platform.TWILIO: + bi.streaming_enabled = st.checkbox( + "**πŸ“‘ Streaming Enabled**", + value=bi.streaming_enabled, + key=f"_bi_streaming_enabled_{bi.id}", + ) + st.caption("Responses will be streamed to the user in real-time if enabled.") + bi.show_feedback_buttons = st.checkbox( + "**πŸ‘πŸΎ πŸ‘ŽπŸ½ Show Feedback Buttons**", + value=bi.show_feedback_buttons, + key=f"_bi_show_feedback_buttons_{bi.id}", + ) + st.caption( + "Users can rate and provide feedback on every copilot response if enabled." + ) + bi.user_language = st.text_input( + "##### 🌐 User Language", + value=bi.user_language, + key=f"_bi_user_language_{bi.id}", + ) st.caption( "Please note that this language is distinct from the one provided in the workflow settings. Hence, this allows you to integrate the same bot in many languages." ) @@ -128,6 +137,728 @@ def render_workflow_url_input(key: str, del_key: str | None, d: dict): st.write("---") +def twilio_specific_settings(bi: BotIntegration): + if st.session_state.get(f"_bi_reset_{bi.id}"): + st.session_state[f"_bi_twilio_default_to_gooey_asr_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_default_to_gooey_asr").default + ) + st.session_state[f"_bi_twilio_default_to_gooey_tts_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_default_to_gooey_tts").default + ) + st.session_state[f"_bi_twilio_voice_{bi.id}"] = BotIntegration._meta.get_field( + "twilio_voice" + ).default + st.session_state[f"_bi_twilio_initial_text_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_initial_text").default + ) + st.session_state[f"_bi_twilio_initial_audio_url_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_initial_audio_url").default + ) + st.session_state[f"_bi_twilio_use_missed_call_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_use_missed_call").default + ) + st.session_state[f"_bi_twilio_asr_language_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_asr_language").default + ) + st.session_state[f"_bi_twilio_waiting_audio_url_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_waiting_audio_url").default + ) + st.session_state[f"_bi_twilio_waiting_text_{bi.id}"] = ( + BotIntegration._meta.get_field("twilio_waiting_text").default + ) + + bi.twilio_voice = ( + st.selectbox( + "##### πŸ—£οΈ Twilio Voice", + [ + "Google.af-ZA-Standard-A", + "Polly.Zeina", + "Google.ar-XA-Standard-A", + "Google.ar-XA-Standard-B", + "Google.ar-XA-Standard-C", + "Google.ar-XA-Standard-D", + "Google.ar-XA-Wavenet-A", + "Google.ar-XA-Wavenet-B", + "Google.ar-XA-Wavenet-C", + "Google.ar-XA-Wavenet-D", + "Polly.Hala-Neural", + "Polly.Zayd-Neural", + "Google.eu-ES-Standard-A", + "Google.bn-IN-Standard-C", + "Google.bn-IN-Standard-D", + "Google.bn-IN-Wavenet-C", + "Google.bn-IN-Wavenet-D", + "Google.bg-BG-Standard-A", + "Polly.Arlet-Neural", + "Google.ca-ES-Standard-A", + "Polly.Hiujin-Neural", + "Google.yue-HK-Standard-A", + "Google.yue-HK-Standard-B", + "Google.yue-HK-Standard-C", + "Google.yue-HK-Standard-D", + "Polly.Zhiyu", + "Polly.Zhiyu-Neural", + "Google.cmn-CN-Standard-A", + "Google.cmn-CN-Standard-B", + "Google.cmn-CN-Standard-C", + "Google.cmn-CN-Standard-D", + "Google.cmn-CN-Wavenet-A", + "Google.cmn-CN-Wavenet-B", + "Google.cmn-CN-Wavenet-C", + "Google.cmn-CN-Wavenet-D", + "Google.cmn-TW-Standard-A", + "Google.cmn-TW-Standard-B", + "Google.cmn-TW-Standard-C", + "Google.cmn-TW-Wavenet-A", + "Google.cmn-TW-Wavenet-B", + "Google.cmn-TW-Wavenet-C", + "Google.cs-CZ-Standard-A", + "Google.cs-CZ-Wavenet-A", + "Polly.Mads", + "Polly.Naja", + "Polly.Sofie-Neural", + "Google.da-DK-Standard-A", + "Google.da-DK-Standard-C", + "Google.da-DK-Standard-D", + "Google.da-DK-Standard-E", + "Google.da-DK-Wavenet-A", + "Google.da-DK-Wavenet-C", + "Google.da-DK-Wavenet-D", + "Google.da-DK-Wavenet-E", + "Polly.Lisa-Neural", + "Google.nl-BE-Standard-A", + "Google.nl-BE-Standard-B", + "Google.nl-BE-Wavenet-A", + "Google.nl-BE-Wavenet-B", + "Polly.Lotte", + "Polly.Ruben", + "Polly.Laura-Neural", + "Google.nl-NL-Standard-A", + "Google.nl-NL-Standard-B", + "Google.nl-NL-Standard-C", + "Google.nl-NL-Standard-D", + "Google.nl-NL-Standard-E", + "Google.nl-NL-Wavenet-A", + "Google.nl-NL-Wavenet-B", + "Google.nl-NL-Wavenet-C", + "Google.nl-NL-Wavenet-D", + "Google.nl-NL-Wavenet-E", + "Polly.Nicole", + "Polly.Russell", + "Polly.Olivia-Neural", + "Google.en-AU-Standard-A", + "Google.en-AU-Standard-B", + "Google.en-AU-Standard-C", + "Google.en-AU-Standard-D", + "Google.en-AU-Wavenet-A", + "Google.en-AU-Wavenet-B", + "Google.en-AU-Wavenet-C", + "Google.en-AU-Wavenet-D", + "Google.en-AU-Neural2-A", + "Google.en-AU-Neural2-B", + "Google.en-AU-Neural2-C", + "Google.en-AU-Neural2-D", + "Polly.Raveena", + "Google.en-IN-Standard-A", + "Google.en-IN-Standard-B", + "Google.en-IN-Standard-C", + "Google.en-IN-Standard-D", + "Google.en-IN-Wavenet-A", + "Google.en-IN-Wavenet-B", + "Google.en-IN-Wavenet-C", + "Google.en-IN-Wavenet-D", + "Google.en-IN-Neural2-A", + "Google.en-IN-Neural2-B", + "Google.en-IN-Neural2-C", + "Google.en-IN-Neural2-D", + "Polly.Niamh-Neural", + "Polly.Aria-Neural", + "Polly.Ayanda-Neural", + "Polly.Amy", + "Polly.Brian", + "Polly.Emma", + "Polly.Amy-Neural", + "Polly.Emma-Neural", + "Polly.Brian-Neural", + "Polly.Arthur-Neural", + "Google.en-GB-Standard-A", + "Google.en-GB-Standard-B", + "Google.en-GB-Standard-C", + "Google.en-GB-Standard-D", + "Google.en-GB-Standard-F", + "Google.en-GB-Wavenet-A", + "Google.en-GB-Wavenet-B", + "Google.en-GB-Wavenet-C", + "Google.en-GB-Wavenet-D", + "Google.en-GB-Wavenet-F", + "Google.en-GB-Neural2-A", + "Google.en-GB-Neural2-B", + "Google.en-GB-Neural2-C", + "Google.en-GB-Neural2-D", + "Google.en-GB-Neural2-F", + "Polly.Ivy", + "Polly.Joanna", + "Polly.Joey", + "Polly.Justin", + "Polly.Kendra", + "Polly.Kimberly", + "Polly.Matthew", + "Polly.Salli", + "child) Polly.Ivy-Neural", + "Polly.Joanna-Neural*", + "Polly.Kendra-Neural", + "child) Polly.Kevin-Neural", + "Polly.Kimberly-Neural", + "Polly.Salli-Neural", + "Polly.Joey-Neural", + "child) Polly.Justin-Neural", + "Polly.Matthew-Neural*", + "Polly.Ruth-Neural", + "Polly.Stephen-Neural", + "Polly.Gregory-Neural", + "Polly.Danielle-Neural", + "Google.en-US-Standard-A", + "Google.en-US-Standard-B", + "Google.en-US-Standard-C", + "Google.en-US-Standard-D", + "Google.en-US-Standard-E", + "Google.en-US-Standard-F", + "Google.en-US-Standard-G", + "Google.en-US-Standard-H", + "Google.en-US-Standard-I", + "Google.en-US-Standard-J", + "Google.en-US-Wavenet-A", + "Google.en-US-Wavenet-B", + "Google.en-US-Wavenet-C", + "Google.en-US-Wavenet-D", + "Google.en-US-Wavenet-E", + "Google.en-US-Wavenet-F", + "Google.en-US-Wavenet-G", + "Google.en-US-Wavenet-H", + "Google.en-US-Wavenet-I", + "Google.en-US-Wavenet-J", + "Google.en-US-Neural2-A", + "Google.en-US-Neural2-C", + "Google.en-US-Neural2-D", + "Google.en-US-Neural2-E", + "Google.en-US-Neural2-F", + "Google.en-US-Neural2-G", + "Google.en-US-Neural2-H", + "Google.en-US-Neural2-I", + "Google.en-US-Neural2-J", + "Polly.Geraint", + "Google.fil-PH-Standard-A", + "Google.fil-PH-Standard-B", + "Google.fil-PH-Standard-C", + "Google.fil-PH-Standard-D", + "Google.fil-PH-Wavenet-A", + "Google.fil-PH-Wavenet-B", + "Google.fil-PH-Wavenet-C", + "Google.fil-PH-Wavenet-D", + "Polly.Suvi-Neural", + "Google.fi-FI-Standard-A", + "Google.fi-FI-Wavenet-A", + "Polly.Isabelle-Neural", + "Polly.Chantal", + "Polly.Gabrielle-Neural", + "Polly.Liam-Neural", + "Google.fr-CA-Standard-A", + "Google.fr-CA-Standard-B", + "Google.fr-CA-Standard-C", + "Google.fr-CA-Standard-D", + "Google.fr-CA-Wavenet-A", + "Google.fr-CA-Wavenet-B", + "Google.fr-CA-Wavenet-C", + "Google.fr-CA-Wavenet-D", + "Google.fr-CA-Neural2-A", + "Google.fr-CA-Neural2-B", + "Google.fr-CA-Neural2-C", + "Google.fr-CA-Neural2-D", + "Polly.Céline/Polly.Celine", + "Polly.Léa/Polly.Lea", + "Polly.Mathieu", + "Polly.Lea-Neural", + "Polly.Remi-Neural", + "Google.fr-FR-Standard-A", + "Google.fr-FR-Standard-B", + "Google.fr-FR-Standard-C", + "Google.fr-FR-Standard-D", + "Google.fr-FR-Standard-E", + "Google.fr-FR-Wavenet-A", + "Google.fr-FR-Wavenet-B", + "Google.fr-FR-Wavenet-C", + "Google.fr-FR-Wavenet-D", + "Google.fr-FR-Wavenet-E", + "Google.fr-FR-Neural2-A", + "Google.fr-FR-Neural2-B", + "Google.fr-FR-Neural2-C", + "Google.fr-FR-Neural2-D", + "Google.fr-FR-Neural2-E", + "Google.gl-ES-Standard-A", + "Polly.Hannah-Neural", + "Polly.Hans", + "Polly.Marlene", + "Polly.Vicki", + "Polly.Vicki-Neural", + "Polly.Daniel-Neural", + "Google.de-DE-Standard-A", + "Google.de-DE-Standard-B", + "Google.de-DE-Standard-C", + "Google.de-DE-Standard-D", + "Google.de-DE-Standard-E", + "Google.de-DE-Standard-F", + "Google.de-DE-Wavenet-A", + "Google.de-DE-Wavenet-B", + "Google.de-DE-Wavenet-C", + "Google.de-DE-Wavenet-D", + "Google.de-DE-Wavenet-E", + "Google.de-DE-Wavenet-F", + "Google.de-DE-Neural2-A", + "Google.de-DE-Neural2-B", + "Google.de-DE-Neural2-C", + "Google.de-DE-Neural2-D", + "Google.de-DE-Neural2-F", + "Google.el-GR-Standard-A", + "Google.el-GR-Wavenet-A", + "Google.gu-IN-Standard-C", + "Google.gu-IN-Standard-D", + "Google.gu-IN-Wavenet-C", + "Google.gu-IN-Wavenet-D", + "Google.he-IL-Standard-A", + "Google.he-IL-Standard-B", + "Google.he-IL-Standard-C", + "Google.he-IL-Standard-D", + "Google.he-IL-Wavenet-A", + "Google.he-IL-Wavenet-B", + "Google.he-IL-Wavenet-C", + "Google.he-IL-Wavenet-D", + "Polly.Aditi", + "Polly.Kajal-Neural", + "Google.hi-IN-Standard-A", + "Google.hi-IN-Standard-B", + "Google.hi-IN-Standard-C", + "Google.hi-IN-Standard-D", + "Google.hi-IN-Wavenet-A", + "Google.hi-IN-Wavenet-B", + "Google.hi-IN-Wavenet-C", + "Google.hi-IN-Wavenet-D", + "Google.hi-IN-Neural2-A", + "Google.hi-IN-Neural2-B", + "Google.hi-IN-Neural2-C", + "Google.hi-IN-Neural2-D", + "Google.hu-HU-Standard-A", + "Google.hu-HU-Wavenet-A", + "Polly.Dóra/Polly.Dora", + "Polly.Karl", + "Google.is-IS-Standard-A", + "Google.id-ID-Standard-A", + "Google.id-ID-Standard-B", + "Google.id-ID-Standard-C", + "Google.id-ID-Standard-D", + "Google.id-ID-Wavenet-A", + "Google.id-ID-Wavenet-B", + "Google.id-ID-Wavenet-C", + "Google.id-ID-Wavenet-D", + "Polly.Bianca", + "Polly.Carla", + "Polly.Giorgio", + "Polly.Bianca-Neural", + "Polly.Adriano-Neural", + "Google.it-IT-Standard-B", + "Google.it-IT-Standard-C", + "Google.it-IT-Standard-D", + "Google.it-IT-Wavenet-B", + "Google.it-IT-Wavenet-C", + "Google.it-IT-Wavenet-D", + "Google.it-IT-Neural2-A", + "Google.it-IT-Neural2-C", + "Polly.Mizuki", + "Polly.Takumi", + "Polly.Takumi-Neural", + "Polly.Kazuha-Neural", + "Polly.Tomoko-Neural", + "Google.ja-JP-Standard-B", + "Google.ja-JP-Standard-C", + "Google.ja-JP-Standard-D", + "Google.ja-JP-Wavenet-B", + "Google.ja-JP-Wavenet-C", + "Google.ja-JP-Wavenet-D", + "Google.kn-IN-Standard-C", + "Google.kn-IN-Standard-D", + "Google.kn-IN-Wavenet-C", + "Google.kn-IN-Wavenet-D", + "Polly.Seoyeon", + "Polly.Seoyeon-Neural", + "Google.ko-KR-Standard-A", + "Google.ko-KR-Standard-B", + "Google.ko-KR-Standard-C", + "Google.ko-KR-Standard-D", + "Google.ko-KR-Wavenet-A", + "Google.ko-KR-Wavenet-B", + "Google.ko-KR-Wavenet-C", + "Google.ko-KR-Wavenet-D", + "Google.ko-KR-Neural2-A", + "Google.ko-KR-Neural2-B", + "Google.ko-KR-Neural2-C", + "Google.lv-LV-Standard-A", + "Google.lt-LT-Standard-A", + "Google.ms-MY-Standard-A", + "Google.ms-MY-Standard-B", + "Google.ms-MY-Standard-C", + "Google.ms-MY-Standard-D", + "Google.ms-MY-Wavenet-A", + "Google.ms-MY-Wavenet-B", + "Google.ms-MY-Wavenet-C", + "Google.ms-MY-Wavenet-D", + "Google.ml-IN-Wavenet-C", + "Google.ml-IN-Wavenet-D", + "Google.mr-IN-Standard-A", + "Google.mr-IN-Standard-B", + "Google.mr-IN-Standard-C", + "Google.mr-IN-Wavenet-A", + "Google.mr-IN-Wavenet-B", + "Google.mr-IN-Wavenet-C", + "Polly.Liv", + "Polly.Ida-Neural", + "Google.nb-NO-Standard-A", + "Google.nb-NO-Standard-B", + "Google.nb-NO-Standard-C", + "Google.nb-NO-Standard-D", + "Google.nb-NO-Standard-E", + "Google.nb-NO-Wavenet-A", + "Google.nb-NO-Wavenet-B", + "Google.nb-NO-Wavenet-C", + "Google.nb-NO-Wavenet-D", + "Google.nb-NO-Wavenet-E", + "Polly.Jacek", + "Polly.Jan", + "Polly.Ewa", + "Polly.Maja", + "Polly.Ola-Neural", + "Google.pl-PL-Standard-A", + "Google.pl-PL-Standard-B", + "Google.pl-PL-Standard-C", + "Google.pl-PL-Standard-D", + "Google.pl-PL-Standard-E", + "Google.pl-PL-Wavenet-A", + "Google.pl-PL-Wavenet-B", + "Google.pl-PL-Wavenet-C", + "Google.pl-PL-Wavenet-D", + "Google.pl-PL-Wavenet-E", + "Polly.Camila", + "Polly.Ricardo", + "Polly.Vitória/Polly.Vitoria", + "Polly.Camila-Neural", + "Polly.Vitoria-Neural", + "Polly.Thiago-Neural", + "Google.pt-BR-Standard-B", + "Google.pt-BR-Standard-C", + "Google.pt-BR-Wavenet-B", + "Google.pt-BR-Wavenet-C", + "Google.pt-BR-Neural2-A", + "Google.pt-BR-Neural2-B", + "Google.pt-BR-Neural2-C", + "Polly.Cristiano", + "Polly.IneΜ‚s/Polly.Ines", + "Polly.Ines-Neural", + "Google.pt-PT-Standard-A", + "Google.pt-PT-Standard-B", + "Google.pt-PT-Standard-C", + "Google.pt-PT-Standard-D", + "Google.pt-PT-Wavenet-A", + "Google.pt-PT-Wavenet-B", + "Google.pt-PT-Wavenet-C", + "Google.pt-PT-Wavenet-D", + "Google.pa-IN-Standard-A", + "Google.pa-IN-Standard-B", + "Google.pa-IN-Standard-C", + "Google.pa-IN-Standard-D", + "Google.pa-IN-Wavenet-A", + "Google.pa-IN-Wavenet-B", + "Google.pa-IN-Wavenet-C", + "Google.pa-IN-Wavenet-D", + "Polly.Carmen", + "Google.ro-RO-Standard-A", + "Google.ro-RO-Wavenet-A", + "Polly.Maxim", + "Polly.Tatyana", + "Google.ru-RU-Standard-A", + "Google.ru-RU-Standard-B", + "Google.ru-RU-Standard-C", + "Google.ru-RU-Standard-D", + "Google.ru-RU-Standard-E", + "Google.ru-RU-Wavenet-A", + "Google.ru-RU-Wavenet-B", + "Google.ru-RU-Wavenet-C", + "Google.ru-RU-Wavenet-D", + "Google.ru-RU-Wavenet-E", + "Google.sr-RS-Standard-A", + "Google.sk-SK-Standard-A", + "Google.sk-SK-Wavenet-A", + "Polly.Mia", + "Polly.Mia-Neural", + "Polly.Andres-Neural", + "Polly.Conchita", + "Polly.Enrique", + "Polly.Lucia", + "Polly.Lucia-Neural", + "Polly.Sergio-Neural", + "Google.es-ES-Standard-B", + "Google.es-ES-Standard-C", + "Google.es-ES-Standard-D", + "Google.es-ES-Wavenet-B", + "Google.es-ES-Wavenet-C", + "Google.es-ES-Wavenet-D", + "Google.es-ES-Neural2-A", + "Google.es-ES-Neural2-B", + "Google.es-ES-Neural2-C", + "Google.es-ES-Neural2-D", + "Google.es-ES-Neural2-E", + "Google.es-ES-Neural2-F", + "man", + "woman", + "Polly.Lupe", + "Polly.Miguel", + "Polly.Penélope/Polly.Penelope", + "Polly.Lupe-Neural", + "Polly.Pedro-Neural", + "Google.es-US-Standard-A", + "Google.es-US-Standard-B", + "Google.es-US-Standard-C", + "Google.es-US-Wavenet-A", + "Google.es-US-Wavenet-B", + "Google.es-US-Wavenet-C", + "Google.es-US-Neural2-A", + "Google.es-US-Neural2-B", + "Google.es-US-Neural2-C", + "Polly.Astrid", + "Polly.Elin-Neural", + "Google.sv-SE-Standard-A", + "Google.sv-SE-Standard-B", + "Google.sv-SE-Standard-C", + "Google.sv-SE-Standard-D", + "Google.sv-SE-Standard-E", + "Google.sv-SE-Wavenet-A", + "Google.sv-SE-Wavenet-B", + "Google.sv-SE-Wavenet-C", + "Google.sv-SE-Wavenet-D", + "Google.sv-SE-Wavenet-E", + "Google.ta-IN-Standard-C", + "Google.ta-IN-Standard-D", + "Google.ta-IN-Wavenet-C", + "Google.ta-IN-Wavenet-D", + "Google.te-IN-Standard-A", + "Google.te-IN-Standard-B", + "Google.th-TH-Standard-A", + "Polly.Filiz", + "Google.tr-TR-Standard-A", + "Google.tr-TR-Standard-B", + "Google.tr-TR-Standard-C", + "Google.tr-TR-Standard-D", + "Google.tr-TR-Standard-E", + "Google.tr-TR-Wavenet-A", + "Google.tr-TR-Wavenet-B", + "Google.tr-TR-Wavenet-C", + "Google.tr-TR-Wavenet-D", + "Google.tr-TR-Wavenet-E", + "Google.uk-UA-Standard-A", + "Google.uk-UA-Wavenet-A", + "Google.vi-VN-Standard-A", + "Google.vi-VN-Standard-B", + "Google.vi-VN-Standard-C", + "Google.vi-VN-Standard-D", + "Google.vi-VN-Wavenet-A", + "Google.vi-VN-Wavenet-B", + "Google.vi-VN-Wavenet-C", + "Google.vi-VN-Wavenet-D", + "Polly.Gwyneth", + ], + value=bi.twilio_voice, + format_func=lambda x: x.capitalize(), + key=f"_bi_twilio_voice_{bi.id}", + ) + or "Woman" + ) + bi.twilio_asr_language = ( + st.selectbox( + "##### 🌐 Twilio ASR Language", + [ + "af-ZA", + "am-ET", + "hy-AM", + "az-AZ", + "id-ID", + "ms-MY", + "bn-BD", + "bn-IN", + "ca-ES", + "cs-CZ", + "da-DK", + "de-DE", + "en-AU", + "en-CA", + "en-GH", + "en-GB", + "en-IN", + "en-IE", + "en-KE", + "en-NZ", + "en-NG", + "en-PH", + "en-ZA", + "en-TZ", + "en-US", + "es-AR", + "es-BO", + "es-CL", + "es-CO", + "es-CR", + "es-EC", + "es-SV", + "es-ES", + "es-US", + "es-GT", + "es-HN", + "es-MX", + "es-NI", + "es-PA", + "es-PY", + "es-PE", + "es-PR", + "es-DO", + "es-UY", + "es-VE", + "eu-ES", + "fil-PH", + "fr-CA", + "fr-FR", + "gl-ES", + "ka-GE", + "gu-IN", + "hr-HR", + "zu-ZA", + "is-IS", + "it-IT", + "jv-ID", + "kn-IN", + "km-KH", + "lo-LA", + "lv-LV", + "lt-LT", + "hu-HU", + "ml-IN", + "mr-IN", + "nl-NL", + "ne-NP", + "nb-NO", + "pl-PL", + "pt-BR", + "pt-PT", + "ro-RO", + "si-LK", + "sk-SK", + "sl-SI", + "su-ID", + "sw-TZ", + "sw-KE", + "fi-FI", + "sv-SE", + "ta-IN", + "ta-SG", + "ta-LK", + "ta-MY", + "te-IN", + "vi-VN", + "tr-TR", + "ur-PK", + "ur-IN", + "el-GR", + "bg-BG", + "ru-RU", + "sr-RS", + "uk-UA", + "he-IL", + "ar-IL", + "ar-JO", + "ar-AE", + "ar-BH", + "ar-DZ", + "ar-SA", + "ar-IQ", + "ar-KW", + "ar-MA", + "ar-TN", + "ar-OM", + "ar-PS", + "ar-QA", + "ar-LB", + "ar-EG", + "fa-IR", + "hi-IN", + "th-TH", + "ko-KR", + "cmn-Hant-TW", + "yue-Hant-HK", + "ja-JP", + "cmn-Hans-HK", + "cmn-Hans-CN", + ], + key=f"_bi_twilio_asr_language_{bi.id}", + value=bi.twilio_asr_language, + ) + or "en-US" + ) + bi.twilio_default_to_gooey_asr = st.checkbox( + "🎀 Default to Gooey ASR", + value=bi.twilio_default_to_gooey_asr, + key=f"_bi_twilio_default_to_gooey_asr_{bi.id}", + ) + st.caption( + "Use Gooey's ASR for transcribing incoming audio messages (must also be enabled on the underlying run). Does not support interruptions." + ) + bi.twilio_default_to_gooey_tts = st.checkbox( + "πŸ“’ Default to Gooey TTS", + value=bi.twilio_default_to_gooey_tts, + key=f"_bi_twilio_default_to_gooey_tts_{bi.id}", + ) + st.caption( + "Use Gooey's TTS for converting text to speech in outgoing messages (must also be enabled on the underlying run)." + ) + bi.twilio_initial_text = st.text_area( + "###### πŸ“ Initial Text (said at the beginning of each call)", + value=bi.twilio_initial_text, + key=f"_bi_twilio_initial_text_{bi.id}", + ) + bi.twilio_initial_audio_url = ( + st.file_uploader( + "###### πŸ”Š Initial Audio (played at the beginning of each call)", + accept=["audio/*"], + key=f"_bi_twilio_initial_audio_url_{bi.id}", + ) + or "" + ) + bi.twilio_waiting_audio_url = ( + st.file_uploader( + "###### 🎡 Waiting Audio (played while waiting for a response -- Voice)", + accept=["audio/*"], + key=f"_bi_twilio_waiting_audio_url_{bi.id}", + ) + or "" + ) + bi.twilio_waiting_text = st.text_area( + "###### πŸ“ Waiting Text (texted while waiting for a response -- SMS)", + key=f"_bi_twilio_waiting_text_{bi.id}", + ) + bi.twilio_use_missed_call = st.checkbox( + "πŸ“ž Use Missed Call", + value=bi.twilio_use_missed_call, + key=f"_bi_twilio_use_missed_call_{bi.id}", + ) + st.caption( + "When enabled, immediately hangs up incoming calls and calls back the user so they don't incur charges (depending on their carrier/plan)." + ) + + def slack_specific_settings(bi: BotIntegration, default_name: str): if st.session_state.get(f"_bi_reset_{bi.id}"): st.session_state[f"_bi_name_{bi.id}"] = default_name @@ -185,20 +916,30 @@ def broadcast_input(bi: BotIntegration): optional=True, accept=["audio/*"], ) - video = st.file_uploader( - "**πŸŽ₯ Video**", - key=key + ":video", - help="Attach a video to this message.", - optional=True, - accept=["video/*"], - ) - documents = st.file_uploader( - "**πŸ“„ Documents**", - key=key + ":documents", - help="Attach documents to this message.", - accept_multiple_files=True, - optional=True, - ) + video = None + documents = None + medium = None + if bi.platform == Platform.TWILIO: + medium = st.selectbox( + "###### πŸ“± Medium", + ["Voice Call", "SMS/MMS"], + key=key + ":medium", + ) + else: + video = st.file_uploader( + "**πŸŽ₯ Video**", + key=key + ":video", + help="Attach a video to this message.", + optional=True, + accept=["video/*"], + ) + documents = st.file_uploader( + "**πŸ“„ Documents**", + key=key + ":documents", + help="Attach documents to this message.", + accept_multiple_files=True, + optional=True, + ) should_confirm_key = key + ":should_confirm" confirmed_send_btn = key + ":confirmed_send" @@ -219,6 +960,7 @@ def broadcast_input(bi: BotIntegration): documents=documents, bi=bi, convo_qs=convos, + medium=medium, ) else: if not convos.exists(): @@ -257,6 +999,8 @@ def get_bot_test_link(bi: BotIntegration) -> str | None: ), ) ) + elif bi.twilio_phone_number_sid: + return f"https://console.twilio.com/us1/develop/phone-numbers/manage/incoming/{bi.twilio_phone_number_sid}/calls" else: return None diff --git a/daras_ai_v2/bots.py b/daras_ai_v2/bots.py index 6096b9808..aeb0a30d8 100644 --- a/daras_ai_v2/bots.py +++ b/daras_ai_v2/bots.py @@ -84,7 +84,7 @@ class BotInterface: page_cls: typing.Type[BasePage] | None query_params: dict - language: str + language: str | None billing_account_uid: str show_feedback_buttons: bool = False streaming_enabled: bool = False @@ -140,37 +140,44 @@ def nice_filename(self, mime_type: str) -> str: return f"{self.platform.name}_{self.input_type}_from_{self.user_id}_to_{self.bot_id}{ext}" def _unpack_bot_integration(self): - bi = self.convo.bot_integration - if bi.published_run: - self.page_cls = Workflow(bi.published_run.workflow).page_cls + self.bi = self.convo.bot_integration + if self.bi.published_run: + self.page_cls = Workflow(self.bi.published_run.workflow).page_cls self.query_params = self.page_cls.clean_query_params( - example_id=bi.published_run.published_run_id, + example_id=self.bi.published_run.published_run_id, run_id="", uid="", ) - saved_run = bi.published_run.saved_run + saved_run = self.bi.published_run.saved_run self.input_glossary = saved_run.state.get("input_glossary_document") self.output_glossary = saved_run.state.get("output_glossary_document") self.language = saved_run.state.get("user_language") - elif bi.saved_run: - self.page_cls = Workflow(bi.saved_run.workflow).page_cls + elif self.bi.saved_run: + self.page_cls = Workflow(self.bi.saved_run.workflow).page_cls self.query_params = self.page_cls.clean_query_params( - example_id=bi.saved_run.example_id, - run_id=bi.saved_run.run_id, - uid=bi.saved_run.uid, + example_id=self.bi.saved_run.example_id, + run_id=self.bi.saved_run.run_id, + uid=self.bi.saved_run.uid, ) - self.input_glossary = bi.saved_run.state.get("input_glossary_document") - self.output_glossary = bi.saved_run.state.get("output_glossary_document") - self.language = bi.saved_run.state.get("user_language") + self.input_glossary = self.bi.saved_run.state.get("input_glossary_document") + self.output_glossary = self.bi.saved_run.state.get( + "output_glossary_document" + ) + self.language = self.bi.saved_run.state.get("user_language") else: self.page_cls = None self.query_params = {} - self.billing_account_uid = bi.billing_account_uid - if should_translate_lang(bi.user_language): - self.language = bi.user_language - self.show_feedback_buttons = bi.show_feedback_buttons - self.streaming_enabled = bi.streaming_enabled + self.billing_account_uid = self.bi.billing_account_uid + if should_translate_lang(self.bi.user_language): + self.language = self.bi.user_language + else: + self.language = None + self.show_feedback_buttons = self.bi.show_feedback_buttons + self.streaming_enabled = self.bi.streaming_enabled + + def translate(self, text: str | None) -> str: + return self.bi.translate(text or "") def _echo(bot, input_text): @@ -199,8 +206,18 @@ def _mock_api_output(input_text): } -@db_middleware def msg_handler(bot: BotInterface): + try: + _msg_handler(bot) + except Exception as e: + bot.send_msg( + text=bot.translate("Sorry, an error occurred. Please try again later."), + ) + capture_exception(e) + + +@db_middleware +def _msg_handler(bot: BotInterface): recieved_time: datetime = timezone.now() if not bot.page_cls: bot.send_msg(text=PAGE_NOT_CONNECTED_ERROR) diff --git a/daras_ai_v2/twilio_bot.py b/daras_ai_v2/twilio_bot.py new file mode 100644 index 000000000..4eb614c7b --- /dev/null +++ b/daras_ai_v2/twilio_bot.py @@ -0,0 +1,223 @@ +from bots.models import BotIntegration, Platform, Conversation +from daras_ai_v2.bots import BotInterface, ReplyButton +from phonenumber_field.phonenumber import PhoneNumber + +from twilio.rest import Client +from twilio.twiml.voice_response import VoiceResponse +from daras_ai_v2.fastapi_tricks import get_route_url + +from uuid import uuid4 +import base64 + + +class TwilioSMS(BotInterface): + platform = Platform.TWILIO + + def __init__(self, *, sid: str, convo: Conversation, text: str, bi: BotIntegration): + self.convo = convo + + self._text = text + self.input_type = "text" + + self.user_msg_id = sid + self.bot_id = bi.id + self.user_id = convo.twilio_phone_number.as_e164 + + self._unpack_bot_integration() + + def get_input_text(self) -> str | None: + return self._text + + def send_msg( + self, + *, + text: str | None = None, + audio: str | None = None, + video: str | None = None, + buttons: list[ReplyButton] | None = None, + documents: list[str] | None = None, + should_translate: bool = False, + update_msg_id: str | None = None, + ) -> str | None: + assert buttons is None, "Interactive mode is not implemented yet" + assert update_msg_id is None, "Twilio does not support un-sms-ing things" + + if should_translate: + text = self.translate(text) + + return send_sms_message( + self.convo, + text=text, + media_url=audio or video or (documents[0] if documents else None), + ).sid + + def mark_read(self): + pass # handled in the webhook + + +class TwilioVoice(BotInterface): + platform = Platform.TWILIO + + def __init__( + self, + *, + incoming_number: str, + queue_name: str, + call_sid: str, + text: str | None = None, + audio: str | None = None, + bi: BotIntegration, + ): + self.user_msg_id = uuid4().hex + self._queue_name = queue_name + self._call_sid = call_sid + self.bot_id = bi.id + self.user_id = incoming_number + + try: + self.convo = Conversation.objects.get( + twilio_phone_number=PhoneNumber.from_string(incoming_number), + bot_integration=bi, + ) + except Conversation.DoesNotExist: + self.convo = Conversation.objects.get_or_create( + twilio_phone_number=PhoneNumber.from_string(incoming_number), + bot_integration=bi, + )[0] + + if audio: + self.input_type = "audio" + else: + self.input_type = "text" + + self._text = text + self._audio = audio + + self._unpack_bot_integration() + + def get_input_text(self) -> str | None: + return self._text + + def get_input_audio(self) -> str | None: + return self._audio + + def send_msg( + self, + *, + text: str | None = None, + audio: str | None = None, + video: str | None = None, + buttons: list[ReplyButton] | None = None, + documents: list[str] | None = None, + should_translate: bool = False, + update_msg_id: str | None = None, + ) -> str | None: + assert documents is None, "Twilio does not support sending documents via Voice" + assert video is None, "Twilio does not support sending videos via Voice" + assert buttons is None, "Interactive mode is not implemented yet" + assert update_msg_id is None, "Twilio does not support un-saying things" + + # don't send both audio and text version of the same message + if self.bi.twilio_default_to_gooey_tts and audio: + text = None + else: + audio = None + + if should_translate: + text = self.translate(text) + + twilio_voice_call_respond( + text=text, + audio_url=audio, + queue_name=self._queue_name, + call_sid=self._call_sid, + bi=self.bi, + ) + + return uuid4().hex + + def mark_read(self): + pass # handled in the webhook + + +def twilio_voice_call_respond( + text: str | None, + audio_url: str | None, + queue_name: str, + call_sid: str, + bi: BotIntegration, +): + """Respond to the user in the queue with the given text and audio URL.""" + from routers.twilio_api import twilio_voice_call_response + + text = text + audio_url = audio_url + text = base64.b64encode(text.encode()).decode() if text else "N" + audio_url = base64.b64encode(audio_url.encode()).decode() if audio_url else "N" + + queue_sid = None + client = Client(bi.twilio_account_sid, bi.twilio_auth_token) + for queue in client.queues.list(): + if queue.friendly_name == queue_name: + queue_sid = queue.sid + break + assert queue_sid, "Queue not found" + + client.queues(queue_sid).members(call_sid).update( + url=get_route_url( + twilio_voice_call_response, + dict(bi_id=bi.id, text=text, audio_url=audio_url), + ), + method="POST", + ) + + return queue_sid + + +def create_voice_call(convo: Conversation, text: str | None, audio_url: str | None): + """Create a new voice call saying the given text and audio URL and then hanging up. Useful for notifications.""" + from routers.twilio_api import say + + assert ( + convo.twilio_phone_number + ), "This is not a Twilio conversation, it has no phone number." + + bi: BotIntegration = convo.bot_integration + client = Client(bi.twilio_account_sid, bi.twilio_auth_token) + + resp = VoiceResponse() + if text: + say(resp, text, bi) + if audio_url: + resp.play(audio_url) + + call = client.calls.create( + twiml=str(resp), + to=convo.twilio_phone_number.as_e164, + from_=bi.twilio_phone_number.as_e164, + ) + + return call + + +def send_sms_message( + convo: Conversation, text: str | None, media_url: str | None = None +): + """Send an SMS message to the given conversation.""" + + assert ( + convo.twilio_phone_number + ), "This is not a Twilio conversation, it has no phone number." + + account_sid = convo.bot_integration.twilio_account_sid + auth_token = convo.bot_integration.twilio_auth_token + client = Client(account_sid, auth_token) + + message = client.messages.create( + body=text or "", + media_url=media_url, + from_=convo.bot_integration.twilio_phone_number.as_e164, + to=convo.twilio_phone_number.as_e164, + ) + + return message diff --git a/poetry.lock b/poetry.lock index 70ce360c0..833f22b76 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "absl-py" @@ -133,6 +133,20 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["Brotli", "aiodns", "cchardet"] +[[package]] +name = "aiohttp-retry" +version = "2.8.3" +description = "Simple retry client for aiohttp" +optional = false +python-versions = ">=3.7" +files = [ + {file = "aiohttp_retry-2.8.3-py3-none-any.whl", hash = "sha256:3aeeead8f6afe48272db93ced9440cf4eda8b6fd7ee2abb25357b7eb28525b45"}, + {file = "aiohttp_retry-2.8.3.tar.gz", hash = "sha256:9a8e637e31682ad36e1ff9f8bcba912fcfc7d7041722bc901a4b948da4d71ea9"}, +] + +[package.dependencies] +aiohttp = "*" + [[package]] name = "aiosignal" version = "1.3.1" @@ -2928,6 +2942,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -4479,6 +4503,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -4486,8 +4511,16 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -4504,6 +4537,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -4511,6 +4545,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -5815,6 +5850,23 @@ torchhub = ["filelock", "huggingface-hub (>=0.16.4,<1.0)", "importlib-metadata", video = ["av (==9.2.0)", "decord (==0.6.0)"] vision = ["Pillow (<10.0.0)"] +[[package]] +name = "twilio" +version = "9.2.3" +description = "Twilio API client and TwiML generator" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "twilio-9.2.3-py2.py3-none-any.whl", hash = "sha256:76bfc39aa8d854510907cb7f9465814dfdea9e91ec199bb44f0785f05746f4cc"}, + {file = "twilio-9.2.3.tar.gz", hash = "sha256:da2255b5f3753cb3bf647fc6c50edbdb367ebc3cde6802806f6f863058a65f75"}, +] + +[package.dependencies] +aiohttp = ">=3.8.4" +aiohttp-retry = ">=2.8.3" +PyJWT = ">=2.0.0,<3.0.0" +requests = ">=2.0.0" + [[package]] name = "typing-extensions" version = "4.8.0" @@ -6438,4 +6490,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "fa29696b70dce0df83af18fb12a0f311dd05dccabfc7d3fe9ffe71630e042f14" +content-hash = "c6d35da21595cdc70dbdeae0f24139e6794331bd0b77333f5a6388a071c7deaf" diff --git a/pyproject.toml b/pyproject.toml index d703612fc..851224671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ emoji = "^2.10.1" pyvespa = "^0.39.0" anthropic = "^0.25.5" azure-cognitiveservices-speech = "^1.37.0" +twilio = "^9.2.3" [tool.poetry.group.dev.dependencies] watchdog = "^2.1.9" diff --git a/recipes/VideoBots.py b/recipes/VideoBots.py index f8a3b8534..f9acb283e 100644 --- a/recipes/VideoBots.py +++ b/recipes/VideoBots.py @@ -33,6 +33,7 @@ from daras_ai_v2.bot_integration_widgets import ( general_integration_settings, slack_specific_settings, + twilio_specific_settings, broadcast_input, get_bot_test_link, web_widget_config, @@ -98,6 +99,7 @@ from recipes.Lipsync import LipsyncPage from recipes.TextToSpeech import TextToSpeechPage, TextToSpeechSettings from url_shortener.models import ShortenedURL +from routers.twilio_api import twilio_connect DEFAULT_COPILOT_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/7a3127ec-1f71-11ef-aa2b-02420a00015d/Copilot.jpg" INTEGRATION_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/c3ba2392-d6b9-11ee-a67b-6ace8d8c9501/image.png" @@ -1138,21 +1140,76 @@ def render_integrations_add(self, label: str, run_title: str): st.newline() pressed_platform = None - with ( - st.tag("table", className="d-flex justify-content-center"), - st.tag("tbody"), + if st.session_state.get("__twilio_redirect_url") == "connecting...": + st.write( + "[Contact Us](sales@gooey.ai) to set up a Twilio phone number for you. Or enter your own [Twilio credentials](twilio.com/console):" + ) + + st.text_input("Account SID", key="twilio_account_sid", type="password") + st.text_input("Auth Token", key="twilio_auth_token", type="password") + st.text_input("Phone Number", key="twilio_phone_number") + st.text_input("Phone Number SID", key="twilio_phone_number_sid") + if st.button( + "Connect", key="twilio_connect", type="primary", style={"width": "100%"} + ): + current_run = self.get_current_sr() + published_run = self.get_current_published_run() + twilio_account_sid = st.session_state.pop("twilio_account_sid") + twilio_auth_token = st.session_state.pop("twilio_auth_token") + twilio_phone_number = st.session_state.pop("twilio_phone_number") + twilio_phone_number_sid = st.session_state.pop( + "twilio_phone_number_sid" + ) + if ( + not current_run + or not published_run + or not twilio_account_sid + or not twilio_auth_token + or not twilio_phone_number + or not twilio_phone_number_sid + ): + st.error("Please fill in all fields.") + return + + bi = twilio_connect( + current_run=current_run, + published_run=published_run, + twilio_account_sid=twilio_account_sid, + twilio_auth_token=twilio_auth_token, + twilio_phone_number=twilio_phone_number, + twilio_phone_number_sid=twilio_phone_number_sid, + billing_user=self.request.user, + ) + st.session_state["__twilio_redirect_url"] = self.current_app_url( + RecipeTabs.integrations, + path_params=dict(integration_id=bi.api_integration_id()), + ) + pressed_platform = Platform.TWILIO + else: + return + + if pressed_platform == None: + with ( + st.tag("table", className="d-flex justify-content-center"), + st.tag("tbody"), + ): + for choice in connect_choices: + with st.tag("tr"): + with st.tag("td"): + if st.button( + f'{choice.platform.name}', + className="p-0 border border-1 border-secondary rounded", + style=dict(width="160px", height="60px"), + ): + pressed_platform = choice.platform + with st.tag("td", className="ps-3"): + st.caption(choice.label) + + if pressed_platform == Platform.TWILIO and not st.session_state.get( + "__twilio_redirect_url" ): - for choice in connect_choices: - with st.tag("tr"): - with st.tag("td"): - if st.button( - f'{choice.platform.name}', - className="p-0 border border-1 border-secondary rounded", - style=dict(width="160px", height="60px"), - ): - pressed_platform = choice.platform - with st.tag("td", className="ps-3"): - st.caption(choice.label) + st.session_state["__twilio_redirect_url"] = "connecting..." + st.experimental_rerun() if pressed_platform: on_connect = self.current_app_url(RecipeTabs.integrations) @@ -1173,6 +1230,8 @@ def render_integrations_add(self, label: str, run_title: str): redirect_url = slack_connect_url(on_connect) case Platform.FACEBOOK: redirect_url = fb_connect_url(on_connect) + case Platform.TWILIO: + redirect_url = st.session_state.pop("__twilio_redirect_url") case _: raise ValueError(f"Unsupported platform: {pressed_platform}") @@ -1248,6 +1307,12 @@ def render_integrations_settings( unsafe_allow_html=True, new_tab=True, ) + if bi.twilio_phone_number: + copy_to_clipboard_button( + f' Copy Phone Number', + value=bi.twilio_phone_number.as_e164, + type="secondary", + ) col1, col2 = st.columns(2, style={"alignItems": "center"}) with col1: @@ -1267,6 +1332,8 @@ def render_integrations_settings( unsafe_allow_html=True, new_tab=True, ) + elif bi.platform == Platform.TWILIO and test_link: + pass elif test_link: st.anchor( f"{icon} Message {bi.get_display_name()}", @@ -1277,6 +1344,24 @@ def render_integrations_settings( else: st.write("Message quicklink not available.") + if bi.twilio_phone_number: + st.anchor( + ' Start Voice Call', + f"tel:{bi.twilio_phone_number.as_e164}", + unsafe_allow_html=True, + new_tab=True, + ) + st.anchor( + ' Send SMS', + f"sms:{bi.twilio_phone_number.as_e164}", + unsafe_allow_html=True, + new_tab=True, + ) + elif bi.platform == Platform.TWILIO: + st.write( + "Phone number incorrectly configured. Please re-add the integration and double check spelling. [Contact Us](support@gooey.ai) if you need help." + ) + if bi.platform == Platform.WEB: embed_code = get_web_widget_embed_code(bi) copy_to_clipboard_button( @@ -1305,6 +1390,13 @@ def render_integrations_settings( ), new_tab=True, ) + if bi.platform == Platform.TWILIO and test_link: + st.anchor( + f"{icon} View Calls/Messages", + test_link, + unsafe_allow_html=True, + new_tab=True, + ) if bi.platform == Platform.WHATSAPP and bi.wa_business_waba_id: col1, col2 = st.columns(2, style={"alignItems": "center"}) @@ -1339,9 +1431,11 @@ def render_integrations_settings( with st.expander("Configure Settings πŸ› οΈ"): if bi.platform == Platform.SLACK: slack_specific_settings(bi, run_title) + if bi.platform == Platform.TWILIO: + twilio_specific_settings(bi) general_integration_settings(bi, self.request.user) - if bi.platform in [Platform.SLACK, Platform.WHATSAPP]: + if bi.platform in [Platform.SLACK, Platform.WHATSAPP, Platform.TWILIO]: st.newline() broadcast_input(bi) st.write("---") @@ -1626,4 +1720,9 @@ class ConnectChoice(typing.NamedTuple): img="https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/9f201a92-1e9d-11ef-884b-02420a000134/thumbs/image_400x400.png", label="Connect to a Facebook Page you own. [Help Guide](https://gooey.ai/docs/guides/copilot/deploy-to-facebook)", ), + ConnectChoice( + platform=Platform.TWILIO, + img="https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/22b4b384-3d54-11ef-88d7-02420a000135/Twilio-logo-red.svg.png", + label="Connect a Twilio phone number to chat via SMS or Voice calls.", + ), ] diff --git a/recipes/VideoBotsStats.py b/recipes/VideoBotsStats.py index a550c368e..b3bed4bee 100644 --- a/recipes/VideoBotsStats.py +++ b/recipes/VideoBotsStats.py @@ -42,6 +42,8 @@ "conversation__ig_account_id", "conversation__wa_phone_number", "conversation__slack_user_id", + "conversation__twilio_phone_number", + "conversation__web_user_id", ] diff --git a/routers/twilio_api.py b/routers/twilio_api.py new file mode 100644 index 000000000..3e456dd6d --- /dev/null +++ b/routers/twilio_api.py @@ -0,0 +1,441 @@ +from twilio.rest import Client +from twilio.twiml.voice_response import VoiceResponse, Gather +from twilio.twiml.messaging_response import MessagingResponse + +from app_users.models import AppUser +from bots.models import Conversation, BotIntegration, SavedRun, PublishedRun, Platform +from phonenumber_field.phonenumber import PhoneNumber + +from fastapi import APIRouter, Response +from starlette.background import BackgroundTasks +from daras_ai_v2.fastapi_tricks import fastapi_request_urlencoded_body, get_route_url +import base64 +from sentry_sdk import capture_exception + +router = APIRouter() + + +def say(resp: VoiceResponse, text: str, bi: BotIntegration): + """Say the given text using the bot integration's voice. If the bot integration is set to use Gooey TTS, use that instead.""" + + if bi.twilio_default_to_gooey_tts and bi.get_active_saved_run(): + from recipes.TextToSpeech import TextToSpeechPage + from routers.api import submit_api_call + from gooeysite.bg_db_conn import get_celery_result_db_safe + + try: + tts_state = TextToSpeechPage.RequestModel.parse_obj( + {**bi.get_active_saved_run().state, "text_prompt": text} + ).dict() + page, result, run_id, uid = submit_api_call( + page_cls=TextToSpeechPage, + user=AppUser.objects.get(uid=bi.billing_account_uid), + request_body=tts_state, + query_params={}, + ) + get_celery_result_db_safe(result) + # get the final state from db + sr = page.run_doc_sr(run_id, uid) + state = sr.to_dict() + resp.play(state["audio_url"]) + except Exception as e: + resp.say(text, voice=bi.twilio_voice) + capture_exception(e) + else: + resp.say(text, voice=bi.twilio_voice) + + +@router.post("/__/twilio/voice/") +def twilio_voice_call( + background_tasks: BackgroundTasks, data: dict = fastapi_request_urlencoded_body +): + """Handle incoming Twilio voice call.""" + + # data = {'AccountSid': ['XXXX'], 'ApiVersion': ['2010-04-01'], 'CallSid': ['XXXX'], 'CallStatus': ['ringing'], 'CallToken': ['XXXX'], 'Called': ['XXXX'], 'CalledCity': ['XXXX'], 'CalledCountry': ['XXXX'], 'CalledState': ['XXXX'], 'CalledZip': ['XXXX'], 'Caller': ['XXXX'], 'CallerCity': ['XXXX'], 'CallerCountry': ['XXXX'], 'CallerState': ['XXXX'], 'CallerZip': ['XXXX'], 'Direction': ['inbound'], 'From': ['XXXX'], 'FromCity': ['XXXX'], 'FromCountry': ['XXXX'], 'FromState': ['XXXX'], 'FromZip': ['XXXX'], 'StirVerstat': ['XXXX'], 'To': ['XXXX'], 'ToCity': ['XXXX'], 'ToCountry': ['XXXX'], 'ToState': ['XXXX'], 'ToZip': ['XXXX']} + + account_sid = data["AccountSid"][0] + phone_number = data["To"][0] + user_phone_number = data["From"][0] + + try: + bi = BotIntegration.objects.get( + twilio_account_sid=account_sid, + twilio_phone_number=PhoneNumber.from_string(phone_number), + ) + except BotIntegration.DoesNotExist as e: + capture_exception(e) + return Response(status_code=404) + + text = bi.twilio_initial_text.strip() + audio_url = bi.twilio_initial_audio_url.strip() + if not text and not audio_url: + text = bi.translate( + f"Welcome to {bi.name}! Please ask your question and press 0 if the end of your question isn't detected.", + ) + + if bi.twilio_use_missed_call: + resp = VoiceResponse() + resp.reject() + + background_tasks.add_task( + start_voice_call_session, text, audio_url, bi, user_phone_number + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + text = base64.b64encode(text.encode()).decode() if text else "N" + audio_url = base64.b64encode(audio_url.encode()).decode() if audio_url else "N" + + resp = VoiceResponse() + resp.redirect( + get_route_url( + twilio_voice_call_response, + dict(bi_id=bi.id, text=text, audio_url=audio_url), + ) + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +@router.post("/__/twilio/voice/asked/") +def twilio_voice_call_asked( + background_tasks: BackgroundTasks, data: dict = fastapi_request_urlencoded_body +): + """After the initial call, the user has asked a question via Twilio ASR. Handle their question.""" + from daras_ai_v2.bots import msg_handler + from daras_ai_v2.twilio_bot import TwilioVoice + + # data = {'AccountSid': ['XXXX'], 'ApiVersion': ['2010-04-01'], 'CallSid': ['XXXX'], 'CallStatus': ['in-progress'], 'Called': ['XXXX'], 'CalledCity': ['XXXX'], 'CalledCountry': ['XXXX'], 'CalledState': ['XXXX'], 'CalledZip': ['XXXX'], 'Caller': ['XXXX'], 'CallerCity': ['XXXX'], 'CallerCountry': ['XXXX'], 'CallerState': ['XXXX'], 'CallerZip': ['XXXX'], 'Confidence': ['0.9128386'], 'Direction': ['inbound'], 'From': ['XXXX'], 'FromCity': ['XXXX'], 'FromCountry': ['XXXX'], 'FromState': ['XXXX'], 'FromZip': ['XXXX'], 'Language': ['en-US'], 'SpeechResult': ['Hello.'], 'To': ['XXXX'], 'ToCity': ['XXXX'], 'ToCountry': ['XXXX'], 'ToState': ['XXXX'], 'ToZip': ['XXXX']} + + account_sid = data["AccountSid"][0] + user_phone_number = data["From"][0] + phone_number = data["To"][0] + text = data["SpeechResult"][0] + call_sid = data["CallSid"][0] + + try: + bi = BotIntegration.objects.get( + twilio_account_sid=account_sid, twilio_phone_number=phone_number + ) + except BotIntegration.DoesNotExist as e: + capture_exception(e) + return Response(status_code=404) + + # start processing the user's question + queue_name = f"{bi.id}-{user_phone_number}" + bot = TwilioVoice( + incoming_number=user_phone_number, + queue_name=queue_name, + call_sid=call_sid, + text=text, + audio=None, + bi=bi, + ) + + background_tasks.add_task(msg_handler, bot) + + # send back waiting audio + resp = VoiceResponse() + say(resp, bot.translate("I heard ") + text, bi) + + resp.enqueue( + name=queue_name, + wait_url=get_route_url(twilio_voice_call_wait, dict(bi_id=bi.id)), + wait_url_method="POST", + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +@router.post("/__/twilio/voice/asked_audio/") +def twilio_voice_call_asked_audio( + background_tasks: BackgroundTasks, data: dict = fastapi_request_urlencoded_body +): + """After the initial call, the user has asked a question via Gooey ASR. Handle their question.""" + from daras_ai_v2.bots import msg_handler + from daras_ai_v2.twilio_bot import TwilioVoice + + # data: {'AccountSid': ['XXXX'], 'ApiVersion': ['2010-04-01'], 'CallSid': ['XXXX'], 'CallStatus': ['in-progress'], 'Called': ['XXXX'], 'CalledCity': ['XXXX'], 'CalledCountry': ['XXXX'], 'CalledState': ['XXXX'], 'CalledZip': ['XXXX'], 'Caller': ['XXXX'], 'CallerCity': ['XXXX'], 'CallerCountry': ['XXXX'], 'CallerState': ['XXXX'], 'CallerZip': ['XXXX'], 'Direction': ['inbound'], 'From': ['XXXX'], 'FromCity': ['XXXX'], 'FromCountry': ['XXXX'], 'FromState': ['XXXX'], 'FromZip': ['XXXX'], 'RecordingDuration': ['XXXX'], 'RecordingSid': ['XXXX'], 'RecordingUrl': ['https://api.twilio.com/2010-04-01/Accounts/AC5bac377df5bf25292fe863b9ddb2db2e/Recordings/RE0a6ed1afa9efaf42eb93c407b89619dd'], 'To': ['XXXX'], 'ToCity': ['XXXX'], 'ToCountry': ['XXXX'], 'ToState': ['XXXX'], 'ToZip': ['XXXX']} + + account_sid = data["AccountSid"][0] + user_phone_number = data["From"][0] + phone_number = data["To"][0] + audio_url = data["RecordingUrl"][0] # wav file + call_sid = data["CallSid"][0] + + try: + bi = BotIntegration.objects.get( + twilio_account_sid=account_sid, twilio_phone_number=phone_number + ) + except BotIntegration.DoesNotExist as e: + capture_exception(e) + return Response(status_code=404) + + # start processing the user's question + queue_name = f"{bi.id}-{user_phone_number}" + bot = TwilioVoice( + incoming_number=user_phone_number, + queue_name=queue_name, + call_sid=call_sid, + text=None, + audio=audio_url, + bi=bi, + ) + + background_tasks.add_task(msg_handler, bot) + + # send back waiting audio + resp = VoiceResponse() + resp.enqueue( + name=queue_name, + wait_url=get_route_url(twilio_voice_call_wait, dict(bi_id=bi.id)), + wait_url_method="POST", + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +@router.post("/__/twilio/voice/wait/{bi_id}/") +def twilio_voice_call_wait(bi_id: int): + """Play the waiting audio for the user in the queue.""" + + try: + bi = BotIntegration.objects.get(id=bi_id) + except BotIntegration.DoesNotExist as e: + capture_exception(e) + return Response(status_code=404) + + resp = VoiceResponse() + + if bi.twilio_waiting_audio_url: + resp.play(bi.twilio_waiting_audio_url) + else: + resp.play("http://com.twilio.sounds.music.s3.amazonaws.com/ClockworkWaltz.mp3") + resp.play("http://com.twilio.sounds.music.s3.amazonaws.com/BusyStrings.mp3") + resp.play( + "http://com.twilio.sounds.music.s3.amazonaws.com/oldDog_-_endless_goodbye_%28instr.%29.mp3" + ) + resp.play( + "http://com.twilio.sounds.music.s3.amazonaws.com/Mellotroniac_-_Flight_Of_Young_Hearts_Flute.mp3" + ) + resp.play( + "http://com.twilio.sounds.music.s3.amazonaws.com/MARKOVICHAMP-Borghestral.mp3" + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +@router.post("/__/twilio/voice/response/{bi_id}/{text}/{audio_url}/") +def twilio_voice_call_response(bi_id: int, text: str, audio_url: str): + """Response is ready, user has been dequeued, send the response and ask for the next one.""" + + text = base64.b64decode(text).decode() if text != "N" else "" + audio_url = base64.b64decode(audio_url).decode() if audio_url != "N" else "" + + try: + bi = BotIntegration.objects.get(id=bi_id) + except BotIntegration.DoesNotExist as e: + capture_exception(e) + return Response(status_code=404) + + resp = VoiceResponse() + + if bi.twilio_default_to_gooey_asr: + # record does not support nesting, so we can't support interrupting the response with the next question + if text: + say(resp, text, bi) + if audio_url: + resp.play(audio_url) + + # try recording 3 times to give the user a chance to start speaking + for _ in range(3): + resp.record( + action=get_route_url(twilio_voice_call_asked_audio), + method="POST", + timeout=3, + finish_on_key="0", + play_beep=False, + ) + else: + gather = Gather( + input="speech", # also supports dtmf (keypad input) and a combination of both + timeout=20, # users get 20 to start speaking + speechTimeout=3, # a 3 second pause ends the input + action=get_route_url( + twilio_voice_call_asked + ), # the URL to send the user's question to + method="POST", + finish_on_key="0", # user can press 0 to end the input + language=bi.twilio_asr_language, + speech_model="phone_call", # optimized for phone call audio + enhanced=True, # only phone_call model supports enhanced + ) + + # by attaching to gather, we allow the user to interrupt with the next question while the response is playing + if text: + gather.say(text, voice=bi.twilio_voice) + if audio_url: + gather.play(audio_url) + + resp.append(gather) + + # if the user doesn't say anything, we'll ask them to call back in a quieter environment + say( + resp, + bi.translate( + "Sorry, I didn't get that. Please call again in a more quiet environment." + ), + bi, + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +# uncomment for debugging: +# @router.post("/__/twilio/voice/status/") +# def twilio_voice_call_status(data: dict = fastapi_request_urlencoded_body): +# """Handle incoming Twilio voice call status update.""" + +# print("Twilio status update", data) + +# return Response(status_code=204) + + +@router.post("/__/twilio/voice/error/") +def twilio_voice_call_error(): + """If an unhandled error occurs in the voice call webhook, return a generic error message.""" + + resp = VoiceResponse() + resp.say( + "Sorry. This number has been incorrectly configured. Contact the bot integration owner or try again later." + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +@router.post("/__/twilio/sms/") +def twilio_sms( + background_tasks: BackgroundTasks, data: dict = fastapi_request_urlencoded_body +): + """Handle incoming Twilio SMS.""" + from daras_ai_v2.twilio_bot import TwilioSMS + from daras_ai_v2.bots import msg_handler + + phone_number = data["To"][0] + user_phone_number = data["From"][0] + + try: + bi = BotIntegration.objects.get( + twilio_phone_number=PhoneNumber.from_string(phone_number) + ) + except BotIntegration.DoesNotExist as e: + capture_exception(e) + return Response(status_code=404) + + convo, created = Conversation.objects.get_or_create( + bot_integration=bi, + twilio_phone_number=PhoneNumber.from_string(user_phone_number), + ) + bot = TwilioSMS( + sid=data["MessageSid"][0], + convo=convo, + text=data["Body"][0], + bi=bi, + ) + background_tasks.add_task(msg_handler, bot) + + resp = MessagingResponse() + + if created and bi.twilio_initial_text.strip(): + resp.message(bi.twilio_initial_text) + + if bi.twilio_waiting_text.strip(): + resp.message(bi.twilio_waiting_text) + else: + resp.message(bot.translate("Please wait while we process your request.")) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +@router.post("/__/twilio/sms/error/") +def twilio_sms_error(): + """If an unhandled error occurs in the SMS webhook, return a generic error message.""" + + resp = MessagingResponse() + resp.message( + "Sorry. This number has been incorrectly configured. Contact the bot integration owner or try again later." + ) + + return Response(str(resp), headers={"Content-Type": "text/xml"}) + + +def start_voice_call_session( + text: str, audio_url: str, bi: BotIntegration, user_phone_number: str +): + client = Client(bi.twilio_account_sid, bi.twilio_auth_token) + + resp = VoiceResponse() + + text = base64.b64encode(text.encode()).decode() if text else "N" + audio_url = base64.b64encode(audio_url.encode()).decode() if audio_url else "N" + + resp.redirect( + get_route_url( + twilio_voice_call_response, + dict(bi_id=bi.id, text=text, audio_url=audio_url), + ) + ) + + call = client.calls.create( + twiml=str(resp), + to=user_phone_number, + from_=bi.twilio_phone_number.as_e164, + ) + + return call + + +def twilio_connect( + current_run: SavedRun, + published_run: PublishedRun, + twilio_account_sid: str, + twilio_auth_token: str, + twilio_phone_number: str, + twilio_phone_number_sid: str, + billing_user: AppUser, +) -> BotIntegration: + """Connect a new bot integration to Twilio and return it. This will also setup the necessary webhooks for voice calls and SMS messages""" + + # setup webhooks so we can receive voice calls and SMS messages + client = Client(twilio_account_sid, twilio_auth_token) + client.incoming_phone_numbers(twilio_phone_number_sid).update( + sms_fallback_method="POST", + sms_fallback_url=get_route_url(twilio_sms_error), + sms_method="POST", + sms_url=get_route_url(twilio_sms), + # status_callback_method="POST", # uncomment for debugging + # status_callback=get_route_url(twilio_voice_call_status), + voice_fallback_method="POST", + voice_fallback_url=get_route_url(twilio_voice_call_error), + voice_method="POST", + voice_url=get_route_url(twilio_voice_call), + ) + + # create bot integration + return BotIntegration.objects.create( + name=published_run.title, + billing_account_uid=billing_user.uid, + platform=Platform.TWILIO, + streaming_enabled=False, + saved_run=current_run, + published_run=published_run, + by_line=billing_user.display_name, + descripton=published_run.notes, + photo_url=billing_user.photo_url, + website_url=billing_user.website_url, + user_language=current_run.state.get("user_language") or "en", + twilio_account_sid=twilio_account_sid, + twilio_auth_token=twilio_auth_token, + twilio_phone_number=PhoneNumber.from_string(twilio_phone_number), + twilio_phone_number_sid=twilio_phone_number_sid, + ) diff --git a/server.py b/server.py index 2d8cc5630..21bc876b7 100644 --- a/server.py +++ b/server.py @@ -45,6 +45,7 @@ stripe, broadcast_api, bots_api, + twilio_api, ) import url_shortener.routers as url_shortener @@ -62,6 +63,7 @@ app.include_router(url_shortener.app, include_in_schema=False) app.include_router(paypal.router, include_in_schema=False) app.include_router(stripe.router, include_in_schema=False) +app.include_router(twilio_api.router, include_in_schema=False) app.add_middleware( CORSMiddleware,