Skip to content

Commit

Permalink
add option to truncate tables, show db path, ensure profile and db op…
Browse files Browse the repository at this point in the history
…tions are mutually exclusive
  • Loading branch information
aprxi committed Aug 9, 2024
1 parent 277baad commit 48147d6
Show file tree
Hide file tree
Showing 7 changed files with 105 additions and 11 deletions.
5 changes: 5 additions & 0 deletions lumni/src/apps/builtin/llm/prompt/src/chat/db/connector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,11 @@ impl DatabaseConnector {
tx.commit()?;
Ok(result)
}

pub fn vacuum(&self) -> Result<(), SqliteError> {
self.connection.execute("VACUUM", [])?;
Ok(())
}
}

#[derive(Debug)]
Expand Down
3 changes: 1 addition & 2 deletions lumni/src/apps/builtin/llm/prompt/src/chat/db/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ mod model;
mod store;
mod user_profiles;

pub use connector::DatabaseOperationError;
pub use conversations::ConversationDbHandler;
pub use lumni::Timestamp;
pub use model::{ModelIdentifier, ModelSpec};
Expand All @@ -19,7 +18,7 @@ pub use user_profiles::UserProfileDbHandler;

pub use super::ConversationCache;
use super::PromptRole;
pub use crate::external as lumni;
use crate::external as lumni;

#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct ModelServerName(pub String);
Expand Down
46 changes: 44 additions & 2 deletions lumni/src/apps/builtin/llm/prompt/src/chat/db/store.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::{Arc, OnceLock};

use lumni::api::error::ApplicationError;
use rusqlite::{Error as SqliteError, OptionalExtension};
use tokio::sync::Mutex as TokioMutex;

Expand All @@ -12,20 +13,36 @@ use super::{
Conversation, ConversationId, ConversationStatus, Message, MessageId,
ModelIdentifier,
};
use crate::external as lumni;

static PROMPT_SQLITE_FILEPATH: OnceLock<PathBuf> = OnceLock::new();

pub struct ConversationDatabase {
db: Arc<TokioMutex<DatabaseConnector>>,
encryption_handler: Option<Arc<EncryptionHandler>>,
}

impl ConversationDatabase {
pub fn new(sqlite_file: &PathBuf) -> Result<Self, SqliteError> {
pub fn new(sqlite_file: &PathBuf) -> Result<Self, DatabaseOperationError> {
PROMPT_SQLITE_FILEPATH
.set(sqlite_file.clone())
.map_err(|_| {
DatabaseOperationError::ApplicationError(
ApplicationError::DatabaseError(
"Failed to set the SQLite filepath".to_string(),
),
)
})?;
Ok(Self {
db: Arc::new(TokioMutex::new(DatabaseConnector::new(sqlite_file)?)),
encryption_handler: None,
})
}

pub fn get_filepath() -> &'static PathBuf {
PROMPT_SQLITE_FILEPATH.get().expect("Filepath not set")
}

pub fn get_conversation_handler(
&self,
conversation_id: Option<ConversationId>,
Expand All @@ -48,6 +65,31 @@ impl ConversationDatabase {
)
}

pub async fn truncate_and_vacuum(&self) -> Result<(), ApplicationError> {
let mut db = self.db.lock().await;
db.process_queue_with_result(|tx| {
// Disable foreign key constraints temporarily
tx.execute("PRAGMA foreign_keys = OFF", [])?;

// Truncate all tables except metadata, user_profiles, and models
tx.execute_batch(
"
DELETE FROM conversation_tags;
DELETE FROM tags;
DELETE FROM attachments;
DELETE FROM messages;
DELETE FROM conversations;
",
)?;

// Re-enable foreign key constraints
tx.execute("PRAGMA foreign_keys = ON", [])?;
Ok(())
})?;
db.vacuum()?; // Reclaim unused space
Ok(())
}

pub async fn fetch_last_conversation_id(
&self,
) -> Result<Option<ConversationId>, DatabaseOperationError> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::sync::Arc;
use base64::engine::general_purpose;
use base64::Engine as _;
use lumni::api::error::{ApplicationError, EncryptionError};
use rusqlite::{params, Error as SqliteError, OptionalExtension};
use rusqlite::{params, OptionalExtension};
use serde_json::{Map, Value as JsonValue};
use sha2::{Digest, Sha256};
use tokio::sync::Mutex as TokioMutex;
Expand Down
3 changes: 1 addition & 2 deletions lumni/src/apps/builtin/llm/prompt/src/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ pub fn parse_cli_arguments(spec: ApplicationSpec) -> Command {
Arg::new("profile")
.long("profile")
.short('p')
.help("Use a specific profile")
.global(true),
.help("Use a specific profile"),
)
.arg(
Arg::new("system")
Expand Down
49 changes: 46 additions & 3 deletions lumni/src/apps/builtin/llm/prompt/src/cli/subcommands/db.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use std::sync::Arc;

use clap::{Arg, ArgMatches, Command};
use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command};
use lumni::api::error::ApplicationError;

use super::ConversationDatabase;
Expand All @@ -9,6 +9,14 @@ use crate::external as lumni;
pub fn create_db_subcommand() -> Command {
Command::new("db")
.about("Query the conversation database")
.arg(
Arg::new("path")
.long("path")
.short('p')
.help("Path to the SQLite database file")
.action(ArgAction::SetTrue)
.value_name("FILE"),
)
.arg(
Arg::new("list")
.long("list")
Expand All @@ -17,28 +25,63 @@ pub fn create_db_subcommand() -> Command {
.num_args(0..=1)
.value_name("LIMIT"),
)
.arg(
Arg::new("last")
.long("last")
.short('L')
.help("Print the last conversation")
.action(ArgAction::SetTrue),
)
.arg(
Arg::new("id")
.long("id")
.short('i')
.help("Fetch a specific conversation by ID")
.num_args(1),
)
.arg(
Arg::new("truncate")
.long("truncate")
.help("Truncate all tables and vacuum the database")
.action(ArgAction::SetTrue),
)
.group(
ArgGroup::new("db_group")
.args(["path", "list", "last", "id", "truncate"])
.required(false)
.multiple(false),
)
}

pub async fn handle_db_subcommand(
db_matches: &ArgMatches,
db_conn: &Arc<ConversationDatabase>,
) -> Result<(), ApplicationError> {
if db_matches.contains_id("list") {
if db_matches.get_flag("truncate") {
let result = db_conn.truncate_and_vacuum().await;
if result.is_ok() {
println!("Database tables truncated");
}
result
} else if db_matches.get_flag("path") {
let filepath = ConversationDatabase::get_filepath();
println!("Sqlite filepath: {:?}", filepath);
Ok(())
} else if db_matches.contains_id("list") {
let limit = match db_matches.get_one::<String>("list") {
Some(value) => value.parse().unwrap_or(20),
None => 20,
};
db_conn.print_conversation_list(limit).await
} else if let Some(id_value) = db_matches.get_one::<String>("id") {
db_conn.print_conversation_by_id(id_value).await
} else {
} else if db_matches.get_flag("last") {
// If any arguments are present but not handled above, print the last conversation
db_conn.print_last_conversation().await
} else {
// If no arguments are provided, print the help message
let mut db_command = create_db_subcommand();
db_command.print_help()?;
Ok(())
}
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use clap::{Arg, ArgAction, ArgMatches, Command};
use clap::{Arg, ArgAction, ArgGroup, ArgMatches, Command};
use lumni::api::error::ApplicationError;
use serde_json::Value as JsonValue;

Expand Down Expand Up @@ -52,6 +52,12 @@ pub fn create_profile_subcommand() -> Command {
.help("List all profiles")
.action(ArgAction::SetTrue),
)
.group(
ArgGroup::new("profile_group")
.args(["set", "get", "show", "delete", "default", "list"])
.required(false)
.multiple(false),
)
}

pub async fn handle_profile_subcommand(
Expand Down

0 comments on commit 48147d6

Please sign in to comment.