From a2d0b27737432a9f0689d7db6e7ba9f92cc42871 Mon Sep 17 00:00:00 2001 From: tison Date: Fri, 9 Aug 2024 16:23:04 +0800 Subject: [PATCH] feat: allow rolling file at size (#32) Signed-off-by: tison --- .github/workflows/ci.yml | 4 +- Cargo.toml | 15 ++-- examples/no_color_stdio.rs | 37 -------- examples/rolling_file.rs | 6 +- src/append/rolling_file/rolling.rs | 140 ++++++++++++++++++----------- 5 files changed, 99 insertions(+), 103 deletions(-) delete mode 100644 examples/no_color_stdio.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a1b9a1..d5d3f80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,10 +63,10 @@ jobs: - name: Run examples run: | cargo run --example simple_stdio - cargo run --example fn_layout_filter - cargo run --features="no-color" --example no_color_stdio + cargo run --features="no-color" --example simple_stdio cargo run --features="json" --example json_stdio cargo run --features="json,rolling_file" --example rolling_file + cargo run --example fn_layout_filter required: name: Required diff --git a/Cargo.toml b/Cargo.toml index 2f88e2b..f4564a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/tisonkun/logforth" rust-version = "1.71.0" -version = "0.7.3" +version = "0.8.0" categories = ["development-tools::debugging"] keywords = ["logging", "log", "opentelemetry", "fastrace"] @@ -80,19 +80,10 @@ optional = true version = "0.24" ## Examples -[[example]] -name = "fn_layout_filter" -path = "examples/fn_layout_filter.rs" - [[example]] name = "simple_stdio" path = "examples/simple_stdio.rs" -[[example]] -name = "no_color_stdio" -path = "examples/no_color_stdio.rs" -required-features = ["no-color"] - [[example]] name = "json_stdio" path = "examples/json_stdio.rs" @@ -102,3 +93,7 @@ required-features = ["json"] name = "rolling_file" path = "examples/rolling_file.rs" required-features = ["rolling_file", "json"] + +[[example]] +name = "fn_layout_filter" +path = "examples/fn_layout_filter.rs" diff --git a/examples/no_color_stdio.rs b/examples/no_color_stdio.rs deleted file mode 100644 index 55cb772..0000000 --- a/examples/no_color_stdio.rs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2024 tison -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use log::LevelFilter; -use logforth::append; -use logforth::layout::TextLayout; -use logforth::Dispatch; -use logforth::Logger; - -fn main() { - Logger::new() - .dispatch( - Dispatch::new() - .filter(LevelFilter::Trace) - .layout(TextLayout::default()) - .append(append::Stdout), - ) - .apply() - .unwrap(); - - log::error!("Hello error!"); - log::warn!("Hello warn!"); - log::info!("Hello info!"); - log::debug!("Hello debug!"); - log::trace!("Hello trace!"); -} diff --git a/examples/rolling_file.rs b/examples/rolling_file.rs index 7244956..c6bef14 100644 --- a/examples/rolling_file.rs +++ b/examples/rolling_file.rs @@ -17,7 +17,9 @@ use logforth::append::rolling_file::NonBlockingBuilder; use logforth::append::rolling_file::RollingFile; use logforth::append::rolling_file::RollingFileWriter; use logforth::append::rolling_file::Rotation; +use logforth::append::Stdout; use logforth::layout::JsonLayout; +use logforth::layout::TextLayout; use logforth::Dispatch; use logforth::Logger; @@ -26,7 +28,8 @@ fn main() { .rotation(Rotation::Minutely) .filename_prefix("example") .filename_suffix("log") - .max_log_files(2) + .max_log_files(10) + .max_file_size(1024 * 1024) .build("logs") .unwrap(); let (writer, _guard) = NonBlockingBuilder::default().finish(rolling); @@ -38,6 +41,7 @@ fn main() { .layout(JsonLayout) .append(RollingFile::new(writer)), ) + .dispatch(Dispatch::new().layout(TextLayout::default()).append(Stdout)) .apply() .unwrap(); diff --git a/src/append/rolling_file/rolling.rs b/src/append/rolling_file/rolling.rs index a8c2cd4..d50da15 100644 --- a/src/append/rolling_file/rolling.rs +++ b/src/append/rolling_file/rolling.rs @@ -19,8 +19,6 @@ use std::io; use std::io::Write; use std::path::Path; use std::path::PathBuf; -use std::sync::atomic::AtomicUsize; -use std::sync::atomic::Ordering; use anyhow::Context; use parking_lot::RwLock; @@ -52,11 +50,19 @@ impl Write for RollingFileWriter { fn write(&mut self, buf: &[u8]) -> io::Result { let now = self.now(); let writer = self.writer.get_mut(); - if let Some(current_time) = self.state.should_rollover(now) { - self.state.advance_date(now, current_time); - self.state.refresh_writer(now, writer); + if self.state.should_rollover_on_date(now) { + self.state.advance_date(now); + self.state.refresh_writer(now, 0, writer); } - writer.write(buf) + if self.state.should_rollover_on_size() { + let cnt = self.state.advance_cnt(); + self.state.refresh_writer(now, cnt, writer); + } + + writer.write(buf).map(|n| { + self.state.current_filesize += n; + n + }) } fn flush(&mut self) -> io::Result<()> { @@ -70,6 +76,7 @@ pub struct RollingFileWriterBuilder { rotation: Rotation, prefix: Option, suffix: Option, + max_size: usize, max_files: Option, } @@ -86,6 +93,7 @@ impl RollingFileWriterBuilder { rotation: Rotation::Never, prefix: None, suffix: None, + max_size: usize::MAX, max_files: None, } } @@ -124,16 +132,26 @@ impl RollingFileWriterBuilder { self } + /// Sets the maximum size of a log file in bytes. + #[must_use] + pub fn max_file_size(mut self, n: usize) -> Self { + self.max_size = n; + self + } + pub fn build(self, dir: impl AsRef) -> anyhow::Result { let Self { rotation, prefix, suffix, + max_size, max_files, } = self; let directory = dir.as_ref().to_path_buf(); let now = OffsetDateTime::now_utc(); - let (state, writer) = State::new(now, rotation, directory, prefix, suffix, max_files)?; + let (state, writer) = State::new( + now, rotation, directory, prefix, suffix, max_size, max_files, + )?; Ok(RollingFileWriter { state, writer }) } } @@ -145,7 +163,11 @@ struct State { log_filename_suffix: Option, date_format: Vec>, rotation: Rotation, - next_date: AtomicUsize, + current_date: OffsetDateTime, + current_count: usize, + current_filesize: usize, + next_date: usize, + max_size: usize, max_files: Option, } @@ -156,6 +178,7 @@ impl State { dir: impl AsRef, log_filename_prefix: Option, log_filename_suffix: Option, + max_size: usize, max_files: Option, ) -> anyhow::Result<(Self, RwLock)> { let log_dir = dir.as_ref().to_path_buf(); @@ -163,26 +186,32 @@ impl State { let next_date = rotation .next_date(&now) .map(|date| date.unix_timestamp() as usize) - .map(AtomicUsize::new) - .unwrap_or(AtomicUsize::new(0)); + .unwrap_or(0); + + let current_date = now; + let current_count = 0; + let current_filesize = 0; let state = State { log_dir, log_filename_prefix, log_filename_suffix, date_format, + current_date, + current_count, + current_filesize, next_date, rotation, + max_size, max_files, }; - let filename = state.join_date(&now); - let file = open_file(state.log_dir.as_ref(), &filename)?; + let file = state.create_log_writer(now, 0)?; let writer = RwLock::new(file); Ok((state, writer)) } - fn join_date(&self, date: &OffsetDateTime) -> String { + fn join_date(&self, date: &OffsetDateTime, cnt: usize) -> String { let date = date.format(&self.date_format).expect( "failed to format OffsetDateTime; this is a bug in logforth rolling file appender", ); @@ -192,16 +221,33 @@ impl State { &self.log_filename_prefix, &self.log_filename_suffix, ) { - (&Rotation::Never, Some(filename), None) => filename.to_string(), - (&Rotation::Never, Some(filename), Some(suffix)) => format!("{}.{}", filename, suffix), - (&Rotation::Never, None, Some(suffix)) => suffix.to_string(), - (_, Some(filename), Some(suffix)) => format!("{}.{}.{}", filename, date, suffix), - (_, Some(filename), None) => format!("{}.{}", filename, date), - (_, None, Some(suffix)) => format!("{}.{}", date, suffix), - (_, None, None) => date, + (&Rotation::Never, Some(filename), None) => format!("{filename}.{cnt}"), + (&Rotation::Never, Some(filename), Some(suffix)) => { + format!("{filename}.{cnt}.{suffix}") + } + (&Rotation::Never, None, Some(suffix)) => format!("{cnt}.{suffix}"), + (_, Some(filename), Some(suffix)) => format!("{filename}.{date}.{cnt}.{suffix}"), + (_, Some(filename), None) => format!("{filename}.{date}.{cnt}"), + (_, None, Some(suffix)) => format!("{date}.{cnt}.{suffix}"), + (_, None, None) => format!("{date}.{cnt}"), } } + fn create_log_writer(&self, now: OffsetDateTime, cnt: usize) -> anyhow::Result { + fs::create_dir_all(&self.log_dir).context("failed to create log directory")?; + let filename = self.join_date(&now, cnt); + if let Some(max_files) = self.max_files { + if let Err(err) = self.delete_oldest_logs(max_files) { + eprintln!("failed to delete oldest logs: {err}"); + } + } + OpenOptions::new() + .append(true) + .create(true) + .open(self.log_dir.join(filename)) + .context("failed to create log file") + } + fn delete_oldest_logs(&self, max_files: usize) -> anyhow::Result<()> { let read_dir = fs::read_dir(&self.log_dir) .with_context(|| format!("failed to read log dir: {}", self.log_dir.display()))?; @@ -261,16 +307,8 @@ impl State { Ok(()) } - fn refresh_writer(&self, now: OffsetDateTime, file: &mut File) { - let filename = self.join_date(&now); - - if let Some(max_files) = self.max_files { - if let Err(err) = self.delete_oldest_logs(max_files) { - eprintln!("failed to delete oldest logs: {err}"); - } - } - - match open_file(&self.log_dir, &filename) { + fn refresh_writer(&self, now: OffsetDateTime, cnt: usize, file: &mut File) { + match self.create_log_writer(now, cnt) { Ok(new_file) => { if let Err(err) = file.flush() { eprintln!("failed to flush previous writer: {err}"); @@ -281,40 +319,37 @@ impl State { } } - fn should_rollover(&self, date: OffsetDateTime) -> Option { - let next_date = self.next_date.load(Ordering::Acquire); - + fn should_rollover_on_date(&self, date: OffsetDateTime) -> bool { + let next_date = self.next_date; if next_date == 0 { - None - } else if date.unix_timestamp() as usize >= next_date { - Some(next_date) + false } else { - None + date.unix_timestamp() as usize >= next_date } } - fn advance_date(&self, now: OffsetDateTime, current: usize) -> bool { - let next_date = self + fn should_rollover_on_size(&self) -> bool { + self.current_filesize >= self.max_size + } + + fn advance_cnt(&mut self) -> usize { + self.current_count += 1; + self.current_filesize = 0; + self.current_count + } + + fn advance_date(&mut self, now: OffsetDateTime) { + self.current_date = now; + self.current_count = 0; + self.current_filesize = 0; + self.next_date = self .rotation .next_date(&now) .map(|date| date.unix_timestamp() as usize) .unwrap_or(0); - self.next_date - .compare_exchange(current, next_date, Ordering::AcqRel, Ordering::Acquire) - .is_ok() } } -fn open_file(dir: &Path, filename: &str) -> anyhow::Result { - fs::create_dir_all(dir).context("failed to create log directory")?; - - let mut open_options = OpenOptions::new(); - open_options.append(true).create(true); - open_options - .open(dir.join(filename)) - .context("failed to create log file") -} - /// Defines a fixed period for rolling of a log file. #[derive(Clone, Eq, PartialEq, Debug)] pub enum Rotation { @@ -326,7 +361,6 @@ pub enum Rotation { Daily, /// No Rotation Never, - // TODO(tisonkun): consider support rotating on file size exceeding a threshold. } impl Rotation {