From 5f5331dd596858aa2944cddaafb773e7da6f86a4 Mon Sep 17 00:00:00 2001 From: quambene Date: Mon, 1 Apr 2024 17:58:28 +0200 Subject: [PATCH 01/11] Refactor write image --- src/cmd/query.rs | 16 ++++++++-- src/sources/image.rs | 69 +++++++++++++------------------------------- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/src/cmd/query.rs b/src/cmd/query.rs index 5d2ebaf..adba193 100644 --- a/src/cmd/query.rs +++ b/src/cmd/query.rs @@ -27,15 +27,25 @@ pub fn query(matches: &ArgMatches) -> Result<(), anyhow::Error> { } if matches.is_present(arg::SAVE) { + let save_dir = Path::new(arg::value(arg::SAVE_DIR, matches)?); + // If argument 'FILE_TYPE' is not present the default value 'csv' will be used match matches.value_of(arg::FILE_TYPE) { Some(file_type) => match file_type { "csv" => { - let save_dir = Path::new(arg::value(arg::SAVE_DIR, matches)?); sources::write_csv(&mut df_query, save_dir, now)?; } - x if x == "jpg" => sources::write_image(matches, df_query, x)?, - x if x == "png" => sources::write_image(matches, df_query, x)?, + "jpg" | "png" => { + let image_column = arg::value(arg::IMAGE_COLUMN, matches)?; + let image_name = arg::value(arg::IMAGE_NAME, matches)?; + sources::write_image( + save_dir, + image_column, + image_name, + df_query, + file_type, + )?; + } _ => { return Err(anyhow!( "Value '{}' not supported for argument '{}'", diff --git a/src/sources/image.rs b/src/sources/image.rs index aff8f75..e1d31d4 100644 --- a/src/sources/image.rs +++ b/src/sources/image.rs @@ -1,24 +1,19 @@ -use crate::arg; -use anyhow::{anyhow, Context}; -use clap::ArgMatches; +use anyhow::Context; use polars::prelude::{DataFrame, DataType, TakeRandom}; use std::{ fs::{self, File}, io::Write, - path::PathBuf, + path::Path, }; use uuid::Uuid; pub fn write_image( - matches: &ArgMatches, + target_dir: &Path, + image_column: &str, + image_name: &str, df: DataFrame, file_type: &str, ) -> Result<(), anyhow::Error> { - let target_dir = match matches.value_of(arg::SAVE_DIR) { - Some(save_dir) => PathBuf::from(save_dir), - None => return Err(anyhow!("Missing value for argument '{}'", arg::SAVE_DIR)), - }; - match target_dir.exists() { true => (), false => fs::create_dir(&target_dir).context(format!( @@ -27,29 +22,15 @@ pub fn write_image( ))?, } - let image_col = match matches.value_of(arg::IMAGE_COLUMN) { - Some(column_name) => column_name, - None => { - return Err(anyhow!( - "Missing value for argument '{}'", - arg::IMAGE_COLUMN - )) - } - }; - let name_col = match matches.value_of(arg::IMAGE_NAME) { - Some(column_name) => column_name, - None => return Err(anyhow!("Missing value for argument '{}'", arg::IMAGE_NAME)), - }; - for i in 0..df.height() { let data_type = df - .column(name_col) + .column(image_name) .context("Can't find column for image name")? .dtype(); let image_name = match data_type { DataType::Utf8 => df - .column(name_col) + .column(image_name) .context("Can't find column for image name")? .utf8() .context("Can't convert series to chunked array")? @@ -57,26 +38,19 @@ pub fn write_image( .map(|str| str.to_string()), DataType::Null => None, _ => Some( - df.column(name_col) + df.column(image_name) .context("Can't find column for image name")? .get(i)? .to_string(), ), }; - let image_name = match image_name { - Some(image_name) => image_name, - None => { - // Indicate the missing image name by a UUID - Uuid::new_v4().to_hyphenated().to_string() - } - }; - + let image_name = image_name.unwrap_or(Uuid::new_v4().to_hyphenated().to_string()); let target_file = image_name + "." + file_type; let target_path = target_dir.join(target_file); let image = df - .column(image_col) + .column(image_column) .context("Can't find column for images")? .list() .context("Can't convert series to chunked array")? @@ -84,20 +58,17 @@ pub fn write_image( println!("Save query result to file: {}", target_path.display()); - match image { - Some(image) => { - let bytes: Vec = image - .u8() - .context("Can't convert series to chunked array")? - .into_iter() - .map(|byte| byte.expect("Can't convert series to bytes")) - .collect(); + if let Some(image) = image { + let bytes = image + .u8() + .context("Can't convert series to chunked array")? + .into_iter() + .map(|byte| byte.expect("Can't convert series to bytes")) + .collect::>(); - let mut file = File::create(target_path).context("Unable to create file")?; - file.write_all(bytes.as_slice()) - .context("Unable to write file.")?; - } - None => continue, + let mut file = File::create(target_path).context("Unable to create file")?; + file.write_all(bytes.as_slice()) + .context("Unable to write file.")?; } } From caa8be3632cea4fbee95a9dcd86115b9ad953c02 Mon Sep 17 00:00:00 2001 From: quambene Date: Mon, 1 Apr 2024 18:00:04 +0200 Subject: [PATCH 02/11] Refactor write image II --- src/sources/image.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sources/image.rs b/src/sources/image.rs index e1d31d4..6942b60 100644 --- a/src/sources/image.rs +++ b/src/sources/image.rs @@ -14,12 +14,11 @@ pub fn write_image( df: DataFrame, file_type: &str, ) -> Result<(), anyhow::Error> { - match target_dir.exists() { - true => (), - false => fs::create_dir(&target_dir).context(format!( + if !target_dir.exists() { + fs::create_dir(&target_dir).context(format!( "Can't create directory: '{}'", target_dir.display() - ))?, + ))?; } for i in 0..df.height() { From add83ca6ceb2ce3c9cc41abaedc4cfbb7e90c1ae Mon Sep 17 00:00:00 2001 From: quambene Date: Tue, 2 Apr 2024 22:56:17 +0200 Subject: [PATCH 03/11] Update gitignore --- .gitignore | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index b1d5069..34a2268 100644 --- a/.gitignore +++ b/.gitignore @@ -17,32 +17,26 @@ rust-toolchain # Environment **/.env -### Keys +# Keys **/*.jks **/*.p8 **/*.pem -### Config files +# Config files **/*.yaml -### Releases -releases - -### Data -sent_emails -my-sent-emails -saved_queries -my-saved-queries +# Data **/*.csv **/*.jpg **/*.png **/*.pdf **/*.odt -### Allow test data +# Allow test data !test_data/* -### IDEs - # VS Code **/.vscode + +# Docker +docker-compose.yaml From 65ed68b73cfb71b2ffb0f0aa66ad4ae44bebb846 Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 01:42:53 +0200 Subject: [PATCH 04/11] Refactor query test --- tests/cmd/test_query.rs | 69 +++++++++++------------------------------ 1 file changed, 18 insertions(+), 51 deletions(-) diff --git a/tests/cmd/test_query.rs b/tests/cmd/test_query.rs index 1f3a345..dd99c18 100644 --- a/tests/cmd/test_query.rs +++ b/tests/cmd/test_query.rs @@ -4,63 +4,36 @@ - DB_USER - DB_PASSWORD - DB_NAME - - TEST_QUERY */ use assert_cmd::Command; use predicates::str; -use std::{env, fs}; +use std::fs; use tempfile::tempdir; #[test] -#[ignore] fn test_query_display() { - let test_query = env::var("TEST_QUERY").expect("Missing environment variable 'TEST_QUERY'"); + let test_query = "select email, first_name, last_name from account"; + println!("Execute 'pigeon query {test_query} --display'"); let mut cmd = Command::cargo_bin("pigeon").unwrap(); - cmd.args(["query", test_query.as_str(), "--display"]); - cmd.assert() - .success() - .stdout(str::contains("Display query result")); + cmd.args(["query", test_query, "--display"]); + cmd.assert().success().stdout(str::contains( + "Display query result: shape: (2, 3) +┌────────────────────────────┬────────────┬──────────────┐ +│ email ┆ first_name ┆ last_name │ +│ --- ┆ --- ┆ --- │ +│ str ┆ str ┆ str │ +╞════════════════════════════╪════════════╪══════════════╡ +│ marie@curie.com ┆ Marie ┆ Curie │ +│ alexandre@grothendieck.com ┆ Alexandre ┆ Grothendieck │ +└────────────────────────────┴────────────┴──────────────┘", + )); } #[test] -#[ignore] fn test_query_save() { - let test_query = env::var("TEST_QUERY").expect("Missing environment variable 'TEST_QUERY'"); - let temp_dir = tempdir().unwrap(); - let temp_path = temp_dir.path(); - assert!(temp_path.exists(), "Missing path: {}", temp_path.display()); - let save_dir = temp_path.to_str().unwrap(); - - println!("Execute 'pigeon query {test_query} --save'"); - let mut cmd = Command::cargo_bin("pigeon").unwrap(); - cmd.current_dir(save_dir); - cmd.args(["query", test_query.as_str(), "--save"]); - cmd.assert() - .success() - .stdout(str::contains("Save query result to file")); - - if let Ok(mut entries) = fs::read_dir(temp_path.join("saved_queries")) { - let dir_entry = entries.find_map(|entry| { - if let Ok(entry) = entry { - if entry.file_name().to_str().is_some_and(|file_name| { - file_name.contains("query") && file_name.ends_with(".csv") - }) { - return Some(entry); - } - } - - None - }); - assert!(dir_entry.is_some()); - } -} - -#[test] -#[ignore] -fn test_query_save_dir() { - let test_query = env::var("TEST_QUERY").expect("Missing environment variable 'TEST_QUERY'"); + let test_query = "select email, first_name, last_name from account"; let temp_dir = tempdir().unwrap(); let temp_path = temp_dir.path(); assert!(temp_path.exists(), "Missing path: {}", temp_path.display()); @@ -68,16 +41,10 @@ fn test_query_save_dir() { println!("Execute 'pigeon query {test_query} --save --save-dir {save_dir}'"); let mut cmd = Command::cargo_bin("pigeon").unwrap(); - cmd.args([ - "query", - test_query.as_str(), - "--save", - "--save-dir", - save_dir, - ]); + cmd.args(["query", test_query, "--save", "--save-dir", save_dir]); cmd.assert() .success() - .stdout(str::contains("Save query result to file")); + .stdout(str::contains("Save query result to file:")); if let Ok(mut entries) = fs::read_dir(temp_path) { let dir_entry = entries.find_map(|entry| { From 980dba3dc546a4483d6089756369cb7ae80ae69c Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 02:02:27 +0200 Subject: [PATCH 05/11] Format code --- src/cmd/query.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cmd/query.rs b/src/cmd/query.rs index adba193..6e99f94 100644 --- a/src/cmd/query.rs +++ b/src/cmd/query.rs @@ -18,7 +18,6 @@ pub fn query(matches: &ArgMatches) -> Result<(), anyhow::Error> { let now = Utc::now(); let conn_vars = ConnVars::from_env()?; let ssh_tunnel = matches.value_of(arg::SSH_TUNNEL); - let connection = DbConnection::new(&conn_vars, ssh_tunnel)?; let mut df_query = sources::query_postgres(&connection, query)?; From a4d4552fdcb2a6f485cc8830989af8902bec9323 Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 02:13:00 +0200 Subject: [PATCH 06/11] Add docker compose yaml --- .gitignore | 6 +++--- README.md | 36 +++++++++++++++++++----------------- docker-compose.yaml | 15 +++++++++++++++ 3 files changed, 37 insertions(+), 20 deletions(-) create mode 100644 docker-compose.yaml diff --git a/.gitignore b/.gitignore index 34a2268..29149cb 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ rust-toolchain # Config files **/*.yaml +# Allow docker-compose.yaml +!docker-compose.yaml + # Data **/*.csv **/*.jpg @@ -37,6 +40,3 @@ rust-toolchain # VS Code **/.vscode - -# Docker -docker-compose.yaml diff --git a/README.md b/README.md index 76e77b8..14c32af 100644 --- a/README.md +++ b/README.md @@ -445,23 +445,25 @@ Sendgrid | equals monthly limit ## Testing -Some integration tests require a locally running database, and an AWS SES account. -Specify the following environment variables: - -- SMTP - - `SMTP_SERVER` - - `SMTP_USERNAME` - - `SMTP_PASSWORD` -- AWS SES - - `AWS_ACCESS_KEY_ID` - - `AWS_SECRET_ACCESS_KEY` - - `AWS_REGION` -- Postgres - - `DB_HOST` - - `DB_PORT` - - `DB_USER` - - `DB_PASSWORD` - - `DB_NAME` +Some integration tests require a locally running database, and an AWS SES +account: + +1. Specify the following environment variables: + - SMTP + - `SMTP_SERVER` + - `SMTP_USERNAME` + - `SMTP_PASSWORD` + - AWS SES + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_REGION` + - Postgres + - `DB_HOST` + - `DB_PORT` + - `DB_USER` + - `DB_PASSWORD` + - `DB_NAME` +2. Set up a temporary postgres db: `docker-compose run --rm --service-ports postgres` ``` bash # Run unit tests and integration tests diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..deb4c2f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,15 @@ +version: '3' +services: + postgres: + env_file: + - .env + volumes: + - ./test_data:/docker-entrypoint-initdb.d + image: postgres:16-alpine + restart: "no" + ports: + - "${DB_PORT}:5432" + environment: + POSTGRES_USER: "${DB_USER}" + POSTGRES_PASSWORD: "${DB_PASSWORD}" + POSTGRES_DB: "${DB_NAME}" From 005fae16ad3203649db41c7826b4733cf633f5f0 Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 02:13:41 +0200 Subject: [PATCH 07/11] Populate test data --- test_data/init.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 test_data/init.sql diff --git a/test_data/init.sql b/test_data/init.sql new file mode 100644 index 0000000..5d146ed --- /dev/null +++ b/test_data/init.sql @@ -0,0 +1,10 @@ +CREATE TABLE account ( + id serial primary key, + first_name character varying, + last_name character varying, + email character varying NOT NULL +); + +COPY account(first_name, last_name, email) +FROM + '/docker-entrypoint-initdb.d/contacts.csv' DELIMITER ',' CSV HEADER; \ No newline at end of file From 167b314fe35002633b77af65177a2bd97ab1491e Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 02:29:24 +0200 Subject: [PATCH 08/11] Fix clippy warnings --- src/sources/image.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sources/image.rs b/src/sources/image.rs index 6942b60..834d261 100644 --- a/src/sources/image.rs +++ b/src/sources/image.rs @@ -15,7 +15,7 @@ pub fn write_image( file_type: &str, ) -> Result<(), anyhow::Error> { if !target_dir.exists() { - fs::create_dir(&target_dir).context(format!( + fs::create_dir(target_dir).context(format!( "Can't create directory: '{}'", target_dir.display() ))?; From 2428836686500aefc9be6350404e0fe3e2b994ef Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 03:04:06 +0200 Subject: [PATCH 09/11] Fix pipeline --- .github/workflows/rust-ci.yml | 14 +++++++++++--- docker-compose.yaml | 2 -- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 2d98362..1785314 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -51,26 +51,34 @@ jobs: - name: cargo test --lib run: cargo test --lib --locked integration-test: + env: + DB_HOST: 127.0.0.1 + DB_PORT: 5432 + DB_USER: pigeon + DB_PASSWORD: pigeon-pw + DB_NAME: pigeon-db runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable + - name: Start container + run: docker compose up -d - name: cargo test --test '*' run: cargo test --test '*' --locked + - name: Stop container + run: docker compose down -v os-test: runs-on: ${{ matrix.os }} name: os-test / ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [macos-latest] + os: [macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - name: cargo test --lib run: cargo test --lib --locked - - name: cargo test --test '*' - run: cargo test --test '*' --locked doc-test: runs-on: ubuntu-latest steps: diff --git a/docker-compose.yaml b/docker-compose.yaml index deb4c2f..2788daf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,8 +1,6 @@ version: '3' services: postgres: - env_file: - - .env volumes: - ./test_data:/docker-entrypoint-initdb.d image: postgres:16-alpine From 706806cbc5c7171b0c0ff68f07a786a5095c3c1b Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 03:17:36 +0200 Subject: [PATCH 10/11] Fix pipeline II --- .github/workflows/rust-ci.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 1785314..abb8b62 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -5,6 +5,11 @@ on: pull_request: env: CARGO_TERM_COLOR: always + DB_HOST: 127.0.0.1 + DB_PORT: 5432 + DB_USER: pigeon + DB_PASSWORD: pigeon-pw + DB_NAME: pigeon-db jobs: check: runs-on: ubuntu-latest @@ -51,12 +56,6 @@ jobs: - name: cargo test --lib run: cargo test --lib --locked integration-test: - env: - DB_HOST: 127.0.0.1 - DB_PORT: 5432 - DB_USER: pigeon - DB_PASSWORD: pigeon-pw - DB_NAME: pigeon-db runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -73,12 +72,18 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest, windows-latest] + os: [macos-latest] steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - name: cargo test --lib run: cargo test --lib --locked + - name: Start container + run: docker compose up -d + - name: cargo test --test '*' + run: cargo test --test '*' --locked + - name: Stop container + run: docker compose down -v doc-test: runs-on: ubuntu-latest steps: @@ -98,8 +103,12 @@ jobs: - name: cargo generate-lockfile if: hashFiles('Cargo.lock') == '' run: cargo generate-lockfile + - name: Start container + run: docker compose up -d - name: cargo llvm-cov run: cargo llvm-cov --locked --all-features --lcov --output-path lcov.info + - name: Stop container + run: docker compose down -v - name: Upload to codecov.io uses: codecov/codecov-action@v3 with: From d0a96fb29a42b93a29cddd158da5de1c8b6de66d Mon Sep 17 00:00:00 2001 From: quambene Date: Wed, 3 Apr 2024 03:23:03 +0200 Subject: [PATCH 11/11] Fix pipeline III --- .github/workflows/rust-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index abb8b62..0d05514 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -78,12 +78,6 @@ jobs: - uses: dtolnay/rust-toolchain@stable - name: cargo test --lib run: cargo test --lib --locked - - name: Start container - run: docker compose up -d - - name: cargo test --test '*' - run: cargo test --test '*' --locked - - name: Stop container - run: docker compose down -v doc-test: runs-on: ubuntu-latest steps: