Rust: make the cli flags override automatic

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "shared/tree-sitter-extractor",
     "ruby/extractor",
     "rust/extractor",
+    "rust/extractor/macros",
 ]
 
 [patch.crates-io]

MODULE.bazel

@@ -60,6 +60,7 @@ r.from_cargo(
         "//:Cargo.toml",
         "//ruby/extractor:Cargo.toml",
         "//rust/extractor:Cargo.toml",
+        "//rust/extractor/macros:Cargo.toml",
         "//shared/tree-sitter-extractor:Cargo.toml",
     ],
 )

ruby/extractor/BUILD.bazel

@@ -12,6 +12,6 @@ codeql_rust_binary(
     deps = all_crate_deps(
         normal = True,
     ) + [
-        "//shared/tree-sitter-extractor:codeql-extractor",
+        "//shared/tree-sitter-extractor",
     ],
 )

rust/extractor/BUILD.bazel

@@ -7,11 +7,13 @@ codeql_rust_binary(
     aliases = aliases(),
     proc_macro_deps = all_crate_deps(
         proc_macro = True,
-    ),
+    ) + [
+        "//rust/extractor/macros",
+    ],
     visibility = ["//rust:__subpackages__"],
     deps = all_crate_deps(
         normal = True,
     ) + [
-        "//shared/tree-sitter-extractor:codeql-extractor",
+        "//shared/tree-sitter-extractor",
     ],
 )

rust/extractor/Cargo.toml

@@ -22,7 +22,6 @@ serde = "1.0.209"
 serde_with = "3.9.0"
 stderrlog = "0.6.0"
 triomphe = "0.1.13"
-# Ideally, we'd like to pull this in via a relative path.
-# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525
-# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory.
+argfile = "0.2.1"
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
+rust-extractor-macros = { path = "macros" }

rust/extractor/macros/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@rules_rust//rust:defs.bzl", "rust_proc_macro")
+load("@tree_sitter_extractors_deps//:defs.bzl", "aliases", "all_crate_deps")
+
+rust_proc_macro(
+    name = "rust_extractor_macros",
+    srcs = glob(["src/**/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    deps = all_crate_deps(
+        normal = True,
+    ),
+)
+
+alias(
+    name = "macros",
+    actual = "rust_extractor_macros",
+    visibility = ["//rust:__subpackages__"],
+)

rust/extractor/macros/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "rust-extractor-macros"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+quote = "1.0.37"
+syn = { version = "2.0.77", features = ["full"] }

rust/extractor/macros/src/lib.rs

@@ -0,0 +1,56 @@
+use proc_macro::TokenStream;
+use quote::{format_ident, quote};
+
+/// Allow all fields in the extractor config to be also overrideable by extractor CLI flags
+#[proc_macro_attribute]
+pub fn extractor_cli_config(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    let ast = syn::parse_macro_input!(item as syn::ItemStruct);
+    let name = &ast.ident;
+    let new_name = format_ident!("Cli{}", name);
+    let fields: Vec<_> = ast
+        .fields
+        .iter()
+        .map(|f| {
+            let id = f.ident.as_ref().unwrap();
+            let ty = &f.ty;
+            if let syn::Type::Path(p) = ty {
+                if p.path.is_ident(&format_ident!("bool")) {
+                    return quote! {
+                        #[arg(long)]
+                        #[serde(skip_serializing_if="<&bool>::not")]
+                        #id: bool,
+                    };
+                }
+            }
+            if id == &format_ident!("verbose") {
+                quote! {
+                    #[arg(long, short, action=clap::ArgAction::Count)]
+                    #[serde(skip_serializing_if="u8::is_zero")]
+                    #id: u8,
+                }
+            } else if id == &format_ident!("inputs") {
+                quote! {
+                    #id: #ty,
+                }
+            } else {
+                quote! {
+                    #[arg(long)]
+                    #id: Option<#ty>,
+                }
+            }
+        })
+        .collect();
+    let gen = quote! {
+        #[serde_with::apply(_ => #[serde(default)])]
+        #[derive(Debug, Deserialize, Default)]
+        #ast
+
+        #[serde_with::skip_serializing_none]
+        #[derive(clap::Parser, Serialize)]
+        #[command(about, long_about = None)]
+        struct #new_name {
+            #(#fields)*
+        }
+    };
+    gen.into()
+}

rust/extractor/src/config.rs

@@ -1,14 +1,17 @@
 use anyhow::Context;
-use clap::{ArgAction, Parser, ValueEnum};
+use clap::Parser;
 use codeql_extractor::trap;
 use figment::{
     providers::{Env, Serialized},
     Figment,
 };
+use num_traits::Zero;
+use rust_extractor_macros::extractor_cli_config;
 use serde::{Deserialize, Serialize};
+use std::ops::Not;
 use std::path::PathBuf;
 
-#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, ValueEnum)]
+#[derive(Debug, PartialEq, Eq, Default, Serialize, Deserialize, Clone, Copy, clap::ValueEnum)]
 #[serde(rename_all = "lowercase")]
 #[clap(rename_all = "lowercase")]
 pub enum Compression {
@@ -26,8 +29,7 @@ impl From<Compression> for trap::Compression {
     }
 }
 
-#[serde_with::apply(_ => #[serde(default)])]
-#[derive(Debug, Deserialize, Default)]
+#[extractor_cli_config]
 pub struct Config {
     pub scratch_dir: PathBuf,
     pub trap_dir: PathBuf,
@@ -38,40 +40,13 @@ pub struct Config {
     pub inputs: Vec<PathBuf>,
 }
 
-#[serde_with::apply(_ => #[serde(skip_serializing_if = "is_default")])]
-#[derive(clap::Parser, Serialize)]
-#[command(about, long_about = None)]
-struct CliArgs {
-    #[arg(long)]
-    scratch_dir: Option<PathBuf>,
-    #[arg(long)]
-    trap_dir: Option<PathBuf>,
-    #[arg(long)]
-    source_archive_dir: Option<PathBuf>,
-    #[arg(long)]
-    compression: Option<Compression>,
-    #[arg(short, long, action = ArgAction::Count)]
-    verbose: u8,
-    #[arg(long)]
-    inputs_file: Option<PathBuf>,
-
-    inputs: Vec<PathBuf>,
-}
-
-fn is_default<T: Default + PartialEq>(t: &T) -> bool {
-    *t == Default::default()
-}
-
 impl Config {
     pub fn extract() -> anyhow::Result<Config> {
-        let mut cli_args = CliArgs::parse();
-        if let Some(inputs_file) = cli_args.inputs_file.take() {
-            let inputs_list = std::fs::read_to_string(inputs_file).context("reading file list")?;
-            cli_args
-                .inputs
-                .extend(inputs_list.split_terminator("\n").map(PathBuf::from));
-        }
+        let args = argfile::expand_args(argfile::parse_fromfile, argfile::PREFIX)
+            .context("expanding parameter files")?;
+        let cli_args = CliConfig::parse_from(args);
         Figment::new()
+            .merge(Env::prefixed("CODEQL_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_"))
             .merge(Env::prefixed("CODEQL_EXTRACTOR_RUST_OPTION_"))
             .merge(Serialized::defaults(cli_args))

rust/lint.py

@@ -5,12 +5,14 @@
 import shutil
 import sys
 
-extractor_dir = pathlib.Path(__file__).resolve().parent / "extractor"
+this_dir = pathlib.Path(__file__).resolve().parent
 
 cargo = shutil.which("cargo")
 assert cargo, "no cargo binary found on `PATH`"
 
-fmt = subprocess.run([cargo, "fmt", "--quiet"], cwd=extractor_dir)
-clippy = subprocess.run([cargo, "clippy", "--fix", "--allow-dirty", "--allow-staged", "--quiet"],
-                        cwd=extractor_dir)
+fmt = subprocess.run([cargo, "fmt", "--all", "--quiet"], cwd=this_dir)
+for manifest in this_dir.rglob("Cargo.toml"):
+    if not manifest.is_relative_to(this_dir / "ql"):
+        clippy = subprocess.run([cargo, "clippy", "--fix", "--allow-dirty", "--allow-staged", "--quiet"],
+                                cwd=manifest.parent)
 sys.exit(fmt.returncode or clippy.returncode)

rust/tools/index-files.sh

@@ -2,4 +2,4 @@
 
 set -eu
 
-exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" --inputs-file="$1"
+exec "$CODEQL_EXTRACTOR_RUST_ROOT/tools/$CODEQL_PLATFORM/extractor" @"$1"

shared/tree-sitter-extractor/BUILD.bazel

@@ -1,8 +1,6 @@
 load("@rules_rust//rust:defs.bzl", "rust_library")
 load("@tree_sitter_extractors_deps//:defs.bzl", "aliases", "all_crate_deps")
 
-package(default_visibility = ["//visibility:public"])
-
 rust_library(
     name = "codeql-extractor",
     srcs = glob([
@@ -15,6 +13,12 @@ rust_library(
     deps = all_crate_deps(),
 )
 
+alias(
+    name = "tree-sitter-extractor",
+    actual = ":codeql-extractor",
+    visibility = ["//visibility:public"],
+)
+
 filegroup(
     name = "dbscheme-prefix",
     srcs = ["src/generator/prefix.dbscheme"],