diff --git a/Gemfile.lock b/Gemfile.lock index 80bd6db..096ae2f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,23 +8,27 @@ GEM specs: ast (2.4.2) json (2.6.3) - parallel (1.22.1) - parser (3.2.2.0) + language_server-protocol (3.17.0.3) + parallel (1.23.0) + parser (3.2.2.3) ast (~> 2.4.1) + racc + racc (1.7.1) rainbow (3.1.1) - regexp_parser (2.8.0) + regexp_parser (2.8.1) rexml (3.2.5) - rubocop (1.50.2) + rubocop (1.53.0) json (~> 2.3) + language_server-protocol (>= 3.17.0) parallel (~> 1.10) - parser (>= 3.2.0.0) + parser (>= 3.2.2.3) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) rubocop-ast (>= 1.28.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 3.0) - rubocop-ast (1.28.0) + rubocop-ast (1.29.0) parser (>= 3.2.1.0) ruby-progressbar (1.13.0) unicode-display_width (2.4.2) diff --git a/README.md b/README.md index 0c9cedf..33bef7b 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ - Remove duplicate queries - Merge all queries in one file - List all available datasets +- Merge existing customqueries.json to the output +- Merge local custom queries files as well as remote ones ## Install @@ -43,8 +45,10 @@ See [INSTALL](INSTALL.md) for options with package managers. Usage: bqm [options] -o, --output-path PATH Path where to store the query file -l, --list List available datasets + -i, --local-sets FILE,... Local custom queries files Example: bqm -o ~/.config/bloodhound/customqueries.json +Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/tmp/b.json ``` Example: diff --git a/bin/bqm b/bin/bqm index 6a49bd0..660f404 100755 --- a/bin/bqm +++ b/bin/bqm @@ -14,7 +14,8 @@ def find_dataset raise IOError, "The dataset file #{source_file} does not exist or is unreadable." end -def merge(source) +# Merge remote sets defined in data/query-sets.json +def merge_remote(source) sets = get_datasets(source) queries = [] sets.each do |s| @@ -25,6 +26,24 @@ def merge(source) queries end +# Merge local sets provided by the user +def merge_local(sources) + queries = [] + sources.each do |source| + if File.file?(source) && File.readable?(source) + begin + data = JSON.load_file(source) + rescue NoMethodError # ruby 2.7 retro-compatibility + data = JSON.parse(File.read(source)) + end + queries += data['queries'] + else + raise IOError, "The dataset file #{source} does not exist or is unreadable." + end + end + queries +end + # Query class just for the sake of having custom comparison class BQMquery attr_accessor :data @@ -72,14 +91,20 @@ if __FILE__ == $PROGRAM_NAME source = find_dataset require 'optparse' - options = {} + options = { + :'local-sets' => [] + } OptionParser.new do |parser| parser.banner = 'Usage: bqm [options]' parser.on('-o', '--output-path PATH', 'Path where to store the query file') parser.on('-l', '--list', 'List available datasets') + parser.on('-i', '--local-sets FILE,...', Array, 'Local custom queries files') do |f| + options[:'local-sets'] += f + end parser.separator '' parser.separator 'Example: bqm -o ~/.config/bloodhound/customqueries.json' + parser.separator 'Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/tmp/b.json' end.parse!(into: options) out = options[:'output-path'] @@ -95,16 +120,20 @@ if __FILE__ == $PROGRAM_NAME if File.file?(out) && File.readable?(out) puts "[+] The output path #{out} already exists" puts '[?] Do you want to overwrite it? [y/n]' - if gets.chomp == 'y' + if STDIN.gets.chomp == 'y' puts '[?] What to do with the existing queries? (merge / discard) [m/d]' - flags[:merge_actual] = true if gets.chomp == 'm' + flags[:merge_actual] = true if STDIN.gets.chomp == 'm' else exit end end puts '[+] Fetching and merging datasets' - data = merge(source) - if flags[:merge_actual] + data = merge_remote(source) + local_set = options[:'local-sets'] + if local_set + data += merge_local(local_set) + end + if flags[:'merge_actual'] puts '[+] Merging your existing queries' data += JSON.parse(File.read(out))['queries'] end diff --git a/bqm.gemspec b/bqm.gemspec index 24e570c..e7b896e 100644 --- a/bqm.gemspec +++ b/bqm.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |s| s.name = 'bqm' - s.version = '1.3.0' + s.version = '1.4.0' s.platform = Gem::Platform::RUBY s.summary = 'Download BloudHound query lists, deduplicate entries and merge them in one file.' s.description = 'Deduplicate custom BloudHound queries from different datasets and merge them in one customqueries.json file.' @@ -17,8 +17,8 @@ Gem::Specification.new do |s| s.metadata = { 'bug_tracker_uri' => 'https://github.com/Acceis/bqm/issues', - # 'changelog_uri' => 'https://github.com/Acceis/bqm/blob/master/docs/CHANGELOG.md', - # 'documentation_uri' => 'https://github.com/Acceis/bqm', + 'changelog_uri' => 'https://github.com/Acceis/bqm/releases', + 'documentation_uri' => 'https://acceis.github.io/bqm/', 'homepage_uri' => 'https://github.com/Acceis/bqm', 'source_code_uri' => 'https://github.com/Acceis/bqm/', 'rubygems_mfa_required' => 'true'