v1.4.0

Gemfile.lock

@@ -8,23 +8,27 @@ GEM
   specs:
     ast (2.4.2)
     json (2.6.3)
-    parallel (1.22.1)
-    parser (3.2.2.0)
+    language_server-protocol (3.17.0.3)
+    parallel (1.23.0)
+    parser (3.2.2.3)
       ast (~> 2.4.1)
+      racc
+    racc (1.7.1)
     rainbow (3.1.1)
-    regexp_parser (2.8.0)
+    regexp_parser (2.8.1)
     rexml (3.2.5)
-    rubocop (1.50.2)
+    rubocop (1.53.0)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.0.0)
+      parser (>= 3.2.2.3)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
       rubocop-ast (>= 1.28.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.28.0)
+    rubocop-ast (1.29.0)
       parser (>= 3.2.1.0)
     ruby-progressbar (1.13.0)
     unicode-display_width (2.4.2)

README.md

@@ -23,6 +23,8 @@
 - Remove duplicate queries
 - Merge all queries in one file
 - List all available datasets
+- Merge existing customqueries.json to the output
+- Merge local custom queries files as well as remote ones
 
 ## Install
 
@@ -43,8 +45,10 @@ See [INSTALL](INSTALL.md) for options with package managers.
 Usage: bqm [options]
     -o, --output-path PATH           Path where to store the query file
     -l, --list                       List available datasets
+    -i, --local-sets FILE,...        Local custom queries files
 
 Example: bqm -o ~/.config/bloodhound/customqueries.json
+Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/tmp/b.json
 ```
 
 Example:

bin/bqm

@@ -14,7 +14,8 @@ def find_dataset
   raise IOError, "The dataset file #{source_file} does not exist or is unreadable."
 end
 
-def merge(source)
+# Merge remote sets defined in data/query-sets.json
+def merge_remote(source)
   sets = get_datasets(source)
   queries = []
   sets.each do |s|
@@ -25,6 +26,24 @@ def merge(source)
   queries
 end
 
+# Merge local sets provided by the user
+def merge_local(sources)
+  queries = []
+  sources.each do |source|
+    if File.file?(source) && File.readable?(source)
+      begin
+        data = JSON.load_file(source)
+      rescue NoMethodError # ruby 2.7 retro-compatibility
+        data = JSON.parse(File.read(source))
+      end
+      queries += data['queries']
+    else
+      raise IOError, "The dataset file #{source} does not exist or is unreadable."
+    end
+  end
+  queries
+end
+
 # Query class just for the sake of having custom comparison
 class BQMquery
   attr_accessor :data
@@ -72,14 +91,20 @@ if __FILE__ == $PROGRAM_NAME
   source = find_dataset
 
   require 'optparse'
-  options = {}
+  options = {
+    :'local-sets' => []
+  }
   OptionParser.new do |parser|
     parser.banner = 'Usage: bqm [options]'
 
     parser.on('-o', '--output-path PATH', 'Path where to store the query file')
     parser.on('-l', '--list', 'List available datasets')
+    parser.on('-i', '--local-sets FILE,...', Array, 'Local custom queries files') do |f|
+      options[:'local-sets'] += f
+    end
     parser.separator ''
     parser.separator 'Example: bqm -o ~/.config/bloodhound/customqueries.json'
+    parser.separator 'Example: bqm -o /tmp/customqueries.json -i /tmp/a.json,/tmp/b.json'
   end.parse!(into: options)
 
   out = options[:'output-path']
@@ -95,16 +120,20 @@ if __FILE__ == $PROGRAM_NAME
     if File.file?(out) && File.readable?(out)
       puts "[+] The output path #{out} already exists"
       puts '[?] Do you want to overwrite it? [y/n]'
-      if gets.chomp == 'y'
+      if STDIN.gets.chomp == 'y'
         puts '[?] What to do with the existing queries? (merge / discard) [m/d]'
-        flags[:merge_actual] = true if gets.chomp == 'm'
+        flags[:merge_actual] = true if STDIN.gets.chomp == 'm'
       else
         exit
       end
     end
     puts '[+] Fetching and merging datasets'
-    data = merge(source)
-    if flags[:merge_actual]
+    data = merge_remote(source)
+    local_set = options[:'local-sets']
+    if local_set
+      data += merge_local(local_set)
+    end
+    if flags[:'merge_actual']
       puts '[+] Merging your existing queries'
       data += JSON.parse(File.read(out))['queries']
     end

bqm.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name          = 'bqm'
-  s.version       = '1.3.0'
+  s.version       = '1.4.0'
   s.platform      = Gem::Platform::RUBY
   s.summary       = 'Download BloudHound query lists, deduplicate entries and merge them in one file.'
   s.description   = 'Deduplicate custom BloudHound queries from different datasets and merge them in one customqueries.json file.'
@@ -17,8 +17,8 @@ Gem::Specification.new do |s|
 
   s.metadata = {
     'bug_tracker_uri'       => 'https://github.com/Acceis/bqm/issues',
-    # 'changelog_uri'         => 'https://github.com/Acceis/bqm/blob/master/docs/CHANGELOG.md',
-    # 'documentation_uri'     => 'https://github.com/Acceis/bqm',
+    'changelog_uri'         => 'https://github.com/Acceis/bqm/releases',
+    'documentation_uri'     => 'https://acceis.github.io/bqm/',
     'homepage_uri'          => 'https://github.com/Acceis/bqm',
     'source_code_uri'       => 'https://github.com/Acceis/bqm/',
     'rubygems_mfa_required' => 'true'