Multi platform

lib/bundix.rb

@@ -17,7 +17,7 @@ class Bundix
 
   SHA256_32 = %r(^[a-z0-9]{52}$)
 
-  attr_reader :options
+  attr_reader :options, :target_platform
 
   attr_accessor :fetcher
 
@@ -32,6 +32,7 @@ def initialize(name, version, options={}, &blk)
 
   def initialize(options)
     @options = { quiet: false, tempfile: nil }.merge(options)
+    @target_platform = options[:platform] ? Gem::Platform.new(options[:platform]) : Gem::Platform::RUBY
     @fetcher = Fetcher.new
   end
 
@@ -40,8 +41,10 @@ def convert
     lock = parse_lockfile
     dep_cache = build_depcache(lock)
 
-    # reverse so git comes last
-    lock.specs.reverse_each.with_object({}) do |spec, gems|
+    lock.specs.group_by(&:name).each.with_object({}) do |(name, specs), gems|
+      # reverse so git/plain-ruby sources come last
+      spec = specs.reverse.find {|s| s.platform == Gem::Platform::RUBY || s.platform =~ target_platform }
+      next unless spec
       gem = find_cached_spec(spec, cache) || convert_spec(spec, cache, dep_cache)
       gems.merge!(gem)
 
@@ -82,15 +85,25 @@ def platforms(spec, dep_cache)
       PLATFORM_MAPPING[platform_name.to_s]
     end.flatten
 
-    {platforms: platforms}
+    # 'platforms' is the Bundler DSL for including a gem if we're on a certain platform.
+    # 'target_platform' is the platform that bundix is currently resolving gem-specs for.
+    # 'gem_platform' is the platform of the resulting spec.
+    # (eg we might be resolving gem-specs for x86_64-darwin, but if there's not a suitable
+    # precompiled gem available, then gem_platform will just be 'ruby')
+    {
+      platforms: platforms,
+      target_platform: target_platform.to_s,
+      gem_platform: spec.platform.to_s,
+    }
   end
 
   def convert_spec(spec, cache, dep_cache)
     {
-      spec.name => {
-        version: spec.version.to_s,
-        source: Source.new(spec, fetcher).convert
-      }.merge(platforms(spec, dep_cache)).merge(groups(spec, dep_cache))
+      spec.name => [
+        platforms(spec, dep_cache),
+        groups(spec, dep_cache),
+        Source.new(spec, fetcher).convert,
+      ].inject(&:merge),
     }
   rescue => ex
     warn "Skipping #{spec.name}: #{ex}"
@@ -102,6 +115,7 @@ def find_cached_spec(spec, cache)
     name, cached = cache.find{|k, v|
       next unless k == spec.name
       next unless cached_source = v['source']
+      next unless target_platform.to_s == v['target_platform']
 
       case spec_source = spec.source
       when Bundler::Source::Git
@@ -169,7 +183,11 @@ def parse_gemset
   end
 
   def parse_lockfile
-    Bundler::LockfileParser.new(File.read(options[:lockfile]))
+    lock = Bundler::LockfileParser.new(File.read(options[:lockfile]))
+    if !lock.platforms.include?(target_platform)
+      raise KeyError, "#{target_platform} not listed in gemfile. Try `bundle lock --add-platform #{target_platform}`"
+    end
+    lock
   end
 
   def self.sh(*args, &block)

lib/bundix/commandline.rb

@@ -15,7 +15,8 @@ class CommandLine
       gemfile: 'Gemfile',
       lockfile: 'Gemfile.lock',
       gemset: 'gemset.nix',
-      project: File.basename(Dir.pwd)
+      project: File.basename(Dir.pwd),
+      platform: 'ruby'
     }
 
     def self.run
@@ -33,8 +34,10 @@ def run
       handle_magic
       handle_lock
       handle_init
-      gemset = build_gemset
-      save_gemset(gemset)
+      platforms_and_paths.each do |platform, path|
+        gemset = build_gemset(path, platform)
+        save_gemset(gemset, path)
+      end
     end
 
     def parse_options
@@ -67,6 +70,14 @@ def parse_options
           options[:gemfile] = File.expand_path(value)
         end
 
+        o.on "--platform=#{options[:platform]}", 'platform version to use' do |value|
+          options[:platform] = value
+        end
+
+        o.on "--platforms=ruby", 'auto-generate gemsets for multiple comma-separated platforms' do |value|
+          options[:platforms] = value
+        end
+
         o.on '-d', '--dependencies', 'include gem dependencies (deprecated)' do
           warn '--dependencies/-d is deprecated because'
           warn 'dependencies will always be fetched'
@@ -145,21 +156,45 @@ def handle_lock
       end
     end
 
-    def build_gemset
-      Bundix.new(options).convert
+    def build_gemset(gemset, platform)
+      Bundix.new(options.merge(gemset: gemset, platform: platform)).convert
     end
 
     def object2nix(obj)
       Nixer.serialize(obj)
     end
 
-    def save_gemset(gemset)
+    # If options[:platforms] is set, autogenerate a list of platforms & paths like
+    # [["ruby", "gemset.nix"], ["x86_64-darwin", "gemset.x86_64-darwin.nix"]]
+    # Otherwise, just rely on the specific platform & path.
+    def platforms_and_paths
+      gemset_path = options[:gemset]
+      if options[:platforms]
+        platforms = options[:platforms].split(",")
+        platforms.map { |p| [p, path_with_platform(gemset_path, p)] }
+      else
+        [[options[:platform], gemset_path]]
+      end
+    end
+
+    # convert a path like "gemset.nix" to a platform-specific one like "gemset.x86_64-linux.nix"
+    def path_with_platform(path, platform)
+      if platform == "ruby"
+        path
+      else
+        path_with_platform = path.sub(/\.nix$/, ".#{platform}.nix")
+        fail "Couldn't add platform to path" unless Regexp.last_match
+        path_with_platform
+      end
+    end
+
+    def save_gemset(gemset, path)
       tempfile = Tempfile.new('gemset.nix', encoding: 'UTF-8')
       begin
         tempfile.write(object2nix(gemset))
         tempfile.flush
-        FileUtils.cp(tempfile.path, options[:gemset])
-        FileUtils.chmod(0644, options[:gemset])
+        FileUtils.cp(tempfile.path, path)
+        FileUtils.chmod(0644, path)
       ensure
         tempfile.close!
         tempfile.unlink

lib/bundix/source.rb

@@ -92,35 +92,65 @@ def format_hash(hash)
     end
 
     def fetch_local_hash(spec)
-      spec.source.caches.each do |cache|
-        path = File.join(cache, "#{spec.full_name}.gem")
-        next unless File.file?(path)
+      has_platform = spec.platform && spec.platform != Gem::Platform::RUBY
+      name_version = "#{spec.name}-#{spec.version}"
+      filename = has_platform ? "#{name_version}-*" : name_version
+
+      paths = spec.source.caches.map(&:to_s)
+      Dir.glob("{#{paths.join(',')}}/#{filename}.gem").each do |path|
+        if has_platform
+          # Find first gem that matches the platform
+          platform = File.basename(path, '.gem')[(name_version.size + 1)..-1]
+          next unless spec.platform =~ platform
+        end
+
         hash = nix_prefetch_url(path)[SHA256_32]
-        return format_hash(hash) if hash
+        return format_hash(hash), platform if hash
       end
 
       nil
     end
 
     def fetch_remotes_hash(spec, remotes)
       remotes.each do |remote|
-        hash = fetch_remote_hash(spec, remote)
-        return remote, format_hash(hash) if hash
+        hash, platform = fetch_remote_hash(spec, remote)
+        return remote, format_hash(hash), platform if hash
       end
 
       nil
     end
 
     def fetch_remote_hash(spec, remote)
+      has_platform = spec.platform && spec.platform != Gem::Platform::RUBY
+      if has_platform
+        # Fetch remote spec to determine the exact platform
+        # Note that we can't simply use the local platform; the platform of the gem might differ.
+        # e.g. universal-darwin-14 covers x86_64-darwin-14
+        spec = spec_for_dependency(remote, spec)
+        return unless spec
+      end
+
       uri = "#{remote}/gems/#{spec.full_name}.gem"
       result = nix_prefetch_url(uri)
       return unless result
-      result[SHA256_32]
+
+      return result[SHA256_32], spec.platform&.to_s
     rescue => e
       puts "ignoring error during fetching: #{e}"
       puts e.backtrace
       nil
     end
+
+    def spec_for_dependency(remote, dependency)
+      sources = Gem::SourceList.from([remote])
+      specs, _errors = Gem::SpecFetcher.new(sources).spec_for_dependency(Gem::Dependency.new(dependency.name, dependency.version), false)
+      specs.each do |spec, source|
+        return spec if dependency.platform == spec.platform
+      end
+      # TODO: When might this happen?
+      puts 'oh, fallback ' + dependency.platform.to_s
+      specs.first.first
+    end
   end
 
   class Source < Struct.new(:spec, :fetcher)
@@ -140,21 +170,35 @@ def convert
 
     def convert_path
       {
-        type: "path",
-        path: spec.source.path
+        version: spec.version.to_s,
+        source: {
+          type: 'path',
+          path: spec.source.path,
+        },
       }
     end
 
     def convert_rubygems
       remotes = spec.source.remotes.map{|remote| remote.to_s.sub(/\/+$/, '') }
-      hash = fetcher.fetch_local_hash(spec)
-      remote, hash = fetcher.fetch_remotes_hash(spec, remotes) unless hash
+      hash, platform = fetcher.fetch_local_hash(spec)
+      remote, hash, platform = fetcher.fetch_remotes_hash(spec, remotes) unless hash
       fail "couldn't fetch hash for #{spec.full_name}" unless hash
-      puts "#{hash} => #{spec.full_name}.gem" if $VERBOSE
 
-      { type: 'gem',
-        remotes: (remote ? [remote] : remotes),
-        sha256: hash }
+      version = spec.version.to_s
+      if platform && platform != Gem::Platform::RUBY
+        version += "-#{platform}"
+      end
+
+      puts "#{hash} => #{spec.name}-#{version}.gem" if $VERBOSE
+
+      {
+        version: version,
+        source: {
+          type: 'gem',
+          remotes: (remote ? [remote] : remotes),
+          sha256: hash
+        },
+      }
     end
 
     def convert_git
@@ -167,11 +211,16 @@ def convert_git
       fail "couldn't fetch hash for #{spec.full_name}" unless hash
       puts "#{hash} => #{uri}" if $VERBOSE
 
-      { type: 'git',
-        url: uri.to_s,
-        rev: revision,
-        sha256: hash,
-        fetchSubmodules: submodules }
+      {
+        version: spec.version.to_s,
+        source: {
+          type: 'git',
+          url: uri.to_s,
+          rev: revision,
+          sha256: hash,
+          fetchSubmodules: submodules,
+        },
+      }
     end
   end
 end

test/convert.rb

@@ -1,22 +1,39 @@
 require 'minitest/autorun'
 require 'bundix'
+require 'digest'
+require 'json'
 
 class TestConvert < Minitest::Test
-  class PrefetchStub
+  class PrefetchStub < Bundix::Fetcher
+    SPECS = {
+      "sorbet-static" => {
+        platform: 'java-123',
+        version: "0.4.4821",
+      },
+    }
+
     def nix_prefetch_url(*args)
-      return "nix_prefetch_url_hash"
+      format_hash(Digest::SHA256.hexdigest(args.to_s))
     end
 
-    def nix_prefetch_git(uri, revision)
-      return '{"sha256": "nix_prefetch_git_hash"}'
+    def nix_prefetch_git(*args)
+      JSON.generate("sha256" => format_hash(Digest::SHA256.hexdigest(args.to_s)))
     end
 
     def fetch_local_hash(spec)
-      return "5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03" #taken from `man nix-hash`
+      # Force to use fetch_remote_hash
+      return nil
     end
 
-    def fetch_remotes_hash(spec, remotes)
-      return "fetch_remotes_hash_hash"
+    def spec_for_dependency(remote, spec)
+      opts = SPECS[spec.name]
+      raise "Unexpected spec query: #{name}" unless opts
+
+      Gem::Specification.new do |s|
+        s.name = spec.name
+        s.version = spec.version
+        s.platform = Gem::Platform.new(opts[:platform]) if opts[:platform]
+      end
     end
   end
 
@@ -36,10 +53,23 @@ def with_gemset(options)
   def test_bundler_dep
     with_gemset(
       :gemfile => File.expand_path("data/bundler-audit/Gemfile", __dir__),
-      :lockfile => File.expand_path("data/bundler-audit/Gemfile.lock", __dir__)
+      :lockfile => File.expand_path("data/bundler-audit/Gemfile.lock", __dir__),
+    ) do |gemset|
+      assert_equal("0.5.0", gemset.dig("bundler-audit", :version))
+      assert_equal("0.19.4", gemset.dig("thor", :version))
+      refute(gemset.has_key?("sorbet-static"))
+    end
+  end
+
+  def test_platform_specific_lookups
+    with_gemset(
+      :gemfile => File.expand_path("data/bundler-audit/Gemfile", __dir__),
+      :lockfile => File.expand_path("data/bundler-audit/Gemfile.lock", __dir__),
+      :platform => 'java'
     ) do |gemset|
       assert_equal("0.5.0", gemset.dig("bundler-audit", :version))
       assert_equal("0.19.4", gemset.dig("thor", :version))
+      assert_equal("0.4.4821-java-unknown", gemset.dig("sorbet-static", :version))
     end
   end
 end

test/data/bundler-audit/Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org' do
   gem 'bundler-audit'
+  gem 'sorbet', '= 0.4.4821'
 end