Merge branch 'release/v4.2.0'

.travis.yml

@@ -3,7 +3,7 @@ rvm:
 - 2.3.8
 - 2.4.5
 - 2.5.3
-- 2.6.0
+- 2.6.1
 addons:
   apt:
     packages:
@@ -13,7 +13,8 @@ addons:
 before_install:
 - pip install --upgrade --user awscli
 - gem update --system
-- gem install bundler
+- gem uninstall -v '>= 2' -i $(rvm gemdir)@global -ax bundler || true
+- gem install bundler -v '< 2'
 before_script:
 - mysql -e 'create database thinking_sphinx;' > /dev/null
 - psql -c 'create database thinking_sphinx;' -U postgres >/dev/null
@@ -32,10 +33,12 @@ env:
   - DATABASE=mysql2 SPHINX_VERSION=3.0.3 SPHINX_ENGINE=sphinx
   - DATABASE=postgresql SPHINX_VERSION=3.0.3 SPHINX_ENGINE=sphinx
   - DATABASE=mysql2 SPHINX_VERSION=3.1.1 SPHINX_ENGINE=sphinx
-  - DATABASE=mysql2 SPHINX_VERSION=2.6.3 SPHINX_ENGINE=manticore
-  - DATABASE=postgresql SPHINX_VERSION=2.6.3 SPHINX_ENGINE=manticore
-  - DATABASE=mysql2 SPHINX_VERSION=2.7.4 SPHINX_ENGINE=manticore
-  - DATABASE=postgresql SPHINX_VERSION=2.7.4 SPHINX_ENGINE=manticore
+  - DATABASE=mysql2 SPHINX_VERSION=2.6.4 SPHINX_ENGINE=manticore
+  - DATABASE=postgresql SPHINX_VERSION=2.6.4 SPHINX_ENGINE=manticore
+  - DATABASE=mysql2 SPHINX_VERSION=2.7.5 SPHINX_ENGINE=manticore
+  - DATABASE=postgresql SPHINX_VERSION=2.7.5 SPHINX_ENGINE=manticore
+  - DATABASE=mysql2 SPHINX_VERSION=2.8.1 SPHINX_ENGINE=manticore
+  - DATABASE=postgresql SPHINX_VERSION=2.8.1 SPHINX_ENGINE=manticore
   # - DATABASE=postgresql SPHINX_VERSION=3.1.1 SPHINX_ENGINE=sphinx
 sudo: false
 addons:

Appraisals

@@ -41,4 +41,10 @@ appraise 'rails_5_2' do
   gem 'rails',  '~> 5.2.0'
   gem 'mysql2', '~> 0.5.0', :platform => :ruby
   gem 'pg',     '~> 1.0',   :platform => :ruby
-end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f >= 2.3
+end if RUBY_PLATFORM != 'java'
+
+appraise 'rails_6_0' do
+  gem 'rails',  '~> 6.0.0.beta1'
+  gem 'mysql2', '~> 0.5.0', :platform => :ruby
+  gem 'pg',     '~> 1.0',   :platform => :ruby
+end if RUBY_PLATFORM != 'java' && RUBY_VERSION.to_f >= 2.5

CHANGELOG.markdown

@@ -2,6 +2,26 @@
 
 All notable changes to this project (at least, from v3.0.0 onwards) are documented in this file.
 
+## 4.2.0 - 2019-03-09
+
+### Added
+
+* Allow changing the default encoding for MySQL database connections from utf8 to something else via the `mysql_encoding` setting in `config/thinking_sphinx.yml`. In the next significant release, the default will change to utf8mb4 (which is supported in MySQL 5.5.3 and newer).
+* Added Rails 6.0 and Manticore 2.8 to the test matrix.
+
+### Changed
+
+* Use Arel's SQL literals for generated order clauses, to avoid warnings from Rails 6.
+
+### Fixed
+
+* Fix usage of alternative primary keys in update and deletion callbacks and attribute access.
+* Ensure `respond_to?` takes Sphinx scopes into account ([Jonathan del Strother](https://github.com/jdelstrother) in [#1124](https://github.com/pat/thinking-sphinx/pull/1124)).
+* Add `:excerpts` as a known option for search requests.
+* Fix depolymorphed association join construction with Rails 6.0.0.beta2.
+* Reset ThinkingSphinx::Configuration's cached values when Rails reloads, to avoid holding onto stale references to ActiveRecord models ([#1125](https://github.com/pat/thinking-sphinx/issues/1125)).
+* Don't join against associations in `sql_query` if they're only used by query-sourced properties ([Hans de Graaff](https://github.com/graaff) in [#1127](https://github.com/pat/thinking-sphinx/pull/1127)).
+
 ## 4.1.0 - 2018-12-28
 
 [Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v4.1.0)

README.textile

@@ -1,6 +1,6 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v4.1.0.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v4.2.0.
 
 h2. Upgrading
 
@@ -14,7 +14,7 @@ It's a gem, so install it like you would any other gem. You will also need to sp
 
 <pre><code>gem 'mysql2',          '~> 0.3',    :platform => :ruby
 gem 'jdbc-mysql',      '~> 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 4.1'</code></pre>
+gem 'thinking-sphinx', '~> 4.2'</code></pre>
 
 The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. If you're using JRuby with a version of Sphinx prior to 2.2.11, there is "currently an issue with Sphinx and jdbc-mysql 5.1.36 or newer":http://sphinxsearch.com/forum/view.html?id=13939, so you'll need to stick to nothing more recent than 5.1.35, or upgrade Sphinx.
 
@@ -29,10 +29,10 @@ h2. Requirements
 The current release of Thinking Sphinx works with the following versions of its dependencies:
 
 |_. Library |_. Minimum |_. Tested Against |
-| Ruby | v2.3 | v2.3.8, v2.4.5, v2.5.3, v2.6.0 |
+| Ruby | v2.3 | v2.3.8, v2.4.5, v2.5.3, v2.6.1 |
 | Sphinx | v2.1.2 | v2.1.9, v2.2.11, v3.0.3, v3.1.1 |
-| Manticore | v2.6.3 | v2.6.3, v2.7.4 |
-| ActiveRecord | v3.2 | v3.2, v4.0, v4.1, v4.2, v5.0, v5.1, v5.2 |
+| Manticore | v2.6.3 | v2.6.4, v2.7.5, v2.8.1 |
+| ActiveRecord | v3.2 | v3.2..v6.0 |
 
 It _might_ work with older versions of Ruby, but it's highly recommended to update to a supported release.
 
@@ -52,7 +52,7 @@ If you want ActiveRecord 3.1 support, then refer to the 3.0.x releases of Thinki
 
 h3. Ruby
 
-You'll need either the standard Ruby (v2.2 or newer) or JRuby (9.1 or newer).
+You'll need either the standard Ruby (v2.3 or newer) or JRuby (9.1 or newer).
 
 h3. Database Versions
 
@@ -81,4 +81,4 @@ You can then run the unit tests with @rake spec:unit@, the acceptance tests with
 
 h2. Licence
 
-Copyright (c) 2007-2018, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.
+Copyright (c) 2007-2019, Thinking Sphinx is developed and maintained by Pat Allan, and is released under the open MIT Licence. Many thanks to "all who have contributed patches":https://github.com/pat/thinking-sphinx/contributors.

lib/thinking_sphinx/active_record/callbacks/delete_callbacks.rb

@@ -19,7 +19,9 @@ def delete_from_sphinx
     return if ThinkingSphinx::Callbacks.suspended? || instance.new_record?
 
     indices.each { |index|
-      ThinkingSphinx::Deletion.perform index, instance.id
+      ThinkingSphinx::Deletion.perform(
+        index, instance.public_send(index.primary_key)
+      )
     }
   end
 

lib/thinking_sphinx/active_record/callbacks/update_callbacks.rb

@@ -55,7 +55,9 @@ def update(index)
     return if attributes.empty?
 
     sphinxql = Riddle::Query.update(
-      index.name, index.document_id_for_key(instance.id), attributes
+      index.name,
+      index.document_id_for_key(instance.public_send(index.primary_key)),
+      attributes
     )
     ThinkingSphinx::Connection.take do |connection|
       connection.execute(sphinxql)

lib/thinking_sphinx/active_record/database_adapters/mysql_adapter.rb

@@ -40,6 +40,12 @@ def time_zone_query_pre
   end
 
   def utf8_query_pre
-    ['SET NAMES utf8']
+    ["SET NAMES #{settings['mysql_encoding']}"]
+  end
+
+  private
+
+  def settings
+    ThinkingSphinx::Configuration.instance.settings
   end
 end

lib/thinking_sphinx/active_record/depolymorph/overridden_reflection.rb

@@ -5,7 +5,7 @@
 class ThinkingSphinx::ActiveRecord::Depolymorph::OverriddenReflection <
   ThinkingSphinx::ActiveRecord::Depolymorph::BaseReflection
 
-  module JoinConstraint
+  module BuildJoinConstraint
     def build_join_constraint(table, foreign_table)
       super.and(
         foreign_table[options[:foreign_type]].eq(
@@ -15,6 +15,16 @@ def build_join_constraint(table, foreign_table)
     end
   end
 
+  module JoinScope
+    def join_scope(table, foreign_table, foreign_klass)
+      super.where(
+        foreign_table[options[:foreign_type]].eq(
+          options[:class_name].constantize.base_class.name
+        )
+      )
+    end
+  end
+
   def self.overridden_classes
     @overridden_classes ||= {}
   end
@@ -28,8 +38,13 @@ def call
   def klass
     self.class.overridden_classes[reflection.class] ||= begin
       subclass = Class.new reflection.class
-      subclass.include JoinConstraint
+      subclass.include extension(reflection)
       subclass
     end
   end
+
+  def extension(reflection)
+    reflection.respond_to?(:build_join_constraint) ?
+      BuildJoinConstraint : JoinScope
+  end
 end

lib/thinking_sphinx/active_record/property_query.rb

@@ -27,6 +27,7 @@ def to_s
   attr_reader :property, :source, :type
 
   delegate :unscoped, :to => :base_association_class, :prefix => true
+  delegate :sql,      :to => Arel
 
   def base_association
     reflections.first
@@ -135,7 +136,7 @@ def to_sql
     relation = relation.joins(joins) if joins.present?
     relation = relation.where("#{quoted_foreign_key} BETWEEN $start AND $end") if ranged?
     relation = relation.where("#{quoted_foreign_key} IS NOT NULL")
-    relation = relation.order("#{quoted_foreign_key} ASC") if type.nil?
+    relation = relation.order(sql("#{quoted_foreign_key} ASC")) if type.nil?
 
     relation.to_sql
   end

lib/thinking_sphinx/active_record/source_joins.rb

@@ -27,7 +27,7 @@ def append_property_associations
   end
 
   def append_column_associations(column)
-    return if column.__stack.empty?
+    return if column.__stack.empty? or column_included_in_queries?(column)
 
     joins.add_join_to column.__stack if column_exists?(column)
   end
@@ -54,4 +54,15 @@ def joins
       joins
     end
   end
+
+  def source_query_properties
+    source.properties.select { |field| field.source_type == :query }
+  end
+
+  # Use "first" here instead of a more intuitive flatten because flatten
+  # will also ask each column to become an Array and that will start
+  # to retrieve data.
+  def column_included_in_queries?(column)
+    source_query_properties.collect(&:columns).collect(&:first).include?(column)
+  end
 end

lib/thinking_sphinx/middlewares/glazier.rb

@@ -16,6 +16,7 @@ def call(contexts)
   class Inner
     def initialize(context)
       @context = context
+      @indices = {}
     end
 
     def call
@@ -31,10 +32,20 @@ def call
 
     attr_reader :context
 
+    def indices_for(model)
+      @indices[model] ||= context[:indices].select do |index|
+        index.model == model
+      end
+    end
+
     def row_for(result)
+      ids = indices_for(result.class).collect do |index|
+        result.send index.primary_key
+      end
+
       context[:raw].detect { |row|
         row['sphinx_internal_class'] == result.class.name &&
-        row['sphinx_internal_id']    == result.id
+        ids.include?(row['sphinx_internal_id'])
       }
     end
   end

lib/thinking_sphinx/middlewares/sphinxql.rb

@@ -82,19 +82,6 @@ def descendants_from_tables
       end.flatten
     end
 
-    def indices_match_classes?
-      indices.collect(&:reference).uniq.sort == classes.collect { |klass|
-        ThinkingSphinx::IndexSet.reference_name(klass)
-      }.sort
-    end
-
-    def inheritance_column_select(klass)
-      <<-SQL
-SELECT DISTINCT #{klass.inheritance_column}
-FROM #{klass.table_name}
-SQL
-    end
-
     def exclusive_filters
       @exclusive_filters ||= (options[:without] || {}).tap do |without|
         without[:sphinx_internal_id] = options[:without_ids] if options[:without_ids].present?
@@ -144,6 +131,19 @@ def indices
       end
     end
 
+    def indices_match_classes?
+      indices.collect(&:reference).uniq.sort == classes.collect { |klass|
+        ThinkingSphinx::IndexSet.reference_name(klass)
+      }.sort
+    end
+
+    def inheritance_column_select(klass)
+      <<-SQL
+SELECT DISTINCT #{klass.inheritance_column}
+FROM #{klass.table_name}
+SQL
+    end
+
     def order_clause
       order_by = options[:order]
       order_by = "#{order_by} ASC" if order_by.is_a? Symbol

lib/thinking_sphinx/railtie.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 class ThinkingSphinx::Railtie < Rails::Railtie
+  config.to_prepare do
+    ThinkingSphinx::Configuration.reset
+  end
+
   initializer 'thinking_sphinx.initialisation' do
     ActiveSupport.on_load(:active_record) do
       ActiveRecord::Base.send :include, ThinkingSphinx::ActiveRecord::Base

lib/thinking_sphinx/scopes.rb

@@ -26,5 +26,9 @@ def method_missing(method, *args, &block)
       query, options = sphinx_scopes[method].call(*args)
       search query, (options || {})
     end
+
+    def respond_to_missing?(method, include_private = false)
+      super || sphinx_scopes.keys.include?(method)
+    end
   end
 end

lib/thinking_sphinx/search.rb

@@ -10,10 +10,11 @@ class ThinkingSphinx::Search < Array
     send class )
   KNOWN_OPTIONS = (
     [
-      :classes, :conditions, :geo, :group_by, :ids_only, :ignore_scopes,
-      :indices, :limit, :masks, :max_matches, :middleware, :offset, :order,
-      :order_group_by, :page, :per_page, :populate, :retry_stale, :select,
-      :skip_sti, :sql, :star, :with, :with_all, :without, :without_ids
+      :classes, :conditions, :excerpts, :geo, :group_by, :ids_only,
+      :ignore_scopes, :indices, :limit, :masks, :max_matches, :middleware,
+      :offset, :order, :order_group_by, :page, :per_page, :populate,
+      :retry_stale, :select, :skip_sti, :sql, :star, :with, :with_all, :without,
+      :without_ids
     ] +
     ThinkingSphinx::Middlewares::SphinxQL::SELECT_OPTIONS
   ).uniq

lib/thinking_sphinx/settings.rb

@@ -17,7 +17,8 @@ class ThinkingSphinx::Settings
     "log"                => "log/ENVIRONMENT.searchd.log",
     "query_log"          => "log/ENVIRONMENT.searchd.query.log",
     "binlog_path"        => "tmp/binlog/ENVIRONMENT",
-    "workers"            => "threads"
+    "workers"            => "threads",
+    "mysql_encoding"     => "utf8"
   }.freeze
 
   def self.call(configuration)

spec/acceptance/attribute_access_spec.rb

@@ -23,6 +23,15 @@
     expect(search.first.weight).to eq(2500)
   end
 
+  it "provides direct access to the weight with alternative primary keys" do
+    album = Album.create! :name => 'Sing to the Moon', :artist => 'Laura Mvula'
+
+    search = Album.search 'sing', :select => "*, weight()"
+    search.context[:panes] << ThinkingSphinx::Panes::WeightPane
+
+    expect(search.first.weight).to be >= 1000
+  end
+
   it "can enumerate with the weight" do
     gods = Book.create! :title => 'American Gods', :year => 2001
     index

spec/acceptance/remove_deleted_records_spec.rb

@@ -35,6 +35,18 @@
       to be_empty
   end
 
+  it "removes records from real-time index results with alternate ids" do
+    album = Album.create! :name => 'Sing to the Moon', :artist => 'Laura Mvula'
+
+    expect(Album.search('Sing', :indices => ['album_real_core']).to_a).
+      to eq([album])
+
+    album.destroy
+
+    expect(Album.search_for_ids('Sing', :indices => ['album_real_core'])).
+      to be_empty
+  end
+
   it "does not remove real-time results when callbacks are disabled" do
     original = ThinkingSphinx::Configuration.instance.
       settings['real_time_callbacks']