Merge branch 'release/v4.3.0'

.travis.yml

@@ -1,15 +1,9 @@
 language: ruby
 rvm:
 - 2.3.8
-- 2.4.5
-- 2.5.3
-- 2.6.1
-addons:
-  apt:
-    packages:
-    - cmake
-    - bison
-    - flex
+- 2.4.6
+- 2.5.5
+- 2.6.2
 before_install:
 - pip install --upgrade --user awscli
 - gem update --system
@@ -23,7 +17,6 @@ before_script:
 script: bundle exec appraisal rspec
 env:
   global:
-  - SPHINX_BIN=ext/sphinx/bin/
   - secure: cUPinkilBafqDSPsTkl/PXYc2aXNKUQKXGK8poBBMqKN9/wjfJx1DWgtowDKalekdZELxDhc85Ye3bL1xlW4nLjOu+U6Tkt8eNw2Nhs1flodHzA/RyENdBLr/tBHt43EjkrDehZx5sBHmWQY4miHs8AJz0oKO9Ae2inTOHx9Iuc=
   matrix:
   - DATABASE=mysql2 SPHINX_VERSION=2.1.9 SPHINX_ENGINE=sphinx
@@ -37,11 +30,12 @@ env:
   - DATABASE=postgresql SPHINX_VERSION=2.6.4 SPHINX_ENGINE=manticore
   - DATABASE=mysql2 SPHINX_VERSION=2.7.5 SPHINX_ENGINE=manticore
   - DATABASE=postgresql SPHINX_VERSION=2.7.5 SPHINX_ENGINE=manticore
-  - DATABASE=mysql2 SPHINX_VERSION=2.8.1 SPHINX_ENGINE=manticore
-  - DATABASE=postgresql SPHINX_VERSION=2.8.1 SPHINX_ENGINE=manticore
+  - DATABASE=mysql2 SPHINX_VERSION=2.8.2 SPHINX_ENGINE=manticore
+  - DATABASE=postgresql SPHINX_VERSION=2.8.2 SPHINX_ENGINE=manticore
   # - DATABASE=postgresql SPHINX_VERSION=3.1.1 SPHINX_ENGINE=sphinx
 sudo: false
 addons:
   postgresql: '9.4'
 services:
+- mysql
 - postgresql

CHANGELOG.markdown

@@ -2,8 +2,24 @@
 
 All notable changes to this project (at least, from v3.0.0 onwards) are documented in this file.
 
+## 4.3.0 - 2019-05-18
+
+[Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v4.3.0)
+
+### Added
+
+* Allow overriding of Sphinx's running state, which is useful when Sphinx commands are interacting with a remote Sphinx daemon. As per discussions in [#1131](https://github.com/pat/thinking-sphinx/pull/1124).
+* Allow skipping of directory creation, as per discussions in [#1131](https://github.com/pat/thinking-sphinx/pull/1131).
+
+### Fixed
+
+* Use ActiveSupport's lock monitor where possible (Rails 5.1.5 onwards) to avoid database deadlocks. Essential investigation by [Jonathan del Strother](https://github.com/jdelstrother) ([#1132](https://github.com/pat/thinking-sphinx/pull/1132)).
+* Allow facet searching on distributed indices ([#1135](https://github.com/pat/thinking-sphinx/pull/1132)).
+
 ## 4.2.0 - 2019-03-09
 
+[Release Notes](https://github.com/pat/thinking-sphinx/releases/tag/v4.2.0)
+
 ### Added
 
 * Allow changing the default encoding for MySQL database connections from utf8 to something else via the `mysql_encoding` setting in `config/thinking_sphinx.yml`. In the next significant release, the default will change to utf8mb4 (which is supported in MySQL 5.5.3 and newer).

README.textile

@@ -1,6 +1,6 @@
 h1. Thinking Sphinx
 
-Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v4.2.0.
+Thinking Sphinx is a library for connecting ActiveRecord to the Sphinx full-text search tool, and integrates closely with Rails (but also works with other Ruby web frameworks). The current release is v4.3.0.
 
 h2. Upgrading
 
@@ -14,7 +14,7 @@ It's a gem, so install it like you would any other gem. You will also need to sp
 
 <pre><code>gem 'mysql2',          '~> 0.3',    :platform => :ruby
 gem 'jdbc-mysql',      '~> 5.1.35', :platform => :jruby
-gem 'thinking-sphinx', '~> 4.2'</code></pre>
+gem 'thinking-sphinx', '~> 4.3'</code></pre>
 
 The MySQL gems mentioned are required for connecting to Sphinx, so please include it even when you're using PostgreSQL for your database. If you're using JRuby with a version of Sphinx prior to 2.2.11, there is "currently an issue with Sphinx and jdbc-mysql 5.1.36 or newer":http://sphinxsearch.com/forum/view.html?id=13939, so you'll need to stick to nothing more recent than 5.1.35, or upgrade Sphinx.
 

bin/loadsphinx

@@ -2,65 +2,57 @@
 
 version=$1
 engine=$2
-name="$engine-$version"
-bucket="thinking-sphinx"
-directory="ext/sphinx"
-prefix="`pwd`/$directory"
-file="ext/$name.tar.gz"
 
-if [ "$engine" == "sphinx" ]; then
-  url="http://sphinxsearch.com/files/$name-release.tar.gz"
-else
-  url="https://github.com/manticoresoftware/manticore.git"
-fi
-
-download_and_compile_source () {
-  if [ "$engine" == "sphinx" ]; then
-    download_and_compile_sphinx
+load_sphinx () {
+  case $version in
+    2.1.9)
+      url="http://sphinxsearch.com/files/sphinxsearch_2.1.9-release-0ubuntu11~trusty_amd64.deb"
+      format="deb";;
+    2.2.11)
+      url="http://sphinxsearch.com/files/sphinxsearch_2.2.11-release-1~trusty_amd64.deb"
+      format="deb";;
+    3.0.3)
+      url="http://sphinxsearch.com/files/sphinx-3.0.3-facc3fb-linux-amd64.tar.gz"
+      format="gz";;
+    3.1.1)
+      url="http://sphinxsearch.com/files/sphinx-3.1.1-612d99f-linux-amd64.tar.gz"
+      format="gz";;
+    *)
+      echo "No Sphinx version $version available"
+      exit 1;;
+  esac
+
+  if [ "$format" == "deb" ]; then
+    curl --location $url -o sphinx.deb
+    sudo apt-get install ./sphinx.deb
   else
-    download_and_compile_manticore
+    curl $url -o sphinx.tar.gz
+    tar -zxvf sphinx.tar.gz
+    sudo mv sphinx-$version/bin/* /usr/local/bin/.
   fi
 }
 
-download_and_compile_sphinx () {
-  curl -O $url
-  tar -zxf $name-release.tar.gz
-  cd $name-release
-  ./configure --with-mysql --with-pgsql --enable-id64 --prefix=$prefix
-  make
-  make install
-  cd ..
-  rm -rf $name-release.tar.gz $name-release
-}
-
-download_and_compile_manticore () {
-  git clone $url $engine
-  cd $engine
-  git checkout $version
-  mkdir build
-  cd build
-  cmake -D WITH_MYSQL=TRUE -D WITH_PGSQL=TRUE -D DISABLE_TESTING=TRUE -D CMAKE_INSTALL_PREFIX=$prefix ..
-  make -j4
-  make install
-  cd ../..
-  rm -rf $engine
+load_manticore () {
+  url="https://github.com/manticoresoftware/manticore/releases/download/$version/manticore_$version.deb"
+
+  case $version in
+    2.6.4)
+      url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.6.4/manticore_2.6.4-180503-37308c3-release-stemmer.trusty_amd64-bin.deb";;
+    2.7.5)
+      url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.7.5/manticore_2.7.5-181204-0f016406-release-stemmer.trusty_amd64-bin.deb";;
+    2.8.2)
+      url="https://github.com/manticoresoftware/manticoresearch/releases/download/2.8.2/manticore_2.8.2-190402-4e81114-release-stemmer.trusty_amd64-bin.deb";;
+    *)
+      echo "No Manticore version $version available"
+      exit 1;;
+  esac
+
+  curl --location $url -o manticore.deb
+  sudo apt-get install ./manticore.deb
 }
 
-load_cache () {
-  mkdir ext
-  curl -o $file http://$bucket.s3.amazonaws.com/bincaches/$name.tar.gz
-  tar -zxf $file
-}
-
-push_cache () {
-  tar -czf $file $directory
-  aws s3 cp $file s3://$bucket/bincaches/$name.tar.gz --acl public-read
-}
-
-if curl -i --head --fail http://$bucket.s3.amazonaws.com/bincaches/$name.tar.gz
-then
-  load_cache
+if [ "$engine" == "sphinx" ]; then
+  load_sphinx
 else
-  download_and_compile_source
-  push_cache
+  load_manticore
 fi

lib/thinking_sphinx/commands/base.rb

@@ -46,4 +46,8 @@ def log(message)
 
     stream.puts message
   end
+
+  def skip_directories?
+    configuration.settings['skip_directory_creation']
+  end
 end

lib/thinking_sphinx/commands/prepare.rb

@@ -2,6 +2,8 @@
 
 class ThinkingSphinx::Commands::Prepare < ThinkingSphinx::Commands::Base
   def call
+    return if skip_directories?
+
     FileUtils.mkdir_p configuration.indices_location
   end
 

lib/thinking_sphinx/commands/running.rb

@@ -2,6 +2,8 @@
 
 class ThinkingSphinx::Commands::Running < ThinkingSphinx::Commands::Base
   def call
+    return true if configuration.settings['skip_running_check']
+
     controller.running?
   end
 

lib/thinking_sphinx/commands/start_attached.rb

@@ -2,7 +2,7 @@
 
 class ThinkingSphinx::Commands::StartAttached < ThinkingSphinx::Commands::Base
   def call
-    FileUtils.mkdir_p configuration.indices_location
+    FileUtils.mkdir_p configuration.indices_location unless skip_directories?
 
     unless pid = fork
       controller.start :verbose => options[:verbose], :nodetach => true

lib/thinking_sphinx/commands/start_detached.rb

@@ -2,7 +2,7 @@
 
 class ThinkingSphinx::Commands::StartDetached < ThinkingSphinx::Commands::Base
   def call
-    FileUtils.mkdir_p configuration.indices_location
+    FileUtils.mkdir_p configuration.indices_location unless skip_directories?
 
     result = controller.start :verbose => options[:verbose]
 

lib/thinking_sphinx/configuration.rb

@@ -10,7 +10,8 @@ class ThinkingSphinx::Configuration < Riddle::Configuration
 
   delegate :environment, :to => :framework
 
-  @@mutex = Mutex.new
+  @@mutex = defined?(ActiveSupport::Concurrency::LoadInterlockAwareMonitor) ?
+    ActiveSupport::Concurrency::LoadInterlockAwareMonitor.new : Mutex.new
 
   def initialize
     super
@@ -105,7 +106,9 @@ def render
   end
 
   def render_to_file
-    FileUtils.mkdir_p searchd.binlog_path unless searchd.binlog_path.blank?
+    unless settings['skip_directory_creation'] || searchd.binlog_path.blank?
+      FileUtils.mkdir_p searchd.binlog_path
+    end
 
     open(configuration_file, 'w') { |file| file.write render }
   end

lib/thinking_sphinx/configuration/distributed_indices.rb

@@ -21,7 +21,7 @@ def append(index)
 
   def distributed_index(reference, indices)
     index = ThinkingSphinx::Distributed::Index.new reference
-    index.local_indices += indices.collect &:name
+    index.local_index_objects = indices
     index
   end
 

lib/thinking_sphinx/core/index.rb

@@ -98,7 +98,10 @@ def pre_render
   end
 
   def set_path
-    FileUtils.mkdir_p path_prefix
+    unless config.settings['skip_directory_creation']
+      FileUtils.mkdir_p path_prefix
+    end
+
     @path = File.join path_prefix, name
   end
 end

lib/thinking_sphinx/distributed/index.rb

@@ -3,11 +3,12 @@
 class ThinkingSphinx::Distributed::Index <
   Riddle::Configuration::DistributedIndex
 
-  attr_reader :reference, :options
+  attr_reader :reference, :options, :local_index_objects
 
   def initialize(reference)
-    @reference = reference
-    @options   = {}
+    @reference           = reference
+    @options             = {}
+    @local_index_objects = []
 
     super reference.to_s.gsub('/', '_')
   end
@@ -20,6 +21,15 @@ def distributed?
     true
   end
 
+  def facets
+    local_index_objects.collect(&:facets).flatten
+  end
+
+  def local_index_objects=(indices)
+    self.local_indices = indices.collect(&:name)
+    @local_index_objects = indices
+  end
+
   def model
     @model ||= reference.to_s.camelize.constantize
   end

spec/acceptance/facets_spec.rb

@@ -124,4 +124,18 @@
       calls += 1
     end
   end
+
+  it "can be called on distributed indices" do
+    blue  = Colour.create! :name => 'blue'
+    green = Colour.create! :name => 'green'
+
+    Tee.create! :colour => blue
+    Tee.create! :colour => blue
+    Tee.create! :colour => green
+    index
+
+    expect(Tee.facets(:indices => ["tee"]).to_hash[:colour_id]).to eq({
+      blue.id => 2, green.id => 1
+    })
+  end
 end

spec/thinking_sphinx/commands/prepare_spec.rb

@@ -7,7 +7,7 @@
     configuration, {}, stream
   ) }
   let(:configuration) { double 'configuration',
-    :indices_location => '/path/to/indices'
+    :indices_location => '/path/to/indices', :settings => {}
   }
   let(:stream)        { double :puts => nil }
 
@@ -20,4 +20,12 @@
 
     command.call
   end
+
+  it "skips directory creation if flag is set" do
+    configuration.settings['skip_directory_creation'] = true
+
+    expect(FileUtils).to_not receive(:mkdir_p)
+
+    command.call
+  end
 end

spec/thinking_sphinx/commands/running_spec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ThinkingSphinx::Commands::Running do
+  let(:command)    { ThinkingSphinx::Commands::Running.new(
+    configuration, {}, stream
+  ) }
+  let(:configuration) {
+    double 'configuration', :controller => controller, :settings => {}
+  }
+  let(:stream)        { double :puts => nil }
+  let(:controller)    { double 'controller', :running? => false }
+
+  it 'returns true when Sphinx is running' do
+    allow(controller).to receive(:running?).and_return(true)
+
+    expect(command.call).to eq(true)
+  end
+
+  it 'returns false when Sphinx is not running' do
+    expect(command.call).to eq(false)
+  end
+
+  it 'returns true if the flag is set' do
+    configuration.settings['skip_running_check'] = true
+
+    expect(command.call).to eq(true)
+  end
+end