Merge branch 'master' into master-ltr-plugin-develop

dev-tools/idea/lucene/join/join.iml

@@ -14,7 +14,6 @@
     <orderEntry type="library" scope="TEST" name="JUnit" level="project" />
     <orderEntry type="module" scope="TEST" module-name="lucene-test-framework" />
     <orderEntry type="module" module-name="grouping" />
-    <orderEntry type="module" module-name="backward-codecs" />
     <orderEntry type="module" module-name="lucene-core" />
   </component>
 </module>

dev-tools/idea/lucene/queryparser/queryparser.iml

@@ -17,6 +17,5 @@
     <orderEntry type="module" module-name="lucene-core" />
     <orderEntry type="module" module-name="queries" />
     <orderEntry type="module" module-name="sandbox" />
-    <orderEntry type="module" module-name="backward-codecs" />
   </component>
 </module>

dev-tools/scripts/addVersion.py

@@ -217,7 +217,9 @@ def main():
   update_changes('lucene/CHANGES.txt', c.version)
   update_changes('solr/CHANGES.txt', c.version, get_solr_init_changes())
 
-  if current_version.is_back_compat_with(c.version):
+  is_back_compat = current_version.major == c.version.major or current_version.is_back_compat_with(c.version)
+
+  if is_back_compat:
     add_constant(c.version, not c.is_latest_version)
   else:
     print('\nNot adding constant for version %s because it is no longer supported' % c.version)
@@ -232,7 +234,7 @@ def main():
     print('\nTODO: ')
     print('  - Move backcompat oldIndexes to unsupportedIndexes in TestBackwardsCompatibility')
     print('  - Update IndexFormatTooOldException throw cases')
-  elif current_version.is_back_compat_with(c.version):
+  elif is_back_compat:
     print('\nTesting changes')
     check_lucene_version_tests()
     check_solr_version_tests()

dev-tools/scripts/buildAndPushRelease.py

@@ -218,7 +218,7 @@ def check_cmdline_tools():  # Fail fast if there are cmdline tool problems
   if os.system('git --version >/dev/null 2>/dev/null'):
     raise RuntimeError('"git --version" returned a non-zero exit code.')
   antVersion = os.popen('ant -version').read().strip()
-  if not antVersion.startswith('Apache Ant(TM) version 1.8'):
+  if not antVersion.startswith('Apache Ant(TM) version 1.8') and not antVersion.startswith('Apache Ant(TM) version 1.9'):
     raise RuntimeError('ant version is not 1.8.X: "%s"' % antVersion)
 
 def main():

dev-tools/scripts/poll-mirrors.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+#
+# vim: softtabstop=2 shiftwidth=2 expandtab
+#
+# Python port of poll-mirrors.pl
+#
+# This script is designed to poll download sites after posting a release
+# and print out notice as each becomes available.  The RM can use this
+# script to delay the release announcement until the release can be
+# downloaded.
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import argparse
+import datetime
+import ftplib
+import re
+import sys
+import time
+
+try:
+  from urllib.parse import urlparse
+except:
+  from urlparse import urlparse
+
+try:
+  import http.client as http
+except ImportError:
+  import httplib as http
+
+def p(s):
+  sys.stdout.write(s)
+  sys.stdout.flush()
+
+def mirror_contains_file(url):
+  url = urlparse(url)
+
+  if url.scheme == 'http':
+    return http_file_exists(url)
+  elif url.scheme == 'ftp':
+    return ftp_file_exists(url)
+
+def http_file_exists(url):
+  exists = False
+
+  try:
+    conn = http.HTTPConnection(url.netloc)
+    conn.request('HEAD', url.path)
+    response = conn.getresponse()
+
+    exists = response.status == 200
+  except:
+    pass
+
+  return exists
+
+def ftp_file_exists(url):
+  listing = []
+  try:
+    conn = ftplib.FTP(url.netloc)
+    conn.login()
+    listing = conn.nlst(url.path)
+    conn.quit()
+  except Exception as e:
+    pass
+
+  return len(listing) > 0
+
+def check_url_list(lst):
+  ret = []
+  for url in lst:
+    if mirror_contains_file(url):
+      p('.')
+    else:
+      p('X')
+      ret.append(url)
+
+  return ret
+
+parser = argparse.ArgumentParser(description='Checks that all Lucene mirrors contain a copy of a release')
+parser.add_argument('-version', '-v', help='Lucene version to check', required=True)
+parser.add_argument('-interval', '-i', help='seconds to wait to query again pending mirrors', type=int, default=300)
+args = parser.parse_args()
+
+try:
+  conn = http.HTTPConnection('www.apache.org')
+  conn.request('GET', '/mirrors/')
+  response = conn.getresponse()
+  html = response.read()
+except Exception as e:
+  p('Unable to fetch the Apache mirrors list!\n')
+  sys.exit(1)
+
+apache_path = 'lucene/java/{}/changes/Changes.html'.format(args.version);
+maven_url = 'http://repo1.maven.org/maven2/' \
+            'org/apache/lucene/lucene-core/{0}/lucene-core-{0}.pom.asc'.format(args.version)
+maven_available = False
+
+pending_mirrors = []
+for match in re.finditer('<TR>(.*?)</TR>', str(html), re.MULTILINE | re.IGNORECASE | re.DOTALL):
+  row = match.group(1)
+  if not '<TD>ok</TD>' in row:
+    # skip bad mirrors
+    continue
+
+  match = re.search('<A\s+HREF\s*=\s*"([^"]+)"\s*>', row, re.MULTILINE | re.IGNORECASE)
+  if match:
+    pending_mirrors.append(match.group(1) + apache_path)
+
+total_mirrors = len(pending_mirrors)
+
+while True:
+  p('\n' + str(datetime.datetime.now()))
+  p('\nPolling {} Apache Mirrors'.format(len(pending_mirrors)))
+  if not maven_available:
+    p(' and Maven Central')
+  p('...\n')
+
+  if not maven_available:
+    maven_available = mirror_contains_file(maven_url)
+
+  start = time.time()
+  pending_mirrors = check_url_list(pending_mirrors)
+  stop = time.time()
+  remaining = args.interval - (stop - start)
+
+  available_mirrors = total_mirrors - len(pending_mirrors)
+
+  p('\n\n{} is{}downloadable from Maven Central\n'.format(args.version, maven_available and ' ' or ' not '))
+  p('{} is downloadable from {}/{} Apache Mirrors ({:.2f}%)\n'.format(args.version, available_mirrors, 
+                                                                      total_mirrors,
+                                                                      available_mirrors * 100 / total_mirrors))
+  if len(pending_mirrors) == 0:
+    break
+
+  if remaining > 0:
+    p('Sleeping for {} seconds...\n'.format(remaining))
+    time.sleep(remaining)
+

dev-tools/scripts/smokeTestRelease.py

@@ -190,16 +190,23 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
       'Implementation-Vendor: The Apache Software Foundation',
       # Make sure 1.8 compiler was used to build release bits:
       'X-Compile-Source-JDK: 8',
-      # Make sure 1.8 ant was used to build release bits: (this will match 1.8+)
-      'Ant-Version: Apache Ant 1.8',
+      # Make sure 1.8 or 1.9 ant was used to build release bits: (this will match 1.8.x, 1.9.x)
+      ('Ant-Version: Apache Ant 1.8', 'Ant-Version: Apache Ant 1.9'),
       # Make sure .class files are 1.8 format:
       'X-Compile-Target-JDK: 8',
       'Specification-Version: %s' % version,
       # Make sure the release was compiled with 1.8:
       'Created-By: 1.8'):
-      if s.find(verify) == -1:
-        raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF' % \
-                           (desc, verify))
+      if type(verify) is not tuple:
+        verify = (verify,)
+      for x in verify:
+        if s.find(x) != -1:
+          break
+      else:
+        if len(verify) == 1:
+          raise RuntimeError('%s is missing "%s" inside its META-INF/MANIFEST.MF' % (desc, verify[0]))
+        else:
+          raise RuntimeError('%s is missing one of "%s" inside its META-INF/MANIFEST.MF' % (desc, verify))
 
     if gitRevision != 'skip':
       # Make sure this matches the version and git revision we think we are releasing: