From e2105279e5ca2e89533656b9c4b83d6f540678f0 Mon Sep 17 00:00:00 2001 From: "markrmiller@gmail.com" Date: Fri, 10 Jul 2020 08:30:18 -0500 Subject: [PATCH] #48 Knock down a couple long tail tests and put the SolrResourceLoader XML doc builder in a thread local. --- .../legacy/LegacyAbstractAnalyticsTest.java | 3 +- .../LegacyAbstractAnalyticsFacetTest.java | 5 +- .../solr/handler/dataimport/DataImporter.java | 17 +----- .../java/org/apache/solr/core/PluginBag.java | 54 +++++++++---------- .../apache/solr/core/SolrResourceLoader.java | 28 ++++++---- .../handler/component/DebugComponent.java | 4 +- .../handler/component/ResponseBuilder.java | 13 ++--- .../solr/handler/component/SearchHandler.java | 2 + .../apache/solr/update/TransactionLog.java | 2 +- .../apache/solr/BasicFunctionalityTest.java | 2 +- .../solr/cloud/CollectionPropsTest.java | 2 +- .../cloud/FullSolrCloudDistribCmdsTest.java | 1 + .../solr/cloud/TestCloudPivotFacet.java | 15 +++--- ...estTolerantUpdateProcessorRandomCloud.java | 10 ++-- .../apache/solr/cloud/ZkShardTermsTest.java | 1 + .../apache/solr/cloud/ZkSolrClientTest.java | 1 + .../CollectionsAPIDistClusterPerZkTest.java | 28 ---------- .../CollectionsAPIDistributedZkTest.java | 29 ++++++++++ .../sim/TestSimDistributedQueue.java | 3 +- .../solr/core/BlobRepositoryCloudTest.java | 2 + .../apache/solr/handler/TestSQLHandler.java | 1 + .../handler/admin/AdminHandlersProxyTest.java | 9 +++- .../handler/admin/StatsReloadRaceTest.java | 2 +- .../component/CustomTermsComponentTest.java | 2 + .../solr/request/TestRemoteStreaming.java | 2 + .../solr/schema/TestUseDocValuesAsStored.java | 1 - .../apache/solr/search/TestRangeQuery.java | 12 ++--- .../facet/SpatialHeatmapFacetsTest.java | 5 +- .../facet/TestCloudJSONFacetSKGEquiv.java | 2 +- .../update/TestInPlaceUpdatesStandalone.java | 5 +- ...cExpirationUpdateProcessorFactoryTest.java | 1 + .../java/org/apache/solr/common/ParWork.java | 9 +++- .../apache/solr/cloud/SolrCloudTestCase.java | 2 +- .../org/apache/solr/util/BaseTestHarness.java | 20 +++++-- 34 files changed, 165 insertions(+), 130 deletions(-) diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsTest.java index ee1cc2e308a0..6133bbca6d4d 100644 --- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsTest.java +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/LegacyAbstractAnalyticsTest.java @@ -42,6 +42,7 @@ import org.apache.solr.analytics.util.OrdinalCalculator; import org.apache.solr.core.XmlConfigFile; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.util.BaseTestHarness; import org.junit.AfterClass; import org.junit.BeforeClass; import org.w3c.dom.Document; @@ -96,7 +97,7 @@ public static void afterClassAbstractAnalysis() { public static void setResponse(String response) throws ParserConfigurationException, IOException, SAXException { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setNamespaceAware(true); // never forget this! - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = BaseTestHarness.getXmlDocumentBuilder(); doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)))); rawResponse = response; } diff --git a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java index 96dcbbb035f5..055ba462dade 100644 --- a/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java +++ b/solr/contrib/analytics/src/test/org/apache/solr/analytics/legacy/facet/LegacyAbstractAnalyticsFacetTest.java @@ -36,6 +36,7 @@ import org.apache.solr.analytics.util.OrdinalCalculator; import org.apache.solr.core.XmlConfigFile; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.util.BaseTestHarness; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -77,9 +78,7 @@ public static void afterClassAbstractAnalysis() { } protected static void setResponse(String response) throws ParserConfigurationException, IOException, SAXException { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); // never forget this! - DocumentBuilder builder = factory.newDocumentBuilder(); + DocumentBuilder builder = BaseTestHarness.getXmlDocumentBuilder(); doc = builder.parse(new InputSource(new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)))); rawResponse = response; } diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java index c5b2f70bf226..15566db853bf 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java @@ -19,6 +19,7 @@ import org.apache.solr.common.EmptyEntityResolver; import org.apache.solr.common.SolrException; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.schema.IndexSchema; import org.apache.solr.util.SystemIdResolver; import org.apache.solr.common.util.NamedList; @@ -192,21 +193,7 @@ public DIHConfiguration loadDataConfig(InputSource configFile) { DIHConfiguration dihcfg = null; try { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - dbf.setValidating(false); - - // only enable xinclude, if XML is coming from safe source (local file) - // and a a SolrCore and SystemId is present (makes no sense otherwise): - if (core != null && configFile.getSystemId() != null) { - try { - dbf.setXIncludeAware(true); - dbf.setNamespaceAware(true); - } catch( UnsupportedOperationException e ) { - log.warn( "XML parser doesn't support XInclude option" ); - } - } - - DocumentBuilder builder = dbf.newDocumentBuilder(); + DocumentBuilder builder = SolrResourceLoader.dbf.newDocumentBuilder(); // only enable xinclude / external entities, if XML is coming from // safe source (local file) and a a SolrCore and SystemId is present: if (core != null && configFile.getSystemId() != null) { diff --git a/solr/core/src/java/org/apache/solr/core/PluginBag.java b/solr/core/src/java/org/apache/solr/core/PluginBag.java index 77419f3d5e6a..419da5a0cfe9 100644 --- a/solr/core/src/java/org/apache/solr/core/PluginBag.java +++ b/solr/core/src/java/org/apache/solr/core/PluginBag.java @@ -300,44 +300,42 @@ void init(Map defaults, SolrCore solrCore, List infos) { List otherPlugins = new ArrayList<>(); List reqHandlerPlugins = new ArrayList<>(); - for (PluginInfo info : infos) { - List list; - System.out.println("plugin clazz:" + meta.clazz); - if (meta.clazz.equals(SolrRequestHandler.class)) { - list = reqHandlerPlugins; - } else { - list = otherPlugins; - } + for (PluginInfo info : infos) { + List list; + if (meta.clazz.equals(SolrRequestHandler.class)) { + list = reqHandlerPlugins; + } else { + list = otherPlugins; + } - list.add(() -> { - System.out.println("load plugin:" + info.className); - PluginHolder o = createPlugin(info); - String name = info.name; - if (meta.clazz.equals(SolrRequestHandler.class)) name = RequestHandlers.normalize(info.name); - PluginHolder old = put(name, o); - if (old != null) { - log.warn("Multiple entries of {} with name {}", meta.getCleanTag(), name); - } - }); + list.add(() -> { + PluginHolder o = createPlugin(info); + String name = info.name; + if (meta.clazz.equals(SolrRequestHandler.class)) name = RequestHandlers.normalize(info.name); + PluginHolder old = put(name, o); + if (old != null) { + log.warn("Multiple entries of {} with name {}", meta.getCleanTag(), name); + } + }); - } + } try (ParWork worker = new ParWork(this)) { worker.collect(otherPlugins); worker.addCollect("initOtherPlugins"); worker.collect(reqHandlerPlugins); worker.addCollect("initReqHandlerPlugins"); } - if (infos.size() > 0) { // Aggregate logging - if (log.isDebugEnabled()) { - log.debug("[{}] Initialized {} plugins of type {}: {}", solrCore.getName(), infos.size(), meta.getCleanTag(), - infos.stream().map(i -> i.name).collect(Collectors.toList())); - } + if (infos.size() > 0) { // Aggregate logging + if (log.isDebugEnabled()) { + log.debug("[{}] Initialized {} plugins of type {}: {}", solrCore.getName(), infos.size(), meta.getCleanTag(), + infos.stream().map(i -> i.name).collect(Collectors.toList())); } - for (Map.Entry e : defaults.entrySet()) { - if (!contains(e.getKey())) { - put(e.getKey(), new PluginHolder(null, e.getValue())); - } + } + for (Map.Entry e : defaults.entrySet()) { + if (!contains(e.getKey())) { + put(e.getKey(), new PluginHolder(null, e.getValue())); } + } } diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 73d78b93375b..9013e38c6ed8 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -33,6 +33,7 @@ import java.nio.file.StandardOpenOption; import java.util.*; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -83,10 +84,9 @@ public class SolrResourceLoader implements ResourceLoader, Closeable { }; private static final Charset UTF_8 = StandardCharsets.UTF_8; - public static final javax.xml.parsers.DocumentBuilderFactory dbf; - private final DocumentBuilder db; + protected final static ThreadLocal THREAD_LOCAL_DB= new ThreadLocal<>(); static { dbf = new DocumentBuilderFactoryImpl(); try { @@ -99,6 +99,8 @@ public class SolrResourceLoader implements ResourceLoader, Closeable { } } + private final SystemIdResolver sysIdResolver; + private static void trySetDOMFeature(DocumentBuilderFactory factory, String feature, boolean enabled) { try { factory.setFeature(feature, enabled); @@ -183,18 +185,22 @@ public SolrResourceLoader(Path instanceDir, ClassLoader parent) { } this.classLoader = URLClassLoader.newInstance(new URL[0], parent); this.resourceClassLoader = URLClassLoader.newInstance(new URL[0], parent); - - try { - db = dbf.newDocumentBuilder(); - } catch (ParserConfigurationException e) { - log.error("Error in parser configuration", e); - throw new RuntimeException(e); - } - db.setEntityResolver(new SystemIdResolver(this)); - db.setErrorHandler(xmllog); + this.sysIdResolver = new SystemIdResolver(this); } public DocumentBuilder getDocumentBuilder() { + DocumentBuilder db = THREAD_LOCAL_DB.get(); + if (db == null) { + try { + db = dbf.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + log.error("Error in parser configuration", e); + throw new RuntimeException(e); + } + db.setErrorHandler(xmllog); + THREAD_LOCAL_DB.set(db); + } + db.setEntityResolver(sysIdResolver); return db; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java index 31fbf99f3bb6..534e378b5a6d 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java @@ -254,7 +254,9 @@ public void finishStage(ResponseBuilder rb) { if (rb.isDebugResults()) { @SuppressWarnings({"rawtypes"}) NamedList sexplain = (NamedList)sdebug.get("explain"); - SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(sexplain, rb.resultIds, arr); + if (sexplain != null) { + SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(sexplain, rb.resultIds, arr); + } } } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java index cf4719817f9b..6ea0ddf64455 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.search.Query; import org.apache.lucene.search.TotalHits; @@ -185,12 +186,12 @@ public void addRequest(SearchComponent me, ShardRequest sreq) { boolean _isOlapAnalytics; // Context fields for grouping - public final Map>> mergedSearchGroups = new HashMap<>(); - public final Map mergedGroupCounts = new HashMap<>(); - public final Map, Set>> searchGroupToShards = new HashMap<>(); - public final Map> mergedTopGroups = new HashMap<>(); - public final Map mergedQueryCommandResults = new HashMap<>(); - public final Map retrievedDocuments = new HashMap<>(); + public final Map>> mergedSearchGroups = new ConcurrentHashMap<>(); + public final Map mergedGroupCounts = new ConcurrentHashMap<>(); + public final Map, Set>> searchGroupToShards = new ConcurrentHashMap<>(); + public final Map> mergedTopGroups = new ConcurrentHashMap<>(); + public final Map mergedQueryCommandResults = new ConcurrentHashMap<>(); + public final Map retrievedDocuments = new ConcurrentHashMap<>(); public int totalHitCount; // Hit count used when distributed grouping is performed. // Used for timeAllowed parameter. First phase elapsed time is subtracted from the time allowed for the second phase. public int firstPhaseElapsedTime; diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index 2051b2044516..dac9c0dd5d8b 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -16,6 +16,7 @@ */ package org.apache.solr.handler.component; +import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; import java.lang.invoke.MethodHandles; @@ -29,6 +30,7 @@ import org.apache.lucene.search.TotalHits; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.cloud.ZkController; +import org.apache.solr.common.ParWork; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; diff --git a/solr/core/src/java/org/apache/solr/update/TransactionLog.java b/solr/core/src/java/org/apache/solr/update/TransactionLog.java index 9a8b4302a5e7..4192c4a2084e 100644 --- a/solr/core/src/java/org/apache/solr/update/TransactionLog.java +++ b/solr/core/src/java/org/apache/solr/update/TransactionLog.java @@ -179,7 +179,7 @@ public boolean writePrimitive(Object val) throws IOException { long start = raf.length(); channel = raf.getChannel(); os = Channels.newOutputStream(channel); - fos = new FastOutputStream(os, new byte[65536], 0); + fos = new FastOutputStream(os, new byte[65536 * 2], 0); // fos = FastOutputStream.wrap(os); if (openExisting) { diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java index 64098c062a59..6b9a2f7dfe49 100644 --- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java +++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java @@ -519,7 +519,7 @@ public void testFieldBoost() throws Exception { String resp = h.query(lrf.makeRequest("q", "text:hello", CommonParams.DEBUG_QUERY, "true")); //System.out.println(resp); // second doc ranked first - assertTrue( resp.indexOf("\"2\"") < resp.indexOf("\"1\"") ); +; assertTrue( resp.indexOf("\"2\"") < resp.indexOf("\"1\"") ); } @Test diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionPropsTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionPropsTest.java index 050ac34608ac..b1a15d90b7e1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CollectionPropsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CollectionPropsTest.java @@ -168,7 +168,7 @@ private void waitForValue(String propertyName, String expectedValue, int timeout return; } lastValueSeen = value; - Thread.sleep(10); + Thread.sleep(250); } String collectionpropsInZk = null; try { diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java index 1fbf0022ac44..af36e7746e7f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java @@ -456,6 +456,7 @@ public void run() { checkShardConsistency(params("q","*:*", "rows", ""+totalDocsExpected, "_trace","batches_done")); } + @Nightly public void testConcurrentIndexing() throws Exception { final CloudSolrClient cloudClient = cluster.getSolrClient(); final String collectionName = createAndSetNewDefaultCollection(); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java index b01e581d8df6..5a7c4b3b88f9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPivotFacet.java @@ -77,7 +77,7 @@ * */ @SuppressSSL // Too Slow -public class TestCloudPivotFacet extends AbstractFullDistribZkTestBase { +public class TestCloudPivotFacet extends SolrCloudBridgeTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -96,6 +96,8 @@ public class TestCloudPivotFacet extends AbstractFullDistribZkTestBase { public TestCloudPivotFacet() { // we need DVs on point fields to compute stats & facets if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true"); + useFieldRandomizedFactor = TestUtil.nextInt(random(), 2, 30); + log.info("init'ing useFieldRandomizedFactor = {}", useFieldRandomizedFactor); } /** @@ -107,8 +109,7 @@ public TestCloudPivotFacet() { @BeforeClass public static void initUseFieldRandomizedFactor() { - useFieldRandomizedFactor = TestUtil.nextInt(random(), 2, 30); - log.info("init'ing useFieldRandomizedFactor = {}", useFieldRandomizedFactor); + } @Test @@ -122,7 +123,7 @@ public void test() throws Exception { final Set fieldNameSet = new HashSet<>(); // build up a randomized index - final int numDocs = atLeast(500); + final int numDocs = atLeast(TEST_NIGHTLY ? 500 : 50); log.info("numDocs: {}", numDocs); for (int i = 1; i <= numDocs; i++) { @@ -142,16 +143,16 @@ public void test() throws Exception { final String[] fieldNames = fieldNameSet.toArray(new String[fieldNameSet.size()]); Arrays.sort(fieldNames); // need determinism when picking random fields - for (int i = 0; i < 5; i++) { + for (int i = 0; i < (TEST_NIGHTLY ? 5 : 2); i++) { String q = "*:*"; if (random().nextBoolean()) { - q = "id:[* TO " + TestUtil.nextInt(random(),300,numDocs) + "]"; + q = "id:[* TO " + TestUtil.nextInt(random(),(TEST_NIGHTLY ? 300 : 30),numDocs) + "]"; } ModifiableSolrParams baseP = params("rows", "0", "q", q); if (random().nextBoolean()) { - baseP.add("fq", "id:[* TO " + TestUtil.nextInt(random(),200,numDocs) + "]"); + baseP.add("fq", "id:[* TO " + TestUtil.nextInt(random(),(TEST_NIGHTLY ? 200 : 20),numDocs) + "]"); } final boolean stats = random().nextBoolean(); diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java index d91078fac823..7fadf7a5d210 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorRandomCloud.java @@ -66,7 +66,7 @@ * and several clients pointed at specific nodes. These are all re-used across multiple test methods, * and assumes that the state of the cluster is healthy between tests. *

- * + *slowest_test_suite=org.apache.solr.cloud.DocValuesNotIndexedTest */ @SuppressSSL(bugUrl="https://issues.apache.org/jira/browse/SOLR-9182 - causes OOM") public class TestTolerantUpdateProcessorRandomCloud extends SolrCloudTestCase { @@ -106,8 +106,6 @@ public static void createMiniSolrCloudCluster() throws Exception { CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor) .setProperties(collectionProperties) .process(CLOUD_CLIENT); - - cluster.waitForActiveCollection(COLLECTION_NAME, numShards, numShards * repFactor); if (NODE_CLIENTS != null) { for (HttpSolrClient client : NODE_CLIENTS) { @@ -271,12 +269,12 @@ public void testRandomUpdates() throws Exception { assertEquals("post update commit failed?", 0, CLOUD_CLIENT.commit().getStatus()); - for (int j = 0; j < 5; j++) { + for (int j = 0; j < 3; j++) { if (expectedDocIds.cardinality() == countDocs(CLOUD_CLIENT)) { break; } log.info("sleeping to give searchers a chance to re-open #{}", j); - Thread.sleep(200); + Thread.sleep(50); } // check the index contents against our expectations @@ -302,7 +300,7 @@ public void testRandomUpdates() throws Exception { * @see #randomUnsetBit */ public void testSanityRandomUnsetBit() { - final int max = atLeast(100); + final int max = atLeast(TEST_NIGHTLY ? 100: 5); BitSet bits = new BitSet(max+1); for (int i = 0; i <= max; i++) { assertFalse("how is bitset already full? iter="+i+" card="+bits.cardinality()+"/max="+max, diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java index 6e3df0d70c99..35780f248bd9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java @@ -196,6 +196,7 @@ public void testRegisterTerm() throws InterruptedException { } @Test + @Nightly public void testRaceConditionOnUpdates() throws InterruptedException { String collection = "raceConditionOnUpdates"; List replicas = Arrays.asList("rep1", "rep2", "rep3", "rep4"); diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java index 282a669abba9..b13cde964ff1 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java @@ -106,6 +106,7 @@ public void testClean() throws Exception { } } + @Nightly public void testReconnect() throws Exception { Path zkDir = createTempDir("zkData"); ZkTestServer server = null; diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistClusterPerZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistClusterPerZkTest.java index 1465b91e6f5c..606d8e5d3756 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistClusterPerZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistClusterPerZkTest.java @@ -161,22 +161,6 @@ public void testMissingNumShards() { }); } - @Test - public void testZeroNumShards() { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - params.set("name", "acollection"); - params.set(REPLICATION_FACTOR, 10); - params.set("numShards", 0); - params.set("collection.configName", "conf"); - - final SolrRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - expectThrows(Exception.class, () -> { - cluster.getSolrClient().request(request); - }); - } - @Test @Ignore // nocommit we can speed this up public void testCreateShouldFailOnExistingCore() throws Exception { @@ -206,18 +190,6 @@ public void testCreateShouldFailOnExistingCore() throws Exception { }); } - @Test - public void testNoConfigSetExist() throws Exception { - expectThrows(Exception.class, () -> { - CollectionAdminRequest.createCollection("noconfig", "conf123", 1, 1) - .process(cluster.getSolrClient()); - }); - - // in both cases, the collection should have default to the core name - //cluster.getSolrClient().getZkStateReader().forceUpdateCollection("noconfig"); - assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains("noconfig")); - } - @Test public void testCoresAreDistributedAcrossNodes() throws Exception { CollectionAdminRequest.createCollection("nodes_used_collection", "conf", 2, 2) diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java index eb3d4ca869b5..e7cbfce597fc 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java @@ -151,6 +151,18 @@ public void testBadActionNames() { }); } + @Test + public void testNoConfigSetExist() throws Exception { + expectThrows(Exception.class, () -> { + CollectionAdminRequest.createCollection("noconfig", "conf123", 1, 1) + .process(cluster.getSolrClient()); + }); + + // in both cases, the collection should have default to the core name + //cluster.getSolrClient().getZkStateReader().forceUpdateCollection("noconfig"); + assertFalse(CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains("noconfig")); + } + @Test public void testMissingRequiredParameters() { ModifiableSolrParams params = new ModifiableSolrParams(); @@ -165,4 +177,21 @@ public void testMissingRequiredParameters() { }); } + + @Test + public void testZeroNumShards() { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("name", "acollection"); + params.set(REPLICATION_FACTOR, 10); + params.set("numShards", 0); + params.set("collection.configName", "conf"); + + final SolrRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + expectThrows(Exception.class, () -> { + cluster.getSolrClient().request(request); + }); + } + } diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistributedQueue.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistributedQueue.java index 81bad1792e9a..9c1aa6439298 100644 --- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistributedQueue.java +++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimDistributedQueue.java @@ -85,6 +85,7 @@ public void testDistributedQueue() throws Exception { } @Test + @Nightly public void testDistributedQueueBlocking() throws Exception { String dqZNode = "/distqueue/test2"; String testData = "hello world"; @@ -107,7 +108,7 @@ public void testDistributedQueueBlocking() throws Exception { assertNull(dq.peek(10)); // Rerun the earlier test make sure updates are still seen, post reconnection. - future = executor.submit(() -> new String(dq.peek(15000), UTF8)); + future = executor.submit(() -> new String(dq.peek(5000), UTF8)); try { future.get(1000, TimeUnit.MILLISECONDS); fail("TimeoutException expected"); diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java index d524d6939850..e508ee0e927c 100644 --- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java +++ b/solr/core/src/test/org/apache/solr/core/BlobRepositoryCloudTest.java @@ -34,8 +34,10 @@ import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.handler.TestBlobHandler; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; +@Ignore // nocommit flakey public class BlobRepositoryCloudTest extends SolrCloudTestCase { public static final Path TEST_PATH = getFile("solr/configsets").toPath(); diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java index b20bec59aa13..2fa50a9e115e 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java @@ -1735,6 +1735,7 @@ public void testTimeSeriesGroupingFacet() throws Exception { } @Test + @Nightly public void testParallelTimeSeriesGrouping() throws Exception { new UpdateRequest() diff --git a/solr/core/src/test/org/apache/solr/handler/admin/AdminHandlersProxyTest.java b/solr/core/src/test/org/apache/solr/handler/admin/AdminHandlersProxyTest.java index 93cbe3e8df4f..6b426f99d7f3 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/AdminHandlersProxyTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/AdminHandlersProxyTest.java @@ -21,6 +21,7 @@ import java.util.Collections; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.lucene.util.IOUtils; @@ -33,6 +34,8 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.TimeOut; +import org.apache.solr.common.util.TimeSource; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -79,11 +82,15 @@ public void proxySystemInfoHandlerAllNodes() throws IOException, SolrServerExcep } @Test - public void proxyMetricsHandlerAllNodes() throws IOException, SolrServerException { + public void proxyMetricsHandlerAllNodes() throws IOException, SolrServerException, TimeoutException, InterruptedException { MapSolrParams params = new MapSolrParams(Collections.singletonMap("nodes", "all")); GenericSolrRequest req = new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/metrics", params); SimpleSolrResponse rsp = req.process(solrClient, null); NamedList nl = rsp.getResponse(); + + TimeOut timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME); + timeout.waitFor("", () -> rsp.getResponse().size() > 2); + assertEquals(3, nl.size()); assertTrue(nl.getName(1).endsWith("_solr")); assertTrue(nl.getName(2).endsWith("_solr")); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/StatsReloadRaceTest.java b/solr/core/src/test/org/apache/solr/handler/admin/StatsReloadRaceTest.java index fc1b496c9f5e..0b85040e0909 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/StatsReloadRaceTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/StatsReloadRaceTest.java @@ -134,7 +134,7 @@ private void requestMetrics(boolean softFail) throws Exception { assertTrue(metrics.get(key) instanceof Long); break; } else { - Thread.sleep(500); + Thread.sleep(250); } } if (softFail && !found) { diff --git a/solr/core/src/test/org/apache/solr/handler/component/CustomTermsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/CustomTermsComponentTest.java index f648535f9ae9..ac5666553583 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/CustomTermsComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/CustomTermsComponentTest.java @@ -30,8 +30,10 @@ import org.apache.solr.cloud.ConfigRequest; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.common.util.NamedList; +import org.junit.Ignore; import org.junit.Test; +@Ignore // nocommit flakey public class CustomTermsComponentTest extends ShardsWhitelistTest { public static class CustomTermsComponent extends TermsComponent { diff --git a/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java b/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java index 20e9a077e8a0..0e6c3806c6de 100644 --- a/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java +++ b/solr/core/src/test/org/apache/solr/request/TestRemoteStreaming.java @@ -41,12 +41,14 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; /** * See SOLR-2854. */ @SuppressSSL // does not yet work with ssl yet - uses raw java.net.URL API rather than HttpClient +@Ignore // nocommit flakey public class TestRemoteStreaming extends SolrJettyTestBase { private static File solrHomeDirectory; diff --git a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java index b1ae50182195..c8c5c9ca2256 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java +++ b/solr/core/src/test/org/apache/solr/schema/TestUseDocValuesAsStored.java @@ -17,7 +17,6 @@ package org.apache.solr.schema; import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import java.io.File; diff --git a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java index da1be69b107b..8c5de7507d3f 100644 --- a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java +++ b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java @@ -255,11 +255,11 @@ public void testRandomRangeQueries() throws Exception { // fields that a value source range query should work on String[] frange_fields = {"foo_i","foo_l","foo_f","foo_d"}; - final int l= -1 * atLeast(TEST_NIGHTLY ? 50 : 15); - final int u= atLeast(TEST_NIGHTLY ? 250 : 50); + final int l= -1 * atLeast(TEST_NIGHTLY ? 50 : 5); + final int u= atLeast(TEST_NIGHTLY ? 250 : 15); // sometimes a very small index, sometimes a very large index - final int numDocs = random().nextBoolean() ? random().nextInt(50) : atLeast(TEST_NIGHTLY ? 1000 : 100); + final int numDocs = random().nextBoolean() ? random().nextInt(TEST_NIGHTLY ? 1000 : 5) : atLeast(TEST_NIGHTLY ? 1000 : 10); createIndex(numDocs, new DocProcessor() { @Override public void process(SolrInputDocument doc) { @@ -319,7 +319,7 @@ public void process(SolrInputDocument doc) { // now build some random queries (against *any* field) and validate that using it in a DBQ changes // the index by the expected number of docs long numDocsLeftInIndex = numDocs; - final int numDBQs= atLeast(10); + final int numDBQs= atLeast(TEST_NIGHTLY ? 10 : 1); for (int i=0; i < numDBQs; i++) { int lower = TestUtil.nextInt(random(), 2 * l, u); int upper = TestUtil.nextInt(random(), lower, 2 * u); @@ -401,7 +401,7 @@ public void testRangeQueryRequiresTO() throws Exception { } public void testCompareTypesRandomRangeQueries() throws Exception { - int cardinality = 10000; + int cardinality = TEST_NIGHTLY ? 10000 : 100; Map types = new HashMap<>(); //single and multivalued field types Map typesMv = new HashMap<>(); // multivalued field types only types.put(NumberType.INTEGER, new String[]{"ti", "ti_dv", "ti_ni_dv", "i_p", "i_ni_p", "i_ndv_p", "tis", "tis_dv", "tis_ni_dv", "is_p", "is_ni_p", "is_ndv_p"}); @@ -415,7 +415,7 @@ public void testCompareTypesRandomRangeQueries() throws Exception { typesMv.put(NumberType.DOUBLE, new String[]{"tds", "tds_dv", "tds_ni_dv", "ds_p", "ds_ni_p", "ds_ndv_p"}); typesMv.put(NumberType.DATE, new String[]{"tdts", "tdts_dv", "tdts_ni_dv", "dts_p", "dts_ni_p", "dts_ndv_p"}); - for (int i = 0; i < atLeast(500); i++) { + for (int i = 0; i < atLeast(TEST_NIGHTLY ? 500 : 50); i++) { if (random().nextInt(50) == 0) { //have some empty docs assertU(adoc("id", String.valueOf(i))); diff --git a/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java b/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java index 5af7bb6a0a6a..5c025659db93 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java +++ b/solr/core/src/test/org/apache/solr/search/facet/SpatialHeatmapFacetsTest.java @@ -314,13 +314,12 @@ private NamedList getHmObj(QueryResponse response) { } @Test - @Repeat(iterations = 3) public void testPng() { //We test via round-trip randomized data: // Make random data - int columns = random().nextInt(100) + 1; - int rows = random().nextInt(100) + 1; + int columns = random().nextInt(TEST_NIGHTLY ? 100 : 10) + 1; + int rows = random().nextInt(TEST_NIGHTLY ? 100 : 10) + 1; int[] counts = new int[columns * rows]; for (int i = 0; i < counts.length; i++) { final int ri = random().nextInt(10); diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java index b3793c2f1b46..06019b425d07 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java @@ -85,7 +85,7 @@ public class TestCloudJSONFacetSKGEquiv extends SolrCloudTestCase { private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT; private static final int MAX_FIELD_NUM = 15; - private static final int UNIQUE_FIELD_VALS = 50; + private static final int UNIQUE_FIELD_VALS = TEST_NIGHTLY ? 50 : 5; /** Multi-Valued string field suffixes that can be randomized for testing diff facet code paths */ private static final String[] MULTI_STR_FIELD_SUFFIXES = new String[] diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java index 90397c12dc92..d0c9f65ad959 100644 --- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java +++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java @@ -541,6 +541,7 @@ public void testUpdateOfNonExistentDVsShouldNotFail() throws Exception { } @Test + @Nightly public void testOnlyPartialUpdatesBetweenCommits() throws Exception { // Full updates long version1 = addAndGetVersion(sdoc("id", "1", "title_s", "first", "val1_i_dvo", "1", "val2_l_dvo", "1"), params()); @@ -783,7 +784,7 @@ public static final RandomUpdate pick(Random r) { public void testReplay_Random_ManyDocsManyUpdates() throws Exception { // build up a random list of updates - final int maxDocId = atLeast(50); + final int maxDocId = atLeast(TEST_NIGHTLY ? 50 : 5); final int numUpdates = maxDocId * 3; checkRandomReplay(maxDocId, numUpdates); } @@ -803,7 +804,7 @@ public void testReplay_Random_FewDocsManyUpdates() throws Exception { public void testReplay_Random_FewDocsManyShortSequences() throws Exception { // build up a random list of updates - final int numIters = atLeast(50); + final int numIters = atLeast(TEST_NIGHTLY ? 50 : 5); for (int i = 0; i < numIters; i++) { final int maxDocId = atLeast(3); diff --git a/solr/core/src/test/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactoryTest.java index 428a94f22c10..c9527c25a3d0 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/DocExpirationUpdateProcessorFactoryTest.java @@ -156,6 +156,7 @@ public void testTTLFieldConversionWithDefaultParam() throws Exception { } + @Nightly public void testAutomaticDeletes() throws Exception { // get a handle on our recorder diff --git a/solr/solrj/src/java/org/apache/solr/common/ParWork.java b/solr/solrj/src/java/org/apache/solr/common/ParWork.java index 384e9182408c..13d16c810c10 100644 --- a/solr/solrj/src/java/org/apache/solr/common/ParWork.java +++ b/solr/solrj/src/java/org/apache/solr/common/ParWork.java @@ -535,6 +535,9 @@ public void close() { if (exp instanceof Error) { throw (Error) exp; } + if (exp instanceof RuntimeException) { + throw (RuntimeException) exp; + } throw new RuntimeException(exp); } } @@ -671,7 +674,11 @@ private void handleObject(String label, AtomicReference exception, fi if (t instanceof Error) { throw (Error) t; } - throw new WorkException(RAN_INTO_AN_ERROR_WHILE_DOING_WORK, t); // TODO, hmm how do I keep zk session timeout and interrupt in play? + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } else { + throw new WorkException(RAN_INTO_AN_ERROR_WHILE_DOING_WORK, t); // TODO, hmm how do I keep zk session timeout and interrupt in play? + } } } } finally { diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java index e6aa4639b77f..f2f3831f51b3 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java @@ -85,7 +85,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final int DEFAULT_TIMEOUT = 15; // this is an important timeout for test stability - can't be too short + public static final int DEFAULT_TIMEOUT = 15; private static SolrQueuedThreadPool qtp; private static class Config { diff --git a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java index aae94be198d0..722f7f4d1c36 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java +++ b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java @@ -27,19 +27,33 @@ import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutorService; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.XML; +import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.core.XmlConfigFile; import org.w3c.dom.Document; import org.xml.sax.SAXException; abstract public class BaseTestHarness { private static final XPath xpath = XmlConfigFile.xpath; - private static final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - public static DocumentBuilder getXmlDocumentBuilder() throws ParserConfigurationException { - return dbf.newDocumentBuilder(); + protected final static ThreadLocal THREAD_LOCAL_DB = new ThreadLocal<>(); + + public synchronized static DocumentBuilder getXmlDocumentBuilder() throws ParserConfigurationException { + DocumentBuilder db = THREAD_LOCAL_DB.get(); + if (db != null) { + return db; + } else { + try { + db = SolrResourceLoader.dbf.newDocumentBuilder(); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } + THREAD_LOCAL_DB.set(db); + } + return db; } public static XPath getXpath() {