Skip to content

Commit

Permalink
#84 Fewer URL's, they make a lot of garbage.
Browse files Browse the repository at this point in the history
  • Loading branch information
markrmiller committed Jul 12, 2020
1 parent f26cbb4 commit 2150014
Show file tree
Hide file tree
Showing 29 changed files with 64 additions and 65 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import java.lang.invoke.MethodHandles;
import java.net.BindException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.EnumSet;
Expand Down Expand Up @@ -156,6 +157,10 @@ public class JettySolrRunner implements Closeable {

private static Scheduler scheduler = new SolrHttpClientScheduler("JettySolrRunnerScheduler", true, null, new ThreadGroup("JettySolrRunnerScheduler"), 1);

public String getContext() {
return config.context;
}

public static class DebugFilter implements Filter {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

Expand Down Expand Up @@ -579,7 +584,7 @@ public void start(boolean reusePort, boolean wait) throws Exception {
if (started) {
proxy.reopen();
} else {
proxy.open(getBaseUrl().toURI());
proxy.open(new URI(getBaseUrl()));
}
}

Expand Down Expand Up @@ -850,32 +855,19 @@ public void setProxyPort(int proxyPort) {
* Returns a base URL consisting of the protocol, host, and port for a
* Connector in use by the Jetty Server contained in this runner.
*/
public URL getBaseUrl() {
try {
return new URL(protocol, host, jettyPort, config.context);
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
public String getBaseUrl() {
return protocol +"://" + host + ":" + jettyPort + config.context;
}

public URL getBaseURLV2(){
try {
return new URL(protocol, host, jettyPort, "/api");
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}

public String getBaseURLV2(){
return protocol +"://" + host + ":" + jettyPort + "/api";
}
/**
* Returns a base URL consisting of the protocol, host, and port for a
* Connector in use by the Jetty Server contained in this runner.
*/
public URL getProxyBaseUrl() {
try {
return new URL(protocol, host, getLocalPort(), config.context);
} catch (MalformedURLException e) {
throw new RuntimeException(e);
}
public String getProxyBaseUrl() {
return protocol +":" + host + ":" + getLocalPort() + config.context;
}

public SolrClient newClient() {
Expand Down Expand Up @@ -924,6 +916,10 @@ public String getSolrHome() {
return solrHome;
}

public String getHost() {
return host;
}

/**
* @return this node's properties
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,8 @@ public NamedList<Object> getToLog() {

/** Returns a string of the form "logid name1=value1 name2=value2 ..." */
public String getToLogAsString(String logid) {
StringBuilder sb = new StringBuilder(logid);
StringBuilder sb = new StringBuilder(logid.length() + 1024);
sb.append(logid);
for (int i=0; i<toLog.size(); i++) {
if (sb.length() > 0) {
sb.append(' ');
Expand Down
2 changes: 1 addition & 1 deletion solr/core/src/test/org/apache/solr/TestTolerantSearch.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public static void createThings() throws Exception {
systemSetPropertySolrDisableShardsWhitelist("true");
solrHome = createSolrHome();
createAndStartJetty(solrHome.getAbsolutePath());
String url = jetty.getBaseUrl().toString();
String url = jetty.getBaseUrl();
collection1 = getHttpSolrClient(url + "/collection1");
collection2 = getHttpSolrClient(url + "/collection2");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,13 @@ public static void setupCluster() throws Exception {
.process(cluster.getSolrClient());
}

private SolrClient buildClient(CloseableHttpClient httpClient, URL url) {
private SolrClient buildClient(CloseableHttpClient httpClient, String url) {
switch (random().nextInt(3)) {
case 0:
// currently only testing with 1 thread
return getConcurrentUpdateSolrClient(url.toString() + "/" + COLLECTION, httpClient, 6, 1);
case 1:
return getHttpSolrClient(url.toString() + "/" + COLLECTION, httpClient);
return getHttpSolrClient(url + "/" + COLLECTION, httpClient);
case 2:
CloudSolrClient client = getCloudSolrClient(cluster.getZkServer().getZkAddress(), random().nextBoolean(), httpClient, 30000, 60000);
client.setDefaultCollection(COLLECTION);
Expand All @@ -83,13 +83,15 @@ private SolrClient buildClient(CloseableHttpClient httpClient, URL url) {
@Test
public void testConnectionReuse() throws Exception {

URL url = cluster.getJettySolrRunners().get(0).getBaseUrl();
String url = cluster.getJettySolrRunners().get(0).getBaseUrl();
String host = cluster.getJettySolrRunners().get(0).getHost();
int port = cluster.getJettySolrRunners().get(0).getLocalPort();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

CloseableHttpClient httpClient = HttpClientUtil.createClient(null, cm);
try (SolrClient client = buildClient(httpClient, url)) {

HttpHost target = new HttpHost(url.getHost(), url.getPort(), isSSLMode() ? "https" : "http");
HttpHost target = new HttpHost(host, port, isSSLMode() ? "https" : "http");
HttpRoute route = new HttpRoute(target);

ConnectionRequest mConn = getClientConnectionRequest(httpClient, route, cm);
Expand Down Expand Up @@ -124,7 +126,7 @@ && random().nextInt(10) > 8) {
}
}

route = new HttpRoute(new HttpHost(url.getHost(), url.getPort(), isSSLMode() ? "https" : "http"));
route = new HttpRoute(new HttpHost(host, port, isSSLMode() ? "https" : "http"));

mConn = cm.requestConnection(route, HttpSolrClient.cacheKey);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import java.lang.invoke.MethodHandles;

import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -187,7 +188,7 @@ public void testThatCantForwardToLeaderFails() throws Exception {
cluster.waitForJettyToStop(leaderToPartition);
leaderToPartition.setProxyPort(proxy.getListenPort());
cluster.startJettySolrRunner(leaderToPartition);
proxy.open(leaderToPartition.getBaseUrl().toURI());
proxy.open(new URI(leaderToPartition.getBaseUrl()));
try {
log.info("leaderToPartition's Proxy: {}", proxy);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public void setupTest() throws Exception {
jetty.setProxyPort(proxy.getListenPort());
cluster.stopJettySolrRunner(jetty);// TODO: Can we avoid this restart
cluster.startJettySolrRunner(jetty);
proxy.open(jetty.getBaseUrl().toURI());
proxy.open(new URI(jetty.getBaseUrl()));
if (log.isInfoEnabled()) {
log.info("Adding proxy for URL: {}. Proxy {}", jetty.getBaseUrl(), proxy.getUrl());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ private JettySolrRunner getOverseerJetty() throws Exception {
int hostPort = overseerUrl.getPort();
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
try {
if (jetty.getBaseUrl().getPort() == hostPort)
if (jetty.getLocalPort() == hostPort)
return jetty;
} catch (IllegalStateException e) {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public void setupCluster() throws Exception {
jetty.setProxyPort(proxy.getListenPort());
cluster.stopJettySolrRunner(jetty);//TODO: Can we avoid this restart
cluster.startJettySolrRunner(jetty);
proxy.open(jetty.getBaseUrl().toURI());
proxy.open(new URI(jetty.getBaseUrl()));
if (log.isInfoEnabled()) {
log.info("Adding proxy for URL: {}. Proxy: {}", jetty.getBaseUrl(), proxy.getUrl());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,9 @@ private static void createMiniSolrCloudCluster() throws Exception {
// inspired by TestMiniSolrCloudCluster
HashMap<String, String> urlMap = new HashMap<>();
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
URL jettyURL = jetty.getBaseUrl();
String nodeKey = jettyURL.getHost() + ":" + jettyURL.getPort() + jettyURL.getPath().replace("/","_");
urlMap.put(nodeKey, jettyURL.toString());
String jettyURL = jetty.getBaseUrl();
String nodeKey = (jetty.getHost() + ":" + jetty.getLocalPort() + jetty.getContext()).replace("/","_");
urlMap.put(nodeKey, jettyURL);
}
ClusterState clusterState = zkStateReader.getClusterState();
for (Slice slice : clusterState.getCollection(COLLECTION_NAME).getSlices()) {
Expand All @@ -148,7 +148,7 @@ private static void createMiniSolrCloudCluster() throws Exception {
assertNotNull("slice has null leader: " + slice.toString(), leader);
assertNotNull("slice leader has null node name: " + slice.toString(), leader.getNodeName());
String leaderUrl = urlMap.remove(leader.getNodeName());
assertNotNull("could not find URL for " + shardName + " leader: " + leader.getNodeName(),
assertNotNull("could not find URL for " + shardName + " leader: " + leader.getNodeName() + " " + urlMap.keySet(),
leaderUrl);
assertEquals("expected two total replicas for: " + slice.getName(),
2, slice.getReplicas().size());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ public void test() throws Exception {
JettySolrRunner replicaJetty = null;
List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
for (JettySolrRunner jettySolrRunner : jettySolrRunners) {
int port = jettySolrRunner.getBaseUrl().getPort();
int port = jettySolrRunner.getLocalPort();
if (replica.getStr(BASE_URL_PROP).contains(":" + port)) {
replicaJetty = jettySolrRunner;
break;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public static void setupCluster() throws Exception {
cluster.stopJettySolrRunner(jetty);//TODO: Can we avoid this restart
cluster.startJettySolrRunner(jetty);
cluster.waitForAllNodes(30);
proxy.open(jetty.getBaseUrl().toURI());
proxy.open(new URI(jetty.getBaseUrl()));
if (log.isInfoEnabled()) {
log.info("Adding proxy for URL: {}. Proxy: {}", jetty.getBaseUrl(), proxy.getUrl());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -522,7 +522,7 @@ private void checkReplicasInactive(List<JettySolrRunner> downJettys) throws Keep

Set<String> downJettyNodes = new TreeSet<>();
for (JettySolrRunner jetty : downJettys) {
downJettyNodes.add(jetty.getBaseUrl().getHost() + ":" + jetty.getBaseUrl().getPort() + "_solr");
downJettyNodes.add(jetty.getHost() + ":" + jetty.getLocalPort() + "_solr");
}
while (timeout.hasTimedOut() == false) {
forceUpdateCollectionStatus();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ private static void createMiniSolrCloudCluster() throws Exception {
CLIENTS.clear();
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
assertNotNull("Cluster contains null jetty?", jetty);
final URL baseUrl = jetty.getBaseUrl();
final String baseUrl = jetty.getBaseUrl();
assertNotNull("Jetty has null baseUrl: " + jetty.toString(), baseUrl);
CLIENTS.add(getHttpSolrClient(baseUrl + "/" + COLLECTION_NAME + "/"));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void setupCluster() throws Exception {
jetty.setProxyPort(proxy.getListenPort());
cluster.stopJettySolrRunner(jetty);//TODO: Can we avoid this restart
cluster.startJettySolrRunner(jetty);
proxy.open(jetty.getBaseUrl().toURI());
proxy.open(new URI(jetty.getBaseUrl()));
if (log.isInfoEnabled()) {
log.info("Adding proxy for URL: {}. Proxy: {}", jetty.getBaseUrl(), proxy.getUrl());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,8 @@ public static void createMiniSolrCloudCluster() throws Exception {
// inspired by TestMiniSolrCloudCluster
HashMap<String, String> urlMap = new HashMap<>();
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
URL jettyURL = jetty.getBaseUrl();
String nodeKey = jettyURL.getHost() + ":" + jettyURL.getPort() + jettyURL.getPath().replace("/","_");
String jettyURL = jetty.getBaseUrl();
String nodeKey = jetty.getHost() + ":" + jetty.getLocalPort() + jetty.getBaseUrl().replace("/","_");
urlMap.put(nodeKey, jettyURL.toString());
}
zkStateReader.forceUpdateCollection(COLLECTION_NAME);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public static void createMiniSolrCloudCluster() throws Exception {
NODE_CLIENTS = new ArrayList<HttpSolrClient>(numServers);

for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
URL jettyURL = jetty.getBaseUrl();
String jettyURL = jetty.getBaseUrl();
NODE_CLIENTS.add(getHttpSolrClient(jettyURL.toString() + "/" + COLLECTION_NAME + "/"));
}
assertEquals(numServers, NODE_CLIENTS.size());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ private void doSplitStaticIndexReplication(SolrIndexSplitter.SplitMethod splitMe
String stoppedNodeName = null;
boolean restarted = false;
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
int port = jetty.getBaseUrl().getPort();
int port = jetty.getLocalPort();
if (replica.getStr(BASE_URL_PROP).contains(":" + port)) {
stoppedNodeName = jetty.getNodeName();
jetty.stop();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ public void testCollectionCreateSearchDelete() throws Exception {
ClusterState clusterState = zkStateReader.getClusterState();
Map<String,JettySolrRunner> jettyMap = new HashMap<>();
for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
String key = jetty.getBaseUrl().toString().substring((jetty.getBaseUrl().getProtocol() + "://").length());
String key = jetty.getBaseUrl().toString().substring((System.getProperty("urlScheme") + "://").length());
jettyMap.put(key, jetty);
}
Collection<Slice> slices = clusterState.getCollection(collectionName).getSlices();
Expand Down Expand Up @@ -257,8 +257,8 @@ public void testStopAllStartAll() throws Exception {
shardLeaderMap.put(slice.getLeader().getNodeName().replace("_solr", "/solr"), Boolean.TRUE);
}
for (int ii = 0; ii < jettys.size(); ++ii) {
final URL jettyBaseUrl = jettys.get(ii).getBaseUrl();
final String jettyBaseUrlString = jettyBaseUrl.toString().substring((jettyBaseUrl.getProtocol() + "://").length());
final String jettyBaseUrl = jettys.get(ii).getBaseUrl();
final String jettyBaseUrlString = jettyBaseUrl.toString().substring((System.getProperty("urlScheme")+ "://").length());
final Boolean isLeader = shardLeaderMap.get(jettyBaseUrlString);
if (Boolean.TRUE.equals(isLeader)) {
leaderIndices.add(ii);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public void testTrigger() throws Exception {
CloudUtil.waitForState(cloudManager, COLL2, 60, TimeUnit.SECONDS, clusterShape(2, 2));

double rate = 1.0;
URL baseUrl = targetNode.getBaseUrl();
String baseUrl = targetNode.getBaseUrl();
long waitForSeconds = 5 + random().nextInt(5);
Map<String, Object> props = createTriggerProps(Arrays.asList(COLL1, COLL2), waitForSeconds, rate, -1);
final List<TriggerEvent> events = new ArrayList<>();
Expand Down
4 changes: 2 additions & 2 deletions solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ public void testHostFragmentRule() throws Exception {
String rulesColl = "hostFragment";

JettySolrRunner jetty = cluster.getRandomJetty(random());
String host = jetty.getBaseUrl().getHost();
String host = jetty.getHost();
String[] ipFragments = host.split("\\.");
String ip_1 = ipFragments[ipFragments.length - 1];
String ip_2 = ipFragments[ipFragments.length - 2];
Expand Down Expand Up @@ -294,7 +294,7 @@ public void testHostFragmentRuleThrowsExceptionWhenIpDoesNotMatch() throws Excep
String rulesColl = "ipRuleColl";

JettySolrRunner jetty = cluster.getRandomJetty(random());
String host = jetty.getBaseUrl().getHost();
String host = jetty.getHost();
String[] ipFragments = host.split("\\.");
String ip_1 = ipFragments[ipFragments.length - 1];
String ip_2 = ipFragments[ipFragments.length - 2];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public TestSystemCollAutoCreate() {

// commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018
public void testAutoCreate() throws Exception {
TestBlobHandler.checkBlobPost(cluster.getJettySolrRunner(0).getBaseUrl().toExternalForm(), cloudClient);
TestBlobHandler.checkBlobPost(cluster.getJettySolrRunner(0).getBaseUrl(), cloudClient);
DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public void tearDown() throws Exception {

@Test
public void testZkread() throws Exception {
URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl();
String baseUrl = cluster.getJettySolrRunner(0).getBaseUrl();
String basezk = baseUrl.toString().replace("/solr", "/api") + "/cluster/zk/data";
String basezkls = baseUrl.toString().replace("/solr", "/api") + "/cluster/zk/ls";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public void tearDown() throws Exception {
*/
@Test
public void monitorZookeeper() throws IOException, SolrServerException, InterruptedException, ExecutionException, TimeoutException {
URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl();
String baseUrl = cluster.getJettySolrRunner(0).getBaseUrl();
HttpSolrClient solr = new HttpSolrClient.Builder(baseUrl.toString()).build();
GenericSolrRequest mntrReq = new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/zookeeper/status", new ModifiableSolrParams());
mntrReq.setResponseParser(new DelegationTokenResponse.JsonMapResponseParser());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,9 +137,8 @@ public void test() throws Exception {
assertThat(getShardHandlerFactory(IMPLICIT_CLUSTER_KEY).getWhitelistHostChecker().hasExplicitWhitelist(), is(false));
for (MiniSolrCloudCluster cluster : clusterId2cluster.values()) {
for (JettySolrRunner runner : cluster.getJettySolrRunners()) {
URI uri = runner.getBaseUrl().toURI();
assertThat(getShardHandlerFactory(EXPLICIT_CLUSTER_KEY).getWhitelistHostChecker().getWhitelistHosts(),
hasItem(uri.getHost() + ":" + uri.getPort()));
hasItem(runner.getHost() + ":" + runner.getLocalPort()));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,7 @@ public void process(HttpRequest request, HttpContext context) throws HttpExcepti
BasicClientCookie cookie = new BasicClientCookie(cookieName, cookieValue);
cookie.setVersion(0);
cookie.setPath("/");
cookie.setDomain(jetty.getBaseUrl().getHost());
cookie.setDomain(jetty.getHost());

CookieStore cookieStore = new BasicCookieStore();
CookieSpec cookieSpec = new SolrPortAwareCookieSpecFactory().create(context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ public class HttpSolrClientSSLAuthConPoolTest extends HttpSolrClientConPoolTest

@BeforeClass
public static void checkUrls() throws Exception {
URL[] urls = new URL[] {
String[] urls = new String[] {
jetty.getBaseUrl(), yetty.getBaseUrl()
};
for (URL u : urls) {
assertEquals("expect https urls ","https", u.getProtocol());
for (String u : urls) {
assertTrue("expect https urls ", u.startsWith("https"));
}
assertFalse("expect different urls "+Arrays.toString(urls),
urls[0].equals(urls[1]));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -849,7 +849,7 @@ protected SocketProxy getProxyForReplica(Replica replica) throws Exception {
runners.add(controlJetty);

for (JettySolrRunner j : runners) {
if (replicaBaseUrl.replaceAll("/$", "").equals(j.getProxyBaseUrl().toExternalForm().replaceAll("/$", ""))) {
if (replicaBaseUrl.replaceAll("/$", "").equals(j.getProxyBaseUrl().replaceAll("/$", ""))) {
return j.getProxy();
}
}
Expand Down
Loading

0 comments on commit 2150014

Please sign in to comment.