Running solr 9 in docker for the integration tests that ran it in jet…

…ty, since it was failing to start. Add solr logging config
UNC-Libraries · Aug 28, 2024 · 42ec81d · 42ec81d
1 parent 421a100
commit 42ec81d
Show file tree

Hide file tree

Showing 8 changed files with 127 additions and 71 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -27,6 +27,12 @@ jobs:
           - /tmp/boxc_test_storage:/tmp/boxc_test_storage
         env:
           CATALINA_OPTS: "-Djava.awt.headless=true -server -Xms512m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -Dfcrepo.external.content.allowed=/config/allowed_external_paths.txt"
+      solr:
+        image: solr:9
+        ports:
+          - "48983:8983"
+        volumes:
+          - /tmp/solr-config:/solr_config
 
     steps:
     - name: Change permissions of /tmp/boxc_test_storage
@@ -47,6 +53,20 @@ jobs:
       run: |
         docker run -d --rm -p 43030:3030 atomgraph/fuseki --mem /test
 
+    # Need to copy the config is outside of the source path, otherwise it produces permission conflicts
+    - name: Copy Solr Config into container
+      run: sudo cp -r ${{ github.workspace }}/etc/solr-config/* /tmp/solr-config/
+
+    - name: Wait for Solr to start
+      run: |
+        until curl -s http://localhost:48983/solr; do
+          echo "Waiting for Solr to start..."
+          sleep 5
+        done
+
+    - name: Precreate Solr core
+      run: docker exec $(docker ps -q -f ancestor=solr:9) solr-precreate access /solr_config
+
     - name: Checkout submodules
       run: git submodule update --init --recursive
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -21,4 +21,14 @@ services:
     image: atomgraph/fuseki
     ports:
       - "43030:3030"
-    command: --mem /test
+    command: --mem /test
+  solr:
+    image: solr:9
+    ports:
+      - "48983:8983"
+    volumes:
+      - ./etc/solr-config:/solr_config/config
+    command:
+      - solr-precreate
+      - access
+      - /solr_config/config
diff --git a/etc/solr-config/log4j2.xml b/etc/solr-config/log4j2.xml
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<!-- Default production configuration is asnychronous logging -->
+<Configuration>
+    <Appenders>
+
+        <Console name="STDOUT" target="SYSTEM_OUT">
+            <PatternLayout>
+                <Pattern>
+                    %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+                </Pattern>
+            </PatternLayout>
+        </Console>
+
+        <RollingRandomAccessFile
+                name="MainLogFile"
+                fileName="${sys:solr.log.dir}/solr.log"
+                filePattern="${sys:solr.log.dir}/solr.log.%i" >
+            <PatternLayout>
+                <Pattern>
+                    %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+                </Pattern>
+            </PatternLayout>
+            <Policies>
+                <OnStartupTriggeringPolicy />
+                <SizeBasedTriggeringPolicy size="32 MB"/>
+            </Policies>
+            <DefaultRolloverStrategy max="10"/>
+        </RollingRandomAccessFile>
+
+        <RollingRandomAccessFile
+                name="SlowLogFile"
+                fileName="${sys:solr.log.dir}/solr_slow_requests.log"
+                filePattern="${sys:solr.log.dir}/solr_slow_requests.log.%i" >
+            <PatternLayout>
+                <Pattern>
+                    %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n
+                </Pattern>
+            </PatternLayout>
+            <Policies>
+                <OnStartupTriggeringPolicy />
+                <SizeBasedTriggeringPolicy size="32 MB"/>
+            </Policies>
+            <DefaultRolloverStrategy max="10"/>
+        </RollingRandomAccessFile>
+
+    </Appenders>
+    <Loggers>
+        <!-- Use <AsyncLogger/<AsyncRoot and <Logger/<Root for asynchronous logging or synchonous logging respectively -->
+        <AsyncLogger name="org.apache.hadoop" level="warn"/>
+        <AsyncLogger name="org.apache.solr.update.LoggingInfoStream" level="off"/>
+        <AsyncLogger name="org.apache.zookeeper" level="warn"/>
+        <!-- HttpSolrCall adds markers denoting the handler class to allow fine grained control, metrics are
+             very noisy so by default the metrics handler is turned off to see metrics logging set DENY to ACCEPT -->
+        <AsyncLogger name="org.apache.solr.servlet.HttpSolrCall" level="info">
+            <MarkerFilter marker="org.apache.solr.handler.admin.MetricsHandler" onMatch="DENY" onMismatch="ACCEPT"/>
+        </AsyncLogger>
+        <AsyncLogger name="org.apache.solr.core.SolrCore.SlowRequest" level="info" additivity="false">
+            <AppenderRef ref="SlowLogFile"/>
+        </AsyncLogger>
+        <AsyncLogger name="org.eclipse.jetty.deploy" level="warn"/>
+        <AsyncLogger name="org.eclipse.jetty.webapp" level="warn"/>
+        <AsyncLogger name="org.eclipse.jetty.server.session" level="warn"/>
+
+        <AsyncRoot level="info">
+            <AppenderRef ref="MainLogFile"/>
+            <AppenderRef ref="STDOUT"/>
+        </AsyncRoot>
+    </Loggers>
+</Configuration>
diff --git a/integration/pom.xml b/integration/pom.xml
@@ -47,67 +47,6 @@
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
-                <groupId>com.googlecode.maven-download-plugin</groupId>
-                <artifactId>download-maven-plugin</artifactId>
-                <version>1.6.8</version>
-                <executions>
-                    <execution>
-                        <id>install-solr</id>
-                        <phase>pre-integration-test</phase>
-                        <goals>
-                            <goal>wget</goal>
-                        </goals>
-                    </execution>
-                </executions>
-                <configuration>
-                    <url>https://www.apache.org/dyn/closer.lua/solr/solr/${solr.version}/solr-${solr.version}.tgz?action=download</url>
-                    <unpack>true</unpack>
-                    <outputDirectory>${project.build.directory}/solr/</outputDirectory>
-                </configuration>
-            </plugin>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>Start solr</id>
-                        <goals>
-                            <goal>exec</goal>
-                        </goals>
-                        <phase>pre-integration-test</phase>
-                        <configuration>
-                            <async>true</async>
-                            <executable>${project.build.directory}/solr/solr-${solr.version}/bin/solr</executable>
-                            <commandlineArgs>start -p 48983 -s ${project.build.directory}/../../etc/solr-config/</commandlineArgs>
-                        </configuration>
-                    </execution>
-                    <execution>
-                        <id>Stop solr</id>
-                        <goals>
-                            <goal>exec</goal>
-                        </goals>
-                        <phase>post-integration-test</phase>
-                        <configuration>
-                            <async>true</async>
-                            <executable>${project.build.directory}/solr/solr-${solr.version}/bin/solr</executable>
-                            <arguments>
-                                <argument>stop</argument>
-                                <argument>-p</argument>
-                                <argument>48983</argument>
-                            </arguments>
-                        </configuration>
-                    </execution>
-                </executions>
-                <configuration>
-                    <systemProperties>
-                        <systemProperty>
-                            <name>solr.solr.home</name>
-                            <value>${project.build.directory}/../../etc/solr-config/</value>
-                        </systemProperty>
-                    </systemProperties>
-                </configuration>
-            </plugin>
             <plugin>
                 <groupId>org.eclipse.jetty</groupId>
                 <artifactId>jetty-maven-plugin</artifactId>

diff --git a/integration/src/test/resources/spring-test/solr-embedded-context.xml b/integration/src/test/resources/spring-test/solr-embedded-context.xml
@@ -50,8 +50,8 @@
         <constructor-arg value="../etc/solr-config/solr.xml" />
     </bean>
 
-    <bean id="coreContainer" class="org.apache.solr.core.CoreContainer" factory-method="createAndLoad" init-method="load">
-        <constructor-arg value="#{solrHomePath.toPath()}" />
-        <constructor-arg value="#{solrConfigPath.toPath()}" />
+    <bean id="coreContainer" class="org.apache.solr.core.CoreContainer" factory-method="createAndLoad" destroy-method="shutdown">
+        <constructor-arg value="#{solrHomePath.toPath().toAbsolutePath()}" />
+        <constructor-arg value="#{solrConfigPath.toPath().toAbsolutePath()}" />
     </bean>
 </beans>
diff --git a/search-solr/src/test/java/edu/unc/lib/boxc/search/solr/test/BaseEmbeddedSolrTest.java b/search-solr/src/test/java/edu/unc/lib/boxc/search/solr/test/BaseEmbeddedSolrTest.java
@@ -85,6 +85,7 @@ protected void index(List<SolrInputDocument> docs) throws Exception {
 
     @AfterEach
     public void tearDown() throws Exception {
+        container.shutdown();
         server.close();
         log.debug("Cleaning up data directory");
         FileUtils.deleteDirectory(dataDir);

diff --git a/services-camel-app/src/test/resources/spring-test/solr-indexing-context.xml b/services-camel-app/src/test/resources/spring-test/solr-indexing-context.xml
@@ -116,9 +116,9 @@
         <constructor-arg value="../etc/solr-config/solr.xml" />
     </bean>
 
-    <bean id="coreContainer" class="org.apache.solr.core.CoreContainer" factory-method="createAndLoad" init-method="load">
-        <constructor-arg value="#{solrHomePath.toPath()}" />
-        <constructor-arg value="#{solrConfigPath.toPath()}" />
+    <bean id="coreContainer" class="org.apache.solr.core.CoreContainer" factory-method="createAndLoad" destroy-method="shutdown">
+        <constructor-arg value="#{solrHomePath.toPath().toAbsolutePath()}" />
+        <constructor-arg value="#{solrConfigPath.toPath().toAbsolutePath()}" />
     </bean>
 
     <bean id="solrSearchService" class="edu.unc.lib.boxc.search.solr.services.SolrSearchService">

diff --git a/web-services-app/src/test/resources/spring-test/solr-indexing-context.xml b/web-services-app/src/test/resources/spring-test/solr-indexing-context.xml
@@ -130,9 +130,9 @@
         <constructor-arg value="../etc/solr-config/solr.xml" />
     </bean>
 
-    <bean id="coreContainer" class="org.apache.solr.core.CoreContainer" factory-method="createAndLoad" init-method="load">
-        <constructor-arg value="#{solrHomePath.toPath()}" />
-        <constructor-arg value="#{solrConfigPath.toPath()}" />
+    <bean id="coreContainer" class="org.apache.solr.core.CoreContainer" factory-method="createAndLoad" destroy-method="shutdown">
+        <constructor-arg value="#{solrHomePath.toPath().toAbsolutePath()}" />
+        <constructor-arg value="#{solrConfigPath.toPath().toAbsolutePath()}" />
     </bean>
 
     <bean id="solrAccessRestrictionUtil" class="edu.unc.lib.boxc.search.solr.utils.AccessRestrictionUtil">