Configurable long timeout for metadata queries (#609)

Co-authored-by: Bryan Keller <[email protected]>

metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/Config.java

@@ -587,6 +587,13 @@ public interface Config {
      */
     int getMetadataQueryTimeout();
 
+    /**
+     * Metadata query timeout in seconds, for longer running queries.
+     *
+     * @return Metadata query timeout in seconds for longer running queries
+     */
+    int getLongMetadataQueryTimeout();
+
     /**
      * Whether to check the existence of the iceberg metadata location before updating the table.
      *

metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/DefaultConfigImpl.java

@@ -664,6 +664,11 @@ public int getMetadataQueryTimeout() {
         return this.metacatProperties.getUsermetadata().getQueryTimeoutInSeconds();
     }
 
+    @Override
+    public int getLongMetadataQueryTimeout() {
+        return this.metacatProperties.getUsermetadata().getLongQueryTimeoutInSeconds();
+    }
+
     @Override
     public boolean isIcebergPreviousMetadataLocationCheckEnabled() {
         return this.metacatProperties.getHive().getIceberg().isIcebergPreviousMetadataLocationCheckEnabled();

metacat-common-server/src/main/java/com/netflix/metacat/common/server/properties/UserMetadata.java

@@ -31,6 +31,7 @@ public class UserMetadata {
     @NonNull
     private Config config = new Config();
     private int queryTimeoutInSeconds = 60;
+    private int longQueryTimeoutInSeconds = 120;
 
     /**
      * config related properties.

metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlServiceUtil.java

@@ -19,6 +19,7 @@
 import org.springframework.dao.EmptyResultDataAccessException;
 import org.springframework.jdbc.core.JdbcTemplate;
 
+import javax.sql.DataSource;
 import java.io.InputStream;
 import java.net.URL;
 import java.nio.file.FileSystems;
@@ -81,6 +82,19 @@ public static void loadMySqlDataSource(final DataSourceManager dataSourceManager
         }
         dataSourceManager.load(UserMetadataService.NAME_DATASOURCE, connectionProperties);
     }
+
+    /**
+     * Create a JDBC template with a query timeout.
+     *
+     * @param dataSource data source
+     * @param timeoutSec query timeout, in sec
+     * @return the JDBC template
+     */
+    public static JdbcTemplate createJdbcTemplate(final DataSource dataSource, final int timeoutSec) {
+        final JdbcTemplate result = new JdbcTemplate(dataSource);
+        result.setQueryTimeout(timeoutSec);
+        return result;
+    }
 }
 
 

metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlTagService.java

@@ -99,7 +99,8 @@ public class MySqlTagService implements TagService {
     private final LookupService lookupService;
     private final MetacatJson metacatJson;
     private final UserMetadataService userMetadataService;
-    private JdbcTemplate jdbcTemplate;
+    private final JdbcTemplate jdbcTemplate;
+    private final JdbcTemplate jdbcTemplateLongTimeout;
 
     /**
      * Constructor.
@@ -119,6 +120,8 @@ public MySqlTagService(
     ) {
         this.config = Preconditions.checkNotNull(config, "config is required");
         this.jdbcTemplate = jdbcTemplate;
+        this.jdbcTemplateLongTimeout = MySqlServiceUtil.createJdbcTemplate(
+            jdbcTemplate.getDataSource(), config.getLongMetadataQueryTimeout());
         this.lookupService = Preconditions.checkNotNull(lookupService, "lookupService is required");
         this.metacatJson = Preconditions.checkNotNull(metacatJson, "metacatJson is required");
         this.userMetadataService = Preconditions.checkNotNull(userMetadataService, "userMetadataService is required");
@@ -320,7 +323,7 @@ public Set<String> getTags() {
      * @return list of qualified names of the items
      */
     @Override
-    @Transactional(readOnly = true, timeout = 120)
+    @Transactional(readOnly = true)
     public List<QualifiedName> list(
         @Nullable final Set<String> includeTags,
         @Nullable final Set<String> excludeTags,
@@ -477,7 +480,7 @@ private List<String> queryTaggedItems(final String name,
             new SqlParameterValue(Types.INTEGER, type == null ? 1 : 0),
             new SqlParameterValue(Types.VARCHAR, type == null ? ".*" : type.getRegexValue())
         ).collect(Collectors.toList()));
-        return jdbcTemplate.query(query,
+        return jdbcTemplateLongTimeout.query(query,
             sqlParams.toArray(),
             (rs, rowNum) -> rs.getString("name"));
     }

metacat-metadata-mysql/src/main/java/com/netflix/metacat/metadata/mysql/MySqlUserMetadataConfig.java

@@ -162,9 +162,7 @@ public DataSourceTransactionManager metadataTxManager(
     public JdbcTemplate metadataJdbcTemplate(
         @Qualifier("metadataDataSource") final DataSource mySqlDataSource,
         final Config config) {
-        final JdbcTemplate result = new JdbcTemplate(mySqlDataSource);
-        result.setQueryTimeout(config.getMetadataQueryTimeout());
-        return result;
+        return MySqlServiceUtil.createJdbcTemplate(
+            mySqlDataSource, config.getMetadataQueryTimeout());
     }
-
 }