From 9750a6f6c24a493195e091b1e7ce8719ece2873c Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 11 Jul 2024 16:10:21 +0200 Subject: [PATCH 01/86] OAK-10341 Tree store --- .../jackrabbit/oak/index/IndexOptions.java | 8 + .../document/DocumentStoreIndexerBase.java | 46 + .../flatfile/FlatFileNodeStoreBuilder.java | 4 +- .../indexer/document/tree/TreeStore.java | 231 +++++ .../document/tree/TreeStoreNodeState.java | 210 ++++ .../document/tree/store/AzureStore.java | 220 +++++ .../document/tree/store/Compression.java | 95 ++ .../document/tree/store/DiskCacheStore.java | 226 +++++ .../document/tree/store/FileStore.java | 279 ++++++ .../tree/store/GarbageCollection.java | 98 ++ .../indexer/document/tree/store/LogStore.java | 122 +++ .../document/tree/store/MemoryStore.java | 99 ++ .../indexer/document/tree/store/PageFile.java | 386 ++++++++ .../indexer/document/tree/store/Session.java | 912 ++++++++++++++++++ .../document/tree/store/SlowStore.java | 140 +++ .../document/tree/store/StatsStore.java | 198 ++++ .../indexer/document/tree/store/Store.java | 125 +++ .../document/tree/store/StoreBuilder.java | 89 ++ .../document/tree/store/utils/Cache.java | 43 + .../document/tree/store/utils/Position.java | 30 + .../tree/store/utils/SortedStream.java | 75 ++ .../document/tree/store/utils/Uuid.java | 181 ++++ .../indexer/document/tree/TreeStoreTest.java | 36 + .../jackrabbit/oak/index/IndexCommand.java | 15 +- 24 files changed, 3862 insertions(+), 6 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Uuid.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java index 90e4b8b7a40..e4b21efd138 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java @@ -56,6 +56,7 @@ public class IndexOptions implements OptionsBean { private final OptionSpec docTraversal; private final OptionSpec enableCowCor; private final OptionSpec buildFlatFileStoreSeparately; + private final OptionSpec useTreeStore; private final OptionSpec consistencyCheck; private final OptionSpec asyncDelay; protected OptionSet options; @@ -113,6 +114,8 @@ public IndexOptions(OptionParser parser){ enableCowCor = parser.accepts("enable-cow-cor", "Enables COW/COR during async indexing using oak-run"); buildFlatFileStoreSeparately = parser.accepts("build-flatfilestore-separately", "Builds FlatFileStore as a separate step and then uses it as part of the doc-traversal-mode for reindexing"); + useTreeStore = parser.accepts("use-tree-store", "Use a pre-built tree store"); + indexImportDir = parser.accepts("index-import-dir", "Directory containing index files. This " + "is required when --index-import operation is selected") .requiredIf(importIndex) @@ -233,6 +236,10 @@ public boolean buildFlatFileStoreSeparately() { return options.has(buildFlatFileStoreSeparately); } + public boolean useTreeStore() { + return options.has(useTreeStore); + } + public String getCheckpoint(){ return checkpoint.value(options); } @@ -272,4 +279,5 @@ private static Set collectionOperationNames(Set actionOpts){ } return result; } + } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 5ba625575d2..a64d05227a1 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -34,6 +34,8 @@ import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.IncrementalStoreBuilder; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.RevisionVector; @@ -313,6 +315,50 @@ public FlatFileStore buildFlatFileStore() throws IOException, CommitFailedExcept return flatFileStore; } + public void reindexUsingTreeStore() throws CommitFailedException, IOException { + NodeStateEntryReader reader = new NodeStateEntryReader(indexHelper.getGCBlobStore()); + TreeStore treeStore = new TreeStore(new File("target/treeStore"), reader); + + // TODO this is mostly a copy of reindex() + + IndexingProgressReporter progressReporter = + new IndexingProgressReporter(IndexUpdateCallback.NOOP, NodeTraversalCallback.NOOP); + configureEstimators(progressReporter); + + NodeState checkpointedState = indexerSupport.retrieveNodeStateForCheckpoint(); + NodeStore copyOnWriteStore = new MemoryNodeStore(checkpointedState); + indexerSupport.switchIndexLanesAndReindexFlag(copyOnWriteStore); + + NodeBuilder builder = copyOnWriteStore.getRoot().builder(); + CompositeIndexer indexer = prepareIndexers(copyOnWriteStore, builder, progressReporter); + if (indexer.isEmpty()) { + return; + } + + closer.register(indexer); + + progressReporter.reset(); + + progressReporter.reindexingTraversalStart("/"); + + preIndexOperations(indexer.getIndexers()); + + Stopwatch indexerWatch = Stopwatch.createStarted(); + + for (NodeStateEntry entry : treeStore) { + reportDocumentRead(entry.getPath(), progressReporter); + indexer.index(entry); + } + + progressReporter.reindexingTraversalEnd(); + progressReporter.logReport(); + log.info("Completed the indexing in {}", indexerWatch); + + copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + + indexerSupport.postIndexWork(copyOnWriteStore); + } + public void reindex() throws CommitFailedException, IOException { INDEXING_PHASE_LOGGER.info("[TASK:FULL_INDEX_CREATION:START] Starting indexing job"); Stopwatch indexJobWatch = Stopwatch.createStarted(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index 294d8f3117d..b06f76ce55d 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -23,6 +23,8 @@ import com.mongodb.client.MongoDatabase; import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.guava.common.collect.Iterables; +import org.apache.jackrabbit.guava.common.collect.Iterables; +import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.index.IndexHelper; import org.apache.jackrabbit.oak.index.IndexerSupport; @@ -256,7 +258,7 @@ public List buildList(IndexHelper indexHelper, IndexerSupport ind private IndexStoreFiles createdSortedStoreFiles() throws IOException, CompositeException { // Check system property defined path String sortedFilePath = System.getProperty(OAK_INDEXER_SORTED_FILE_PATH); - if (StringUtils.isNotBlank(sortedFilePath)) { + if (sortedFilePath != null && !sortedFilePath.isEmpty()) { File sortedDir = new File(sortedFilePath); log.info("Attempting to read from provided sorted files directory [{}] (via system property '{}')", sortedDir.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java new file mode 100644 index 00000000000..1d25e939df8 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry.NodeStateEntryBuilder; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +public class TreeStore implements Iterable, Closeable { + + public static void main(String... args) throws IOException { + String dir = args[0]; + MemoryBlobStore blobStore = new MemoryBlobStore(); + NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); + try (TreeStore treeStore = new TreeStore(new File(dir), entryReader)) { + Session session = treeStore.session; + Store store = treeStore.store; + if (store.keySet().isEmpty()) { + session.init(); + String fileName = args[1]; + try (BufferedReader lineReader = new BufferedReader( + new FileReader(fileName, StandardCharsets.UTF_8))) { + int count = 0; + long start = System.nanoTime(); + while (true) { + String line = lineReader.readLine(); + if (line == null) { + break; + } + count++; + if (count % 1000000 == 0) { + long time = System.nanoTime() - start; + System.out.println(count + " " + (time / count) + " ns/entry"); + } + int index = line.indexOf('|'); + if (index < 0) { + throw new IllegalArgumentException("| is missing: " + line); + } + String path = line.substring(0, index); + String value = line.substring(index + 1); + session.put(path, value); + + if (!path.equals("/")) { + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + session.put(parentPath + "\t" + nodeName, ""); + } + + } + } + session.flush(); + store.close(); + } + Iterator it = treeStore.iterator(); + long nodeCount = 0; + long childNodeCount = 0; + long start = System.nanoTime(); + while (it.hasNext()) { + NodeStateEntry e = it.next(); + childNodeCount += e.getNodeState().getChildNodeCount(Long.MAX_VALUE); + nodeCount++; + if (nodeCount % 1000000 == 0) { + long time = System.nanoTime() - start; + System.out.println("Node count: " + nodeCount + + " child node count: " + childNodeCount + + " speed: " + (time / nodeCount) + " ns/entry"); + } + } + System.out.println("Node count: " + nodeCount + " Child node count: " + childNodeCount); + } + } + + private final Store store; + private final Session session; + private final NodeStateEntryReader entryReader; + private final Cache nodeStateCache = new Cache<>(10000); + + public TreeStore(File directory, NodeStateEntryReader entryReader) { + this.entryReader = entryReader; + String storeConfig = System.getProperty("oak.treeStoreConfig", + "type=file\n" + + "cacheSizeMB=4096\n" + + "maxFileSize=64000000\n" + + "dir=" + directory.getAbsolutePath()); + this.store = StoreBuilder.build(storeConfig); + this.session = new Session(store); + } + + @Override + public void close() throws IOException { + session.flush(); + store.close(); + } + + @Override + public Iterator iterator() { + Iterator> it = session.iterator(); + return new Iterator() { + + NodeStateEntry current; + + { + fetch(); + } + + private void fetch() { + while (it.hasNext()) { + Entry e = it.next(); + if (e.getValue().isEmpty()) { + continue; + } + current = getNodeStateEntry(e.getKey(), e.getValue()); + return; + } + current = null; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public NodeStateEntry next() { + NodeStateEntry result = current; + fetch(); + return result; + } + + }; + } + + NodeStateEntry getNodeStateEntry(String path) { + return new NodeStateEntryBuilder(getNodeState(path), path).build(); + } + + NodeStateEntry getNodeStateEntry(String path, String value) { + return new NodeStateEntryBuilder(getNodeState(path, value), path).build(); + } + + NodeState getNodeState(String path) { + NodeState result = nodeStateCache.get(path); + if (result != null) { + return result; + } + String value = session.get(path); + if (value == null || value.isEmpty()) { + result = EmptyNodeState.MISSING_NODE; + } else { + result = getNodeState(path, value); + } + nodeStateCache.put(path, result); + return result; + } + + NodeState getNodeState(String path, String value) { + NodeState result = nodeStateCache.get(path); + if (result != null) { + return result; + } + String line = path + "|" + value; + NodeStateEntry entry = entryReader.read(line); + result = new TreeStoreNodeState(entry.getNodeState(), path, this); + nodeStateCache.put(path, result); + return result; + } + + /** + * The child node entry for the given path. + * + * @param path the path, e.g. /hello/world + * @return the child node entry, e.g. /helloworld + */ + public static String toChildNodeEntry(String path) { + if (path.equals("/")) { + return "\t"; + } + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + return parentPath + "\t" + nodeName; + } + + /** + * The child node entry for the given parent and child. + * + * @param path the parentPath, e.g. /hello + * @param childName the name of the child node, e.g. world + * @return the child node entry, e.g. /helloworld + */ + public static String toChildNodeEntry(String parentPath, String childName) { + return parentPath + "\t" + childName; + } + + public Session getSession() { + return session; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java new file mode 100644 index 00000000000..a09c49d743f --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.apache.jackrabbit.guava.common.collect.Iterators.transform; + +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class TreeStoreNodeState implements NodeState { + private final NodeState delegate; + private final String path; + private final TreeStore treeStore; + + public TreeStoreNodeState(NodeState delegate, String path, TreeStore treeStore) { + this.delegate = delegate; + this.path = path; + this.treeStore = treeStore; + } + + @Override + public boolean exists() { + return delegate.exists(); + } + + @Override + public boolean hasProperty(@NotNull String name) { + return delegate.hasProperty(name); + } + + @Nullable + @Override + public PropertyState getProperty(@NotNull String name) { + return delegate.getProperty(name); + } + + @Override + public boolean getBoolean(@NotNull String name) { + return delegate.getBoolean(name); + } + + @Override + public long getLong(String name) { + return delegate.getLong(name); + } + + @Nullable + @Override + public String getString(String name) { + return delegate.getString(name); + } + + @NotNull + @Override + public Iterable getStrings(@NotNull String name) { + return delegate.getStrings(name); + } + + @Nullable + @Override + public String getName(@NotNull String name) { + return delegate.getName(name); + } + + @NotNull + @Override + public Iterable getNames(@NotNull String name) { + return delegate.getNames(name); + } + + @Override + public long getPropertyCount() { + return delegate.getPropertyCount(); + } + + @NotNull + @Override + public Iterable getProperties() { + return delegate.getProperties(); + } + + @NotNull + @Override + public NodeBuilder builder() { + return delegate.builder(); + } + + @Override + public boolean compareAgainstBaseState(NodeState base, NodeStateDiff diff) { + return AbstractNodeState.compareAgainstBaseState(this, base, diff); + } + + // ~-------------------------------< child node access > + + @Override + public boolean hasChildNode(@NotNull String name) { + return treeStore.getNodeState(PathUtils.concat(path, name)).exists(); + } + + @NotNull + @Override + public NodeState getChildNode(@NotNull String name) throws IllegalArgumentException { + return treeStore.getNodeState(PathUtils.concat(path, name)); + } + + @Override + public long getChildNodeCount(long max) { + long result = 0; + Iterator it = getChildNodeNamesIterator(); + while (it.hasNext()) { + result++; + if (result > max) { + return Long.MAX_VALUE; + } + it.next(); + } + return result; + } + + @Override + public Iterable getChildNodeNames() { + return new Iterable() { + public Iterator iterator() { + return getChildNodeNamesIterator(); + } + }; + } + + @NotNull + @Override + public Iterable getChildNodeEntries() { + return () -> transform(getChildNodeIterator(), + s -> new MemoryChildNodeEntry(PathUtils.getName(s.getPath()), s.getNodeState())); + } + + private Iterator getChildNodeIterator() { + return transform(getChildNodeNamesIterator(), + s -> treeStore.getNodeStateEntry(PathUtils.concat(path, s))); + } + + Iterator getChildNodeNamesIterator() { + Iterator> it = treeStore.getSession().iterator(path); + return new Iterator() { + String current; + { + fetch(); + } + + private void fetch() { + if (!it.hasNext()) { + current = null; + } else { + Entry e = it.next(); + if (!e.getValue().isEmpty()) { + current = null; + } else { + String key = e.getKey(); + int index = key.lastIndexOf('\t'); + if (index < 0) { + throw new IllegalArgumentException(key); + } + current = key.substring(index + 1); + } + } + } + + public boolean hasNext() { + return current != null; + } + + public String next() { + String result = current; + if (result == null) { + throw new IllegalStateException(); + } + fetch(); + return result; + } + }; + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java new file mode 100644 index 00000000000..31d90f01733 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Uuid; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.blob.CloudAppendBlob; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlobDirectory; +import com.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.blob.ListBlobItem; + +public class AzureStore implements Store { + + private final Properties config; + private final CloudStorageAccount cloud; + private final CloudBlobClient cloudBlobClient; + private final CloudBlobContainer container; + private final CloudBlobDirectory dir; + private Compression compression = Compression.NO; + private long writeCount; + private long readCount; + + public String toString() { + return "azure"; + } + + public AzureStore(Properties config) { + this.config = config; + try { + cloud = CloudStorageAccount.parse( + config.getProperty("storageAccount")); + cloudBlobClient = cloud.createCloudBlobClient(); + String maximumExecutionTimeInMs = config.getProperty("maximumExecutionTimeInMs"); + if (maximumExecutionTimeInMs != null) { + cloudBlobClient.getDefaultRequestOptions(). + setMaximumExecutionTimeInMs(Integer.parseInt(maximumExecutionTimeInMs)); + } + container = cloudBlobClient.getContainerReference( + config.getProperty("container")); + container.createIfNotExists(); + dir = container.getDirectoryReference( + config.getProperty("directory")); + } catch (Exception e) { +; // TODO proper logging +e.printStackTrace(); + throw new IllegalArgumentException(config.toString(), e); + } + } + + @Override + public void setWriteCompression(Compression compression) { + this.compression = compression; + } + + @Override + public PageFile getIfExists(String key) { + try { + readCount++; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + CloudAppendBlob blob = dir.getAppendBlobReference(key); + blob.download(out); + byte[] data = out.toByteArray(); + Compression c = Compression.getCompressionFromData(data[0]); + data = c.expand(data); + return PageFile.fromBytes(data); + } catch (URISyntaxException | StorageException e) { +; // TODO proper logging +e.printStackTrace(); + throw new IllegalArgumentException(key, e); + } + } + + @Override + public boolean supportsByteOperations() { + return true; + } + + @Override + public byte[] getBytes(String key) { + try { + readCount++; + CloudBlockBlob blob = dir.getBlockBlobReference(key); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + blob.download(out); + return out.toByteArray(); + } catch (URISyntaxException | StorageException e) { +; // TODO proper logging +e.printStackTrace(); + throw new IllegalArgumentException(key, e); + } + } + + @Override + public void putBytes(String key, byte[] data) { + try { + writeCount++; + CloudBlockBlob blob = dir.getBlockBlobReference(key); + blob.upload(new ByteArrayInputStream(data), data.length); + } catch (URISyntaxException | StorageException | IOException e) { +; // TODO proper logging +e.printStackTrace(); + throw new IllegalArgumentException(e); + } + } + + @Override + public void put(String key, PageFile value) { + byte[] data = value.toBytes(); + data = compression.compress(data); + CloudBlockBlob blob; + try { + writeCount++; + blob = dir.getBlockBlobReference(key); +long start = System.nanoTime(); + blob.uploadFromByteArray(data, 0, data.length); +long time = System.nanoTime() - start; +System.out.println("Azure upload " + key + " size " + data.length + " nanos " + time); + + } catch (URISyntaxException | StorageException | IOException e) { +; // TODO proper logging +e.printStackTrace(); + throw new IllegalArgumentException(e); + } + } + + @Override + public String newFileName() { + return Uuid.timeBasedVersion7().toShortString(); + } + + @Override + public Set keySet() { + try { + HashSet set = new HashSet<>(); + for (ListBlobItem item : dir.listBlobs()) { + if (item instanceof CloudBlockBlob) { + String name = ((CloudBlockBlob) item).getName(); + int index = name.lastIndexOf('/'); + if (index >= 0) { + name = name.substring(index + 1); + } + set.add(name); + } + } + return set; + } catch (StorageException | URISyntaxException e) { +; // TODO proper logging +e.printStackTrace(); + throw new IllegalArgumentException(e); + } + } + + @Override + public void remove(Set set) { + try { + for(String key : set) { + CloudBlockBlob blob = dir.getBlockBlobReference(key); + writeCount++; + blob.deleteIfExists(); + } + } catch (StorageException | URISyntaxException e) { +; // TODO proper logging +e.printStackTrace(); + throw new IllegalArgumentException(e); + } + } + + @Override + public void removeAll() { + remove(keySet()); + } + + @Override + public long getWriteCount() { + return writeCount; + } + + @Override + public long getReadCount() { + return readCount; + } + + @Override + public void close() { + } + + @Override + public Properties getConfig() { + return config; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java new file mode 100644 index 00000000000..83a14363a52 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Arrays; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + +public enum Compression { + NO { + @Override + byte[] compress(byte[] data) { + return data; + } + + @Override + byte[] expand(byte[] data) { + return data; + } + }, + LZ4 { + private final LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor(); + private final LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + private byte[] compressBuffer = new byte[1024 * 1024]; + + @Override + byte[] compress(byte[] data) { + // synchronization is needed because we share the buffer + synchronized (compressor) { + byte[] buffer = compressBuffer; + if (buffer.length < 2 * data.length) { + // increase the size + buffer = new byte[2 * data.length]; + compressBuffer = buffer; + } + buffer[0] = '4'; + writeInt(buffer, 1, data.length); + int len = 5 + compressor.compress(data, 0, data.length, buffer, 5, buffer.length - 5); + return Arrays.copyOf(buffer, len); + } + } + + @Override + byte[] expand(byte[] data) { + int len = readInt(data, 1); + byte[] target = new byte[len]; + decompressor.decompress(data, 5, target, 0, len); + return target; + } + }; + + abstract byte[] compress(byte[] data); + abstract byte[] expand(byte[] data); + + + public static void writeInt(byte[] buff, int pos, int x) { + buff[pos++] = (byte) (x >> 24); + buff[pos++] = (byte) (x >> 16); + buff[pos++] = (byte) (x >> 8); + buff[pos] = (byte) x; + } + + public static int readInt(byte[] buff, int pos) { + return (buff[pos++] << 24) + ((buff[pos++] & 0xff) << 16) + ((buff[pos++] & 0xff) << 8) + (buff[pos] & 0xff); + } + + public static Compression getCompressionFromData(byte data) { + switch (data) { + case '4': + return Compression.LZ4; + default: + return Compression.NO; + } + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java new file mode 100644 index 00000000000..e25ff91a77d --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache; + +public class DiskCacheStore implements Store { + + private final Properties config; + + // the frontend (local disk) + private Store frontend; + + // the backend (azure or s3) + private Store backend; + + // set of entries that are not yet uploaded + private ConcurrentHashMap uploading = new ConcurrentHashMap<>(); + + // executor service that uploads files + private ExecutorService executor; + + // cache for entries that were downloaded + // (doesn't contain entries that are not yet uploaded) + private Cache cache; + + private int readAhead; + + public String toString() { + return "diskCache(" + frontend + ", " + backend + ")"; + } + + public DiskCacheStore(Properties config) { + this.config = config; + frontend = StoreBuilder.build(StoreBuilder.subProperties(config, "front.")); + + // cache size, in number of files on disk + int cacheSize = Integer.parseInt(config.getProperty("cacheSize", "1000000000")); + + // read ahead this number of children + this.readAhead = Integer.parseInt(config.getProperty("readAhead", "1000000")); + + // using this number of threads at most + int threads = Integer.parseInt(config.getProperty("threads", "20")); + + cache = new Cache(cacheSize) { + private static final long serialVersionUID = 1L; + + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + boolean result = super.removeEldestEntry(eldest); + if (result) { + frontend.remove(Collections.singleton(eldest.getKey())); + } + return result; + } + }; + executor = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), + r -> { + Thread thread = new Thread(r, "DiskCacheStore"); + thread.setDaemon(true); + return thread; + }); + backend = StoreBuilder.build(StoreBuilder.subProperties(config, "back.")); + } + + @Override + public PageFile getIfExists(String key) { + PageFile result = frontend.getIfExists(key); + if (result != null) { + return result; + } + if (backend.supportsByteOperations() && frontend.supportsByteOperations()) { + byte[] data = backend.getBytes(key); + frontend.putBytes(key, data); + result = frontend.get(key); + } else { + result = backend.get(key); + frontend.put(key, result); + } + synchronized (cache) { + cache.put(key, key); + } + if (result.isInnerNode()) { + for (int i = 0; i < readAhead && i < result.getValueCount(); i++) { + String childKey = result.getChildValue(i); + // System.out.println(" prefetching " + childKey + " because we read " + key); + executor.submit(new Runnable() { + + @Override + public void run() { + // this will put the entry in the cache + // and also read the children if needed + getIfExists(childKey); + // System.out.println(" prefetch done for " + childKey + " because we read " + key); + } + + }); + } + } + return result; + } + + @Override + public void put(String key, PageFile value) { + if (uploading.containsKey(key)) { + System.out.println("WARNING: upload is in progress: " + key + + " and a new upload is scheduled. waiting for the upload to finish, to avoid concurrency issues."); + while (uploading.containsKey(key)) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + uploading.put(key, key); + frontend.put(key, value); + executor.submit(new Runnable() { + + @Override + public void run() { + try { + if (frontend.supportsByteOperations() && backend.supportsByteOperations()) { + byte[] data = frontend.getBytes(key); + backend.putBytes(key, data); + } else { + backend.put(key, frontend.get(key)); + } + } catch (Exception e) { + e.printStackTrace(); + } + uploading.remove(key); + } + + }); + } + + @Override + public String newFileName() { + return frontend.newFileName(); + } + + @Override + public Set keySet() { + return backend.keySet(); + } + + @Override + public void remove(Set set) { + synchronized (cache) { + set.forEach(k -> cache.remove(k)); + } + frontend.remove(set); + backend.remove(set); + } + + @Override + public void removeAll() { + frontend.removeAll(); + backend.removeAll(); + } + + @Override + public long getWriteCount() { + return frontend.getWriteCount(); + } + + @Override + public long getReadCount() { + return frontend.getReadCount(); + } + + @Override + public void setWriteCompression(Compression compression) { + frontend.setWriteCompression(compression); + backend.setWriteCompression(compression); + } + + @Override + public void close() { + try { + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.DAYS); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + frontend.close(); + backend.close(); + } + + @Override + public Properties getConfig() { + return config; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java new file mode 100644 index 00000000000..9c72f8d6f75 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Uuid; + +public class FileStore implements Store { + + private final Properties config; + private final String directory; + private Compression compression = Compression.NO; + private long writeCount, readCount; + private Thread backgroundThread; + private ConcurrentHashMap pendingWrites = new ConcurrentHashMap<>(); + private LinkedBlockingQueue queue = new LinkedBlockingQueue<>(100); + + private static final WriteOperation STOP = new WriteOperation(); + + static class WriteOperation { + String key; + byte[] value; + } + + public String toString() { + return "file(" + directory + ")"; + } + + public FileStore(Properties config) { + this.config = config; + this.directory = config.getProperty("dir"); + new File(directory).mkdirs(); + boolean asyncWrite = Boolean.parseBoolean(config.getProperty("async", "false")); + if (asyncWrite) { + startAsyncWriter(); + } + } + + private void startAsyncWriter() { + backgroundThread = new Thread(new Runnable() { + + @Override + public void run() { + try { + for (int i = 0;; i++) { + WriteOperation op = queue.take(); + if (i % 200 == 0) { + // System.out.println(" file writer queue size " + queue.size()); + } + if (op == STOP) { + break; + } + writeFile(op.key, op.value); + pendingWrites.remove(op.key); + } + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + }); + backgroundThread.setDaemon(true); + backgroundThread.start(); + } + + @Override + public void close() { + try { + if (backgroundThread != null) { + queue.put(STOP); + backgroundThread.join(); + } + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Override + public void setWriteCompression(Compression compression) { + this.compression = compression; + } + + @Override + public PageFile getIfExists(String key) { + PageFile pending = pendingWrites.get(key); + if (pending != null) { + return pending; + } + readCount++; + File f = getFile(key); + if (!f.exists()) { + return null; + } + try (RandomAccessFile file = new RandomAccessFile(f, "r")) { + long length = file.length(); + if (length == 0) { + // deleted in the meantime + return null; + } + byte[] data = new byte[(int) length]; + file.readFully(data); + Compression c = Compression.getCompressionFromData(data[0]); + data = c.expand(data); + return PageFile.fromBytes(data); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public void put(String key, PageFile value) { + writeCount++; + if (backgroundThread != null) { + writeFileAsync(key, value.copy()); + } else { + writeFile(key, value.toBytes()); + } + } + + private void writeFileAsync(String key, PageFile value) { + pendingWrites.put(key, value); + WriteOperation op = new WriteOperation(); + op.key = key; + op.value = value.toBytes(); + try { + queue.put(op); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + @Override + public boolean supportsByteOperations() { + return true; + } + + @Override + public byte[] getBytes(String key) { + File f = getFile(key); + if (!f.exists()) { + return null; + } + try { + readCount++; + try (RandomAccessFile file = new RandomAccessFile(f, "r")) { + long length = file.length(); + if (length == 0) { + // deleted in the meantime + return null; + } + byte[] data = new byte[(int) length]; + file.readFully(data); + return data; + } + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public void putBytes(String key, byte[] data) { + try (FileOutputStream out = new FileOutputStream(getFile(key))) { + out.write(data); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private void writeFile(String key, byte[] data) { + data = compression.compress(data); + putBytes(key, data); + + /* + File tempFile = getFile(key, true); + File targetFile = getFile(key); + // https://stackoverflow.com/questions/595631/how-to-atomically-rename-a-file-in-java-even-if-the-dest-file-already-exists + try (RandomAccessFile file = new RandomAccessFile(tempFile, "rw")) { + file.write(data); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + try { + Files.move(tempFile.toPath(), targetFile.toPath(), StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + */ + } + + private File getFile(String key) { + return new File(directory, key); + } + + @Override + public String newFileName() { + return Uuid.timeBasedVersion7().toShortString(); + } + + @Override + public Set keySet() { + File dir = new File(directory); + if (!dir.exists()) { + return Collections.emptySet(); + } + String[] list = dir.list(new FilenameFilter() { + + @Override + public boolean accept(File dir, String name) { + return new File(dir, name).isFile(); + } + + }); + return new HashSet<>(Arrays.asList(list)); + } + + @Override + public void remove(Set set) { + // TODO keep for some time if the file is relatively new? + for (String key : set) { + writeCount++; + getFile(key).delete(); + } + } + + @Override + public void removeAll() { + File dir = new File(directory); + for(File f: dir.listFiles()) { + f.delete(); + } + } + + @Override + public long getWriteCount() { + return writeCount; + } + + @Override + public long getReadCount() { + return readCount; + } + + @Override + public Properties getConfig() { + return config; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java new file mode 100644 index 00000000000..bcf8c8d6151 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.HashSet; +import java.util.List; + +/** + * Remove unreferenced files from the store. + */ +public class GarbageCollection { + + private final Store store; + + public GarbageCollection(Store store) { + this.store = store; + } + + /** + * Run garbage collection. + * + * @param rootFiles + * @return the result + */ + public GarbageCollectionResult run(List rootFiles) { + HashSet used = mark(rootFiles); + return sweep(used); + } + + HashSet mark(List rootFiles) { + HashSet used = new HashSet<>(); + for(String root : rootFiles) { + mark(root, used); + } + return used; + } + + private void mark(String fileName, HashSet used) { + used.add(fileName); + if (fileName.startsWith(Session.LEAF_PREFIX)) { + return; + } + // root or inner node + PageFile file = store.get(fileName); + if (file.isInnerNode()) { + for (int i = 0; i < file.getValueCount(); i++) { + mark(file.getChildValue(i), used); + } + } + } + + private GarbageCollectionResult sweep(HashSet used) { + GarbageCollectionResult result = new GarbageCollectionResult(); + // TODO keep files that were very recently updated + // (don't remove files that are part of a concurrent flush) + HashSet removeSet = new HashSet(); + for (String key: new HashSet<>(store.keySet())) { + if (!used.contains(key)) { + removeSet.add(key); + result.countRemoved++; + } else { + result.countKept++; + } + } + store.remove(removeSet); + return result; + } + + /** + * Garbage collection results. + */ + public static class GarbageCollectionResult { + public long sizeInBytes; + public long countRemoved; + public long countKept; + + public String toString() { + return "removed: " + countRemoved + " kept: " + countKept + " size: " + sizeInBytes; + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java new file mode 100644 index 00000000000..1d9824673cb --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Arrays; +import java.util.Properties; +import java.util.Set; + +public class LogStore implements Store { + + private final Properties config; + private final Store backend; + + public String toString() { + return "log(" + backend + ")"; + } + + LogStore(Store backend) { + this.config = backend.getConfig(); + this.backend = backend; + } + + private void log(String message, Object... args) { + System.out.println(backend + "." + message + " " + Arrays.toString(args)); + } + + @Override + public PageFile getIfExists(String key) { + log("getIfExists", key); + return backend.getIfExists(key); + } + + @Override + public void put(String key, PageFile value) { + log("put", key); + backend.put(key, value); + } + + @Override + public String newFileName() { + return backend.newFileName(); + } + + @Override + public Set keySet() { + log("keySet"); + return backend.keySet(); + } + + @Override + public void remove(Set set) { + log("remove", set); + backend.remove(set); + } + + @Override + public void removeAll() { + log("removeAll"); + backend.removeAll(); + } + + @Override + public long getWriteCount() { + return backend.getWriteCount(); + } + + @Override + public long getReadCount() { + return backend.getReadCount(); + } + + @Override + public void setWriteCompression(Compression compression) { + log("setWriteCompression", compression); + backend.setWriteCompression(compression); + } + + @Override + public void close() { + log("close"); + backend.close(); + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public boolean supportsByteOperations() { + return backend.supportsByteOperations(); + } + + @Override + public byte[] getBytes(String key) { + log("getBytes", key); + return backend.getBytes(key); + } + + @Override + public void putBytes(String key, byte[] data) { + log("putBytes", key, data.length); + backend.putBytes(key, data); + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java new file mode 100644 index 00000000000..7892dfcf2dd --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.HashMap; +import java.util.Properties; +import java.util.Set; + +public class MemoryStore implements Store { + + private final Properties config; + private final HashMap map = new HashMap<>(); + private long nextFileName; + private long writeCount, readCount; + + public MemoryStore() { + this(new Properties()); + } + + public MemoryStore(Properties config) { + this.config = config; + } + + @Override + public void setWriteCompression(Compression compression) { + // ignore + } + + public PageFile getIfExists(String key) { + readCount++; + return map.get(key); + } + + public void put(String key, PageFile file) { + writeCount++; + map.put(key, file); + } + + public String toString() { + return "files: " + map.size(); + } + + public String newFileName() { + return "f" + nextFileName++; + } + + public Set keySet() { + return map.keySet(); + } + + public void remove(Set set) { + for (String key : set) { + writeCount++; + map.remove(key); + } + } + + @Override + public void removeAll() { + map.clear(); + nextFileName = 0; + } + + @Override + public long getWriteCount() { + return writeCount; + } + + @Override + public long getReadCount() { + return readCount; + } + + @Override + public void close() { + } + + @Override + public Properties getConfig() { + return config; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java new file mode 100644 index 00000000000..cf366c5d75c --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * A B-tree page (leaf, or inner node). + * An inner node contains one more value than keys. + * A leaf page has the same number of keys and values. + */ +public class PageFile { + + private static final boolean VERIFY_SIZE = false; + private static final int INITIAL_SIZE_IN_BYTES = 24; + + private String fileName; + + private final boolean innerNode; + + private static ByteBuffer REUSED_BUFFER = ByteBuffer.allocate(1024 * 1024); + + private ArrayList keys = new ArrayList<>(); + private ArrayList values = new ArrayList<>(); + private long update; + private String nextRoot; + private int sizeInBytes = INITIAL_SIZE_IN_BYTES; + + // -1: beginning; 0: middle; 1: end + private int lastSearchIndex; + + // contains unwritten modifications + private boolean modified; + + public PageFile(boolean innerNode) { + this.innerNode = innerNode; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getFileName() { + return fileName; + } + + public void setUpdate(long update) { + modified = true; + this.update = update; + } + + public static PageFile fromBytes(byte[] data) { + ByteBuffer buff = ByteBuffer.wrap(data); + int type = buff.get(); + String nextRoot = readString(buff); + long update = buff.getLong(); + String prefix = readString(buff); + int len = buff.getInt(); + PageFile result; + if (type == 0) { + result = new PageFile(true); + for (int i = 0; i < len; i++) { + result.appendRecord(prefix + readString(buff), readString(buff)); + } + result.values.add(readString(buff)); + } else { + result = new PageFile(false); + for (int i = 0; i < len; i++) { + result.appendRecord(prefix + readString(buff), readString(buff)); + } + } + if (!nextRoot.isEmpty()) { + result.setNextRoot(nextRoot); + } + result.setUpdate(update); + result.modified = false; + return result; + } + + public byte[] toBytes() { + // TODO synchronization is needed because we share the buffer + synchronized (PageFile.class) { + ByteBuffer buff = REUSED_BUFFER; + if (buff.capacity() < sizeInBytes * 2) { + buff = REUSED_BUFFER = ByteBuffer.allocate(sizeInBytes * 2); + } + buff.rewind(); + // first byte may not be '4', as that is used for LZ4 compression + buff.put((byte) (innerNode ? 0 : 1)); + writeString(buff, nextRoot == null ? "" : nextRoot); + buff.putLong(update); + String prefix = keys.size() < 2 ? "" : commonPrefix(keys.get(0), keys.get(keys.size() - 1)); + writeString(buff, prefix); + buff.putInt(keys.size()); + if (innerNode) { + for (int i = 0; i < keys.size(); i++) { + writeString(buff, keys.get(i).substring(prefix.length())); + writeString(buff, values.get(i)); + } + writeString(buff, values.get(values.size() - 1)); + } else { + for (int i = 0; i < keys.size(); i++) { + writeString(buff, keys.get(i).substring(prefix.length())); + writeString(buff, values.get(i)); + } + } + buff.flip(); + buff.rewind(); + byte[] array = new byte[buff.remaining()]; + buff.get(array); + // reset the limit + REUSED_BUFFER = ByteBuffer.wrap(buff.array()); + return array; + } + } + + private void writeString(ByteBuffer buff, String s) { + if (s == null) { + buff.putShort((short) -2); + return; + } + byte[] data = s.getBytes(StandardCharsets.UTF_8); + if (data.length < Short.MAX_VALUE) { + // could get a bit larger, but some negative values are reserved + buff.putShort((short) data.length); + } else { + buff.putShort((short) -1); + buff.putInt(data.length); + } + buff.put(data); + } + + private static String readString(ByteBuffer buff) { + int len = buff.getShort(); + if (len == -2) { + return null; + } else if (len == -1) { + len = buff.getInt(); + int pos = buff.position(); + buff.position(buff.position() + len); + return new String(buff.array(), pos, len, StandardCharsets.UTF_8); + } else { + int pos = buff.position(); + buff.position(buff.position() + len); + return new String(buff.array(), pos, len, StandardCharsets.UTF_8); + } + } + + private static String commonPrefix(String prefix, String x) { + if (prefix == null) { + return x; + } + int i = 0; + for (; i < prefix.length() && i < x.length(); i++) { + if (prefix.charAt(i) != x.charAt(i)) { + break; + } + } + return prefix.substring(0, i); + } + + public String toString() { + return keys + "" + values; + } + + public PageFile copy() { + PageFile result = new PageFile(innerNode); + result.modified = modified; + result.keys = new ArrayList<>(keys); + result.values = new ArrayList<>(values); + result.sizeInBytes = sizeInBytes; + result.nextRoot = nextRoot; + return result; + } + + public void addChild(int index, String childKey, String newChildFileName) { + modified = true; + if (index > 0) { + keys.add(index - 1, childKey); + sizeInBytes += childKey.length(); + } + values.add(index, newChildFileName); + sizeInBytes += 4; + sizeInBytes += newChildFileName.length(); + } + + public void setValue(int index, String value) { + modified = true; + sizeInBytes -= sizeInBytes(values.get(index)); + sizeInBytes += sizeInBytes(value); + values.set(index, value); + } + + private long sizeInBytes(String obj) { + if (obj == null) { + return 5; + } else if (obj instanceof String) { + return ((String) obj).length() + 2; + } else { + throw new IllegalStateException(); + } + } + + public void removeRecord(int index) { + modified = true; + String key = keys.remove(index); + String value = values.remove(index); + sizeInBytes -= 4; + sizeInBytes -= key.length(); + sizeInBytes -= sizeInBytes(value); + } + + public void appendRecord(String k, String v) { + modified = true; + keys.add(k); + values.add(v); + sizeInBytes += 4; + sizeInBytes += k.length(); + sizeInBytes += sizeInBytes(v); + } + + public void insertRecord(int index, String key, String value) { + modified = true; + keys.add(index, key); + values.add(index, value); + sizeInBytes += 4; + sizeInBytes += key.length(); + sizeInBytes += sizeInBytes(value); + } + + public long getUpdate() { + return update; + } + + public int sizeInBytes() { + if (VERIFY_SIZE) { + int size = 24; + for (String p : keys) { + size += p.length(); + size += 4; + } + for (String o : values) { + size += sizeInBytes(o); + } + if (size != sizeInBytes) { + throw new AssertionError(); + } + } + return sizeInBytes; + } + + public boolean canSplit() { + if (innerNode) { + return keys.size() > 2; + } else { + return keys.size() > 1; + } + } + + public List getKeys() { + return keys; + } + + public int getKeyIndex(String key) { + int index; + if (keys.isEmpty()) { + return -1; + } + if (lastSearchIndex == 1) { + if (key.compareTo(keys.get(keys.size() - 1)) > 0) { + index = -(keys.size() + 1); + // index = Collections.binarySearch(recordKeys, key); + } else { + index = Collections.binarySearch(keys, key); + } + } else if (lastSearchIndex == -1) { + if (key.compareTo(keys.get(0)) < 0) { + index = -1; + // index = Collections.binarySearch(recordKeys, key); + } else { + index = Collections.binarySearch(keys, key); + } + } else { + index = Collections.binarySearch(keys, key); + } + if (index == -(keys.size() + 1)) { + lastSearchIndex = 1; + } else if (index == -1) { + lastSearchIndex = -1; + } else { + lastSearchIndex = 0; + } + return index; + } + + public String getValue(int index) { + return values.get(index); + } + + public String getChildValue(int index) { + return (String) values.get(index); + } + + public String getNextKey(String largerThan) { + int index; + if (largerThan == null) { + index = 0; + } else { + index = getKeyIndex(largerThan); + if (index < 0) { + index = -index - 1; + } else { + index++; + } + } + if (index < 0 || index >= keys.size()) { + return null; + } + return keys.get(index); + } + + public String getNextRoot() { + return nextRoot; + } + + public void setNextRoot(String nextRoot) { + modified = true; + this.nextRoot = nextRoot; + } + + public void removeKey(int index) { + modified = true; + String key = keys.get(index); + sizeInBytes -= key.length(); + sizeInBytes -= 4; + keys.remove(index); + } + + public void removeValue(int index) { + modified = true; + String x = (String) values.get(index); + sizeInBytes -= x.length(); + values.remove(index); + } + + public boolean isInnerNode() { + return innerNode; + } + + public int getValueCount() { + return values.size(); + } + + public String getKey(int index) { + return keys.get(index); + } + + public void setModified(boolean modified) { + this.modified = modified; + } + + public boolean isModified() { + return modified; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java new file mode 100644 index 00000000000..47aec1fb605 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -0,0 +1,912 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Properties; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Position; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SortedStream; + +/** + * Read and write keys and values. + */ +public class Session { + + private static final int DEFAULT_CACHE_SIZE = 128; + private static final int DEFAULT_MAX_FILE_SIZE = 16 * 1024; + private static final int DEFAULT_CACHE_SIZE_MB = 16; + private static final int DEFAULT_MAX_ROOTS = 10; + + public static final String ROOT_NAME = "root"; + public static final String LEAF_PREFIX = "data_"; + static final String INNER_NODE_PREFIX = "node_"; + static final String DELETED = new String("DELETED"); + + static final boolean MULTI_ROOT = true; + + private final Store store; + private final Cache cache = new Cache<>(DEFAULT_CACHE_SIZE) { + private static final long serialVersionUID = 1L; + + public boolean removeEldestEntry(Map.Entry eldest) { + boolean result = super.removeEldestEntry(eldest); + if (result) { + String key = eldest.getKey(); + PageFile value = eldest.getValue(); + if(value.isModified()) { + store.put(key, value); + // not strictly needed as it's no longer referenced + value.setModified(false); + } + } + return result; + } + }; + private long updateId; + private int maxFileSize; + private int cacheSizeMB; + private int maxRoots = DEFAULT_MAX_ROOTS; + private long fileReadCount; + + public Session() { + this(new MemoryStore(new Properties())); + } + + public Session(Store store) { + this.store = store; + maxFileSize = Integer.parseInt(store.getConfig().getProperty("maxFileSize", "" + DEFAULT_MAX_FILE_SIZE)); + cacheSizeMB = Integer.parseInt(store.getConfig().getProperty("cacheSizeMB", "" + DEFAULT_CACHE_SIZE_MB)); + changeCacheSize(); + } + + /** + * Set the maximum number of roots. + * + * @param maxRoots the new value + */ + public void setMaxRoots(int maxRoots) { + this.maxRoots = maxRoots; + } + + public int getMaxRoots() { + return maxRoots; + } + + /** + * Set the cache size in MB. + * + * @param mb the value + */ + public void setCacheSizeMB(int mb) { + this.cacheSizeMB = mb; + } + + /** + * Set the maximum file size. Files might be slightly larger than that, but not a lot. + * + * @param sizeBytes the file size in bytes + */ + public void setMaxFileSize(int sizeBytes) { + this.maxFileSize = sizeBytes; + changeCacheSize(); + } + + private void changeCacheSize() { + int cacheEntryCount = (int) (cacheSizeMB * 1024L * 1024 / maxFileSize); + cache.setSize(cacheEntryCount); + } + + /** + * Get the number of files read from the cache or storage. + * + * @return the result + */ + public long getFileReadCount() { + return fileReadCount; + } + + private List getRootFileNames() { + LinkedHashSet result = new LinkedHashSet<>(); + String nextRoot = ROOT_NAME; + do { + boolean isNew = result.add(nextRoot); + if (!isNew) { + throw new IllegalStateException("Linked list contains a loop"); + } + PageFile root = getFile(nextRoot); + nextRoot = root.getNextRoot(); + } while (nextRoot != null); + return new ArrayList<>(result); + } + + private void mergeRootsIfNeeded() { + List roots = getRootFileNames(); + if (roots.size() > maxRoots) { + mergeRoots(Integer.MAX_VALUE); + } + } + + /** + * Initialize the storage, creating a new root if needed. + */ + public void init() { + PageFile root = store.getIfExists(ROOT_NAME); + if (root == null) { + root = newPageFile(false); + putFile(ROOT_NAME, root); + } + } + + private PageFile copyPageFile(PageFile old) { + PageFile result = old.copy(); + result.setUpdate(updateId); + return result; + } + + private PageFile newPageFile(boolean isInternalNode) { + PageFile result = new PageFile(isInternalNode); + result.setUpdate(updateId); + return result; + } + + /** + * Get an entry. + * + * @param key the key + * @return the value, or null + */ + public String get(String key) { + if (key == null) { + throw new NullPointerException(); + } + String fileName = ROOT_NAME; + do { + PageFile file = getFile(fileName); + String nextRoot = file.getNextRoot(); + String result = get(file, key); + if (result != null) { + return result == DELETED ? null : result; + } + fileName = nextRoot; + } while (fileName != null); + return null; + } + + /** + * Get the entry if it exists. + * + * @param root the root file + * @param k the key + * @return null if not found, DELETED if removed, or the value + */ + private String get(PageFile root, String k) { + while (true) { + if (!root.isInnerNode()) { + int index = root.getKeyIndex(k); + if (index >= 0) { + String result = root.getValue(index); + return result == null ? DELETED : result; + } + return null; + } + int index = root.getKeyIndex(k); + if (index < 0) { + index = -index - 2; + } + index++; + String fileName = root.getChildValue(index); + root = getFile(fileName); + // continue with the new file + } + } + + /** + * Put a value. + * To remove an entry, the value needs to be null. + * + * @param key the key + * @param value the value + */ + public void put(String key, String value) { + if (key == null) { + throw new NullPointerException(); + } + if (value == null) { + PageFile root = getFile(ROOT_NAME); + if (root.getNextRoot() != null) { + value = DELETED; + } + } + put(ROOT_NAME, key, value); + } + + /** + * Put a value. + * + * @param rootFileName + * @param key + * @param value (DELETED if we need to put that, or null to remove the entry) + * @return the file name of the root (different than the passed file name, if the file was copied) + */ + private void put(String rootFileName, String key, String value) { + String fileName = rootFileName; + PageFile file = getFile(fileName); + if (file.getUpdate() < updateId) { + fileName = store.newFileName(); + file = copyPageFile(file); + putFile(fileName, file); + } + ArrayList parents = new ArrayList<>(); + String k = key; + while (true) { + int index = file.getKeyIndex(k); + if (!file.isInnerNode()) { + if (index >= 0) { + if (value == null) { + file.removeRecord(index); + } else { + file.setValue(index, value == DELETED ? null : value); + } + } else { + // not found + if (value == null) { + // nothing to do + return; + } + file.insertRecord(-index - 1, k, value == DELETED ? null : value); + } + break; + } + parents.add(fileName); + if (index < 0) { + index = -index - 2; + } + index++; + fileName = file.getChildValue(index); + file = getFile(fileName); + // continue with the new file + } + putFile(fileName, file); + splitOrMerge(fileName, file, parents); + } + + private void splitOrMerge(String fileName, PageFile file, ArrayList parents) { + int size = file.sizeInBytes(); + if (size > maxFileSize && file.canSplit()) { + split(fileName, file, parents); + } else if (file.getKeys().size() == 0) { + merge(fileName, file, parents); + } + } + + private void merge(String fileName, PageFile file, ArrayList parents) { + if (file.getValueCount() > 0) { + return; + } + if (parents.isEmpty()) { + // root: ignore + return; + } + String parentFileName = parents.remove(parents.size() - 1); + PageFile parentFile = getFile(parentFileName); + for (int i = 0; i < parentFile.getValueCount(); i++) { + String pf = parentFile.getChildValue(i); + if (pf.equals(fileName)) { + if (parentFile.getValueCount() == 1) { + parentFile = newPageFile(false); + if (!parentFileName.startsWith(ROOT_NAME)) { + String newParentFileName = LEAF_PREFIX + parentFileName.substring(INNER_NODE_PREFIX.length()); + putFile(newParentFileName, parentFile); + updateChildFileName(parents.get(parents.size() - 1), parentFileName, newParentFileName); + parentFileName = newParentFileName; + } + } else if (i == parentFile.getValueCount() - 1) { + // remove the last entry + parentFile.removeKey(i - 1); + parentFile.removeValue(i); + } else { + parentFile.removeKey(i); + parentFile.removeValue(i); + } + putFile(parentFileName, parentFile); + merge(parentFileName, parentFile, parents); + break; + } + } + } + + private void updateChildFileName(String fileName, String oldChild, String newChild) { + PageFile file = getFile(fileName); + for (int i = 0; i < file.getValueCount(); i++) { + if (file.getChildValue(i).equals(oldChild)) { + file.setValue(i, newChild); + putFile(fileName, file); + return; + } + } + } + + private void split(String fileName, PageFile file, ArrayList parents) { + List keys = new ArrayList<>(file.getKeys()); + String parentFileName, newFileName1, newFileName2; + PageFile parentFile, newFile1, newFile2; + boolean isInternalNode = file.isInnerNode(); + if (parents.isEmpty()) { + // new root + parentFileName = fileName; + parentFile = newPageFile(true); + parentFile.setNextRoot(file.getNextRoot()); + newFileName1 = (isInternalNode ? INNER_NODE_PREFIX : LEAF_PREFIX) + + store.newFileName(); + parentFile.addChild(0, null, newFileName1); + } else { + parentFileName = parents.remove(parents.size() - 1); + parentFile = getFile(parentFileName); + newFileName1 = fileName; + } + newFile1 = newPageFile(isInternalNode); + newFileName2 = (isInternalNode ? INNER_NODE_PREFIX : LEAF_PREFIX) + + store.newFileName(); + newFile2 = newPageFile(isInternalNode); + int sentinelIndex = keys.size() / 2; + String sentinel = keys.get(sentinelIndex); + // shorten the sentinel if possible + String beforeSentinal = keys.get(sentinelIndex - 1); + while (sentinel.length() > 0 && !isInternalNode) { + // for internal nodes, there might be other keys on the left side + // that might be shoter than the entry before the sentinel + String oneShorter = sentinel.substring(0, sentinel.length() - 1); + if (beforeSentinal.compareTo(oneShorter) >= 0) { + break; + } + sentinel = oneShorter; + } + if (!isInternalNode) { + // leaf + for (int i = 0; i < keys.size() / 2; i++) { + String k = keys.get(i); + String v = file.getValue(i); + newFile1.appendRecord(k, v); + } + for (int i = keys.size() / 2; i < keys.size(); i++) { + String k = keys.get(i); + String v = file.getValue(i); + newFile2.appendRecord(k, v); + } + } else { + // inner node + newFile1.addChild(0, null, file.getChildValue(0)); + for (int i = 1; i <= keys.size() / 2; i++) { + String p = keys.get(i - 1); + newFile1.appendRecord(p, file.getChildValue(i)); + } + newFile2.addChild(0, null, file.getChildValue(keys.size() / 2 + 1)); + for (int i = keys.size() / 2 + 2; i <= keys.size(); i++) { + String p = keys.get(i - 1); + newFile2.appendRecord(p, file.getChildValue(i)); + } + } + // insert sentinel into parent + int index = parentFile.getKeyIndex(sentinel); + parentFile.addChild(-index, sentinel, newFileName2); + putFile(newFileName1, newFile1); + putFile(newFileName2, newFile2); + putFile(parentFileName, parentFile); + splitOrMerge(parentFileName, parentFile, parents); + } + + private void putFile(String fileName, PageFile file) { + if (!file.isModified()) { + throw new AssertionError(); + } + file.setFileName(fileName); + cache.put(fileName, file); + } + + private PageFile getFile(String key) { + fileReadCount++; + PageFile result = cache.get(key); + if (result == null) { + result = store.get(key); + result.setFileName(key); + cache.put(key, result); + } + return result; + } + + /** + * Merge a number of roots. Merging will create a new root, which contains the + * data of the previous roots. If there are less roots, this method returns without merging. + * + * @param max the number of roots to merge (Integer.MAX_VALUE for all) + */ + public void mergeRoots(int max) { + List list = getRootFileNames(); + PageFile root = getFile(ROOT_NAME); + String rootFileCopy = ROOT_NAME + "_" + updateId; + root = copyPageFile(root); + root.setModified(true); + putFile(rootFileCopy, root); + Iterator> it = iterator(null, max); + PageFile newRoot = newPageFile(false); + newRoot.setNextRoot(rootFileCopy); + putFile(ROOT_NAME, newRoot); + while(it.hasNext()) { + Entry e = it.next(); + put(e.getKey(), e.getValue()); + // TODO we can remove files that are processed + } + newRoot = getFile(ROOT_NAME); + if (max < list.size()) { + newRoot.setNextRoot(list.get(max)); + } else { + newRoot.setNextRoot(null); + } + flush(); + } + + /** + * Get the number of roots. + * + * @return the number of roots + */ + public int getRootCount() { + return getRootFileNames().size(); + } + + /** + * Make the current tree read-only and switch to a new root. + * If there are already too many roots, then they will be merged. + * All changes are flushed to storage. + */ + public void checkpoint() { + flush(); + mergeRootsIfNeeded(); + List roots = getRootFileNames(); + if (roots.size() > 1) { + // get the last root + for (String s : roots) { + int index = s.lastIndexOf('_'); + if (index >= 0) { + updateId = Math.max(updateId, Long.parseLong(s.substring(index + 1))); + } + } + updateId++; + } + PageFile root = getFile(ROOT_NAME); + cache.remove(ROOT_NAME); + String rootFileCopy = ROOT_NAME + "_" + updateId; + root = copyPageFile(root); + root.setFileName(rootFileCopy); + putFile(rootFileCopy, root); + updateId++; + if (MULTI_ROOT) { + root = newPageFile(false); + root.setNextRoot(rootFileCopy); + putFile(ROOT_NAME, root); + // need to flush here + // so that GC does not collect rootFileCopy + flush(); + root = copyPageFile(root); + putFile(ROOT_NAME, root); + + } else { + flush(); + root = copyPageFile(root); + putFile(ROOT_NAME, root); + } + } + + /** + * Flush all changes to storage. + */ + public void flush() { + // we store all the pages except for the root, and the root at the very end + // this is to get a smaller chance that the root is stored, + // and points to a page that doesn't exist yet - + // but we don't try to ensure this completely; + // stored inner nodes might point to pages that are not stored yet + Entry changedRoot = null; + for(Entry e : cache.entrySet()) { + String k = e.getKey(); + PageFile v = e.getValue(); + if (!v.isModified()) { + continue; + } + if (k.equals(ROOT_NAME)) { + // don't store the changed root yet + changedRoot = e; + } else { + store.put(k, v); + // here we have to reset the flag + v.setModified(false); + } + } + // now store the changed root + if (changedRoot != null) { + String k = changedRoot.getKey(); + PageFile v = changedRoot.getValue(); + store.put(k, v); + // here we have to reset the flag + v.setModified(false); + } + } + + // =============================================================== + // iteration over entries + // this is fast: internally, a stack of Position object is kept + + /** + * Get all entries. Do not add or move entries while + * iterating. + * + * @return the result + */ + public Iterator> iterator() { + return iterator(null); + } + + /** + * Get all entries. Do not add or move entries while iterating. + * + * @param largerThan all returned keys are larger than this; null to start at + * the beginning + * @return the result + */ + public Iterator> iterator(String largerThan) { + return iterator(largerThan, Integer.MAX_VALUE); + } + + private Iterator> iterator(String largerThan, int maxRootCount) { + ArrayList streams = new ArrayList<>(); + String next = ROOT_NAME; + for (int i = 0; i < maxRootCount; i++) { + streams.add(new SortedStream(next, immutableRootIterator(next, largerThan))); + next = getFile(next).getNextRoot(); + if (next == null) { + break; + } + } + PriorityQueue pq = new PriorityQueue<>(streams); + return new Iterator>() { + + Entry current; + String lastKey; + + { + fetchNext(); + } + + private void fetchNext() { + while (pq.size() > 0) { + SortedStream s = pq.poll(); + if (s.currentKey == null) { + // if this is null, it must be the last stream + break; + } + String key = s.currentKey; + if (key.equals(lastKey)) { + continue; + } + String value = s.currentValue; + s.next(); + pq.add(s); + if (value == DELETED) { + continue; + } + lastKey = key; + current = new Entry<>() { + + @Override + public String getKey() { + return key; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String setValue(String value) { + throw new UnsupportedOperationException(); + } + + }; + return; + } + current = null; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public Entry next() { + Entry result = current; + fetchNext(); + return result; + } + + }; + } + + private Iterator immutableRootIterator(String rootFileName, String largerThan) { + + return new Iterator() { + private final ArrayList stack = new ArrayList<>(); + private Position current; + + { + current = new Position(); + current.file = getFile(rootFileName); + current.valuePos = index(current.file, largerThan); + down(largerThan); + if (current.valuePos >= current.file.getValueCount()) { + next(); + } + } + + private int index(PageFile file, String largerThan) { + if (largerThan == null) { + return 0; + } + int index = file.getKeyIndex(largerThan); + if (file.isInnerNode()) { + if (index < 0) { + index = -index - 2; + } + index++; + } else { + if (index < 0) { + index = -index - 1; + } else { + index++; + } + } + return index; + } + + @Override + public String toString() { + return stack + " " + current; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public Position next() { + if (current == null) { + throw new NoSuchElementException(); + } + Position result = current; + current = new Position(); + current.file = result.file; + current.valuePos = result.valuePos + 1; + while (true) { + if (!current.file.isInnerNode() && current.valuePos < result.file.getValueCount()) { + break; + } + if (stack.size() == 0) { + current = null; + break; + } + current = stack.remove(stack.size() - 1); + current.valuePos++; + if (current.valuePos < current.file.getValueCount()) { + down(null); + break; + } + } + return result; + } + + private void down(String largerThan) { + while (current.file.isInnerNode()) { + stack.add(current); + Position pos = new Position(); + PageFile file = getFile(current.file.getChildValue(current.valuePos)); + pos.file = file; + pos.valuePos = index(pos.file, largerThan); + current = pos; + } + } + }; + } + + // =============================================================== + // iteration over keys, over all roots + // this is a bit slow: internally, *for each key* + // it will traverse from all roots down to the leaf + + /** + * Return all keys in sorted order. Roots don't need to be merged. + * + * @return all keys + */ + public Iterable keys() { + return keys(null); + } + + /** + * Return all keys in sorted order. + * + * @param largerThan all returned keys are larger than this; null to start at + * the beginning + * @return the keys + */ + public Iterable keys(String largerThan) { + final String next = getNextKey(largerThan); + return new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + private String current = next; + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public String next() { + if (current == null) { + throw new NoSuchElementException(); + } + String result = current; + current = getNextKey(current); + return result; + } + + }; + } + + }; + } + + private String getNextKey(String largerThan) { + if (MULTI_ROOT) { + String fileName = ROOT_NAME; + String result = null; + do { + String next = getNextKey(largerThan, fileName); + if (result == null) { + result = next; + } else if (next != null && next.compareTo(result) < 0) { + result = next; + } + PageFile file = getFile(fileName); + fileName = file.getNextRoot(); + } while (fileName != null); + return result; + } else { + return getNextKey(largerThan, ROOT_NAME); + } + } + + private String getNextKey(String largerThan, String fileName) { + PageFile file = getFile(fileName); + if (!file.isInnerNode()) { + String nextKey = file.getNextKey(largerThan); + if (nextKey != null) { + return nextKey; + } + return null; + } + int index; + index = largerThan == null ? -1 : file.getKeyIndex(largerThan); + if (index < 0) { + index = -index - 2; + } + index++; + for (; index < file.getValueCount(); index++) { + fileName = file.getChildValue(index); + String result = getNextKey(largerThan, fileName); + if (result != null) { + return result; + } + } + return null; + } + + // =============================================================== + // garbage collection + + public void runGC() { + new GarbageCollection(store).run(getRootFileNames()); + } + + // =============================================================== + // info + + public String getInfo() { + StringBuilder buff = new StringBuilder(); + GarbageCollection gc = new GarbageCollection(store); + int rootId = 0; + for (String r : getRootFileNames()) { + buff.append(String.format("root #%d contains %d files (file name %s)\n", + rootId, gc.mark(Collections.singletonList(r)).size(), r)); + rootId++; + } + return buff.toString(); + } + + // =============================================================== + // partitioning + + public String getMinKey() { + return getNextKey(null); + } + + public String getMaxKey() { + if (getFile(ROOT_NAME).getNextRoot() != null) { + throw new UnsupportedOperationException("Not fully merged"); + } + String fileName = ROOT_NAME; + while (true) { + PageFile file = getFile(fileName); + if (!file.isInnerNode()) { + return file.getKey(file.getKeys().size() - 1); + } + fileName = file.getChildValue(file.getValueCount() - 1); + } + } + + public String getApproximateMedianKey(String low, String high) { + if (getFile(ROOT_NAME).getNextRoot() != null) { + throw new UnsupportedOperationException("Not fully merged"); + } + String fileName = ROOT_NAME; + while (true) { + PageFile file = getFile(fileName); + if (!file.isInnerNode()) { + return file.getKey(0); + } + int i1 = file.getKeyIndex(low); + int i2 = file.getKeyIndex(high); + if (i1 < 0) { + i1 = -i1 - 1; + } + if (i2 < 0) { + i2 = -i2 - 1; + } + if (i2 != i1) { + int middle = (i1 + i2) / 2; + return file.getKey(middle); + } + fileName = file.getChildValue(i1); + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java new file mode 100644 index 00000000000..8169091d21d --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Properties; +import java.util.Set; + +public class SlowStore implements Store { + + private final Properties config; + private final Store backend; + private final long mbOverhead; + private final long mbPerSecond; + + public String toString() { + return "slow(" + backend + ")"; + } + + SlowStore(Store backend) { + this.config = backend.getConfig(); + this.backend = backend; + this.mbOverhead = Integer.parseInt(config.getProperty("mbOverhead", "3")); + this.mbPerSecond = Integer.parseInt(config.getProperty("mbPerSecond", "70")); + } + + private void delay(long nanos, int sizeInBytes) { + long bytes = sizeInBytes + 1_000_000 * mbOverhead; + long nanosRequired = 1_000 * bytes / mbPerSecond; + long delayNanos = nanosRequired - nanos; + long delayMillis = delayNanos / 1_000_000; + if (delayMillis > 0) { + try { + Thread.sleep(delayMillis); + } catch (InterruptedException e) { + // ignore + } + } + } + + @Override + public PageFile getIfExists(String key) { + long start = System.nanoTime(); + PageFile result = backend.getIfExists(key); + long time = System.nanoTime() - start; + delay(time, result == null ? 0 : result.sizeInBytes()); + return result; + } + + @Override + public void put(String key, PageFile value) { + long start = System.nanoTime(); + backend.put(key, value); + long time = System.nanoTime() - start; + delay(time, value.sizeInBytes()); + } + + @Override + public String newFileName() { + return backend.newFileName(); + } + + @Override + public Set keySet() { + return backend.keySet(); + } + + @Override + public void remove(Set set) { + backend.remove(set); + } + + @Override + public void removeAll() { + backend.removeAll(); + } + + @Override + public long getWriteCount() { + return backend.getWriteCount(); + } + + @Override + public long getReadCount() { + return backend.getReadCount(); + } + + @Override + public void setWriteCompression(Compression compression) { + backend.setWriteCompression(compression); + } + + @Override + public void close() { + backend.close(); + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public boolean supportsByteOperations() { + return backend.supportsByteOperations(); + } + + @Override + public byte[] getBytes(String key) { + long start = System.nanoTime(); + byte[] result = backend.getBytes(key); + long time = System.nanoTime() - start; + delay(time, result.length); + return result; + } + + @Override + public void putBytes(String key, byte[] data) { + long start = System.nanoTime(); + backend.putBytes(key, data); + long time = System.nanoTime() - start; + delay(time, data.length); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java new file mode 100644 index 00000000000..ce5b39dc963 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +public class StatsStore implements Store { + + private final Properties config; + private final Store backend; + private long lastLog; + private AtomicLong pending = new AtomicLong(); + + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + + public String toString() { + return "stats(" + backend + ")"; + } + + StatsStore(Store backend) { + this.config = backend.getConfig(); + this.backend = backend; + } + + @Override + public PageFile getIfExists(String key) { + long start = start(); + long sizeInBytes = 0; + try { + PageFile result = backend.getIfExists(key); + sizeInBytes = result == null ? 0 : result.sizeInBytes(); + return result; + } finally { + add("getIfExists", start, sizeInBytes); + } + } + + private long start() { + pending.incrementAndGet(); + return System.nanoTime(); + } + + private void add(String key, long start, long bytes) { + long now = System.nanoTime(); + pending.decrementAndGet(); + long nanos = now - start; + Stats stats = map.computeIfAbsent(key, s -> new Stats(key)); + stats.count++; + stats.nanosMax = Math.max(stats.nanosMax, nanos); + stats.nanosMin = Math.min(stats.nanosMin, nanos); + stats.nanosTotal += nanos; + stats.bytesMax = Math.max(stats.bytesMax, bytes); + stats.bytesMin = Math.min(stats.bytesMin, nanos); + stats.bytesTotal += bytes; + if (lastLog == 0) { + lastLog = start; + } + if (now - lastLog > 10_000_000_000L) { + TreeMap sorted = new TreeMap<>(map); + System.out.print(backend.toString()); + System.out.println(sorted.values().toString() + " pending " + pending); + lastLog = now; + } + } + + @Override + public void put(String key, PageFile value) { + long start = start(); + try { + backend.put(key, value); + } finally { + add("put", start, value.sizeInBytes()); + } + } + + @Override + public String newFileName() { + return backend.newFileName(); + } + + @Override + public Set keySet() { + return backend.keySet(); + } + + @Override + public void remove(Set set) { + backend.remove(set); + } + + @Override + public void removeAll() { + backend.removeAll(); + } + + @Override + public long getWriteCount() { + return backend.getWriteCount(); + } + + @Override + public long getReadCount() { + return backend.getReadCount(); + } + + @Override + public void setWriteCompression(Compression compression) { + backend.setWriteCompression(compression); + } + + @Override + public void close() { + backend.close(); + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public boolean supportsByteOperations() { + return backend.supportsByteOperations(); + } + + @Override + public byte[] getBytes(String key) { + long start = start(); + long len = 0; + try { + byte[] result = backend.getBytes(key); + len = result.length; + return result; + } finally { + add("getBytes", start, len); + } + } + + @Override + public void putBytes(String key, byte[] data) { + long start = start(); + try { + backend.putBytes(key, data); + } finally { + add("putBytes", start, data.length); + } + } + + static class Stats { + final String key; + long count; + long bytesMin; + long bytesMax; + long bytesTotal; + long nanosMin; + long nanosMax; + long nanosTotal; + + public Stats(String key) { + this.key = key; + } + + public String toString() { + if (count == 0) { + return ""; + } + String result = key; + result += " " + count + " calls"; + if (bytesTotal > 0 && nanosTotal > 0) { + result += " " + (bytesTotal / count / 1_000_000) + " avgMB" + + (bytesTotal == 0 ? "" : (" " + ((bytesTotal * 1_000) / nanosTotal) + " MB/s")); + } + return result; + } + + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java new file mode 100644 index 00000000000..37e70560ac6 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Properties; +import java.util.Set; + +/** + * Storage for files. + */ +public interface Store { + + /** + * Get a file + * + * @param key the file name + * @return the file + */ + default PageFile get(String key) { + PageFile result = getIfExists(key); + if (result == null) { + throw new IllegalStateException("Not found: " + key); + } + return result; + } + + /** + * Get a file if it exists + * + * @param key the file name + * @return the file, or null + */ + PageFile getIfExists(String key); + + /** + * Storage a file. + * + * @param key the file name + * @param value the file + */ + void put(String key, PageFile value); + + /** + * Generate a new file name. + * + * @return + */ + String newFileName(); + + /** + * Get the list of files. + * + * @return the result + */ + Set keySet(); + + /** + * Remove a number of files. + * + * @param set the result + */ + void remove(Set set); + + /** + * Remove all files. + */ + void removeAll(); + + /** + * Get the number of files written. + * + * @return the result + */ + long getWriteCount(); + + /** + * Get the number of files read. + * + * @return the result + */ + long getReadCount(); + + /** + * Set the compression algorithm used for writing from now on. + * + * @param compression the compression algorithm + */ + void setWriteCompression(Compression compression); + + /** + * Close the store + */ + void close(); + + Properties getConfig(); + + default boolean supportsByteOperations() { + return false; + } + + default byte[] getBytes(String key) { + throw new UnsupportedOperationException(); + } + + default void putBytes(String key, byte[] data) { + throw new UnsupportedOperationException(); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java new file mode 100644 index 00000000000..f6aeced1444 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Properties; + +public class StoreBuilder { + + /** + * Build a store. The configuration options are passed as a list of properties. + * + * - empty string or null: in-memory. + * - "type=file": file system, with "dir" directory + * - "type=azure": Azure + * - "type=s3": Amazon S3 + * - "type=diskCache": disk cache, with "dir" and "backend=azure" or "s3" + * + * @param config the config + * @return a store + * @throws IllegalArgumentException + */ + public static Store build(String config) throws IllegalArgumentException { + if (config == null || config.isEmpty()) { + return new MemoryStore(new Properties()); + } + config = config.replace(' ', '\n'); + Properties prop = new Properties(); + try { + prop.load(new StringReader(config)); + } catch (IOException e) { + throw new IllegalArgumentException(config, e); + } + return build(prop); + } + + public static Store build(Properties config) { + String type = config.getProperty("type"); + switch (type) { + case "memory": + return new MemoryStore(config); + case "file": + return new FileStore(config); + case "azure": + return new AzureStore(config); + case "diskCache": + return new DiskCacheStore(config); + } + if (type.startsWith("stats.")) { + config.put("type", type.substring("stats.".length())); + return new StatsStore(build(config)); + } else if (type.startsWith("slow.")) { + config.put("type", type.substring("slow.".length())); + return new SlowStore(build(config)); + } else if (type.startsWith("log.")) { + config.put("type", type.substring("log.".length())); + return new LogStore(build(config)); + } + throw new IllegalArgumentException(config.toString()); + } + + public static Properties subProperties(Properties p, String prefix) { + Properties p2 = new Properties(); + for (Object k : p.keySet()) { + if (k.toString().startsWith(prefix)) { + p2.put(k.toString().substring(prefix.length()), p.get(k)); + } + } + return p2; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java new file mode 100644 index 00000000000..fcb2e4bf494 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class Cache extends LinkedHashMap { + + private static final long serialVersionUID = 1L; + private int maxSize; + + public Cache(int maxSize) { + super(16, 0.75f, true); + this.maxSize = maxSize; + } + + public void setSize(int maxSize) { + this.maxSize = maxSize; + } + + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + return size() > maxSize; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java new file mode 100644 index 00000000000..2fc6b6e51ad --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.PageFile; + +public class Position { + public PageFile file; + public int valuePos; + + public String toString() { + return (file.isInnerNode() ? "internal" : "leaf") + " " + valuePos + " " + file.getKeys() + "\n"; + } +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java new file mode 100644 index 00000000000..8659becb193 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.util.Iterator; + +public class SortedStream implements Comparable { + + private final String rootFileName; + private Iterator it; + public String currentKey; + public String currentValue; + + public SortedStream(String rootFileName, Iterator it) { + this.rootFileName = rootFileName; + this.it = it; + next(); + } + + public String toString() { + return "file " + rootFileName + " key " + currentKey + " value " + currentValue; + } + + String currentKeyOrNull() { + return currentKey; + } + + Object currentValue() { + return currentValue; + } + + public void next() { + if (it.hasNext()) { + Position pos = it.next(); + currentKey = pos.file.getKey(pos.valuePos); + currentValue = pos.file.getValue(pos.valuePos); + } else { + currentKey = null; + currentValue = null; + } + } + + @Override + public int compareTo(SortedStream o) { + if (currentKey == null) { + if (o.currentKey == null) { + return rootFileName.compareTo(o.rootFileName); + } + return 1; + } else if (o.currentKey == null) { + return -1; + } + int comp = currentKey.compareTo(o.currentKey); + if (comp == 0) { + return rootFileName.compareTo(o.rootFileName); + } + return comp; + } +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Uuid.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Uuid.java new file mode 100644 index 00000000000..2e2873f6ae0 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Uuid.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.security.SecureRandom; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A UUID implementation. + * + * It supports creating version 7 UUIDs, which are time-ordered. + * See also draft-ietf-uuidrev-rfc4122bis-00 + * + * Unlike java.util.UUID, the comparison is unsigned, + * so that the string comparison yields the same result. + */ +public class Uuid implements Comparable { + + private static final AtomicLong UUID_LAST_MILLIS_AND_COUNT = new AtomicLong(0); + + private static final SecureRandom RANDOM = new SecureRandom(); + + // most significant bits + private final long msb; + + // least significant bits + private final long lsb; + + Uuid(long msb, long lsb) { + this.msb = msb; + this.lsb = lsb; + } + + @Override + public String toString() { + return String.format("%08x-%04x-%04x-%04x-%012x", + msb >>> 32, (msb >>> 16) & 0xffff, msb & 0xffff, + (lsb >>> 48) & 0xffff, lsb & 0xffffffffffffL); + } + + public String toShortString() { + return String.format("%012x%03x%016x", + getTimestampPart(), getCounterPart(), getRandomPart()); + } + + public String toHumanReadableString() { + Instant instant = Instant.ofEpochMilli(getTimestampPart()); + return String.format("%s %03x %016x", + instant.toString(), getCounterPart(), getRandomPart()); + } + + /** + * Get the timestamp part (48 bits). + * + * @return the timestamp part + */ + public long getTimestampPart() { + return msb >>> 16; + } + + /** + * Get the counter part (12 bits). + * + * @return counter part + */ + public long getCounterPart() { + return msb & ((1L << 12) - 1); + } + + /** + * Get the random part (62 bits). + * The first 2 bits are fixed. + * + * @return the random part + */ + public long getRandomPart() { + return lsb; + } + + /** + * Unlike java.util.UUID, the comparison is unsigned, + * so that the string comparison yields the same result. + */ + @Override + public int compareTo(Uuid o) { + if (o.msb != msb) { + return Long.compareUnsigned(msb, o.msb); + } + return Long.compareUnsigned(lsb, o.lsb); + } + + @Override + public int hashCode() { + long x = lsb ^ msb; + return (int) ((x >>> 32) ^ x); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Uuid other = (Uuid) obj; + return lsb == other.lsb && msb == other.msb; + } + + /** + * Get the next timestamp (in milliseconds) and counter. + * The lowest 12 bits of the returned value is the counter. + * The milliseconds are shifted by 12 bits. + * + * @param now the milliseconds + * @param lastMillisAndCount the last returned value + * @return the new value + */ + static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisAndCount) { + long result = now << 12; + while (true) { + long last = lastMillisAndCount.get(); + if (result <= last) { + // ensure it is non-decrementing + result = last + 1; + } + long got = lastMillisAndCount.compareAndExchange(last, result); + if (got == last) { + return result; + } + } + } + + static Uuid timeBasedVersion7(long millisAndCount, + long random) { + long millis = millisAndCount >>> 12; + long counter = millisAndCount & ((1L << 12) - 1); + long version = 7; + long variant = 2; + long msb = (millis << 16) | (version << 12) | counter; + long lsb = (variant << 62) | (random & ((1L << 62) - 1)); + return new Uuid(msb, lsb); + } + + public static Uuid timeBasedVersion7() { + long millisAndCount = getMillisAndCountIncreasing( + System.currentTimeMillis(), + UUID_LAST_MILLIS_AND_COUNT); + long random = RANDOM.nextLong(); + return timeBasedVersion7(millisAndCount, random); + } + + public long getMostSignificantBits() { + return msb; + } + + public long getLeastSignificantBits() { + return lsb; + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java new file mode 100644 index 00000000000..b9d2c591cae --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class TreeStoreTest { + + @Test + public void test() { + assertEquals("\t", TreeStore.toChildNodeEntry("/")); + assertEquals("/\tabc", TreeStore.toChildNodeEntry("/abc")); + assertEquals("/hello\tworld", TreeStore.toChildNodeEntry("/hello/world")); + + assertEquals("/\tabc", TreeStore.toChildNodeEntry("/", "abc")); + assertEquals("/hello\tworld", TreeStore.toChildNodeEntry("/hello", "world")); + } +} diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java index 515930d5324..82876eebc22 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java @@ -31,6 +31,7 @@ import org.apache.jackrabbit.oak.index.async.AsyncIndexerLucene; import org.apache.jackrabbit.oak.index.indexer.document.DocumentStoreIndexer; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater; import org.apache.jackrabbit.oak.run.cli.CommonOptions; import org.apache.jackrabbit.oak.run.cli.DocumentBuilderCustomizer; @@ -251,12 +252,16 @@ private File reindex(IndexOptions idxOpts, ExtendedIndexHelper extendedIndexHelp if (opts.getCommonOpts().isMongo() && idxOpts.isDocTraversalMode()) { log.info("Using Document order traversal to perform reindexing"); try (DocumentStoreIndexer indexer = new DocumentStoreIndexer(extendedIndexHelper, indexerSupport)) { - if (idxOpts.buildFlatFileStoreSeparately()) { - FlatFileStore ffs = indexer.buildFlatFileStore(); - String pathToFFS = ffs.getFlatFileStorePath(); - System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToFFS); + if (idxOpts.useTreeStore()) { + indexer.reindexUsingTreeStore(); + } else { + if (idxOpts.buildFlatFileStoreSeparately()) { + FlatFileStore ffs = indexer.buildFlatFileStore(); + String pathToFFS = ffs.getFlatFileStorePath(); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToFFS); + } + indexer.reindex(); } - indexer.reindex(); } } else { try (OutOfBandIndexer indexer = new OutOfBandIndexer(extendedIndexHelper, indexerSupport)) { From 72ab6b9ebeec4d0e3a8a9e8d4b835c9244596456 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 11 Jul 2024 16:35:57 +0200 Subject: [PATCH 02/86] OAK-10341 Tree store --- .../indexer/document/tree/TreeStore.java | 70 +----- .../indexer/document/tree/TreeStoreUtils.java | 99 ++++++++ .../document/tree/store/AzureStore.java | 220 ----------------- .../document/tree/store/DiskCacheStore.java | 226 ------------------ .../document/tree/store/FileStore.java | 99 +------- .../tree/store/GarbageCollection.java | 2 - .../indexer/document/tree/store/PageFile.java | 4 +- .../indexer/document/tree/store/Session.java | 6 +- .../document/tree/store/StoreBuilder.java | 11 +- .../document/tree/store/utils/Cache.java | 1 + .../tree/store/utils/SortedStream.java | 8 +- .../store/utils/{Uuid.java => TimeUuid.java} | 18 +- 12 files changed, 129 insertions(+), 635 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java delete mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java delete mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java rename oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/{Uuid.java => TimeUuid.java} (91%) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 1d25e939df8..93bd5aefd35 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -18,12 +18,9 @@ */ package org.apache.jackrabbit.oak.index.indexer.document.tree; -import java.io.BufferedReader; import java.io.Closeable; import java.io.File; -import java.io.FileReader; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.Map.Entry; @@ -36,73 +33,10 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; -import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; import org.apache.jackrabbit.oak.spi.state.NodeState; public class TreeStore implements Iterable, Closeable { - public static void main(String... args) throws IOException { - String dir = args[0]; - MemoryBlobStore blobStore = new MemoryBlobStore(); - NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); - try (TreeStore treeStore = new TreeStore(new File(dir), entryReader)) { - Session session = treeStore.session; - Store store = treeStore.store; - if (store.keySet().isEmpty()) { - session.init(); - String fileName = args[1]; - try (BufferedReader lineReader = new BufferedReader( - new FileReader(fileName, StandardCharsets.UTF_8))) { - int count = 0; - long start = System.nanoTime(); - while (true) { - String line = lineReader.readLine(); - if (line == null) { - break; - } - count++; - if (count % 1000000 == 0) { - long time = System.nanoTime() - start; - System.out.println(count + " " + (time / count) + " ns/entry"); - } - int index = line.indexOf('|'); - if (index < 0) { - throw new IllegalArgumentException("| is missing: " + line); - } - String path = line.substring(0, index); - String value = line.substring(index + 1); - session.put(path, value); - - if (!path.equals("/")) { - String nodeName = PathUtils.getName(path); - String parentPath = PathUtils.getParentPath(path); - session.put(parentPath + "\t" + nodeName, ""); - } - - } - } - session.flush(); - store.close(); - } - Iterator it = treeStore.iterator(); - long nodeCount = 0; - long childNodeCount = 0; - long start = System.nanoTime(); - while (it.hasNext()) { - NodeStateEntry e = it.next(); - childNodeCount += e.getNodeState().getChildNodeCount(Long.MAX_VALUE); - nodeCount++; - if (nodeCount % 1000000 == 0) { - long time = System.nanoTime() - start; - System.out.println("Node count: " + nodeCount + - " child node count: " + childNodeCount + - " speed: " + (time / nodeCount) + " ns/entry"); - } - } - System.out.println("Node count: " + nodeCount + " Child node count: " + childNodeCount); - } - } - private final Store store; private final Session session; private final NodeStateEntryReader entryReader; @@ -228,4 +162,8 @@ public Session getSession() { return session; } + public Store getStore() { + return store; + } + } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java new file mode 100644 index 00000000000..c345200c672 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -0,0 +1,99 @@ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import java.util.Iterator; + +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; + +public class TreeStoreUtils { + + public static void main(String... args) throws IOException { + String dir = args[0]; + MemoryBlobStore blobStore = new MemoryBlobStore(); + NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); + try (TreeStore treeStore = new TreeStore(new File(dir), entryReader)) { + Session session = treeStore.getSession(); + Store store = treeStore.getStore(); + if (store.keySet().isEmpty()) { + session.init(); + String fileName = args[1]; + try (BufferedReader lineReader = new BufferedReader( + new FileReader(fileName, StandardCharsets.UTF_8))) { + int count = 0; + long start = System.nanoTime(); + while (true) { + String line = lineReader.readLine(); + if (line == null) { + break; + } + count++; + if (count % 1000000 == 0) { + long time = System.nanoTime() - start; + System.out.println(count + " " + (time / count) + " ns/entry"); + } + int index = line.indexOf('|'); + if (index < 0) { + throw new IllegalArgumentException("| is missing: " + line); + } + String path = line.substring(0, index); + String value = line.substring(index + 1); + session.put(path, value); + + if (!path.equals("/")) { + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + session.put(parentPath + "\t" + nodeName, ""); + } + + } + } + session.flush(); + store.close(); + } + Iterator it = treeStore.iterator(); + long nodeCount = 0; + long childNodeCount = 0; + long start = System.nanoTime(); + while (it.hasNext()) { + NodeStateEntry e = it.next(); + childNodeCount += e.getNodeState().getChildNodeCount(Long.MAX_VALUE); + nodeCount++; + if (nodeCount % 1000000 == 0) { + long time = System.nanoTime() - start; + System.out.println("Node count: " + nodeCount + + " child node count: " + childNodeCount + + " speed: " + (time / nodeCount) + " ns/entry"); + } + } + System.out.println("Node count: " + nodeCount + " Child node count: " + childNodeCount); + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java deleted file mode 100644 index 31d90f01733..00000000000 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/AzureStore.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.index.indexer.document.tree.store; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.HashSet; -import java.util.Properties; -import java.util.Set; - -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Uuid; - -import com.microsoft.azure.storage.CloudStorageAccount; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.blob.CloudAppendBlob; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlobDirectory; -import com.microsoft.azure.storage.blob.CloudBlockBlob; -import com.microsoft.azure.storage.blob.ListBlobItem; - -public class AzureStore implements Store { - - private final Properties config; - private final CloudStorageAccount cloud; - private final CloudBlobClient cloudBlobClient; - private final CloudBlobContainer container; - private final CloudBlobDirectory dir; - private Compression compression = Compression.NO; - private long writeCount; - private long readCount; - - public String toString() { - return "azure"; - } - - public AzureStore(Properties config) { - this.config = config; - try { - cloud = CloudStorageAccount.parse( - config.getProperty("storageAccount")); - cloudBlobClient = cloud.createCloudBlobClient(); - String maximumExecutionTimeInMs = config.getProperty("maximumExecutionTimeInMs"); - if (maximumExecutionTimeInMs != null) { - cloudBlobClient.getDefaultRequestOptions(). - setMaximumExecutionTimeInMs(Integer.parseInt(maximumExecutionTimeInMs)); - } - container = cloudBlobClient.getContainerReference( - config.getProperty("container")); - container.createIfNotExists(); - dir = container.getDirectoryReference( - config.getProperty("directory")); - } catch (Exception e) { -; // TODO proper logging -e.printStackTrace(); - throw new IllegalArgumentException(config.toString(), e); - } - } - - @Override - public void setWriteCompression(Compression compression) { - this.compression = compression; - } - - @Override - public PageFile getIfExists(String key) { - try { - readCount++; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - CloudAppendBlob blob = dir.getAppendBlobReference(key); - blob.download(out); - byte[] data = out.toByteArray(); - Compression c = Compression.getCompressionFromData(data[0]); - data = c.expand(data); - return PageFile.fromBytes(data); - } catch (URISyntaxException | StorageException e) { -; // TODO proper logging -e.printStackTrace(); - throw new IllegalArgumentException(key, e); - } - } - - @Override - public boolean supportsByteOperations() { - return true; - } - - @Override - public byte[] getBytes(String key) { - try { - readCount++; - CloudBlockBlob blob = dir.getBlockBlobReference(key); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - blob.download(out); - return out.toByteArray(); - } catch (URISyntaxException | StorageException e) { -; // TODO proper logging -e.printStackTrace(); - throw new IllegalArgumentException(key, e); - } - } - - @Override - public void putBytes(String key, byte[] data) { - try { - writeCount++; - CloudBlockBlob blob = dir.getBlockBlobReference(key); - blob.upload(new ByteArrayInputStream(data), data.length); - } catch (URISyntaxException | StorageException | IOException e) { -; // TODO proper logging -e.printStackTrace(); - throw new IllegalArgumentException(e); - } - } - - @Override - public void put(String key, PageFile value) { - byte[] data = value.toBytes(); - data = compression.compress(data); - CloudBlockBlob blob; - try { - writeCount++; - blob = dir.getBlockBlobReference(key); -long start = System.nanoTime(); - blob.uploadFromByteArray(data, 0, data.length); -long time = System.nanoTime() - start; -System.out.println("Azure upload " + key + " size " + data.length + " nanos " + time); - - } catch (URISyntaxException | StorageException | IOException e) { -; // TODO proper logging -e.printStackTrace(); - throw new IllegalArgumentException(e); - } - } - - @Override - public String newFileName() { - return Uuid.timeBasedVersion7().toShortString(); - } - - @Override - public Set keySet() { - try { - HashSet set = new HashSet<>(); - for (ListBlobItem item : dir.listBlobs()) { - if (item instanceof CloudBlockBlob) { - String name = ((CloudBlockBlob) item).getName(); - int index = name.lastIndexOf('/'); - if (index >= 0) { - name = name.substring(index + 1); - } - set.add(name); - } - } - return set; - } catch (StorageException | URISyntaxException e) { -; // TODO proper logging -e.printStackTrace(); - throw new IllegalArgumentException(e); - } - } - - @Override - public void remove(Set set) { - try { - for(String key : set) { - CloudBlockBlob blob = dir.getBlockBlobReference(key); - writeCount++; - blob.deleteIfExists(); - } - } catch (StorageException | URISyntaxException e) { -; // TODO proper logging -e.printStackTrace(); - throw new IllegalArgumentException(e); - } - } - - @Override - public void removeAll() { - remove(keySet()); - } - - @Override - public long getWriteCount() { - return writeCount; - } - - @Override - public long getReadCount() { - return readCount; - } - - @Override - public void close() { - } - - @Override - public Properties getConfig() { - return config; - } - -} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java deleted file mode 100644 index e25ff91a77d..00000000000 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/DiskCacheStore.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.index.indexer.document.tree.store; - -import java.util.Collections; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache; - -public class DiskCacheStore implements Store { - - private final Properties config; - - // the frontend (local disk) - private Store frontend; - - // the backend (azure or s3) - private Store backend; - - // set of entries that are not yet uploaded - private ConcurrentHashMap uploading = new ConcurrentHashMap<>(); - - // executor service that uploads files - private ExecutorService executor; - - // cache for entries that were downloaded - // (doesn't contain entries that are not yet uploaded) - private Cache cache; - - private int readAhead; - - public String toString() { - return "diskCache(" + frontend + ", " + backend + ")"; - } - - public DiskCacheStore(Properties config) { - this.config = config; - frontend = StoreBuilder.build(StoreBuilder.subProperties(config, "front.")); - - // cache size, in number of files on disk - int cacheSize = Integer.parseInt(config.getProperty("cacheSize", "1000000000")); - - // read ahead this number of children - this.readAhead = Integer.parseInt(config.getProperty("readAhead", "1000000")); - - // using this number of threads at most - int threads = Integer.parseInt(config.getProperty("threads", "20")); - - cache = new Cache(cacheSize) { - private static final long serialVersionUID = 1L; - - @Override - public boolean removeEldestEntry(Map.Entry eldest) { - boolean result = super.removeEldestEntry(eldest); - if (result) { - frontend.remove(Collections.singleton(eldest.getKey())); - } - return result; - } - }; - executor = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(), - r -> { - Thread thread = new Thread(r, "DiskCacheStore"); - thread.setDaemon(true); - return thread; - }); - backend = StoreBuilder.build(StoreBuilder.subProperties(config, "back.")); - } - - @Override - public PageFile getIfExists(String key) { - PageFile result = frontend.getIfExists(key); - if (result != null) { - return result; - } - if (backend.supportsByteOperations() && frontend.supportsByteOperations()) { - byte[] data = backend.getBytes(key); - frontend.putBytes(key, data); - result = frontend.get(key); - } else { - result = backend.get(key); - frontend.put(key, result); - } - synchronized (cache) { - cache.put(key, key); - } - if (result.isInnerNode()) { - for (int i = 0; i < readAhead && i < result.getValueCount(); i++) { - String childKey = result.getChildValue(i); - // System.out.println(" prefetching " + childKey + " because we read " + key); - executor.submit(new Runnable() { - - @Override - public void run() { - // this will put the entry in the cache - // and also read the children if needed - getIfExists(childKey); - // System.out.println(" prefetch done for " + childKey + " because we read " + key); - } - - }); - } - } - return result; - } - - @Override - public void put(String key, PageFile value) { - if (uploading.containsKey(key)) { - System.out.println("WARNING: upload is in progress: " + key + - " and a new upload is scheduled. waiting for the upload to finish, to avoid concurrency issues."); - while (uploading.containsKey(key)) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } - uploading.put(key, key); - frontend.put(key, value); - executor.submit(new Runnable() { - - @Override - public void run() { - try { - if (frontend.supportsByteOperations() && backend.supportsByteOperations()) { - byte[] data = frontend.getBytes(key); - backend.putBytes(key, data); - } else { - backend.put(key, frontend.get(key)); - } - } catch (Exception e) { - e.printStackTrace(); - } - uploading.remove(key); - } - - }); - } - - @Override - public String newFileName() { - return frontend.newFileName(); - } - - @Override - public Set keySet() { - return backend.keySet(); - } - - @Override - public void remove(Set set) { - synchronized (cache) { - set.forEach(k -> cache.remove(k)); - } - frontend.remove(set); - backend.remove(set); - } - - @Override - public void removeAll() { - frontend.removeAll(); - backend.removeAll(); - } - - @Override - public long getWriteCount() { - return frontend.getWriteCount(); - } - - @Override - public long getReadCount() { - return frontend.getReadCount(); - } - - @Override - public void setWriteCompression(Compression compression) { - frontend.setWriteCompression(compression); - backend.setWriteCompression(compression); - } - - @Override - public void close() { - try { - executor.shutdown(); - executor.awaitTermination(1, TimeUnit.DAYS); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - frontend.close(); - backend.close(); - } - - @Override - public Properties getConfig() { - return config; - } - -} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java index 9c72f8d6f75..ccf035d9bd0 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java @@ -28,10 +28,8 @@ import java.util.HashSet; import java.util.Properties; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Uuid; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.TimeUuid; public class FileStore implements Store { @@ -39,17 +37,7 @@ public class FileStore implements Store { private final String directory; private Compression compression = Compression.NO; private long writeCount, readCount; - private Thread backgroundThread; - private ConcurrentHashMap pendingWrites = new ConcurrentHashMap<>(); - private LinkedBlockingQueue queue = new LinkedBlockingQueue<>(100); - - private static final WriteOperation STOP = new WriteOperation(); - - static class WriteOperation { - String key; - byte[] value; - } - + public String toString() { return "file(" + directory + ")"; } @@ -58,51 +46,10 @@ public FileStore(Properties config) { this.config = config; this.directory = config.getProperty("dir"); new File(directory).mkdirs(); - boolean asyncWrite = Boolean.parseBoolean(config.getProperty("async", "false")); - if (asyncWrite) { - startAsyncWriter(); - } - } - - private void startAsyncWriter() { - backgroundThread = new Thread(new Runnable() { - - @Override - public void run() { - try { - for (int i = 0;; i++) { - WriteOperation op = queue.take(); - if (i % 200 == 0) { - // System.out.println(" file writer queue size " + queue.size()); - } - if (op == STOP) { - break; - } - writeFile(op.key, op.value); - pendingWrites.remove(op.key); - } - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - }); - backgroundThread.setDaemon(true); - backgroundThread.start(); } @Override public void close() { - try { - if (backgroundThread != null) { - queue.put(STOP); - backgroundThread.join(); - } - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } } @Override @@ -112,10 +59,6 @@ public void setWriteCompression(Compression compression) { @Override public PageFile getIfExists(String key) { - PageFile pending = pendingWrites.get(key); - if (pending != null) { - return pending; - } readCount++; File f = getFile(key); if (!f.exists()) { @@ -140,24 +83,7 @@ public PageFile getIfExists(String key) { @Override public void put(String key, PageFile value) { writeCount++; - if (backgroundThread != null) { - writeFileAsync(key, value.copy()); - } else { - writeFile(key, value.toBytes()); - } - } - - private void writeFileAsync(String key, PageFile value) { - pendingWrites.put(key, value); - WriteOperation op = new WriteOperation(); - op.key = key; - op.value = value.toBytes(); - try { - queue.put(op); - } catch (InterruptedException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } + writeFile(key, value.toBytes()); } @Override @@ -200,22 +126,6 @@ public void putBytes(String key, byte[] data) { private void writeFile(String key, byte[] data) { data = compression.compress(data); putBytes(key, data); - - /* - File tempFile = getFile(key, true); - File targetFile = getFile(key); - // https://stackoverflow.com/questions/595631/how-to-atomically-rename-a-file-in-java-even-if-the-dest-file-already-exists - try (RandomAccessFile file = new RandomAccessFile(tempFile, "rw")) { - file.write(data); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } - try { - Files.move(tempFile.toPath(), targetFile.toPath(), StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } - */ } private File getFile(String key) { @@ -224,7 +134,7 @@ private File getFile(String key) { @Override public String newFileName() { - return Uuid.timeBasedVersion7().toShortString(); + return TimeUuid.newUuid().toShortString(); } @Override @@ -246,7 +156,6 @@ public boolean accept(File dir, String name) { @Override public void remove(Set set) { - // TODO keep for some time if the file is relatively new? for (String key : set) { writeCount++; getFile(key).delete(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java index bcf8c8d6151..ff6f3bbe48c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java @@ -67,8 +67,6 @@ private void mark(String fileName, HashSet used) { private GarbageCollectionResult sweep(HashSet used) { GarbageCollectionResult result = new GarbageCollectionResult(); - // TODO keep files that were very recently updated - // (don't remove files that are part of a concurrent flush) HashSet removeSet = new HashSet(); for (String key: new HashSet<>(store.keySet())) { if (!used.contains(key)) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java index cf366c5d75c..787ac2c204b 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java @@ -98,7 +98,7 @@ public static PageFile fromBytes(byte[] data) { } public byte[] toBytes() { - // TODO synchronization is needed because we share the buffer + // synchronization is needed because we share the buffer synchronized (PageFile.class) { ByteBuffer buff = REUSED_BUFFER; if (buff.capacity() < sizeInBytes * 2) { @@ -289,14 +289,12 @@ public int getKeyIndex(String key) { if (lastSearchIndex == 1) { if (key.compareTo(keys.get(keys.size() - 1)) > 0) { index = -(keys.size() + 1); - // index = Collections.binarySearch(recordKeys, key); } else { index = Collections.binarySearch(keys, key); } } else if (lastSearchIndex == -1) { if (key.compareTo(keys.get(0)) < 0) { index = -1; - // index = Collections.binarySearch(recordKeys, key); } else { index = Collections.binarySearch(keys, key); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index 47aec1fb605..5fd5b9f71dd 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -604,15 +604,15 @@ private Iterator> iterator(String largerThan, int maxRootC private void fetchNext() { while (pq.size() > 0) { SortedStream s = pq.poll(); - if (s.currentKey == null) { + String key = s.currentKeyOrNull(); + if (key == null) { // if this is null, it must be the last stream break; } - String key = s.currentKey; if (key.equals(lastKey)) { continue; } - String value = s.currentValue; + String value = s.currentValue(); s.next(); pq.add(s); if (value == DELETED) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java index f6aeced1444..ee04b263a1c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java @@ -28,10 +28,11 @@ public class StoreBuilder { * Build a store. The configuration options are passed as a list of properties. * * - empty string or null: in-memory. + * - "type=memory" * - "type=file": file system, with "dir" directory - * - "type=azure": Azure - * - "type=s3": Amazon S3 - * - "type=diskCache": disk cache, with "dir" and "backend=azure" or "s3" + * - "type=stats.another": statistics wrapper around another + * - "type=slow.another": slow wrapper around another (to simulate slowness) + * - "type=log.another": log wrapper around another (to analyze problems) * * @param config the config * @return a store @@ -58,10 +59,6 @@ public static Store build(Properties config) { return new MemoryStore(config); case "file": return new FileStore(config); - case "azure": - return new AzureStore(config); - case "diskCache": - return new DiskCacheStore(config); } if (type.startsWith("stats.")) { config.put("type", type.substring("stats.".length())); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java index fcb2e4bf494..8c12413d689 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java @@ -21,6 +21,7 @@ import java.util.LinkedHashMap; import java.util.Map; +// TODO limit by memory usage, not number of entries public class Cache extends LinkedHashMap { private static final long serialVersionUID = 1L; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java index 8659becb193..95df3ac239b 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java @@ -24,8 +24,8 @@ public class SortedStream implements Comparable { private final String rootFileName; private Iterator it; - public String currentKey; - public String currentValue; + private String currentKey; + private String currentValue; public SortedStream(String rootFileName, Iterator it) { this.rootFileName = rootFileName; @@ -37,11 +37,11 @@ public String toString() { return "file " + rootFileName + " key " + currentKey + " value " + currentValue; } - String currentKeyOrNull() { + public String currentKeyOrNull() { return currentKey; } - Object currentValue() { + public String currentValue() { return currentValue; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Uuid.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java similarity index 91% rename from oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Uuid.java rename to oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java index 2e2873f6ae0..5cbae014936 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Uuid.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java @@ -31,7 +31,7 @@ * Unlike java.util.UUID, the comparison is unsigned, * so that the string comparison yields the same result. */ -public class Uuid implements Comparable { +public class TimeUuid implements Comparable { private static final AtomicLong UUID_LAST_MILLIS_AND_COUNT = new AtomicLong(0); @@ -43,7 +43,7 @@ public class Uuid implements Comparable { // least significant bits private final long lsb; - Uuid(long msb, long lsb) { + private TimeUuid(long msb, long lsb) { this.msb = msb; this.lsb = lsb; } @@ -99,7 +99,7 @@ public long getRandomPart() { * so that the string comparison yields the same result. */ @Override - public int compareTo(Uuid o) { + public int compareTo(TimeUuid o) { if (o.msb != msb) { return Long.compareUnsigned(msb, o.msb); } @@ -123,7 +123,7 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) { return false; } - Uuid other = (Uuid) obj; + TimeUuid other = (TimeUuid) obj; return lsb == other.lsb && msb == other.msb; } @@ -136,7 +136,7 @@ public boolean equals(Object obj) { * @param lastMillisAndCount the last returned value * @return the new value */ - static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisAndCount) { + private static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisAndCount) { long result = now << 12; while (true) { long last = lastMillisAndCount.get(); @@ -151,7 +151,7 @@ static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisAndCount) } } - static Uuid timeBasedVersion7(long millisAndCount, + private static TimeUuid newUuid(long millisAndCount, long random) { long millis = millisAndCount >>> 12; long counter = millisAndCount & ((1L << 12) - 1); @@ -159,15 +159,15 @@ static Uuid timeBasedVersion7(long millisAndCount, long variant = 2; long msb = (millis << 16) | (version << 12) | counter; long lsb = (variant << 62) | (random & ((1L << 62) - 1)); - return new Uuid(msb, lsb); + return new TimeUuid(msb, lsb); } - public static Uuid timeBasedVersion7() { + public static TimeUuid newUuid() { long millisAndCount = getMillisAndCountIncreasing( System.currentTimeMillis(), UUID_LAST_MILLIS_AND_COUNT); long random = RANDOM.nextLong(); - return timeBasedVersion7(millisAndCount, random); + return newUuid(millisAndCount, random); } public long getMostSignificantBits() { From d49b6c7fced0e9f58a0d78645c762aaca9f2e201 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 11 Jul 2024 20:43:02 +0200 Subject: [PATCH 03/86] OAK-10341 Tree store --- .../indexer/document/tree/TreeStore.java | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 93bd5aefd35..0a0bde2d63a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -18,7 +18,6 @@ */ package org.apache.jackrabbit.oak.index.indexer.document.tree; -import java.io.Closeable; import java.io.File; import java.io.IOException; import java.util.Iterator; @@ -28,6 +27,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry.NodeStateEntryBuilder; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; @@ -35,14 +35,19 @@ import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.spi.state.NodeState; -public class TreeStore implements Iterable, Closeable { +public class TreeStore implements IndexStore { + + private static final String STORE_TYPE = "TreeStore"; private final Store store; + private final File directory; private final Session session; private final NodeStateEntryReader entryReader; private final Cache nodeStateCache = new Cache<>(10000); + private long entryCount; public TreeStore(File directory, NodeStateEntryReader entryReader) { + this.directory = directory; this.entryReader = entryReader; String storeConfig = System.getProperty("oak.treeStoreConfig", "type=file\n" + @@ -166,4 +171,28 @@ public Store getStore() { return store; } + @Override + public String getStorePath() { + return directory.getAbsolutePath(); + } + + @Override + public long getEntryCount() { + return entryCount; + } + + public void setEntryCount(long entryCount) { + this.entryCount = entryCount; + } + + @Override + public String getIndexStoreType() { + return STORE_TYPE; + } + + @Override + public boolean isIncremental() { + return true; + } + } From 2404d146d34dfc372b84d6b7bf6e1513628f6f90 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 12 Jul 2024 11:26:48 +0200 Subject: [PATCH 04/86] OAK-10341 Tree store --- .../indexer/document/tree/TreeStore.java | 24 +++-- .../document/tree/TreeStoreNodeState.java | 12 ++- .../indexer/document/tree/store/PageFile.java | 18 +++- .../indexer/document/tree/store/Session.java | 8 +- .../document/tree/store/utils/Cache.java | 44 --------- .../tree/store/utils/MemoryBoundCache.java | 93 +++++++++++++++++++ 6 files changed, 135 insertions(+), 64 deletions(-) delete mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 0a0bde2d63a..5f12f0d56a2 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -31,7 +31,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -39,11 +39,17 @@ public class TreeStore implements IndexStore { private static final String STORE_TYPE = "TreeStore"; + private static final long CACHE_SIZE_NODE_MB = 128; + private static final long CACHE_SIZE_TREE_STORE_MB = 1024; + private static final long MAX_FILE_SIZE_MB = 64; + private static final long MB = 1024 * 1024; + private final Store store; private final File directory; private final Session session; private final NodeStateEntryReader entryReader; - private final Cache nodeStateCache = new Cache<>(10000); + private final MemoryBoundCache nodeStateCache = + new MemoryBoundCache<>(CACHE_SIZE_NODE_MB * MB); private long entryCount; public TreeStore(File directory, NodeStateEntryReader entryReader) { @@ -51,8 +57,8 @@ public TreeStore(File directory, NodeStateEntryReader entryReader) { this.entryReader = entryReader; String storeConfig = System.getProperty("oak.treeStoreConfig", "type=file\n" + - "cacheSizeMB=4096\n" + - "maxFileSize=64000000\n" + + "cacheSizeMB=" + CACHE_SIZE_TREE_STORE_MB + "\n" + + "maxFileSize=" + MAX_FILE_SIZE_MB * MB + "\n" + "dir=" + directory.getAbsolutePath()); this.store = StoreBuilder.build(storeConfig); this.session = new Session(store); @@ -111,13 +117,13 @@ NodeStateEntry getNodeStateEntry(String path, String value) { } NodeState getNodeState(String path) { - NodeState result = nodeStateCache.get(path); + TreeStoreNodeState result = nodeStateCache.get(path); if (result != null) { return result; } String value = session.get(path); if (value == null || value.isEmpty()) { - result = EmptyNodeState.MISSING_NODE; + result = new TreeStoreNodeState(EmptyNodeState.MISSING_NODE, path, this, path.length()); } else { result = getNodeState(path, value); } @@ -125,14 +131,14 @@ NodeState getNodeState(String path) { return result; } - NodeState getNodeState(String path, String value) { - NodeState result = nodeStateCache.get(path); + TreeStoreNodeState getNodeState(String path, String value) { + TreeStoreNodeState result = nodeStateCache.get(path); if (result != null) { return result; } String line = path + "|" + value; NodeStateEntry entry = entryReader.read(line); - result = new TreeStoreNodeState(entry.getNodeState(), path, this); + result = new TreeStoreNodeState(entry.getNodeState(), path, this, line.length()); nodeStateCache.put(path, result); return result; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java index a09c49d743f..3e875b44902 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java @@ -34,16 +34,24 @@ import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; + +public class TreeStoreNodeState implements NodeState, MemoryBoundCache.MemoryObject { -public class TreeStoreNodeState implements NodeState { private final NodeState delegate; private final String path; private final TreeStore treeStore; + private final long estimatedMemory; - public TreeStoreNodeState(NodeState delegate, String path, TreeStore treeStore) { + public TreeStoreNodeState(NodeState delegate, String path, TreeStore treeStore, long estimatedMemory) { this.delegate = delegate; this.path = path; this.treeStore = treeStore; + this.estimatedMemory = estimatedMemory; + } + + public long estimatedMemory() { + return estimatedMemory; } @Override diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java index 787ac2c204b..902ff033e37 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java @@ -23,18 +23,20 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; /** * A B-tree page (leaf, or inner node). * An inner node contains one more value than keys. * A leaf page has the same number of keys and values. */ -public class PageFile { +public class PageFile implements MemoryBoundCache.MemoryObject { private static final boolean VERIFY_SIZE = false; private static final int INITIAL_SIZE_IN_BYTES = 24; private String fileName; + private final long maxFileSize; private final boolean innerNode; @@ -52,8 +54,14 @@ public class PageFile { // contains unwritten modifications private boolean modified; - public PageFile(boolean innerNode) { + public PageFile(boolean innerNode, long maxFileSize) { this.innerNode = innerNode; + this.maxFileSize = maxFileSize; + } + + @Override + public long estimatedMemory() { + return maxFileSize; } public void setFileName(String fileName) { @@ -78,13 +86,13 @@ public static PageFile fromBytes(byte[] data) { int len = buff.getInt(); PageFile result; if (type == 0) { - result = new PageFile(true); + result = new PageFile(true, data.length); for (int i = 0; i < len; i++) { result.appendRecord(prefix + readString(buff), readString(buff)); } result.values.add(readString(buff)); } else { - result = new PageFile(false); + result = new PageFile(false, data.length); for (int i = 0; i < len; i++) { result.appendRecord(prefix + readString(buff), readString(buff)); } @@ -184,7 +192,7 @@ public String toString() { } public PageFile copy() { - PageFile result = new PageFile(innerNode); + PageFile result = new PageFile(innerNode, maxFileSize); result.modified = modified; result.keys = new ArrayList<>(keys); result.values = new ArrayList<>(values); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index 5fd5b9f71dd..8d0b272b59f 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -27,7 +27,7 @@ import java.util.PriorityQueue; import java.util.Properties; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Cache; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Position; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SortedStream; @@ -49,7 +49,7 @@ public class Session { static final boolean MULTI_ROOT = true; private final Store store; - private final Cache cache = new Cache<>(DEFAULT_CACHE_SIZE) { + private final MemoryBoundCache cache = new MemoryBoundCache<>(DEFAULT_CACHE_SIZE) { private static final long serialVersionUID = 1L; public boolean removeEldestEntry(Map.Entry eldest) { @@ -168,7 +168,7 @@ private PageFile copyPageFile(PageFile old) { } private PageFile newPageFile(boolean isInternalNode) { - PageFile result = new PageFile(isInternalNode); + PageFile result = new PageFile(isInternalNode, maxFileSize); result.setUpdate(updateId); return result; } @@ -458,7 +458,7 @@ public void mergeRoots(int max) { while(it.hasNext()) { Entry e = it.next(); put(e.getKey(), e.getValue()); - // TODO we can remove files that are processed + // we can remove files that are processed } newRoot = getFile(ROOT_NAME); if (max < list.size()) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java deleted file mode 100644 index 8c12413d689..00000000000 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Cache.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; - -import java.util.LinkedHashMap; -import java.util.Map; - -// TODO limit by memory usage, not number of entries -public class Cache extends LinkedHashMap { - - private static final long serialVersionUID = 1L; - private int maxSize; - - public Cache(int maxSize) { - super(16, 0.75f, true); - this.maxSize = maxSize; - } - - public void setSize(int maxSize) { - this.maxSize = maxSize; - } - - @Override - public boolean removeEldestEntry(Map.Entry eldest) { - return size() > maxSize; - } - -} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java new file mode 100644 index 00000000000..d5619e569e5 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class MemoryBoundCache + extends LinkedHashMap { + + private static final long serialVersionUID = 1L; + private long maxMemoryBytes; + private long memoryUsed; + + public MemoryBoundCache(long maxMemoryBytes) { + super(16, 0.75f, true); + this.maxMemoryBytes = maxMemoryBytes; + } + + public void setSize(int maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + } + + @Override + public V put(K key, V value) { + V old = super.put(key, value); + if (old != null) { + memoryUsed -= old.estimatedMemory(); + } + if (value != null) { + memoryUsed += value.estimatedMemory(); + } + return old; + } + + @Override + public void putAll(Map map) { + for(Map.Entry e : map.entrySet()) { + put(e.getKey(), e.getValue()); + } + } + + @Override + public V remove(Object key ) { + V old = super.remove(key); + if (old != null) { + memoryUsed -= old.estimatedMemory(); + } + return old; + } + + @Override + public void clear() { + super.clear(); + memoryUsed = 0; + } + + @Override + public boolean removeEldestEntry(Map.Entry eldest) { + boolean removeEldest = size() > maxMemoryBytes; + if (removeEldest) { + memoryUsed -= eldest.getValue().estimatedMemory(); + } + return removeEldest; + } + + public interface MemoryObject { + /** + * Get the estimate memory size. The value must not change afterwards, otherwise + * the memory calculation is wrong. + * + * @return the memory in bytes + */ + long estimatedMemory(); + } + +} From 1761ddbe5955768f998d429b61e0b7eb35ccb1a8 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 12 Jul 2024 15:07:36 +0200 Subject: [PATCH 05/86] OAK-10341 Tree store --- .../indexer/document/tree/TreeStore.java | 9 ++- .../document/tree/store/FileStore.java | 12 ++- .../indexer/document/tree/store/LogStore.java | 5 ++ .../document/tree/store/MemoryStore.java | 8 ++ .../indexer/document/tree/store/PageFile.java | 16 ++-- .../indexer/document/tree/store/Session.java | 76 ++++++------------- .../document/tree/store/SlowStore.java | 5 ++ .../document/tree/store/StatsStore.java | 5 ++ .../indexer/document/tree/store/Store.java | 10 +++ .../document/tree/store/StoreBuilder.java | 1 + 10 files changed, 82 insertions(+), 65 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 5f12f0d56a2..cafc6bd0d5a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -39,7 +39,7 @@ public class TreeStore implements IndexStore { private static final String STORE_TYPE = "TreeStore"; - private static final long CACHE_SIZE_NODE_MB = 128; + private static final long CACHE_SIZE_NODE_MB = 64; private static final long CACHE_SIZE_TREE_STORE_MB = 1024; private static final long MAX_FILE_SIZE_MB = 64; private static final long MB = 1024 * 1024; @@ -57,8 +57,8 @@ public TreeStore(File directory, NodeStateEntryReader entryReader) { this.entryReader = entryReader; String storeConfig = System.getProperty("oak.treeStoreConfig", "type=file\n" + - "cacheSizeMB=" + CACHE_SIZE_TREE_STORE_MB + "\n" + - "maxFileSize=" + MAX_FILE_SIZE_MB * MB + "\n" + + Session.CACHE_SIZE_MB + "=" + CACHE_SIZE_TREE_STORE_MB + "\n" + + Store.MAX_FILE_SIZE_BYTES + "=" + MAX_FILE_SIZE_MB * MB + "\n" + "dir=" + directory.getAbsolutePath()); this.store = StoreBuilder.build(storeConfig); this.session = new Session(store); @@ -198,7 +198,8 @@ public String getIndexStoreType() { @Override public boolean isIncremental() { - return true; + // TODO support diff and use Session.checkpoint() / flush(). + return false; } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java index ccf035d9bd0..2483684ed61 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java @@ -37,7 +37,8 @@ public class FileStore implements Store { private final String directory; private Compression compression = Compression.NO; private long writeCount, readCount; - + private final long maxFileSizeBytes; + public String toString() { return "file(" + directory + ")"; } @@ -45,6 +46,8 @@ public String toString() { public FileStore(Properties config) { this.config = config; this.directory = config.getProperty("dir"); + this.maxFileSizeBytes = Long.parseLong(config.getProperty( + Store.MAX_FILE_SIZE_BYTES, "" + Store.DEFAULT_MAX_FILE_SIZE_BYTES)); new File(directory).mkdirs(); } @@ -74,7 +77,7 @@ public PageFile getIfExists(String key) { file.readFully(data); Compression c = Compression.getCompressionFromData(data[0]); data = c.expand(data); - return PageFile.fromBytes(data); + return PageFile.fromBytes(data, maxFileSizeBytes); } catch (IOException e) { throw new IllegalArgumentException(e); } @@ -185,4 +188,9 @@ public Properties getConfig() { return config; } + @Override + public long getMaxFileSizeBytes() { + return maxFileSizeBytes; + } + } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java index 1d9824673cb..ae47cdfebf0 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java @@ -119,4 +119,9 @@ public void putBytes(String key, byte[] data) { backend.putBytes(key, data); } + @Override + public long getMaxFileSizeBytes() { + return backend.getMaxFileSizeBytes(); + } + } \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java index 7892dfcf2dd..146c04fcdee 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java @@ -28,6 +28,7 @@ public class MemoryStore implements Store { private final HashMap map = new HashMap<>(); private long nextFileName; private long writeCount, readCount; + private final long maxFileSizeBytes; public MemoryStore() { this(new Properties()); @@ -35,6 +36,8 @@ public MemoryStore() { public MemoryStore(Properties config) { this.config = config; + this.maxFileSizeBytes = Long.parseLong(config.getProperty( + Store.MAX_FILE_SIZE_BYTES, "" + Store.DEFAULT_MAX_FILE_SIZE_BYTES)); } @Override @@ -96,4 +99,9 @@ public Properties getConfig() { return config; } + @Override + public long getMaxFileSizeBytes() { + return maxFileSizeBytes; + } + } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java index 902ff033e37..260519dfe76 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java @@ -36,7 +36,7 @@ public class PageFile implements MemoryBoundCache.MemoryObject { private static final int INITIAL_SIZE_IN_BYTES = 24; private String fileName; - private final long maxFileSize; + private final long maxFileSizeBytes; private final boolean innerNode; @@ -54,14 +54,14 @@ public class PageFile implements MemoryBoundCache.MemoryObject { // contains unwritten modifications private boolean modified; - public PageFile(boolean innerNode, long maxFileSize) { + public PageFile(boolean innerNode, long maxFileSizeBytes) { this.innerNode = innerNode; - this.maxFileSize = maxFileSize; + this.maxFileSizeBytes = maxFileSizeBytes; } @Override public long estimatedMemory() { - return maxFileSize; + return maxFileSizeBytes; } public void setFileName(String fileName) { @@ -77,7 +77,7 @@ public void setUpdate(long update) { this.update = update; } - public static PageFile fromBytes(byte[] data) { + public static PageFile fromBytes(byte[] data, long maxFileSizeBytes) { ByteBuffer buff = ByteBuffer.wrap(data); int type = buff.get(); String nextRoot = readString(buff); @@ -86,13 +86,13 @@ public static PageFile fromBytes(byte[] data) { int len = buff.getInt(); PageFile result; if (type == 0) { - result = new PageFile(true, data.length); + result = new PageFile(true, maxFileSizeBytes); for (int i = 0; i < len; i++) { result.appendRecord(prefix + readString(buff), readString(buff)); } result.values.add(readString(buff)); } else { - result = new PageFile(false, data.length); + result = new PageFile(false, maxFileSizeBytes); for (int i = 0; i < len; i++) { result.appendRecord(prefix + readString(buff), readString(buff)); } @@ -192,7 +192,7 @@ public String toString() { } public PageFile copy() { - PageFile result = new PageFile(innerNode, maxFileSize); + PageFile result = new PageFile(innerNode, maxFileSizeBytes); result.modified = modified; result.keys = new ArrayList<>(keys); result.values = new ArrayList<>(values); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index 8d0b272b59f..510c2fb0c08 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -36,9 +36,8 @@ */ public class Session { - private static final int DEFAULT_CACHE_SIZE = 128; - private static final int DEFAULT_MAX_FILE_SIZE = 16 * 1024; - private static final int DEFAULT_CACHE_SIZE_MB = 16; + public static final String CACHE_SIZE_MB = "cacheSizeMB"; + private static final int DEFAULT_CACHE_SIZE_MB = 256; private static final int DEFAULT_MAX_ROOTS = 10; public static final String ROOT_NAME = "root"; @@ -49,26 +48,8 @@ public class Session { static final boolean MULTI_ROOT = true; private final Store store; - private final MemoryBoundCache cache = new MemoryBoundCache<>(DEFAULT_CACHE_SIZE) { - private static final long serialVersionUID = 1L; - - public boolean removeEldestEntry(Map.Entry eldest) { - boolean result = super.removeEldestEntry(eldest); - if (result) { - String key = eldest.getKey(); - PageFile value = eldest.getValue(); - if(value.isModified()) { - store.put(key, value); - // not strictly needed as it's no longer referenced - value.setModified(false); - } - } - return result; - } - }; + private final MemoryBoundCache cache; private long updateId; - private int maxFileSize; - private int cacheSizeMB; private int maxRoots = DEFAULT_MAX_ROOTS; private long fileReadCount; @@ -78,9 +59,26 @@ public Session() { public Session(Store store) { this.store = store; - maxFileSize = Integer.parseInt(store.getConfig().getProperty("maxFileSize", "" + DEFAULT_MAX_FILE_SIZE)); - cacheSizeMB = Integer.parseInt(store.getConfig().getProperty("cacheSizeMB", "" + DEFAULT_CACHE_SIZE_MB)); - changeCacheSize(); + long cacheSizeMB = Long.parseLong(store.getConfig().getProperty( + CACHE_SIZE_MB, "" + DEFAULT_CACHE_SIZE_MB)); + long cacheSizeBytes = cacheSizeMB * 1024 * 1024; + this.cache = new MemoryBoundCache<>(cacheSizeBytes) { + private static final long serialVersionUID = 1L; + + public boolean removeEldestEntry(Map.Entry eldest) { + boolean result = super.removeEldestEntry(eldest); + if (result) { + String key = eldest.getKey(); + PageFile value = eldest.getValue(); + if(value.isModified()) { + store.put(key, value); + // not strictly needed as it's no longer referenced + value.setModified(false); + } + } + return result; + } + }; } /** @@ -96,30 +94,6 @@ public int getMaxRoots() { return maxRoots; } - /** - * Set the cache size in MB. - * - * @param mb the value - */ - public void setCacheSizeMB(int mb) { - this.cacheSizeMB = mb; - } - - /** - * Set the maximum file size. Files might be slightly larger than that, but not a lot. - * - * @param sizeBytes the file size in bytes - */ - public void setMaxFileSize(int sizeBytes) { - this.maxFileSize = sizeBytes; - changeCacheSize(); - } - - private void changeCacheSize() { - int cacheEntryCount = (int) (cacheSizeMB * 1024L * 1024 / maxFileSize); - cache.setSize(cacheEntryCount); - } - /** * Get the number of files read from the cache or storage. * @@ -168,7 +142,7 @@ private PageFile copyPageFile(PageFile old) { } private PageFile newPageFile(boolean isInternalNode) { - PageFile result = new PageFile(isInternalNode, maxFileSize); + PageFile result = new PageFile(isInternalNode, store.getMaxFileSizeBytes()); result.setUpdate(updateId); return result; } @@ -296,7 +270,7 @@ private void put(String rootFileName, String key, String value) { private void splitOrMerge(String fileName, PageFile file, ArrayList parents) { int size = file.sizeInBytes(); - if (size > maxFileSize && file.canSplit()) { + if (size > store.getMaxFileSizeBytes() && file.canSplit()) { split(fileName, file, parents); } else if (file.getKeys().size() == 0) { merge(fileName, file, parents); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java index 8169091d21d..f931e4beca7 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java @@ -137,4 +137,9 @@ public void putBytes(String key, byte[] data) { delay(time, data.length); } + @Override + public long getMaxFileSizeBytes() { + return backend.getMaxFileSizeBytes(); + } + } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java index ce5b39dc963..601ba775555 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java @@ -166,6 +166,11 @@ public void putBytes(String key, byte[] data) { } } + @Override + public long getMaxFileSizeBytes() { + return backend.getMaxFileSizeBytes(); + } + static class Stats { final String key; long count; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java index 37e70560ac6..5696b826f7b 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java @@ -26,6 +26,9 @@ */ public interface Store { + public static final String MAX_FILE_SIZE_BYTES = "maxFileSizeBytes"; + public static final int DEFAULT_MAX_FILE_SIZE_BYTES = 8 * 1024 * 1024; + /** * Get a file * @@ -110,6 +113,13 @@ default PageFile get(String key) { Properties getConfig(); + /** + * Get the maximum file size configured. + * + * @return the file size, in bytes + */ + long getMaxFileSizeBytes(); + default boolean supportsByteOperations() { return false; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java index ee04b263a1c..c390c2a496b 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java @@ -33,6 +33,7 @@ public class StoreBuilder { * - "type=stats.another": statistics wrapper around another * - "type=slow.another": slow wrapper around another (to simulate slowness) * - "type=log.another": log wrapper around another (to analyze problems) + * - "maxFileSizeBytes=16000000": the maximum file size in bytes * * @param config the config * @return a store From 3d6d009d654fc8061baac1b66502e47f71fa0921 Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Thu, 11 Jul 2024 12:59:31 +0200 Subject: [PATCH 06/86] OAK-10944: oak-auth-ldap: update commons-pool2 dependency to 2.12.0 (#1576) --- oak-auth-ldap/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-auth-ldap/pom.xml b/oak-auth-ldap/pom.xml index 024817e0781..2f52d9a4c74 100644 --- a/oak-auth-ldap/pom.xml +++ b/oak-auth-ldap/pom.xml @@ -85,7 +85,7 @@ org.apache.commons commons-pool2 - 2.8.0 + 2.12.0 org.apache.commons From dc10a1206566d27004aea9aa7f9612547d2078cd Mon Sep 17 00:00:00 2001 From: mbaedke Date: Thu, 11 Jul 2024 14:26:16 +0200 Subject: [PATCH 07/86] OAK-10705: oak-standalone: update dependencies (#1411) Updated dependencies. --- oak-examples/standalone/pom.xml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/oak-examples/standalone/pom.xml b/oak-examples/standalone/pom.xml index 7ca0f1e9211..941cef1e26c 100644 --- a/oak-examples/standalone/pom.xml +++ b/oak-examples/standalone/pom.xml @@ -90,7 +90,7 @@ org.fusesource.jansi jansi - 1.11 + 2.4.1 true @@ -187,18 +187,17 @@ org.apache.felix org.apache.felix.http.proxy - 2.3.2 + 3.0.6 org.apache.felix org.apache.felix.http.bridge - 2.3.2 + 5.1.6 org.apache.felix org.apache.felix.webconsole - 4.2.10 - all + 5.0.0 org.apache.felix From f12ccb1716e6677b54ad3f06955cc007479584bf Mon Sep 17 00:00:00 2001 From: nit0906 Date: Fri, 12 Jul 2024 08:31:31 +0530 Subject: [PATCH 08/86] OAK-10905 | Add a configurable async checkpoint creator service (#1560) --- .../plugins/index/AsyncCheckpointCreator.java | 143 ++++++++++++++++++ .../plugins/index/AsyncCheckpointService.java | 104 +++++++++++++ .../oak/plugins/index/IndexUtils.java | 45 ++++++ .../index/AsyncCheckpointCreatorTest.java | 60 ++++++++ .../index/AsyncCheckpointServiceTest.java | 131 ++++++++++++++++ .../oak/plugins/index/IndexUtilsTest.java | 99 +++++++++++- 6 files changed, 579 insertions(+), 3 deletions(-) create mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java create mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java create mode 100644 oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java create mode 100644 oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java new file mode 100644 index 00000000000..d4a1ab6de8a --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index; + + +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.util.ISO8601; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.Calendar; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; + +/** + * This class is responsible for creating and deleting checkpoints asynchronously. + * The number of minimum concurrent checkpoints to keep in the system, along with the default lifetime of a checkpoint + * can be configured. + * When executed, this class should create one checkpoint in a single run with a configurable name. + * Following the creation of the checkpoint, it should try to delete checkpoints with the given name, + * in case the total number of such checkpoints is greater than the configured minimum concurrent checkpoints. + * By default, this task is registered using AsyncCheckpointService + */ +public class AsyncCheckpointCreator implements Runnable { + + /** + * Name of service property which determines the name of this Async task + */ + public static final String PROP_ASYNC_NAME = "oak.checkpoint.async"; + + private final String name; + private final long checkpointLifetimeInSeconds; + private final long minConcurrentCheckpoints; + private final long maxConcurrentCheckpoints; + private final NodeStore store; + public static final String CHECKPOINT_CREATOR_KEY = "creator"; + public static final String CHECKPOINT_CREATED_KEY = "created"; + public static final String CHECKPOINT_CREATED_TIMESTAMP_KEY= "created-epoch"; + public static final String CHECKPOINT_THREAD_KEY = "thread"; + public static final String CHECKPOINT_NAME_KEY = "name"; + private static final Logger log = LoggerFactory + .getLogger(AsyncCheckpointCreator.class); + + public AsyncCheckpointCreator(@NotNull NodeStore store, @NotNull String name, long checkpointLifetimeInSeconds, long minConcurrentCheckpoints, long maxConcurrentCheckpoints) { + this.store = store; + this.name = name; + this.checkpointLifetimeInSeconds = checkpointLifetimeInSeconds; + this.minConcurrentCheckpoints = minConcurrentCheckpoints; + // maxConcurrentCheckpoints should at least be 1 more than minConcurrentCheckpoints + if (maxConcurrentCheckpoints <= minConcurrentCheckpoints) { + log.warn("[{}] Max concurrent checkpoints is less than or equal to min concurrent checkpoints. " + + "Setting max concurrent checkpoints to min concurrent checkpoints + 1.", this.name); + this.maxConcurrentCheckpoints = minConcurrentCheckpoints + 1; + } else { + this.maxConcurrentCheckpoints = maxConcurrentCheckpoints; + } + } + + public String getName() { + return name; + } + + protected long getCheckpointLifetimeInSeconds() { + return checkpointLifetimeInSeconds; + } + + protected long getMinConcurrentCheckpoints() { + return minConcurrentCheckpoints; + } + + protected long getMaxConcurrentCheckpoints() { + return maxConcurrentCheckpoints; + } + + @Override + public void run() { + Map> filteredCheckpointMap = IndexUtils.getFilteredCheckpoints(store, + entry -> name.equals(entry.getValue().get(CHECKPOINT_NAME_KEY))); + // If the number of checkpoints created by this task are equal to or greater than the maxConcurrentCheckpoints, skip + // creation of a new checkpoint. + // This could happen in case the deletion of older checkpoints failed in multiple previous runs. + if (filteredCheckpointMap.size() >= maxConcurrentCheckpoints) { + log.warn("[{}] Skipping checkpoint creation as the number of concurrent checkpoints is already at max limit {}", name, maxConcurrentCheckpoints); + } else { + Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); + String creationTimeStamp = String.valueOf(cal.getTimeInMillis()); + String creationTimeISOFormat = ISO8601.format(cal); + String checkpoint = store.checkpoint(checkpointLifetimeInSeconds * 1000, Map.of( + CHECKPOINT_CREATOR_KEY, AsyncCheckpointCreator.class.getSimpleName(), + CHECKPOINT_CREATED_KEY, creationTimeISOFormat, + CHECKPOINT_CREATED_TIMESTAMP_KEY, creationTimeStamp, + CHECKPOINT_THREAD_KEY, Thread.currentThread().getName(), + CHECKPOINT_NAME_KEY, name)); + log.info("[{}] Created checkpoint {} with creation time {}", name, checkpoint, creationTimeISOFormat); + } + + // Get a list of checkpoints filtered on the basis of CHECKPOINT_NAME_KEY (name). This is done using the + // getFilteredCheckpoints in the IndexUtils, which gets all the checkpoints in the node store and then filters the list based on + // the provided predicate using the checkpoint info map associated with the checkpoints. + // The filtered checkpoints list is then sorted based on the CHECKPOINT_CREATED_TIMESTAMP_KEY (created-epoch). + // Both the initial filtering and sorting is done based on the information from the associated checkpoint info map of a given checkpoint. + Set sortedCheckpointSet = IndexUtils.getSortedCheckpointMap(IndexUtils.getFilteredCheckpoints(store, + entry -> name.equals(entry.getValue().get(CHECKPOINT_NAME_KEY))), CHECKPOINT_CREATED_TIMESTAMP_KEY).keySet(); + int counter = sortedCheckpointSet.size(); + // Iterate over the sortedCheckpointSet which is sorted based on the creation timestamp -> the oldest first + // We try and delete the checkpoints as long as the counter is greater than minConcurrentCheckpoints + // This ensures that the system always has concurrent checkpoints equal to or greater than the configured minConcurrentCheckpoints + for (String cp : sortedCheckpointSet) { + // Delete the checkpoint as long as the checkpoint count is greater than concurrentCheckpoints + if (counter > minConcurrentCheckpoints) { + if(store.release(cp) ) { + log.info("[{}] Deleted checkpoint {}", name, cp); + } else { + log.warn("[{}] Unable to delete checkpoint {}. Removal will be attempted again in next run.", name, cp); + } + } else { + break; + } + // Decrement the counter irrespective of the outcome of the checkpoint deletion. + // If we don't decrement the counter in case of a failure while trying to delete a checkpoint, + // the next iteration will try to delete a comparatively newer checkpoint and keep the older one in the system (which we don't want). + // The checkpoint that failed to get deleted in this case should get deleted in the next run. + counter --; + } + + } + +} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java new file mode 100644 index 00000000000..4bc3f90bdac --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java @@ -0,0 +1,104 @@ +package org.apache.jackrabbit.oak.plugins.index; + +import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.CompositeRegistration; +import org.apache.jackrabbit.oak.spi.whiteboard.Registration; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.osgi.framework.BundleContext; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.ConfigurationPolicy; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.metatype.annotations.AttributeDefinition; +import org.osgi.service.metatype.annotations.Designate; +import org.osgi.service.metatype.annotations.ObjectClassDefinition; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.ScheduleExecutionInstanceTypes.RUN_ON_LEADER; +import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.scheduleWithFixedDelay; + +@Component( + configurationPolicy = ConfigurationPolicy.REQUIRE) +@Designate(ocd = AsyncCheckpointService.Configuration.class, factory = true) +public class AsyncCheckpointService { + + @ObjectClassDefinition( + name = "Apache Jackrabbit Oak Async Checkpoint Service", + description = "Configures the async checkpoint services which performs periodic creation and deletion of checkpoints" + ) + @interface Configuration { + + @AttributeDefinition( + name = "Checkpoint Creator Identifier", + description = "Unique identifier to be used for creating checkpoints" + ) + String name() default "checkpoint-async"; + + @AttributeDefinition( + name = "Enable", + description = "Flag to enable/disable the checkpoints creation task" + ) + boolean enable() default false; + + @AttributeDefinition( + name = "Minimum Concurrent Checkpoints", + description = "Minimum number of concurrent checkpoints to keep in system" + ) + long minConcurrentCheckpoints() default 2; + + @AttributeDefinition( + name = "Maximum Concurrent Checkpoints", + description = "Maximum number of concurrent checkpoints to keep in system. " + + "This limit is to prevent multiple checkpoint creation in case the deletion of older " + + "checkpoints fails multiple times. This should always be greater than Minimum Concurrent Checkpoints" + ) + long maxConcurrentCheckpoints() default 10; + + @AttributeDefinition( + name = "Checkpoint Lifetime", + description = "Lifetime of a checkpoint in seconds" + ) + long checkpointLifetime() default 60 * 60 * 24; + + @AttributeDefinition( + name = "Time Interval", + description = "Time interval between consecutive job runs in seconds. This would be the time interval between two consecutive checkpoints creation." + ) + long timeIntervalBetweenCheckpoints() default 60 * 15; + + } + + private final List regs = new ArrayList<>(); + @Reference + private NodeStore nodeStore; + + @Activate + public void activate(BundleContext bundleContext, AsyncCheckpointService.Configuration config) { + Whiteboard whiteboard = new OsgiWhiteboard(bundleContext); + if (config.enable()) { + AsyncCheckpointCreator task = new AsyncCheckpointCreator(nodeStore, config.name(), config.checkpointLifetime(), config.minConcurrentCheckpoints(), config.maxConcurrentCheckpoints()); + registerAsyncCheckpointCreator(whiteboard, task, config.timeIntervalBetweenCheckpoints()); + } + } + + private void registerAsyncCheckpointCreator(Whiteboard whiteboard, AsyncCheckpointCreator task, long delayInSeconds) { + Map config = Map.of( + AsyncCheckpointCreator.PROP_ASYNC_NAME, task.getName(), + "scheduler.name", AsyncCheckpointCreator.class.getName() + "-" + task.getName() + ); + regs.add(scheduleWithFixedDelay(whiteboard, task, config, delayInSeconds, RUN_ON_LEADER, true)); + } + + @Deactivate + public void deactivate() throws IOException { + new CompositeRegistration(regs).unregister(); + } + + +} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java index a6f64f0f42f..076ef54029b 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java @@ -31,9 +31,14 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.UNIQUE_PROPERTY_NAME; import java.util.Collection; +import java.util.Comparator; import java.util.Map; import java.util.Set; +import java.util.LinkedHashMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import javax.jcr.RepositoryException; @@ -49,6 +54,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.plugins.tree.TreeUtil; +import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -281,4 +287,43 @@ public static String getCaller(@Nullable String[] ignoredJavaPackages) { // if every element is ignored, we assume it's an internal request return "(internal)"; } + + /** + * Returns a Map consisting of checkpoints and checkpointInfoMap filtered based on a given predicate + * which can utilise checkpoint info map for filtering + * @param store + * @param predicate predicate used for filtering of checkpoints + * @return Map> filteredCheckpoint Map + */ + public static Map> getFilteredCheckpoints(NodeStore store, Predicate>> predicate) { + return StreamSupport.stream(store.checkpoints().spliterator(), false) + .collect(Collectors.toMap(str -> str, + store::checkpointInfo)) + .entrySet() + .stream() + .filter(predicate) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * Returns a map with checkpoint name as key and checkpoint metadata map as value, sorted on the basis of particular key in the metadata map. + * For example - given the following checkpoints in the system along with their associated metadata maps - + * checkpoint3 - {created-epoch=123458, creator=creator1} + * checkpoint1 - {created-epoch=123456, creator=creator2} + * checkpoint2 - {created-epoch=123457, creator=creator3} + * This method should return - + * {checkpoint1={created-epoch=123456, creator=creator2}, + * checkpoint2={created-epoch=123457, creator=creator3}, + * checkpoint3={created-epoch=123458, creator=creator1}} + * @param checkpointMap - the map consisting of checkpoints as keys and checkpoint metadata map as values + * @param keyForComparator - key in the metadata map of the checkpoint that can be used as comparator to sort on checkpoint creation time. + * @return Map> sorted checkpoint map + */ + public static Map> getSortedCheckpointMap(Map> checkpointMap, + String keyForComparator) { + return checkpointMap.entrySet().stream() + .filter(entry -> entry.getValue().containsKey(keyForComparator)) + .sorted(Comparator.comparingLong(entry -> Long.parseLong(entry.getValue().get(keyForComparator)))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (o, n) -> n, LinkedHashMap::new)); + } } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java new file mode 100644 index 00000000000..c569c78b598 --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index; + +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.Assert; +import org.junit.Test; + +import java.util.*; + +public class AsyncCheckpointCreatorTest { + + @Test + public void testAsync() { + NodeStore store = new MemoryNodeStore(); + int minConcurrentCheckpoints = 3; + int maxConcurrentCheckpoints = 5; + AsyncCheckpointCreator task = new AsyncCheckpointCreator(store, "test", 1000, minConcurrentCheckpoints, maxConcurrentCheckpoints); + Map checkpointMap = new LinkedHashMap<>(); + for (int i = 0; i < minConcurrentCheckpoints; i++) { + List checkpointList = new ArrayList<>(); + task.run(); + for(String checkpoint : store.checkpoints()) { + if (!checkpointMap.containsValue(checkpoint)) { + checkpointMap.put(i + 1, checkpoint); + } + checkpointList.add(checkpoint); + } + Assert.assertEquals(i + 1, checkpointList.size()); + } + // Task run post the minConcurrentCheckpoints should not result in additional + // checkpoints since the oldest checkpoint should get deleted + for (int j = 0; j < 2 ; j++) { + List checkpointList = new ArrayList<>(); + task.run(); + for(String checkpoint : store.checkpoints()) { + checkpointList.add(checkpoint); + } + Assert.assertFalse(checkpointList.contains(checkpointMap.get(j + 1))); + Assert.assertEquals(minConcurrentCheckpoints, checkpointList.size()); + } + } +} diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java new file mode 100644 index 00000000000..0bb148d010c --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index; + +import org.apache.jackrabbit.guava.common.collect.ImmutableMap; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.state.Clusterable; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.sling.testing.mock.osgi.MockOsgi; +import org.apache.sling.testing.mock.osgi.junit.OsgiContext; +import org.jetbrains.annotations.NotNull; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class AsyncCheckpointServiceTest { + + @Rule + public final OsgiContext context = new OsgiContext(); + + private final MemoryNodeStore nodeStore = new AsyncCheckpointServiceTest.FakeClusterableMemoryNodeStore(); + private final AsyncCheckpointService service = new AsyncCheckpointService(); + + /** + * This test is used to verify the registration of the AsyncCheckpointService with the OSGI context. + * The test verifies this with 3 different configurations of the AsyncCheckpointService, 2 of them enabled and 1 disabled. + */ + @Test + public void asyncReg() { + injectDefaultServices(); + // Create 3 configurations of the AsyncCheckpointService, 2 of them enabled and 1 disabled. + String name1 = "checkpoint-async-test-1"; + String name2 = "checkpoint-async-test-2"; + String name3 = "checkpoint-async-test-3"; + Map config1 = ImmutableMap.of( + "name", "checkpoint-async-test-1", + "enable", true, + "minConcurrentCheckpoints", 3L, + "maxConcurrentCheckpoints", 10L, + "checkpointLifetime", 60 * 60 * 24L, + "timeIntervalBetweenCheckpoints", 60 * 15L + ); + + Map config2 = ImmutableMap.of( + "name", "checkpoint-async-test-2", + "enable", false, + "minConcurrentCheckpoints", 3L, + "maxConcurrentCheckpoints", 10L, + "checkpointLifetime", 60 * 60 * 24L, + "timeIntervalBetweenCheckpoints", 60 * 15L + ); + + Map config3 = ImmutableMap.of( + "name", "checkpoint-async-test-3", + "enable", true, + "minConcurrentCheckpoints", 4L, + "maxConcurrentCheckpoints", 2L, + "checkpointLifetime", 60 * 24L, + "timeIntervalBetweenCheckpoints", 60 * 15L + ); + // Activate the service with the above 3 configurations. + MockOsgi.activate(service, context.bundleContext(), config1); + MockOsgi.activate(service, context.bundleContext(), config2); + MockOsgi.activate(service, context.bundleContext(), config3); + // Verify that the configs that are enabled are registered with the OSGI context and the one that is disabled is not. + assertEquals(1, context.getServices(Runnable.class, "(oak.checkpoint.async="+name1+")").length); + assertEquals(0, context.getServices(Runnable.class, "(oak.checkpoint.async="+name2+")").length); + assertEquals(1, context.getServices(Runnable.class, "(oak.checkpoint.async="+name3+")").length); + + // Get the instances fo the async tasks that are registered as per the enabled configs and verify the values of the + // configured minConcurrentCheckpoints and checkpointLifetimeInSeconds. + AsyncCheckpointCreator checkpointCreator1 = getCheckpointCreator("checkpoint-async-test-1"); + assertEquals(3, checkpointCreator1.getMinConcurrentCheckpoints()); + assertEquals(10, checkpointCreator1.getMaxConcurrentCheckpoints()); + assertEquals(60 * 60 * 24, checkpointCreator1.getCheckpointLifetimeInSeconds()); + AsyncCheckpointCreator checkpointCreator3 = getCheckpointCreator("checkpoint-async-test-3"); + assertEquals(4, checkpointCreator3.getMinConcurrentCheckpoints()); + assertEquals(5, checkpointCreator3.getMaxConcurrentCheckpoints()); + assertEquals(60 * 24, checkpointCreator3.getCheckpointLifetimeInSeconds()); + MockOsgi.deactivate(service, context.bundleContext()); + assertNull(context.getService(Runnable.class)); + } + + + private AsyncCheckpointCreator getCheckpointCreator(String name) { + return (AsyncCheckpointCreator) context.getServices(Runnable.class, "(oak.checkpoint.async="+name+")")[0]; + } + + private void injectDefaultServices() { + context.registerService(NodeStore.class, nodeStore); + MockOsgi.injectServices(service, context.bundleContext()); + } + + private static class FakeClusterableMemoryNodeStore extends MemoryNodeStore implements Clusterable { + @NotNull + @Override + public String getInstanceId() { + return "foo"; + } + + @Override + public String getVisibilityToken() { + return ""; + } + + @Override + public boolean isVisible(String visibilityToken, long maxWaitMillis) throws InterruptedException { + return true; + } + } +} diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java index 5894946773d..a5b9036de95 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java @@ -18,14 +18,27 @@ */ package org.apache.jackrabbit.oak.plugins.index; - import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.Assert; import org.junit.Test; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.Calendar; + import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + public class IndexUtilsTest { @@ -35,6 +48,10 @@ public class IndexUtilsTest { // all packages used with Oak private static final String[] OAK_CLASSES_IGNORED = new String[] {"org.apache.jackrabbit", "java.lang", "sun.reflect", "jdk"}; + private static final String CHECKPOINT_CREATOR_KEY = "creator"; + + private static final String CHECKPOINT_CREATED_TIMESTAMP_KEY = "created"; + @Test public void asyncName() throws Exception { assertNull(IndexUtils.getAsyncLaneName(EMPTY_NODE, "/fooIndex")); @@ -56,5 +73,81 @@ public void getCaller() { String caller = IndexUtils.getCaller(OAK_CLASSES_IGNORED); assertTrue(caller.startsWith("org.junit.runners")); - } + } + + @Test + public void checkpointFilterAndSorting() throws Exception { + NodeStore store = null; + Set checkpointSet = new LinkedHashSet<>(); + try { + store = new MemoryNodeStore(); + + // Check all works fines if no checkpoints present + Map> noCheckpointMap = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process-non-existing".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY))); + assertEquals(0, noCheckpointMap.size()); + + // Create checkpoints + String checkpoint1 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process")); + checkpointSet.add(checkpoint1); + Thread.sleep(1000); + String checkpoint2 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process-2")); + checkpointSet.add(checkpoint2); + Thread.sleep(1000); + String checkpoint3 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process")); + checkpointSet.add(checkpoint3); + Thread.sleep(1000); + String checkpoint4 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process")); + checkpointSet.add(checkpoint4); + + // Check for happy case + Map> filteredCheckpointMap = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process".equals(entry.getValue().get("creator"))); + + assertEquals(3, filteredCheckpointMap.size()); + for (String checkpoint : filteredCheckpointMap.keySet()) { + assertEquals("checkpoint-helper-test-process",filteredCheckpointMap.get(checkpoint).get(CHECKPOINT_CREATOR_KEY)); + } + // Check sorting now + Map> sortedCheckpointMap = IndexUtils.getSortedCheckpointMap(filteredCheckpointMap, CHECKPOINT_CREATED_TIMESTAMP_KEY); + assertEquals(3, sortedCheckpointMap.size()); + Iterator sortedCheckpointIt = sortedCheckpointMap.keySet().iterator(); + assertEquals(checkpoint1, sortedCheckpointIt.next()); + assertEquals(checkpoint3, sortedCheckpointIt.next()); + assertEquals(checkpoint4, sortedCheckpointIt.next()); + + // Check for negative edge cases + Map> filteredCheckpointMap2 = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process-non-existing".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY))); + assertEquals(0, filteredCheckpointMap2.size()); + + // Create a checkpoint with incorrect metadata + Map checkpointMetadata = getMetaDataMap("checkpoint-helper-test-process"); + checkpointMetadata.remove(CHECKPOINT_CREATED_TIMESTAMP_KEY); + String latestFilteredCheckpointWithNoTimestamp = store.checkpoint(1800000L, checkpointMetadata); + checkpointSet.add(latestFilteredCheckpointWithNoTimestamp); + + // Modify the predicate in the filter method here to also include the check that created exists in the checkpoint metadata + Map> filteredCheckpointMap3 = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY)) && entry.getValue().containsKey(CHECKPOINT_CREATED_TIMESTAMP_KEY)); + assertEquals(3, filteredCheckpointMap3.size()); + Assert.assertFalse(filteredCheckpointMap3.containsKey(latestFilteredCheckpointWithNoTimestamp)); + + Map checkpointMetadata2 = getMetaDataMap("checkpoint-helper-test-process-3"); + checkpointMetadata2.remove(CHECKPOINT_CREATED_TIMESTAMP_KEY); + String latestFilteredCheckpointWithNoTimestamp2 = store.checkpoint(1800000L, checkpointMetadata2); + checkpointSet.add(latestFilteredCheckpointWithNoTimestamp2); + Map> filteredCheckpointMap4 = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process-3".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY))); + assertEquals(1, filteredCheckpointMap4.size()); + Assert.assertTrue(filteredCheckpointMap4.containsKey(latestFilteredCheckpointWithNoTimestamp2)); + } finally { + for (String checkpoint : checkpointSet) { + store.release(checkpoint); + } + } + + } + + public static Map getMetaDataMap(String creator) { + Map checkpointMetaData = new LinkedHashMap<>(); + checkpointMetaData.put(CHECKPOINT_CREATOR_KEY, creator); + checkpointMetaData.put(CHECKPOINT_CREATED_TIMESTAMP_KEY, String.valueOf(Calendar.getInstance().getTimeInMillis())); + return checkpointMetaData; + } } \ No newline at end of file From 7f112faca966859e8bf117bcbd4665892f00a330 Mon Sep 17 00:00:00 2001 From: nit0906 Date: Fri, 12 Jul 2024 15:43:50 +0530 Subject: [PATCH 09/86] OAK-10905 | Add license header to AsyncCheckpointService (#1579) --- .../plugins/index/AsyncCheckpointService.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java index 4bc3f90bdac..460e5fe93f2 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.jackrabbit.oak.plugins.index; import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; From ba530fe46563757474a4ed1740aa3396e5f051bd Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Mon, 15 Jul 2024 16:12:47 +0200 Subject: [PATCH 10/86] OAK-10848: commons: remove use of slf4j.event.Level in SystemPropertySupplier implementation (#1580) --- .../properties/SystemPropertySupplier.java | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java index c1b0da0e592..30ad2c59616 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java @@ -25,7 +25,6 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.slf4j.event.Level; /** * Utility class for consistent handling of system properties. @@ -50,7 +49,7 @@ public class SystemPropertySupplier implements Supplier { private final Function parser; private Logger log = LOG; - private Level successLogLevel = Level.INFO; + private String successLogLevel = "INFO"; private Predicate validator = (a) -> true; private Function sysPropReader = System::getProperty; private BiFunction setMessageFormatter = (a, b) -> { @@ -100,22 +99,14 @@ public SystemPropertySupplier formatSetMessage(@NotNull BiFunction logSuccessAs(String successLogLevel) { - Level newLevel; + String newLevel; switch (Objects.requireNonNull(successLogLevel)) { case "DEBUG": - newLevel = Level.DEBUG; - break; case "ERROR": - newLevel = Level.ERROR; - break; case "INFO": - newLevel = Level.INFO; - break; case "TRACE": - newLevel = Level.TRACE; - break; case "WARN": - newLevel = Level.WARN; + newLevel = successLogLevel; break; default: throw new IllegalArgumentException("unsupported log level: " + successLogLevel); @@ -162,19 +153,19 @@ public T get() { if (!ret.equals(defaultValue)) { String msg = setMessageFormatter.apply(propName, ret); switch (successLogLevel) { - case INFO: + case "INFO": log.info(msg); break; - case DEBUG: + case "DEBUG": log.debug(msg); break; - case ERROR: + case "ERROR": log.error(msg); break; - case TRACE: + case "TRACE": log.trace(msg); break; - case WARN: + case "WARN": log.warn(msg); break; default: From 36cfcfa08e66ef2aecc5cec3c08e6dd08db9038e Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Tue, 16 Jul 2024 14:00:37 +0100 Subject: [PATCH 11/86] OAK-10685: remove unused import of java.io.UnsupportedEncodingException --- .../oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java | 1 - 1 file changed, 1 deletion(-) diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java index 0eba902e7fc..33add8dafa4 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java @@ -29,7 +29,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; From cafbd29d520c247a74e88314b1f518c95fb74443 Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Thu, 18 Jul 2024 06:47:10 +0200 Subject: [PATCH 12/86] OAK-10949: blob-cloud, segment-aws: update aws SDK to 1.12.761 (dependencies reference vulnerable amazon ion-java version) (#1581) --- oak-blob-cloud/pom.xml | 2 +- oak-segment-aws/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml index dd7d4479db3..b6fc94844d2 100644 --- a/oak-blob-cloud/pom.xml +++ b/oak-blob-cloud/pom.xml @@ -32,7 +32,7 @@ 4.1.107.Final - 1.12.353 + 1.12.761 diff --git a/oak-segment-aws/pom.xml b/oak-segment-aws/pom.xml index e6ed369c425..06105369ab6 100644 --- a/oak-segment-aws/pom.xml +++ b/oak-segment-aws/pom.xml @@ -33,7 +33,7 @@ Oak Segment AWS - 1.12.353 + 1.12.761 1.0.392 From b098fd8930ec91cd4ea15769c21db8e4ee47864d Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Thu, 18 Jul 2024 07:04:03 +0200 Subject: [PATCH 13/86] OAK-10954: Update spotbugs plugin to 4.8.6.2 (#1588) --- oak-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml index 29598811d04..483409b135c 100644 --- a/oak-parent/pom.xml +++ b/oak-parent/pom.xml @@ -331,7 +331,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.8.5.0 + 4.8.6.2 org.apache.maven.plugins From 5fb55e05e0ba137024e07d66dc3856864b87eacf Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Thu, 18 Jul 2024 19:51:10 +0200 Subject: [PATCH 14/86] OAK-10959: webapp: update Tomcat dependency to 9.0.90 (#1589) --- oak-examples/webapp/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-examples/webapp/pom.xml b/oak-examples/webapp/pom.xml index c07b8d1f6a2..8af0f35cf8d 100644 --- a/oak-examples/webapp/pom.xml +++ b/oak-examples/webapp/pom.xml @@ -35,7 +35,7 @@ Web application that hosts and serves a Jackrabbit Oak content repository - 9.0.89 + 9.0.90 true From cd35db705fe768ede6991c6a5d25de034081114b Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Fri, 19 Jul 2024 13:48:02 +0200 Subject: [PATCH 15/86] OAK-10960: blob-cloud, segment: update netty version to 4.1.111 (#1590) --- oak-blob-cloud-azure/pom.xml | 2 +- oak-blob-cloud/pom.xml | 2 +- oak-segment-azure/pom.xml | 2 +- oak-segment-tar/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/oak-blob-cloud-azure/pom.xml b/oak-blob-cloud-azure/pom.xml index 4d102b1a723..3a1a12fcced 100644 --- a/oak-blob-cloud-azure/pom.xml +++ b/oak-blob-cloud-azure/pom.xml @@ -31,7 +31,7 @@ bundle - 4.1.109.Final + 4.1.111.Final diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml index b6fc94844d2..4515d7138ab 100644 --- a/oak-blob-cloud/pom.xml +++ b/oak-blob-cloud/pom.xml @@ -31,7 +31,7 @@ bundle - 4.1.107.Final + 4.1.111.Final 1.12.761 diff --git a/oak-segment-azure/pom.xml b/oak-segment-azure/pom.xml index a975c5988de..ea5b073be4b 100644 --- a/oak-segment-azure/pom.xml +++ b/oak-segment-azure/pom.xml @@ -33,7 +33,7 @@ Oak Segment Azure - 4.1.109.Final + 4.1.111.Final diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml index 848cf0af680..b9e732bde2d 100644 --- a/oak-segment-tar/pom.xml +++ b/oak-segment-tar/pom.xml @@ -33,7 +33,7 @@ Oak Segment Tar - 4.1.104.Final + 4.1.111.Final 1.4.2 From 5d56db1224f91992db8fb2978b2a4a482a7ca705 Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Fri, 19 Jul 2024 16:07:48 +0200 Subject: [PATCH 16/86] OAK-10945: Remove usage of Guava Function interface (#1578) * OAK-10945: Remove usage of Guava Function interface (oak-upgrade) * OAK-10945: Remove usage of Guava Function interface (oak-store-spi) * OAK-10945: Remove usage of Guava Function interface (oak-it) * OAK-10945: Remove usage of Guava Function interface (oak-store-document) * OAK-10945: Remove usage of Guava Function interface (oak-store-composite) * OAK-10945: Remove usage of Guava Function interface (oak-segment-tar) * OAK-10945: Remove usage of Guava Function interface (oak-segment-azure) * OAK-10945: Remove usage of Guava Function interface (oak-security-spi) * OAK-10945: Remove usage of Guava Function interface (oak-security-search) * OAK-10945: Remove usage of Guava Function interface (oak-run-commons) * OAK-10945: Remove usage of Guava Function interface (oak-run) * OAK-10945: Remove usage of Guava Function interface (oak-lucene) * OAK-10945: Remove usage of Guava Function interface (oak-jcr) * OAK-10945: Remove usage of Guava Function interface (oak-exercise) * OAK-10945: Remove usage of Guava Function interface (oak-core-spi) * OAK-10945: Remove usage of Guava Function interface (oak-core) * OAK-10945: Remove usage of Guava Function interface (oak-commons) * OAK-10945: Remove usage of Guava Function interface (oak-blob-plugins) * OAK-10945: Remove usage of Guava Function interface (oak-blob-cloud, oak-blob-cloud-azure) * OAK-10945: Remove usage of Guava Function interface (oak-upgrade) - cleanup * OAK-10945: Remove usage of Guava Function interface (oak-store-spi) - cleanup * OAK-10945: Remove usage of Guava Function interface (oak-store-document) - cleanup * OAK-10945: Remove usage of Guava Function interface (oak-store-composite) - cleanup * OAK-10945: Remove usage of Guava Function interface (oak-segment-tar) - cleanup * OAK-10945: Remove usage of Guava Function interface (oak-lucene) - cleanup * OAK-10945: Remove usage of Guava Function interface (oak-jcr) - cleanup --- .../blobstorage/AzureBlobStoreBackend.java | 10 +-- .../oak/blob/cloud/s3/S3Backend.java | 20 +---- .../blob/AbstractSharedCachingDataStore.java | 7 +- .../blob/MarkSweepGarbageCollector.java | 27 ++---- .../plugins/blob/datastore/BlobIdTracker.java | 9 +- .../blob/datastore/DataStoreBlobStore.java | 17 ++-- .../blob/datastore/SharedDataStoreUtils.java | 24 ++--- .../blob/SharedDataStoreUtilsTest.java | 14 +-- .../datastore/DataStoreBlobStoreTest.java | 10 +-- .../blob/datastore/OakFileDataStoreTest.java | 9 +- .../blob/datastore/SharedDataStoreTest.java | 9 +- .../oak/commons/FileIOUtilsTest.java | 7 +- .../jackrabbit/oak/spi/mount/MountInfo.java | 5 +- .../java/org/apache/jackrabbit/oak/Oak.java | 8 +- .../jackrabbit/oak/core/SecureNodeState.java | 6 +- .../oak/management/RepositoryManager.java | 3 +- .../plugins/identifier/IdentifierManager.java | 9 +- .../plugins/index/IndexInfoServiceImpl.java | 23 ++--- .../index/aggregate/AggregateIndex.java | 10 +-- .../plugins/index/cursor/AncestorCursor.java | 17 +--- .../index/reference/ReferenceIndex.java | 8 +- .../migration/AbstractDecoratedNodeState.java | 36 +++----- .../nodetype/EffectiveNodeTypeImpl.java | 11 +-- .../oak/plugins/tree/impl/AbstractTree.java | 8 +- .../oak/plugins/version/VersionHook.java | 11 +-- .../oak/query/ast/LowerCaseImpl.java | 13 +-- .../principal/PrincipalProviderImpl.java | 4 +- .../security/user/AuthorizableIterator.java | 5 +- .../security/user/UserPrincipalProvider.java | 4 +- .../user/autosave/AuthorizableWrapper.java | 6 +- .../user/query/ResultRowToAuthorizable.java | 5 +- .../security/user/query/UserQueryManager.java | 3 +- .../plugins/tree/impl/ImmutableTreeTest.java | 11 +-- .../evaluation/ChildOrderPropertyTest.java | 10 +-- .../oak/security/privilege/JcrAllTest.java | 9 +- .../security/privilege/PrivilegeImplTest.java | 10 +-- .../apache/jackrabbit/oak/util/NodeUtil.java | 8 +- .../CustomExternalIdentityProvider.java | 11 +-- .../privilege/L4_CustomPrivilegeTest.java | 11 +-- .../oak/composite/CompositeNodeStoreTest.java | 2 +- .../oak/jcr/delegate/NodeDelegate.java | 16 +--- .../jcr/delegate/VersionHistoryDelegate.java | 9 +- .../jackrabbit/oak/jcr/session/NodeImpl.java | 23 ++--- .../oak/jcr/version/VersionHistoryImpl.java | 15 +--- .../jackrabbit/oak/jcr/xml/ImporterImpl.java | 17 +--- .../oak/jcr/version/VersionableTest.java | 18 ++-- .../oak/plugins/index/lucene/IndexCopier.java | 16 +--- .../index/lucene/hybrid/IndexedPaths.java | 31 +++---- .../lucene/hybrid/LuceneDocumentHolder.java | 9 +- .../lucene/ResultCountingIndexProvider.java | 17 ++-- .../document/DocumentNodeStoreHelper.java | 12 +-- .../AbstractSegmentTarExplorerBackend.java | 9 +- .../tika/CSVFileBinaryResourceProvider.java | 5 +- .../tika/NodeStoreBinaryResourceProvider.java | 5 +- .../oak/run/DataStoreCheckCommand.java | 2 +- .../jackrabbit/oak/run/PrintingDiff.java | 5 +- .../oak/plugins/tika/BinarySourceMapper.java | 3 +- .../CSVFileBinaryResourceProviderTest.java | 5 +- .../oak/run/DataStoreCheckTest.java | 18 ++-- .../oak/run/DataStoreCommandTest.java | 16 +--- .../plugins/index/search/AggregateTest.java | 3 +- .../spi/security/CompositeConfiguration.java | 36 +++----- .../credentials/SimpleCredentialsSupport.java | 10 +-- .../token/CompositeTokenConfiguration.java | 10 +-- .../privilege/PrivilegeBitsProvider.java | 5 +- .../oak/segment/azure/tool/ToolUtils.java | 10 +-- .../oak/segment/CachingSegmentReader.java | 19 +--- .../jackrabbit/oak/segment/ReaderCache.java | 3 +- .../jackrabbit/oak/segment/Revisions.java | 3 +- .../oak/segment/WriterCacheManager.java | 9 +- .../oak/segment/file/ReadOnlyRevisions.java | 2 +- .../oak/segment/file/TarRevisions.java | 3 +- .../segment/file/tooling/RevisionHistory.java | 10 +-- .../segment/memory/MemoryStoreRevisions.java | 2 +- .../oak/segment/tool/PrintingDiff.java | 4 +- .../jackrabbit/oak/segment/tool/Utils.java | 12 +-- .../oak/segment/ReaderCacheTest.java | 29 ++----- .../oak/segment/file/TarRevisionsTest.java | 2 +- .../composite/CompositeChildrenCountTest.java | 18 +--- .../oak/plugins/document/Branch.java | 15 +--- .../oak/plugins/document/Commit.java | 11 +-- .../plugins/document/DocumentNodeState.java | 18 +--- .../plugins/document/DocumentNodeStore.java | 27 ++---- .../document/DocumentNodeStoreBranch.java | 8 +- .../document/DocumentNodeStoreMBeanImpl.java | 22 +---- .../plugins/document/MissingBcSweeper2.java | 4 +- .../oak/plugins/document/NodeDocument.java | 28 ++---- .../plugins/document/NodeDocumentSweeper.java | 20 ++--- .../oak/plugins/document/PropertyHistory.java | 10 +-- .../oak/plugins/document/SplitOperations.java | 4 +- .../document/UnsavedModifications.java | 8 +- .../document/VersionGarbageCollector.java | 17 +--- .../document/mongo/MongoDocumentStore.java | 22 +---- .../document/mongo/MongoVersionGCSupport.java | 21 ++--- .../document/persistentCache/NodeCache.java | 11 +-- .../persistentCache/PersistentCache.java | 11 ++- .../document/rdb/RDBDocumentStore.java | 15 +--- .../document/rdb/RDBDocumentStoreJDBC.java | 10 +-- .../DelegatingDocumentNodeState.java | 11 +-- .../oak/plugins/document/util/Utils.java | 8 +- ...goVersionGCSupportDefaultNoBranchTest.java | 2 +- .../oak/plugins/document/TestUtils.java | 3 +- .../document/VersionGarbageCollectorIT.java | 8 +- .../plugins/memory/MemoryChildNodeEntry.java | 13 +-- .../oak/plugins/memory/ModifiedNodeState.java | 19 +--- .../plugins/memory/MultiPropertyState.java | 87 +++---------------- .../commit/ProgressNotificationEditor.java | 3 +- .../oak/spi/state/AbstractNodeState.java | 10 +-- .../oak/spi/state/ChildNodeEntry.java | 12 +-- .../oak/upgrade/RepositoryUpgrade.java | 14 +-- .../oak/upgrade/SameNameSiblingsEditor.java | 8 +- .../checkpoint/CheckpointRetriever.java | 13 +-- .../oak/upgrade/cli/AbstractOak2OakTest.java | 11 +-- 113 files changed, 319 insertions(+), 1056 deletions(-) diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java index 33add8dafa4..044301fb33c 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java @@ -45,8 +45,8 @@ import java.util.Queue; import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.cache.Cache; import org.apache.jackrabbit.guava.common.cache.CacheBuilder; @@ -462,13 +462,7 @@ public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException @Override public Iterator getAllIdentifiers() throws DataStoreException { return new RecordsIterator( - new Function() { - @Override - public DataIdentifier apply(AzureBlobInfo input) { - return new DataIdentifier(getIdentifierName(input.getName())); - } - } - ); + input -> new DataIdentifier(getIdentifierName(input.getName()))); } diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java index 37f28d36ced..81f910ab1a9 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java @@ -38,6 +38,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; @@ -89,7 +90,6 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; import com.amazonaws.util.StringUtils; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.cache.Cache; @@ -441,13 +441,7 @@ public InputStream read(DataIdentifier identifier) @Override public Iterator getAllIdentifiers() throws DataStoreException { - return new RecordsIterator( - new Function() { - @Override - public DataIdentifier apply(S3ObjectSummary input) { - return new DataIdentifier(getIdentifierName(input.getKey())); - } - }); + return new RecordsIterator(input -> new DataIdentifier(getIdentifierName(input.getKey()))); } @Override @@ -642,14 +636,8 @@ public void deleteAllMetadataRecords(String prefix) { public Iterator getAllRecords() { final AbstractSharedBackend backend = this; return new RecordsIterator( - new Function() { - @Override - public DataRecord apply(S3ObjectSummary input) { - return new S3DataRecord(backend, s3service, bucket, - new DataIdentifier(getIdentifierName(input.getKey())), - input.getLastModified().getTime(), input.getSize(), s3ReqDecorator); - } - }); + input -> new S3DataRecord(backend, s3service, bucket, new DataIdentifier(getIdentifierName(input.getKey())), + input.getLastModified().getTime(), input.getSize(), s3ReqDecorator)); } @Override diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java index 6f81f95064d..70fb3e0bfc7 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java @@ -56,7 +56,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -277,11 +276,7 @@ public DataRecord addRecord(InputStream inputStream, BlobOptions blobOptions) @Override public Iterator getAllIdentifiers() throws DataStoreException { return Iterators.concat(Iterators.transform(cache.getStagingCache().getAllIdentifiers(), - new Function() { - @Nullable @Override public DataIdentifier apply(@Nullable String id) { - return new DataIdentifier(id); - } - }), backend.getAllIdentifiers()); + id -> new DataIdentifier(id)), backend.getAllIdentifiers()); } @Override diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java index 9f016f161fb..45b3edb16ad 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java @@ -51,7 +51,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.StandardSystemProperty; import org.apache.jackrabbit.guava.common.base.Stopwatch; @@ -264,22 +263,14 @@ public List getStats() throws Exception { List refFiles = ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ImmutableListMultimap references = - FluentIterable.from(refFiles).index(new Function() { - @Override public String apply(DataRecord input) { - return SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString()); - - } - }); + FluentIterable.from(refFiles).index( + input -> SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString())); // Get all the markers available List markerFiles = ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType()); - Map markers = Maps.uniqueIndex(markerFiles, new Function() { - @Override - public String apply(DataRecord input) { - return input.getIdentifier().toString().substring(SharedStoreRecordType.MARKED_START_MARKER.getType().length() + 1); - } - }); + Map markers = Maps.uniqueIndex(markerFiles, + input -> input.getIdentifier().toString().substring(SharedStoreRecordType.MARKED_START_MARKER.getType().length() + 1)); // Get all the repositories registered List repoFiles = @@ -645,16 +636,12 @@ public void addReference(String blobId, final String nodeId) { final Joiner delimJoiner = Joiner.on(DELIM).skipNulls(); Iterator> partitions = Iterators.partition(idIter, getBatchCount()); while (partitions.hasNext()) { - List idBatch = Lists.transform(partitions.next(), new Function() { - @Nullable @Override - public String apply(@Nullable String id) { + List idBatch = Lists.transform(partitions.next(), id -> { if (logPath) { return delimJoiner.join(id, nodeId); } return id; - } - }); + }); if (debugMode) { LOG.trace("chunkIds : {}", idBatch); } @@ -891,7 +878,7 @@ long mergeAllMarkedReferences(GarbageCollectableBlobStore blobStore, GarbageColl List repoFiles = ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); LOG.info("Repositories registered {}", repoFiles); - + // Retrieve repos for which reference files have not been created Set unAvailRepos = SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles); diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java index ad4f653c577..402d573d5f1 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java @@ -24,14 +24,12 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.nio.file.Path; import java.util.Iterator; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -77,7 +75,6 @@ import static org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker.BlobIdStore.Type.IN_PROCESS; import static org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker.BlobIdStore.Type.REFS; - /** * Tracks the blob ids available or added in the blob store using the {@link BlobIdStore} . * @@ -271,8 +268,7 @@ private void globalMerge() throws IOException { Iterable refRecords = datastore.getAllMetadataRecords(fileNamePrefix); // Download all the corresponding files for the records - List refFiles = newArrayList(transform(refRecords, new Function() { - @Override public File apply(DataRecord input) { + List refFiles = newArrayList(transform(refRecords, input -> { InputStream inputStream = null; try { inputStream = input.getStream(); @@ -283,8 +279,7 @@ private void globalMerge() throws IOException { closeQuietly(inputStream); } return null; - } - })); + })); LOG.info("Retrieved all blob id files in [{}]", watch.elapsed(TimeUnit.MILLISECONDS)); // Merge all the downloaded files in to the local store diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java index d057007bc57..5536e2dac7b 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java @@ -78,7 +78,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -525,15 +525,12 @@ public boolean apply(@Nullable DataRecord input) { } return false; } - }), new Function() { - @Override - public String apply(DataRecord input) { + }), input -> { if (encodeLengthInId) { return BlobId.of(input).encodedValue(); } return input.getIdentifier().toString(); - } - }); + }); } @Override @@ -775,18 +772,14 @@ public Iterator getAllRecords() throws DataStoreException { Iterator result = delegate instanceof SharedDataStore ? ((SharedDataStore) delegate).getAllRecords() : Iterators.transform(delegate.getAllIdentifiers(), - new Function() { - @Nullable - @Override - public DataRecord apply(@Nullable DataIdentifier input) { + input -> { try { return delegate.getRecord(input); } catch (DataStoreException e) { log.warn("Error occurred while fetching DataRecord for identifier {}", input, e); } return null; - } - }); + }); if (stats instanceof ExtendedBlobStatsCollector) { ((ExtendedBlobStatsCollector) stats).getAllRecordsCalled(System.nanoTime() - start, TimeUnit.NANOSECONDS); diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java index 8c0a076b17f..294040b0bdf 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java @@ -18,9 +18,9 @@ import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.collect.FluentIterable; @@ -61,7 +61,7 @@ public static DataRecord getEarliestRecord(List recs) { public Long apply(@NotNull DataRecord input) { return input.getLastModified(); } - }).min(recs); + }::apply).min(recs); } /** @@ -73,22 +73,10 @@ public Long apply(@NotNull DataRecord input) { */ public static Set refsNotAvailableFromRepos(List repos, List refs) { - return Sets.difference(FluentIterable.from(repos).uniqueIndex( - new Function() { - @Override - @Nullable - public String apply(@NotNull DataRecord input) { - return SharedStoreRecordType.REPOSITORY.getIdFromName(input.getIdentifier().toString()); - } - }).keySet(), - FluentIterable.from(refs).index( - new Function() { - @Override - @Nullable - public String apply(@NotNull DataRecord input) { - return SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString()); - } - }).keySet()); + return Sets.difference(FluentIterable.from(repos) + .uniqueIndex(input -> SharedStoreRecordType.REPOSITORY.getIdFromName(input.getIdentifier().toString())).keySet(), + FluentIterable.from(refs) + .index(input -> SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString())).keySet()); } /** diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java index 48eafa75ba4..4a125a0d34a 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java @@ -33,7 +33,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.guava.common.collect.Maps; @@ -49,7 +48,6 @@ import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils; import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType; import org.apache.jackrabbit.oak.stats.Clock; -import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -481,11 +479,7 @@ public void testGetAllRecords() throws Exception { } Set retrieved = newHashSet(Iterables.transform(newHashSet(dataStore.getAllRecords()), - new Function() { - @Nullable @Override public String apply(@Nullable DataRecord input) { - return input.getIdentifier().toString(); - } - })); + input -> input.getIdentifier().toString())); assertEquals(added, retrieved); } @@ -515,11 +509,7 @@ public void testGetAllRecordsWithMeta() throws Exception { } Set retrieved = newHashSet(Iterables.transform(newHashSet(dataStore.getAllRecords()), - new Function() { - @Nullable @Override public String apply(@Nullable DataRecord input) { - return input.getIdentifier().toString(); - } - })); + input -> input.getIdentifier().toString())); assertEquals(added, retrieved); } diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java index 42a896dacb9..8893928c20a 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java @@ -29,7 +29,6 @@ import java.util.Set; import java.util.UUID; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -44,7 +43,6 @@ import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStoreTest; import org.apache.jackrabbit.oak.spi.blob.BlobStoreInputStream; import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector; -import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -174,13 +172,7 @@ public void testGetAllChunks() throws Exception{ DataIdentifier d30 = new DataIdentifier("d-30"); List dis = ImmutableList.of(d10, d20, d30); List recs = Lists.newArrayList( - Iterables.transform(dis, new Function() { - @Nullable - @Override - public DataRecord apply(@Nullable DataIdentifier input) { - return new TimeDataRecord(input); - } - })); + Iterables.transform(dis, input -> new TimeDataRecord(input))); OakFileDataStore mockedDS = mock(OakFileDataStore.class); when(mockedDS.getAllRecords()).thenReturn(recs.iterator()); when(mockedDS.getRecord(new DataIdentifier("d-10"))).thenReturn(new TimeDataRecord(d10)); diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java index 8f58a64e89f..e9e533f536f 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java @@ -24,14 +24,12 @@ import java.util.Map; import java.util.Set; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.jackrabbit.core.data.DataIdentifier; import org.apache.jackrabbit.core.data.FileDataStore; -import org.jetbrains.annotations.Nullable; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -77,12 +75,7 @@ private void testGetAllIdentifiers(String path, String unnormalizedPath) throws fds.init(null); Iterator dis = fds.getAllIdentifiers(); - Set fileNames = Sets.newHashSet(Iterators.transform(dis, new Function() { - @Override - public String apply(@Nullable DataIdentifier input) { - return input.toString(); - } - })); + Set fileNames = Sets.newHashSet(Iterators.transform(dis, input -> input.toString())); Set expectedNames = Sets.newHashSet("abcdef","bcdefg","cdefgh"); assertEquals(expectedNames, fileNames); diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java index 87804e28c12..9f0514bc464 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java @@ -31,7 +31,6 @@ import java.util.Properties; import java.util.Set; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -47,7 +46,6 @@ import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore; import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreTest.FixtureHelper.DATA_STORE; -import org.jetbrains.annotations.Nullable; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -146,12 +144,7 @@ private void testGetAllIdentifiers(String path, String unnormalizedPath) throws fds.init(null); Iterator dis = fds.getAllIdentifiers(); - Set fileNames = Sets.newHashSet(Iterators.transform(dis, new Function() { - @Override - public String apply(@Nullable DataIdentifier input) { - return input.toString(); - } - })); + Set fileNames = Sets.newHashSet(Iterators.transform(dis, input-> input.toString())); Set expectedNames = Sets.newHashSet("abcdef","bcdefg","cdefgh"); assertEquals(expectedNames, fileNames); diff --git a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java index c172dd429fb..3b52f24b3ff 100644 --- a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java +++ b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java @@ -60,7 +60,6 @@ import java.util.Set; import org.apache.commons.io.FileUtils; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -192,11 +191,7 @@ public void sortLargeFileWithCustomComparatorTest() throws IOException { } Iterator boxedEntries = Longs.asList(entries).iterator(); - Iterator hexEntries = Iterators.transform(boxedEntries, new Function() { - @Nullable @Override public String apply(@Nullable Long input) { - return Long.toHexString(input); - } - }); + Iterator hexEntries = Iterators.transform(boxedEntries, input -> Long.toHexString(input)); File f = assertWrite(hexEntries, false, numEntries); Comparator prefixComparator = new Comparator() { diff --git a/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java b/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java index 8f927824033..c7fe29eabe4 100644 --- a/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java +++ b/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java @@ -26,13 +26,12 @@ import java.util.NavigableSet; import java.util.Set; import java.util.TreeSet; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; -import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; import static org.apache.jackrabbit.guava.common.collect.Sets.newTreeSet; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.commons.PathUtils.isAncestor; @@ -141,7 +140,7 @@ public String getPathFragmentName() { private static TreeSet cleanCopy(Collection includedPaths) { // ensure that paths don't have trailing slashes - this triggers an assertion in PathUtils isAncestor - return newTreeSet(transform(includedPaths, SANITIZE_PATH)); + return newTreeSet(transform(includedPaths, SANITIZE_PATH::apply)); } public Set getPathsSupportingFragments() { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java index 4f2beb318de..8d9309f37b8 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java @@ -50,7 +50,6 @@ import javax.management.StandardMBean; import javax.security.auth.login.LoginException; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -701,16 +700,13 @@ private void initialContent(IndexEditorProvider indexEditors, QueryIndexProvider // FIXME: OAK-810 move to proper workspace initialization // initialize default workspace Iterable workspaceInitializers = Iterables.transform(securityProvider.getConfigurations(), - new Function() { - @Override - public WorkspaceInitializer apply(SecurityConfiguration sc) { + sc -> { WorkspaceInitializer wi = sc.getWorkspaceInitializer(); if (wi instanceof QueryIndexProviderAware) { ((QueryIndexProviderAware) wi).setQueryIndexProvider(indexProvider); } return wi; - } - }); + }); OakInitializer.initialize(workspaceInitializers, store, defaultWorkspaceName, initHook); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java index 51a772fc108..db9dbca1397 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.core; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry; @@ -32,6 +31,9 @@ import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.guava.common.collect.Iterables.filter; import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; + +import java.util.function.Function; + import static java.util.Collections.emptyList; class SecureNodeState extends AbstractNodeState { @@ -144,7 +146,7 @@ public Iterable getChildNodeEntries() { } else if (treePermission.canRead()) { Iterable readable = transform( state.getChildNodeEntries(), - new WrapChildEntryFunction()); + new WrapChildEntryFunction()::apply); return filter(readable, new IterableNodePredicate()); } else { return emptyList(); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java index 4ecf4c6740d..9e256f6e5a2 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.management; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; @@ -32,11 +31,11 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; import javax.management.openmbean.CompositeData; import javax.management.openmbean.TabularData; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.jmx.FileStoreBackupRestoreMBean; import org.apache.jackrabbit.oak.api.jmx.RepositoryManagementMBean; import org.apache.jackrabbit.oak.api.jmx.SessionMBean; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java index 02f119feaab..f9fe4b19524 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java @@ -20,10 +20,11 @@ import java.util.Collections; import java.util.Iterator; import java.util.Map; +import java.util.function.Function; + import javax.jcr.PropertyType; import javax.jcr.query.Query; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.JcrConstants; @@ -237,7 +238,7 @@ private Iterable findPaths(@NotNull final Result result, @NotNull final return new Iterable() { @Override public Iterator iterator() { - return Iterators.concat(transform(result.getRows().iterator(), new RowToPaths())); + return Iterators.concat(transform(result.getRows().iterator(), new RowToPaths()::apply)); } class RowToPaths implements Function> { @@ -271,7 +272,7 @@ public String apply(PropertyState pState) { if (!rowPath.startsWith(VersionConstants.VERSION_STORE_PATH)) { if (propertyName == null) { return filter( - transform(root.getTree(rowPath).getProperties().iterator(), new PropertyToPath()), + transform(root.getTree(rowPath).getProperties().iterator(), new PropertyToPath()::apply), notNull()); } else { // for a fixed property name, we don't need to look for it, but just assume that @@ -318,7 +319,7 @@ public Iterable getReferences(@NotNull Tree tree, @NotNull final String pr QueryEngine.INTERNAL_SQL2_QUERY, Query.JCR_SQL2, bindings, NO_MAPPINGS); - Iterable resultTrees = Iterables.transform(result.getRows(), (Function) row -> row.getTree(null)); + Iterable resultTrees = Iterables.transform(result.getRows(), row -> row.getTree(null)); return Iterables.filter(resultTrees, tree1 -> !tree1.getPath().startsWith(VersionConstants.VERSION_STORE_PATH) ); } catch (ParseException e) { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java index c263bc767a3..756bef928c2 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.index; import java.io.IOException; @@ -24,7 +23,6 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; @@ -71,19 +69,16 @@ public Iterable getAllIndexInfo() { } else { activeIndexes.addAll(allIndexes); } - return Iterables.filter(Iterables.transform(indexPathService.getIndexPaths(), new Function() { - @Override - public IndexInfo apply(String indexPath) { - try { - IndexInfo info = getInfo(indexPath); - if (info != null) { - info.setActive(activeIndexes.contains(indexPath)); - } - return info; - } catch (Exception e) { - log.warn("Error occurred while capturing IndexInfo for path {}", indexPath, e); - return null; + return Iterables.filter(Iterables.transform(indexPathService.getIndexPaths(), indexPath -> { + try { + IndexInfo info = getInfo(indexPath); + if (info != null) { + info.setActive(activeIndexes.contains(indexPath)); } + return info; + } catch (Exception e) { + log.warn("Error occurred while capturing IndexInfo for path {}", indexPath, e); + return null; } }), notNull()); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java index b22cfbe2189..da5fe35b5d0 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java @@ -35,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvanceFulltextQueryIndex; @@ -208,13 +207,8 @@ public boolean visit(FullTextAnd and) { public boolean visit(FullTextOr or) { final int[] index = new int[1]; List cursors = Lists.transform(or.list, - new Function() { - @Override - public Cursor apply(FullTextExpression input) { - return flatten(input, plan, filter, state, - path + " or(" + index[0]++ + ")"); - } - }); + input -> flatten(input, plan, filter, state, + path + " or(" + index[0]++ + ")")); result.set(Cursors.newConcatCursor(cursors, filter.getQueryLimits())); return true; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java index c24a8de67d4..2842e8202ea 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java @@ -20,11 +20,9 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.spi.query.Cursor; -import org.apache.jackrabbit.oak.spi.query.IndexRow; import org.apache.jackrabbit.oak.spi.query.QueryLimits; import org.jetbrains.annotations.Nullable; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -39,12 +37,7 @@ public AncestorCursor(Cursor cursor, int level, QueryLimits settings) { private static Iterator transform(Cursor cursor, final int level) { Iterator unfiltered = Iterators.transform(cursor, - new Function() { - @Override - public String apply(@Nullable IndexRow input) { - return input != null ? input.getPath() : null; - } - }); + input -> input != null ? input.getPath() : null); Iterator filtered = Iterators.filter(unfiltered, new Predicate() { @Override @@ -52,11 +45,7 @@ public boolean apply(@Nullable String input) { return input != null && PathUtils.getDepth(input) >= level; } }); - return Iterators.transform(filtered, new Function() { - @Override - public String apply(String input) { - return PathUtils.getAncestorPath(input, level); - } - }); + return Iterators.transform(filtered, + input -> PathUtils.getAncestorPath(input, level)); } } \ No newline at end of file diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java index 438ede13c03..089ed8ff6d7 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java @@ -45,7 +45,6 @@ import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -147,12 +146,7 @@ public boolean apply(String path) { } }); } - paths = transform(paths, new Function() { - @Override - public String apply(String path) { - return getParentPath(path); - } - }); + paths = transform(paths, path -> getParentPath(path)); return newPathCursor(paths, filter.getQueryLimits()); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java index 374d29a464b..9b513bf141d 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.migration; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.api.PropertyState; @@ -38,6 +37,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; + import static org.apache.jackrabbit.guava.common.base.Predicates.notNull; import static org.apache.jackrabbit.oak.plugins.tree.TreeConstants.OAK_CHILD_ORDER; @@ -139,23 +139,16 @@ public NodeState getChildNode(@NotNull final String name) throws IllegalArgument @Override @NotNull public Iterable getChildNodeEntries() { - final Iterable transformed = Iterables.transform( - delegate.getChildNodeEntries(), - new Function() { - @Nullable - @Override - public ChildNodeEntry apply(@Nullable final ChildNodeEntry childNodeEntry) { - if (childNodeEntry != null) { - final String name = childNodeEntry.getName(); - final NodeState nodeState = decorate(name, childNodeEntry.getNodeState()); - if (nodeState.exists()) { - return new MemoryChildNodeEntry(name, nodeState); - } - } - return null; - } + final Iterable transformed = Iterables.transform(delegate.getChildNodeEntries(), childNodeEntry -> { + if (childNodeEntry != null) { + final String name = childNodeEntry.getName(); + final NodeState nodeState = decorate(name, childNodeEntry.getNodeState()); + if (nodeState.exists()) { + return new MemoryChildNodeEntry(name, nodeState); } - ); + } + return null; + }); return Iterables.filter(transformed, notNull()); } @@ -179,14 +172,7 @@ public PropertyState getProperty(@NotNull String name) { public Iterable getProperties() { final Iterable propertyStates = Iterables.transform( delegate.getProperties(), - new Function() { - @Override - @Nullable - public PropertyState apply(@Nullable final PropertyState propertyState) { - return decorate(propertyState); - } - } - ); + propertyState -> decorate(propertyState)); return Iterables.filter(Iterables.concat(propertyStates, getNewPropertyStates()), notNull()); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java index 61dbab78e31..1b34ec9608c 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java @@ -44,7 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Maps; @@ -227,14 +226,8 @@ public boolean apply(PropertyDefinition propertyDefinition) { @NotNull @Override public Iterable getNamedNodeDefinitions(@NotNull final String oakName) { - return Iterables.concat(Iterables.transform( - nodeTypes.values(), - new Function>() { - @Override - public Iterable apply(NodeTypeImpl input) { - return input.getDeclaredNamedNodeDefinitions(oakName); - } - })); + return Iterables.concat(Iterables.transform(nodeTypes.values(), + input -> input.getDeclaredNamedNodeDefinitions(oakName))); } /** diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java index 4ba4f6734d0..62eced8d187 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java @@ -35,7 +35,6 @@ import java.util.List; import java.util.Set; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.oak.api.PropertyState; @@ -320,13 +319,10 @@ public long getChildrenCount(long max) { @NotNull public Iterable getChildren() { Iterable children = transform(getChildNames(), - new Function() { - @Override - public Tree apply(String name) { + name -> { AbstractTree child = createChild(name); return child.exists() ? child : null; - } - }); + }); return filter(children, notNull()); } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java index 2bce99afae6..5c46f93a4a2 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java @@ -18,7 +18,6 @@ */ package org.apache.jackrabbit.oak.plugins.version; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.spi.commit.CommitHook; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; @@ -26,11 +25,11 @@ import org.apache.jackrabbit.oak.spi.commit.EditorProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.osgi.service.component.annotations.Component; import java.util.List; import java.util.Set; + import static org.apache.jackrabbit.guava.common.collect.Collections2.transform; import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; @@ -77,12 +76,6 @@ public NodeState processCommit(NodeState before, NodeState after, CommitInfo inf providers.add(new VersionableCollector.Provider(existingVersionables)); providers.add(new OrphanedVersionCleaner.Provider(existingVersionables)); - return compose(transform(providers, new Function() { - @Nullable - @Override - public CommitHook apply(@Nullable EditorProvider input) { - return new EditorHook(input); - } - })).processCommit(before, after, info); + return compose(transform(providers, input -> new EditorHook(input))).processCommit(before, after, info); } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java index d306eac10e5..df46dd82735 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java @@ -30,8 +30,6 @@ import org.apache.jackrabbit.oak.spi.query.QueryConstants; import org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry; -import org.apache.jackrabbit.guava.common.base.Function; - import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.api.Type.STRINGS; @@ -60,12 +58,12 @@ boolean accept(AstVisitor v) { public String toString() { return "lower(" + operand + ')'; } - + @Override public PropertyExistenceImpl getPropertyExistence() { return operand.getPropertyExistence(); } - + @Override public Set getSelectors() { return operand.getSelectors(); @@ -80,12 +78,7 @@ public PropertyValue currentProperty() { // TODO toLowerCase(): document the Turkish locale problem if (p.getType().isArray()) { Iterable lowerCase = transform(p.getValue(STRINGS), - new Function() { - @Override - public String apply(String input) { - return input.toLowerCase(); - } - }); + input -> input.toLowerCase()); return PropertyValues.newString(lowerCase); } else { String value = p.getValue(STRING); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java index 598f7702eb3..0fd19da1cd5 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.security.principal; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.principal.ItemBasedPrincipal; import org.apache.jackrabbit.api.security.user.Authorizable; @@ -45,6 +44,7 @@ import java.util.Iterator; import java.util.Objects; import java.util.Set; +import java.util.function.Function; /** * The {@code PrincipalProviderImpl} is a principal provider implementation @@ -144,7 +144,7 @@ public Iterator findPrincipals(@Nullable final String nameH } try { Iterator authorizables = findAuthorizables(nameHint, searchType, offset, limit); - Iterator principals = Iterators.filter(Iterators.transform(authorizables, new AuthorizableToPrincipal()), Objects::nonNull); + Iterator principals = Iterators.filter(Iterators.transform(authorizables, new AuthorizableToPrincipal()::apply), Objects::nonNull); return EveryoneFilter.filter(principals, nameHint, searchType, offset, limit); } catch (RepositoryException e) { log.debug(e.getMessage()); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java index 632bdbd3be1..9179bcfd7da 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.security.user; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.user.Authorizable; import org.apache.jackrabbit.oak.api.Tree; @@ -33,6 +32,7 @@ import java.util.Iterator; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.function.Predicate; /** @@ -50,7 +50,8 @@ final class AuthorizableIterator implements Iterator { static AuthorizableIterator create(@NotNull Iterator authorizableTrees, @NotNull UserManagerImpl userManager, @NotNull AuthorizableType authorizableType) { - Iterator it = Iterators.transform(authorizableTrees, new TreeToAuthorizable(userManager, authorizableType)); + Iterator it = Iterators.transform(authorizableTrees, + new TreeToAuthorizable(userManager, authorizableType)::apply); long size = getSize(authorizableTrees); return new AuthorizableIterator(it, size, false); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java index 2d48d311928..3b65a8e3886 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.security.user; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.principal.ItemBasedPrincipal; @@ -51,6 +50,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Set; +import java.util.function.Function; import static org.apache.jackrabbit.oak.api.QueryEngine.NO_BINDINGS; import static org.apache.jackrabbit.oak.api.Type.STRING; @@ -184,7 +184,7 @@ public Iterator findPrincipals(@Nullable final String nameH limit, offset, NO_BINDINGS, namePathMapper.getSessionLocalMappings()); Iterator principals = Iterators.filter( - Iterators.transform(result.getRows().iterator(), new ResultRowToPrincipal()), + Iterators.transform(result.getRows().iterator(), new ResultRowToPrincipal()::apply), Predicates.notNull()); // everyone is injected only in complete set, not on pages diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java index 378297fc48b..9ae0dd6db4e 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java @@ -17,8 +17,8 @@ package org.apache.jackrabbit.oak.security.user.autosave; import java.util.Iterator; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.user.Authorizable; import org.apache.jackrabbit.api.security.user.Group; @@ -41,10 +41,10 @@ public T apply(T authorizable) { } static Iterator createIterator(Iterator dlgs, AutoSaveEnabledManager mgr) { - return Iterators.transform(dlgs, new AuthorizableWrapper<>(mgr)); + return Iterators.transform(dlgs, new AuthorizableWrapper(mgr)::apply); } static Iterator createGroupIterator(Iterator dlgs, AutoSaveEnabledManager mgr) { - return Iterators.transform(dlgs, new AuthorizableWrapper<>(mgr)); + return Iterators.transform(dlgs, new AuthorizableWrapper(mgr)::apply); } } \ No newline at end of file diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java index aeab5feca05..b9a5dbcadf3 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java @@ -16,14 +16,11 @@ */ package org.apache.jackrabbit.oak.security.user.query; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.api.security.user.Authorizable; import org.apache.jackrabbit.oak.api.ResultRow; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.api.Tree; -import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.security.user.UserManagerImpl; -import org.apache.jackrabbit.oak.spi.query.QueryConstants; import org.apache.jackrabbit.oak.spi.security.user.AuthorizableType; import org.apache.jackrabbit.oak.spi.security.user.util.UserUtil; import org.jetbrains.annotations.NotNull; @@ -31,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.function.Function; + import javax.jcr.RepositoryException; /** diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java index c14982565f8..5a76e75d0f1 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java @@ -273,7 +273,8 @@ private Iterator findAuthorizables(@NotNull String statement, statement, javax.jcr.query.Query.XPATH, limit, offset, NO_BINDINGS, namePathMapper.getSessionLocalMappings()); Iterable resultRows = query.getRows(); - Iterator authorizables = Iterators.transform(resultRows.iterator(), new ResultRowToAuthorizable(userManager, root, type, query.getSelectorNames())); + Iterator authorizables = Iterators.transform(resultRows.iterator(), + new ResultRowToAuthorizable(userManager, root, type, query.getSelectorNames())::apply); return Iterators.filter(authorizables, new UniqueResultPredicate()); } catch (ParseException e) { throw new RepositoryException("Invalid user query "+statement, e); diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java index 25082d51dcb..2ff4ce0ffbe 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java @@ -19,7 +19,7 @@ package org.apache.jackrabbit.oak.plugins.tree.impl; import java.util.List; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.JcrConstants; @@ -34,7 +34,6 @@ import org.apache.jackrabbit.oak.util.NodeUtil; import org.apache.jackrabbit.util.Text; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -264,13 +263,7 @@ private static ImmutableTree getHiddenTree(@NotNull ImmutableTree immutable) { } private static void assertSequence(Iterable trees, String... names) { - List actual = Lists.newArrayList(Iterables.transform(trees, new Function() { - @Nullable - @Override - public String apply(Tree input) { - return input.getName(); - } - })); + List actual = Lists.newArrayList(Iterables.transform(trees, input -> input.getName())); assertEquals(Lists.newArrayList(names), actual); } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java index f01bbf8bb4d..9b79e8686a3 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Set; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -34,7 +33,6 @@ import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; -import org.jetbrains.annotations.Nullable; import org.junit.Before; import org.junit.Test; @@ -101,13 +99,7 @@ public void testChildOrderWithoutPropertyReadAccess() throws Exception { assertFalse(aTree.hasProperty(JcrConstants.JCR_PRIMARYTYPE)); List expected = ImmutableList.of("/a/bb", "/a/b"); - Iterable childPaths = Iterables.transform(aTree.getChildren(), new Function() { - @Nullable - @Override - public String apply(Tree input) { - return input.getPath(); - } - }); + Iterable childPaths = Iterables.transform(aTree.getChildren(), input -> input.getPath()); assertTrue(childPaths.toString(), Iterables.elementsEqual(expected, childPaths)); } } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java index f053db0eea9..ff42518973d 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java @@ -20,7 +20,6 @@ import java.util.Collections; import javax.jcr.security.Privilege; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.api.security.authorization.PrivilegeManager; import org.apache.jackrabbit.oak.AbstractSecurityTest; @@ -30,7 +29,6 @@ import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeBitsProvider; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConfiguration; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; -import org.jetbrains.annotations.Nullable; import org.junit.Test; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; @@ -71,12 +69,7 @@ public void testAllAggregation() throws Exception { Iterable declaredAggr = Arrays.asList(pMgr.getPrivilege(JCR_ALL).getDeclaredAggregatePrivileges()); String[] allAggregates = Iterables.toArray(Iterables.transform( declaredAggr, - new Function() { - @Override - public String apply(@Nullable Privilege privilege) { - return checkNotNull(privilege).getName(); - } - }), String.class); + privilege -> checkNotNull(privilege).getName()), String.class); PrivilegeBits all2 = bitsProvider.getBits(allAggregates); assertEquals(all, all2); diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java index bf71f9e2c1b..52a641b537a 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java @@ -19,7 +19,6 @@ import java.util.Set; import javax.jcr.security.Privilege; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -32,7 +31,6 @@ import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeDefinition; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -69,13 +67,7 @@ private static void assertAggregation(@NotNull Privilege[] aggr, @NotNull String assertEquals(expectedNames.length, aggr.length); Set expected = Sets.newHashSet(expectedNames); - Set result = Sets.newHashSet(Iterables.transform(ImmutableSet.copyOf(aggr), new Function() { - @Nullable - @Override - public String apply(Privilege input) { - return input.getName(); - } - })); + Set result = Sets.newHashSet(Iterables.transform(ImmutableSet.copyOf(aggr), input -> input.getName())); assertEquals(expected, result); } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java index 809043ccbb5..e5ab83aba58 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java @@ -22,7 +22,6 @@ import javax.jcr.RepositoryException; import javax.jcr.Value; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.PropertyState; @@ -221,12 +220,7 @@ public void setName(String propertyName, String value) { } public void setNames(String propertyName, String... values) { - tree.setProperty(propertyName, Lists.transform(Arrays.asList(values), new Function() { - @Override - public String apply(String jcrName) { - return getOakName(jcrName); - } - }), NAMES); + tree.setProperty(propertyName, Lists.transform(Arrays.asList(values), jcrName -> getOakName(jcrName)), NAMES); } public void setDate(String name, long time) { diff --git a/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java b/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java index 6b52903668a..2bacc21c2c4 100644 --- a/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java +++ b/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.exercise.security.authentication.external; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableMap; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -28,7 +27,6 @@ import org.apache.jackrabbit.oak.spi.security.authentication.external.ExternalUser; import org.apache.jackrabbit.util.Text; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.osgi.service.component.annotations.Activate; import org.osgi.service.component.annotations.Component; import org.osgi.service.component.annotations.ConfigurationPolicy; @@ -155,13 +153,8 @@ public Iterable getDeclaredGroups() { if (groupIds == null || groupIds.isEmpty()) { return ImmutableSet.of(); } else { - return Iterables.transform(groupIds, new Function() { - @Nullable - @Override - public ExternalIdentityRef apply(String input) { - return new ExternalIdentityRef(input, getName()); - } - }); + return Iterables.transform(groupIds, + input -> new ExternalIdentityRef(input, getName())); } } diff --git a/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java b/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java index 5f1cac95442..3287dba151b 100644 --- a/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java +++ b/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java @@ -21,7 +21,6 @@ import java.util.UUID; import javax.jcr.security.Privilege; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -29,7 +28,6 @@ import org.apache.jackrabbit.oak.AbstractSecurityTest; import org.apache.jackrabbit.oak.spi.security.principal.EveryonePrincipal; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; -import org.jetbrains.annotations.Nullable; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -118,13 +116,8 @@ private static void assertEqualPrivileges(Set expectedNames, Privilege[] fail(); } - Iterable resultNames = Iterables.transform(Sets.newHashSet(result), new Function() { - @Nullable - @Override - public String apply(Privilege input) { - return input.toString(); - } - }); + Iterable resultNames = Iterables.transform(Sets.newHashSet(result), + input -> input.toString()); Iterables.removeAll(resultNames, expectedNames); assertFalse(resultNames.iterator().hasNext()); diff --git a/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java b/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java index 52494fd01ae..9d0fd3654ad 100644 --- a/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java +++ b/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java @@ -658,7 +658,7 @@ public void duplicatedChildren() throws CommitFailedException { deepMountBuilder.child("new").setProperty("store", "deepMounted", Type.STRING); deepMountedStore.merge(deepMountBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); - List children = newArrayList(filter(store.getRoot().getChildNodeEntries(), compose(Predicates.equalTo("new"), GET_NAME))); + List children = newArrayList(filter(store.getRoot().getChildNodeEntries(), compose(Predicates.equalTo("new"), GET_NAME::apply))); assertEquals(1, children.size()); assertEquals("global", children.get(0).getNodeState().getString("store")); diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java index 4bed1fb2464..9316bf10a73 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java @@ -61,6 +61,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.function.Function; import javax.jcr.InvalidItemStateException; import javax.jcr.ItemNotFoundException; @@ -71,7 +72,6 @@ import javax.jcr.nodetype.NoSuchNodeTypeException; import javax.jcr.security.AccessControlException; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.JcrConstants; @@ -299,12 +299,7 @@ public PropertyDelegate getProperty(String relPath) throws RepositoryException { @NotNull public Iterator getProperties() throws InvalidItemStateException { return transform(getTree().getProperties().iterator(), - new Function() { - @Override - public PropertyDelegate apply(PropertyState propertyState) { - return new PropertyDelegate(sessionDelegate, tree, propertyState.getName()); - } - }); + propertyState -> new PropertyDelegate(sessionDelegate, tree, propertyState.getName())); } /** @@ -355,12 +350,7 @@ public boolean apply(Tree tree) { return tree.exists(); } }), - new Function() { - @Override - public NodeDelegate apply(Tree tree) { - return new NodeDelegate(sessionDelegate, tree); - } - }); + tree -> new NodeDelegate(sessionDelegate, tree)); } public void orderBefore(String source, String target) diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java index 95e05e13b3f..ab67996295b 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java @@ -29,7 +29,6 @@ import javax.jcr.version.LabelExistsVersionException; import javax.jcr.version.VersionException; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.JcrConstants; @@ -162,12 +161,8 @@ public int compare(NodeDelegate n1, NodeDelegate n2) { } }); final Tree thisTree = getTree(); - return Iterators.transform(versions.iterator(), new Function() { - @Override - public VersionDelegate apply(NodeDelegate nd) { - return VersionDelegate.create(sessionDelegate, thisTree.getChild(nd.getName())); - } - }); + return Iterators.transform(versions.iterator(), + nd -> VersionDelegate.create(sessionDelegate, thisTree.getChild(nd.getName()))); } @NotNull diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java index b9b992c6368..ead1b743023 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java @@ -64,7 +64,7 @@ import javax.jcr.version.VersionException; import javax.jcr.version.VersionHistory; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -823,14 +823,11 @@ public PropertyIterator perform() throws InvalidItemStateException { Iterable propertyOakPaths = idManager.getReferences(weak, node.getTree(), name); // TODO: oak name? Iterable properties = Iterables.transform( propertyOakPaths, - new Function() { - @Override - public Property apply(String oakPath) { + oakPath -> { PropertyDelegate pd = sessionDelegate.getProperty(oakPath); return pd == null ? null : new PropertyImpl(pd, sessionContext); } - } - ); + ); return new PropertyIteratorAdapter(sessionDelegate.sync(properties.iterator())); } @@ -1374,23 +1371,13 @@ private EffectiveNodeType getEffectiveNodeType() throws RepositoryException { private Iterator nodeIterator(Iterator childNodes) { return sessionDelegate.sync(transform( childNodes, - new Function() { - @Override - public Node apply(NodeDelegate nodeDelegate) { - return new NodeImpl(nodeDelegate, sessionContext); - } - })); + nodeDelegate -> new NodeImpl(nodeDelegate, sessionContext))); } private Iterator propertyIterator(Iterator properties) { return sessionDelegate.sync(transform( properties, - new Function() { - @Override - public Property apply(PropertyDelegate propertyDelegate) { - return new PropertyImpl(propertyDelegate, sessionContext); - } - })); + propertyDelegate -> new PropertyImpl(propertyDelegate, sessionContext))); } private void checkValidWorkspace(String workspaceName) diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java index e5bd7a3f0cf..e1160a78a5b 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java @@ -34,7 +34,6 @@ import javax.jcr.version.VersionHistory; import javax.jcr.version.VersionIterator; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.commons.iterator.FrozenNodeIteratorAdapter; import org.apache.jackrabbit.commons.iterator.VersionIteratorAdapter; import org.apache.jackrabbit.oak.jcr.delegate.VersionDelegate; @@ -88,12 +87,7 @@ public VersionIterator getAllLinearVersions() throws RepositoryException { @Override public VersionIterator perform() throws RepositoryException { Iterator versions = transform(dlg.getAllLinearVersions(), - new Function() { - @Override - public Version apply(VersionDelegate input) { - return new VersionImpl(input, sessionContext); - } - }); + input -> new VersionImpl(input, sessionContext)); return new VersionIteratorAdapter(sessionDelegate.sync(versions)); } }); @@ -106,12 +100,7 @@ public VersionIterator getAllVersions() throws RepositoryException { @Override public VersionIterator perform() throws RepositoryException { Iterator versions = transform(dlg.getAllVersions(), - new Function() { - @Override - public Version apply(VersionDelegate input) { - return new VersionImpl(input, sessionContext); - } - }); + input -> new VersionImpl(input, sessionContext)); return new VersionIteratorAdapter(sessionDelegate.sync(versions)); } }); diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java index d53ac86d11a..30beec363cf 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java @@ -36,7 +36,6 @@ import javax.jcr.nodetype.PropertyDefinition; import javax.jcr.version.VersionException; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -299,31 +298,23 @@ private void importProperties(@NotNull Tree tree, } private Iterable getPropertyImporters() { - return Iterables.filter(Iterables.transform(pItemImporters, new Function() { - @Nullable - @Override - public ProtectedPropertyImporter apply(@Nullable ProtectedItemImporter importer) { + return Iterables.filter(Iterables.transform(pItemImporters, importer -> { if (importer instanceof ProtectedPropertyImporter) { return (ProtectedPropertyImporter) importer; } else { return null; } - } - }), Predicates.notNull()); + }), Predicates.notNull()); } private Iterable getNodeImporters() { - return Iterables.filter(Iterables.transform(pItemImporters, new Function() { - @Nullable - @Override - public ProtectedNodeImporter apply(@Nullable ProtectedItemImporter importer) { + return Iterables.filter(Iterables.transform(pItemImporters, importer -> { if (importer instanceof ProtectedNodeImporter) { return (ProtectedNodeImporter) importer; } else { return null; } - } - }), Predicates.notNull()); + }), Predicates.notNull()); } //-----------------------------------------------------------< Importer >--- diff --git a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java index 2ffc9328157..822cf62f87f 100644 --- a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java +++ b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java @@ -28,15 +28,12 @@ import javax.jcr.version.VersionHistory; import javax.jcr.version.VersionManager; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.spi.security.principal.EveryonePrincipal; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; import org.apache.jackrabbit.test.AbstractJCRTest; -import org.jetbrains.annotations.Nullable; -import org.junit.Test; import java.util.Set; @@ -442,17 +439,12 @@ private static void assertSuccessors(VersionHistory history, Set expecte } private static Set getNames(Version[] versions) { - return newHashSet(transform(asList(versions), new Function() { - @Nullable - @Override - public String apply(@Nullable Version input) { - try { - return input.getName(); - } catch (RepositoryException e) { - return null; - } + return newHashSet(transform(asList(versions), input -> { + try { + return input.getName(); + } catch (RepositoryException e) { + return null; } })); } - } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java index 7d9d7094fa9..a18f560a0ef 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.index.lucene; import java.io.Closeable; @@ -41,7 +40,6 @@ import javax.management.openmbean.TabularDataSupport; import javax.management.openmbean.TabularType; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.guava.common.util.concurrent.Monitor; @@ -612,12 +610,7 @@ public long getLocalIndexDirSize() { @Override public String[] getGarbageDetails() { return toArray(transform(failedToDeleteFiles.values(), - new Function() { - @Override - public String apply(LocalIndexFile input) { - return input.deleteLog(); - } - }), String.class); + input -> input.deleteLog()), String.class); } @Override @@ -661,12 +654,7 @@ public String getSkippedFromUploadSize() { @Override public String[] getCopyInProgressDetails() { return toArray(transform(copyInProgressFiles, - new Function() { - @Override - public String apply(LocalIndexFile input) { - return input.copyLog(); - } - }), String.class); + input -> input.copyLog()), String.class); } @Override diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java index 5d0f373f581..374afe09432 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java @@ -16,14 +16,10 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid; -import java.util.Collection; import java.util.Iterator; -import java.util.Map; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Multimap; import org.apache.jackrabbit.oak.plugins.document.spi.JournalProperty; @@ -39,23 +35,18 @@ public IndexedPaths(Multimap indexedPaths) { @Override public Iterator iterator() { - return Iterators.transform(indexedPaths.asMap().entrySet().iterator(), - new Function>, IndexedPathInfo>() { - @Override - public IndexedPathInfo apply(final Map.Entry> input) { - return new IndexedPathInfo() { - @Override - public String getPath() { - return input.getKey(); - } + return Iterators.transform(indexedPaths.asMap().entrySet().iterator(), input -> + new IndexedPathInfo() { + @Override + public String getPath() { + return input.getKey(); + } - @Override - public Iterable getIndexPaths() { - return input.getValue(); - } - }; - } - }); + @Override + public Iterable getIndexPaths() { + return input.getValue(); + } + }); } @Override diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java index 83375129d86..462bc28f4ec 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java @@ -16,13 +16,11 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid; import java.util.Collection; import java.util.Map; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ArrayListMultimap; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.ListMultimap; @@ -117,9 +115,7 @@ private boolean queueSizeWithinLimits(){ } private static Iterable asLuceneDocInfo(ListMultimap docs) { - return Iterables.transform(docs.entries(), new Function, LuceneDocInfo>() { - @Override - public LuceneDocInfo apply(final Map.Entry input) { + return Iterables.transform(docs.entries(), input -> { return new LuceneDocInfo() { @Override public String getIndexPath() { @@ -131,7 +127,6 @@ public String getDocPath() { return input.getValue(); } }; - } - }); + }); } } diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java index 6f0946ccf5c..ac7a4d5459c 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java @@ -16,10 +16,8 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.index.lucene; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.api.Result; import org.apache.jackrabbit.oak.spi.query.Cursor; @@ -32,6 +30,7 @@ import org.jetbrains.annotations.NotNull; import java.util.List; + class ResultCountingIndexProvider implements QueryIndexProvider { private final QueryIndexProvider delegate; private final CountingCursorFactory cursorFactory; @@ -65,15 +64,11 @@ void incrementCount() { @Override public List getQueryIndexes(NodeState nodeState) { if (shouldCount) { - return Lists.transform(delegate.getQueryIndexes(nodeState), new Function() { - @NotNull - @Override - public QueryIndex apply(@NotNull QueryIndex input) { - if (input instanceof AdvanceFulltextQueryIndex) { - return new CountingIndex((AdvanceFulltextQueryIndex)input, cursorFactory); - } else { - return input; - } + return Lists.transform(delegate.getQueryIndexes(nodeState), input -> { + if (input instanceof AdvanceFulltextQueryIndex) { + return new CountingIndex((AdvanceFulltextQueryIndex) input, cursorFactory); + } else { + return input; } }); } else { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java index 9cb26f2e0b6..448a0d644dc 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java @@ -35,15 +35,12 @@ import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStoreHelper; import org.apache.jackrabbit.oak.plugins.document.util.Utils; import org.bson.conversions.Bson; -import org.jetbrains.annotations.Nullable; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.guava.common.primitives.Longs; import com.mongodb.BasicDBObject; -import com.mongodb.DBObject; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.model.Filters; @@ -156,13 +153,8 @@ private static Iterable getDocuments(DocumentStore store) { mds, Collection.NODES); Bson query = Filters.eq(NodeDocument.HAS_BINARY_FLAG, NodeDocument.HAS_BINARY_VAL); FindIterable cursor = dbCol.find(query); - return Iterables.transform(cursor, new Function() { - @Nullable - @Override - public NodeDocument apply(DBObject input) { - return MongoDocumentStoreHelper.convertFromDBObject(mds, Collection.NODES, input); - } - }); + return Iterables.transform(cursor, + input -> MongoDocumentStoreHelper.convertFromDBObject(mds, Collection.NODES, input)); } else { return Utils.getSelectedDocuments(store, NodeDocument.HAS_BINARY_FLAG, NodeDocument.HAS_BINARY_VAL); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java index 2c929dfd4ca..55696e8f809 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java @@ -18,7 +18,6 @@ */ package org.apache.jackrabbit.oak.explorer; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Maps; import org.apache.jackrabbit.oak.api.Blob; @@ -29,7 +28,6 @@ import org.apache.jackrabbit.oak.segment.SegmentNodeState; import org.apache.jackrabbit.oak.segment.SegmentNodeStateHelper; import org.apache.jackrabbit.oak.segment.SegmentPropertyState; -import org.apache.jackrabbit.oak.segment.file.JournalEntry; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; @@ -84,12 +82,7 @@ public List readRevisions() { try { journalReader = new JournalReader(journal); Iterator revisionIterator = Iterators.transform(journalReader, - new Function() { - @Override - public String apply(JournalEntry entry) { - return entry.getRevision(); - } - }); + entry -> entry.getRevision()); try { revs = newArrayList(revisionIterator); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java index 5b46b310936..2066a69d35b 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java @@ -16,15 +16,14 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.tika; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.FluentIterable; import org.apache.jackrabbit.guava.common.io.Closer; @@ -75,7 +74,7 @@ public FluentIterable getBinaries(final String path) throws IOEx CSVParser parser = CSVParser.parse(dataFile, StandardCharsets.UTF_8, FORMAT); closer.register(parser); return FluentIterable.from(parser) - .transform(new RecordTransformer()) + .transform(new RecordTransformer()::apply) .filter(notNull()) .filter(new Predicate() { @Override diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java index 53669591820..658787a73be 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java @@ -19,7 +19,6 @@ package org.apache.jackrabbit.oak.plugins.tika; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.FluentIterable; import org.apache.jackrabbit.guava.common.collect.TreeTraverser; import org.apache.jackrabbit.JcrConstants; @@ -37,6 +36,8 @@ import static org.apache.jackrabbit.oak.plugins.tree.factories.TreeFactory.createReadOnlyTree; import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode; +import java.util.function.Function; + class NodeStoreBinaryResourceProvider implements BinaryResourceProvider { private static final Logger log = LoggerFactory.getLogger(NodeStoreBinaryResourceProvider.class); private final NodeStore nodeStore; @@ -50,7 +51,7 @@ public NodeStoreBinaryResourceProvider(NodeStore nodeStore, BlobStore blobStore) public FluentIterable getBinaries(String path) { return new OakTreeTraverser() .preOrderTraversal(createReadOnlyTree(getNode(nodeStore.getRoot(), path))) - .transform(new TreeToBinarySource()) + .transform(new TreeToBinarySource()::apply) .filter(notNull()); } diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java index 7d3e4c53cc8..6107f2d9608 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java @@ -45,7 +45,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.base.Stopwatch; diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java index 1e400b19ebe..1bda8da88e1 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.jackrabbit.oak.run; import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; @@ -28,8 +27,8 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; import java.io.PrintWriter; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -127,7 +126,7 @@ private static String toString(PropertyState ps) { String v = BLOB_LENGTH.apply(ps.getValue(BINARY)); val.append(" = {").append(v).append("}"); } else if (ps.getType() == BINARIES) { - String v = transform(ps.getValue(BINARIES), BLOB_LENGTH).toString(); + String v = transform(ps.getValue(BINARIES), BLOB_LENGTH::apply).toString(); val.append("[").append(ps.count()).append("] = ").append(v); } else if (ps.isArray()) { val.append("[").append(ps.count()).append("] = ").append(ps.getValue(STRINGS)); diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java index 25fa44927d7..1a1d526b88f 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java @@ -16,10 +16,9 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.tika; -import org.apache.jackrabbit.guava.common.base.Function; +import java.util.function.Function; public enum BinarySourceMapper implements Function { BY_BLOBID { diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java index ebd3a7077e8..62e4c676c5f 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.tika; import java.io.File; @@ -52,12 +51,12 @@ public void testGetBinaries() throws Exception { CSVFileBinaryResourceProvider provider = new CSVFileBinaryResourceProvider(dataFile, new MemoryBlobStore()); - Map binaries = provider.getBinaries("/").uniqueIndex(BinarySourceMapper.BY_BLOBID); + Map binaries = provider.getBinaries("/").uniqueIndex(BinarySourceMapper.BY_BLOBID::apply); assertEquals(3, binaries.size()); assertEquals("a", binaries.get("a").getBlobId()); assertEquals("/a", binaries.get("a").getPath()); - binaries = provider.getBinaries("/a").uniqueIndex(BinarySourceMapper.BY_BLOBID); + binaries = provider.getBinaries("/a").uniqueIndex(BinarySourceMapper.BY_BLOBID::apply); assertEquals(1, binaries.size()); provider.close(); diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java index 60e720cf311..17f454e6cb3 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java @@ -39,7 +39,6 @@ import java.util.Random; import java.util.Set; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -67,7 +66,6 @@ import org.apache.jackrabbit.oak.spi.commit.EmptyHook; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -444,20 +442,14 @@ private static String createTempConfig(File cfgFile, Properties props) throws IO } private static Set encodedIds(Set ids, String dsOption) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { - @Nullable @Override public String apply(@Nullable String input) { - return DataStoreCheckCommand.encodeId(input, "--"+dsOption); - } - })); + return Sets.newHashSet(Iterators.transform(ids.iterator(), + input -> DataStoreCheckCommand.encodeId(input, "--" + dsOption))); } private static Set encodedIdsAndPath(Set ids, String dsOption, Map blobsAddedWithNodes) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { - @Nullable @Override public String apply(@Nullable String input) { - return Joiner.on(",").join( + return Sets.newHashSet(Iterators.transform(ids.iterator(), + input -> Joiner.on(",").join( DataStoreCheckCommand.encodeId(input, "--"+dsOption), - blobsAddedWithNodes.get(input)); - } - })); + blobsAddedWithNodes.get(input)))); } } diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java index 63def135d06..d77b05e1cfa 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java @@ -38,7 +38,6 @@ import java.util.stream.StreamSupport; import ch.qos.logback.classic.Level; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.base.Strings; @@ -87,7 +86,6 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.stats.Clock; -import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Assert; import org.junit.Assume; @@ -936,19 +934,13 @@ private static String createTempConfig(File cfgFile, Properties props) throws IO private static Set encodedIdsAndPath(Set ids, Type dsOption, Map idToNodes, boolean encodeId) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { - @Nullable @Override public String apply(@Nullable String input) { - return Joiner.on(",").join(encodeId ? encodeId(input, dsOption) : input, idToNodes.get(input)); - } - })); + return Sets.newHashSet(Iterators.transform(ids.iterator(), + input -> Joiner.on(",").join(encodeId ? encodeId(input, dsOption) : input, idToNodes.get(input)))); } private static Set encodeIds(Set ids, Type dsOption) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { - @Nullable @Override public String apply(@Nullable String input) { - return encodeId(input, dsOption); - } - })); + return Sets.newHashSet(Iterators.transform(ids.iterator(), + input -> encodeId(input, dsOption))); } diff --git a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java index bd369b42529..38ac728f152 100644 --- a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java +++ b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java @@ -28,7 +28,6 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ArrayListMultimap; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.ListMultimap; @@ -106,7 +105,7 @@ public void noOfChildNodeRead() { NodeState state = nb.getNodeState(); final AtomicInteger counter = new AtomicInteger(); Iterable countingIterator = Iterables.transform(state.getChildNodeEntries(), - (Function) input -> { + input -> { counter.incrementAndGet(); return input; }); diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java index 092275b7d02..8a131765e68 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java @@ -18,7 +18,6 @@ */ package org.apache.jackrabbit.oak.spi.security; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -51,6 +50,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Function; /** * Abstract base implementation for {@link SecurityConfiguration}s that can @@ -205,34 +205,22 @@ public ConfigurationParameters getParameters() { @NotNull @Override public WorkspaceInitializer getWorkspaceInitializer() { - return new CompositeWorkspaceInitializer(Lists.transform(getConfigurations(), new Function() { - @Override - public WorkspaceInitializer apply(T securityConfiguration) { - return securityConfiguration.getWorkspaceInitializer(); - } - })); + return new CompositeWorkspaceInitializer(Lists.transform(getConfigurations(), + securityConfiguration -> securityConfiguration.getWorkspaceInitializer())); } @NotNull @Override public RepositoryInitializer getRepositoryInitializer() { - return new CompositeInitializer(Lists.transform(getConfigurations(), new Function() { - @Override - public RepositoryInitializer apply(T securityConfiguration) { - return securityConfiguration.getRepositoryInitializer(); - } - })); + return new CompositeInitializer(Lists.transform(getConfigurations(), + securityConfiguration -> securityConfiguration.getRepositoryInitializer())); } @NotNull @Override public List getCommitHooks(@NotNull final String workspaceName) { - Iterable t = Iterables.concat(Lists.transform(getConfigurations(), new Function>() { - @Override - public List apply(T securityConfiguration) { - return securityConfiguration.getCommitHooks(workspaceName); - } - })); + Iterable t = Iterables.concat(Lists.transform(getConfigurations(), + securityConfiguration -> securityConfiguration.getCommitHooks(workspaceName))); return ImmutableList.copyOf(t); } @@ -252,12 +240,8 @@ public List getConflictHandlers() { @NotNull @Override public List getProtectedItemImporters() { - Iterable t = Iterables.concat(Lists.transform(getConfigurations(), new Function>() { - @Override - public List apply(T securityConfiguration) { - return securityConfiguration.getProtectedItemImporters(); - } - })); + Iterable t = Iterables.concat(Lists.transform(getConfigurations(), + securityConfiguration -> securityConfiguration.getProtectedItemImporters())); return ImmutableList.copyOf(t); } @@ -307,7 +291,7 @@ private static final class CompositeContext implements Context { private void refresh(@NotNull List configurations) { Set s = Sets.newLinkedHashSetWithExpectedSize(configurations.size()); - for (Context c : Iterables.transform(configurations, ContextFunction.INSTANCE)) { + for (Context c : Iterables.transform(configurations, ContextFunction.INSTANCE::apply)) { if (DEFAULT != c) { s.add(c); } diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java index b5ea558753a..47011218563 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java @@ -23,7 +23,6 @@ import javax.jcr.Credentials; import javax.jcr.SimpleCredentials; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Maps; @@ -66,13 +65,8 @@ public String getUserId(@NotNull Credentials credentials) { public Map getAttributes(@NotNull Credentials credentials) { if (credentials instanceof SimpleCredentials) { final SimpleCredentials sc = (SimpleCredentials) credentials; - return Maps.asMap(ImmutableSet.copyOf(sc.getAttributeNames()), new Function() { - @Nullable - @Override - public Object apply(String input) { - return sc.getAttribute(input); - } - }); + return Maps.asMap(ImmutableSet.copyOf(sc.getAttributeNames()), + input -> sc.getAttribute(input)); } else { return Collections.emptyMap(); } diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java index 0af35372189..fffe95cfb71 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java @@ -17,7 +17,7 @@ package org.apache.jackrabbit.oak.spi.security.authentication.token; import java.util.List; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.spi.security.CompositeConfiguration; @@ -40,12 +40,8 @@ public CompositeTokenConfiguration(@NotNull SecurityProvider securityProvider) { @NotNull @Override public TokenProvider getTokenProvider(@NotNull final Root root) { - List providers = Lists.transform(getConfigurations(), new Function() { - @Override - public TokenProvider apply(TokenConfiguration tokenConfiguration) { - return tokenConfiguration.getTokenProvider(root); - } - }); + List providers = Lists.transform(getConfigurations(), + tokenConfiguration -> tokenConfiguration.getTokenProvider(root)); return CompositeTokenProvider.newInstance(providers); } } diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java index b8cfb4cabdd..1e0bb2def92 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java @@ -22,10 +22,11 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Function; + import javax.jcr.security.AccessControlException; import javax.jcr.security.Privilege; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.FluentIterable; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; @@ -261,7 +262,7 @@ public Iterable getAggregatedPrivilegeNames(@NotNull String... privilege @NotNull private Iterable extractAggregatedPrivileges(@NotNull Iterable privilegeNames) { - return FluentIterable.from(privilegeNames).transformAndConcat(new ExtractAggregatedPrivileges()); + return FluentIterable.from(privilegeNames).transformAndConcat(new ExtractAggregatedPrivileges()::apply); } @NotNull diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java index f1845bc0a64..20283ef77b8 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java @@ -61,7 +61,6 @@ import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCache; import org.apache.jackrabbit.guava.common.base.Stopwatch; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import com.microsoft.azure.storage.StorageCredentials; @@ -212,13 +211,8 @@ public static List readRevisions(String uri) { if (journal.exists()) { try (JournalReader journalReader = new JournalReader(journal)) { - Iterator revisionIterator = Iterators.transform(journalReader, new Function() { - @NotNull - @Override - public String apply(JournalEntry entry) { - return entry.getRevision(); - } - }); + Iterator revisionIterator = Iterators.transform(journalReader, + entry -> entry.getRevision()); return newArrayList(revisionIterator); } catch (Exception e) { e.printStackTrace(); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java index 9f6fa1d81ab..c69bb62b531 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java @@ -23,7 +23,6 @@ import java.io.UnsupportedEncodingException; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Supplier; import org.apache.jackrabbit.oak.cache.CacheStats; import org.apache.jackrabbit.oak.segment.util.SafeEncode; @@ -98,13 +97,8 @@ public String readString(@NotNull RecordId id) { final SegmentId segmentId = id.getSegmentId(); long msb = segmentId.getMostSignificantBits(); long lsb = segmentId.getLeastSignificantBits(); - return stringCache.get(msb, lsb, id.getRecordNumber(), new Function() { - @NotNull - @Override - public String apply(Integer offset) { - return segmentId.getSegment().readString(offset); - } - }); + return stringCache.get(msb, lsb, id.getRecordNumber(), + offset -> segmentId.getSegment().readString(offset)); } @NotNull @@ -122,13 +116,8 @@ public Template readTemplate(@NotNull RecordId id) { final SegmentId segmentId = id.getSegmentId(); long msb = segmentId.getMostSignificantBits(); long lsb = segmentId.getLeastSignificantBits(); - return templateCache.get(msb, lsb, id.getRecordNumber(), new Function() { - @NotNull - @Override - public Template apply(Integer offset) { - return segmentId.getSegment().readTemplate(offset); - } - }); + return templateCache.get(msb, lsb, id.getRecordNumber(), + offset -> segmentId.getSegment().readTemplate(offset)); } private static String safeEncode(String value) { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java index 0d4e489c1ae..a6e41a4bca1 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java @@ -23,6 +23,7 @@ import static org.apache.jackrabbit.oak.segment.CacheWeights.OBJECT_HEADER_SIZE; import java.util.Arrays; +import java.util.function.Function; import org.apache.jackrabbit.guava.common.cache.Weigher; import org.apache.jackrabbit.oak.cache.CacheLIRS; @@ -30,7 +31,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.apache.jackrabbit.guava.common.base.Function; + /** * A cache consisting of a fast and slow component. The fast cache for small items is based diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java index d17d0f0f60b..92b88b7ce04 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java @@ -16,10 +16,9 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.segment; -import org.apache.jackrabbit.guava.common.base.Function; +import java.util.function.Function; import org.jetbrains.annotations.NotNull; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java index 0db45bc6e5a..799c716e93a 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java @@ -36,7 +36,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Supplier; @@ -281,12 +280,8 @@ T getGeneration(final int generation) { @NotNull @Override public Iterator iterator() { - return transform(generations.values().iterator(), new Function, T>() { - @Nullable @Override - public T apply(Supplier cacheFactory) { - return cacheFactory.get(); - } - }); + return transform(generations.values().iterator(), + cacheFactory -> cacheFactory.get()); } void evictGenerations(@NotNull Predicate evict) { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java index 985853de231..32e229daf01 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java @@ -25,8 +25,8 @@ import java.io.Closeable; import java.io.IOException; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; import org.apache.jackrabbit.oak.segment.SegmentIdProvider; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java index b484629e0a0..485b4384e4a 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.segment.file; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; @@ -32,8 +31,8 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Supplier; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java index 1241ffb214c..3618761da04 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java @@ -28,14 +28,12 @@ import java.io.IOException; import java.util.Iterator; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.oak.json.BlobSerializer; import org.apache.jackrabbit.oak.json.JsonSerializer; import org.apache.jackrabbit.oak.segment.SegmentNodeState; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; -import org.apache.jackrabbit.oak.segment.file.JournalEntry; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -80,15 +78,11 @@ public Iterator getHistory(@NotNull JournalFile journal, @NotNul checkNotNull(path); try (JournalReader journalReader = new JournalReader(checkNotNull(journal))) { - return Iterators.transform(journalReader, - new Function() { - @NotNull @Override - public HistoryElement apply(JournalEntry entry) { + return Iterators.transform(journalReader, entry -> { store.setRevision(entry.getRevision()); NodeState node = getNode(store.getHead(), path); return new HistoryElement(entry.getRevision(), node); - } - }); + }); } } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java index 53c3b92d7d2..dab99d8c072 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java @@ -23,8 +23,8 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import java.io.IOException; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java index 7920bc36a49..3cea0f777d8 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java @@ -28,8 +28,8 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; import java.io.PrintWriter; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -112,7 +112,7 @@ private static String toString(PropertyState ps) { String v = BLOB_LENGTH.apply(ps.getValue(BINARY)); val.append(" = {").append(v).append("}"); } else if (ps.getType() == BINARIES) { - String v = transform(ps.getValue(BINARIES), BLOB_LENGTH).toString(); + String v = transform(ps.getValue(BINARIES), BLOB_LENGTH::apply).toString(); val.append("[").append(ps.count()).append("] = ").append(v); } else if (ps.isArray()) { val.append("[").append(ps.count()).append("] = ").append(ps.getValue(STRINGS)); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java index 06ea7b87f26..4c73ac90669 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java @@ -26,20 +26,17 @@ import java.util.Iterator; import java.util.List; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.oak.commons.json.JsonObject; import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; import org.apache.jackrabbit.oak.segment.SegmentId; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; -import org.apache.jackrabbit.oak.segment.file.JournalEntry; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile; import org.apache.jackrabbit.oak.segment.file.tooling.BasicReadOnlyBlobStore; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; import org.apache.jackrabbit.oak.spi.blob.BlobStore; -import org.jetbrains.annotations.NotNull; public final class Utils { @@ -77,13 +74,8 @@ public static List readRevisions(File store) { if (journal.exists()) { try (JournalReader journalReader = new JournalReader(journal)) { - Iterator revisionIterator = Iterators.transform(journalReader, new Function() { - @NotNull - @Override - public String apply(JournalEntry entry) { - return entry.getRevision(); - } - }); + Iterator revisionIterator = Iterators.transform(journalReader, + entry -> entry.getRevision()); return newArrayList(revisionIterator); } catch (Exception e) { e.printStackTrace(); diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java index 502b69e5a5b..da7f6ecdf1d 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java @@ -26,24 +26,18 @@ import java.util.ArrayList; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.Test; -import org.apache.jackrabbit.guava.common.base.Function; - public class ReaderCacheTest { @Test public void empty() { final AtomicInteger counter = new AtomicInteger(); - Function loader = new Function() { - @Override @NotNull - public String apply(@Nullable Integer input) { + Function loader = input -> { counter.incrementAndGet(); return valueOf(input); - } }; StringCache c = new StringCache(0); for (int repeat = 0; repeat < 10; repeat++) { @@ -61,12 +55,9 @@ public String apply(@Nullable Integer input) { public void largeEntries() { final AtomicInteger counter = new AtomicInteger(); final String large = new String(new char[1024]); - Function loader = new Function() { - @Override @Nullable - public String apply(@Nullable Integer input) { + Function loader = input -> { counter.incrementAndGet(); return large + input; - } }; StringCache c = new StringCache(1024); for (int repeat = 0; repeat < 10; repeat++) { @@ -84,12 +75,7 @@ public String apply(@Nullable Integer input) { @Test public void clear() { final AtomicInteger counter = new AtomicInteger(); - Function uniqueLoader = new Function() { - @Override @Nullable - public String apply(@Nullable Integer input) { - return valueOf(counter.incrementAndGet()); - } - }; + Function uniqueLoader = input -> valueOf(counter.incrementAndGet()); StringCache c = new StringCache(0); // load a new entry assertEquals("1", c.get(0, 0, 0, uniqueLoader)); @@ -107,12 +93,7 @@ public void randomized() { int segmentCount = 10; for (int i = 0; i < segmentCount; i++) { final int x = i; - Function loader = new Function() { - @Override @Nullable - public String apply(@Nullable Integer input) { - return "loader #" + x + " offset " + input; - } - }; + Function loader = input -> "loader #" + x + " offset " + input; loaderList.add(loader); } StringCache c = new StringCache(10); diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java index 0d4a5f89b07..05cbe069265 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java @@ -34,8 +34,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Functions; import org.apache.jackrabbit.guava.common.util.concurrent.ListenableFuture; import org.apache.jackrabbit.guava.common.util.concurrent.ListeningExecutorService; diff --git a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java index 8469917df50..f25fdb9938c 100644 --- a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java +++ b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java @@ -18,7 +18,6 @@ */ package org.apache.jackrabbit.oak.composite; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -35,7 +34,6 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.junit.Test; import java.util.List; @@ -224,13 +222,7 @@ public Iterable getChildNodeEntries() { Iterable childrenIterable = cycle(new MemoryChildNodeEntry("child", EMPTY_NODE)); return asCountingIterable(limit(childrenIterable, childrenCount == MAX_VALUE ? 1000 : (int) childrenCount)); } else { - return asCountingIterable(transform(asList(children), new Function() { - @Nullable - @Override - public ChildNodeEntry apply(@Nullable String input) { - return new MemoryChildNodeEntry(input, EMPTY_NODE); - } - })); + return asCountingIterable(transform(asList(children), input -> new MemoryChildNodeEntry(input, EMPTY_NODE))); } } @@ -246,13 +238,9 @@ public NodeBuilder builder() { } private Iterable asCountingIterable(Iterable input) { - return Iterables.transform(input, new Function() { - @Nullable - @Override - public T apply(@Nullable T input) { + return Iterables.transform(input, inp -> { fetchedChildren++; - return input; - } + return inp; }); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java index 56040cf770a..ea64bb6cc89 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java @@ -31,7 +31,6 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentSkipListMap; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -279,12 +278,7 @@ public boolean apply(Map.Entry input) { return !input.getValue().isRebase() && input.getKey().compareRevisionTime(r) <= 0; } - }), new Function, Iterable>() { - @Override - public Iterable apply(Map.Entry input) { - return input.getValue().getModifiedPaths(); - } - }); + }), input -> input.getValue().getModifiedPaths()); return Iterables.concat(paths); } @@ -412,12 +406,7 @@ protected boolean isRebase() { @Override Iterable getModifiedPaths() { Iterable> paths = transform(previous.values(), - new Function>() { - @Override - public Iterable apply(BranchCommit branchCommit) { - return branchCommit.getModifiedPaths(); - } - }); + branchCommit -> branchCommit.getModifiedPaths()); return Iterables.concat(paths); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java index b3016219cdd..1f2214542b1 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java @@ -29,7 +29,6 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.oak.commons.json.JsopStream; @@ -891,16 +890,8 @@ private boolean isBundled(Path path) { return bundledNodes.containsKey(path); } - private static final Function KEY_TO_NAME = - new Function() { - @Override - public String apply(UpdateOp.Key input) { - return input.getName(); - } - }; - private static boolean hasContentChanges(UpdateOp op) { return filter(transform(op.getChanges().keySet(), - KEY_TO_NAME), Utils.PROPERTY_OR_DELETED).iterator().hasNext(); + input -> input.getName()), Utils.PROPERTY_OR_DELETED).iterator().hasNext(); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java index 215c5007014..a9cc56fb4d0 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java @@ -47,7 +47,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -573,24 +572,19 @@ private AbstractDocumentNodeState getSecondaryNodeState(){ private Iterable getChildNodeEntries(@NotNull String name, int limit) { Iterable children = store.getChildNodes(this, name, limit); - return Iterables.transform(children, new Function() { - @Override - public ChildNodeEntry apply(final AbstractDocumentNodeState input) { + return Iterables.transform(children, input -> { return new AbstractChildNodeEntry() { - @NotNull @Override public String getName() { return input.getPath().getName(); } - @NotNull @Override public NodeState getNodeState() { return input; } }; - } - }); + }); } private static Map asMap(Iterable props){ @@ -754,24 +748,18 @@ private AbstractDocumentNodeState createBundledState(String childNodeName, Match } private Iterator getBundledChildren(){ - return Iterators.transform(bundlingContext.getBundledChildNodeNames().iterator(), - new Function() { - @Override - public ChildNodeEntry apply(final String childNodeName) { + return Iterators.transform(bundlingContext.getBundledChildNodeNames().iterator(), childNodeName -> { return new AbstractChildNodeEntry() { - @NotNull @Override public String getName() { return childNodeName; } - @NotNull @Override public NodeState getNodeState() { return createBundledState(childNodeName, bundlingContext.matcher.next(childNodeName)); } }; - } }); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java index 88e7d142085..f5ed5650755 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java @@ -67,6 +67,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Function; import javax.jcr.PropertyType; @@ -126,7 +127,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.base.Strings; @@ -492,19 +493,6 @@ public String serialize(Blob blob) { } }; - /** - * A predicate, which takes a String and returns {@code true} if the String - * is a serialized binary value of a {@link DocumentPropertyState}. The - * apply method will throw an IllegalArgumentException if the String is - * malformed. - */ - private final Function binarySize = new Function() { - @Override - public Long apply(@Nullable String input) { - return getBinarySize(input); - } - }; - private final Clock clock; private final Checkpoints checkpoints; @@ -1629,7 +1617,7 @@ private String docAsString(String id, boolean cached) { return e.toString(); } } - }); + }::apply); } @Nullable @@ -2229,12 +2217,7 @@ public Iterable checkpoints() { public boolean apply(Map.Entry cp) { return cp.getValue().getExpiryTime() > now; } - }), new Function, String>() { - @Override - public String apply(Map.Entry cp) { - return cp.getKey().toString(); - } - }); + }), cp -> cp.getKey().toString()); } @Nullable @@ -2673,7 +2656,7 @@ private void backgroundSplit() { continue; } cleanCollisions(doc, collisionGarbageBatchSize); - Iterator it = doc.split(this, head, binarySize).iterator(); + Iterator it = doc.split(this, head, input -> getBinarySize(input)).iterator(); while(it.hasNext()) { UpdateOp op = it.next(); Path path = doc.getPath(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java index 1b04f62fa54..0378c78624a 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java @@ -31,7 +31,6 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -752,12 +751,7 @@ private void checkForConflicts() throws CommitFailedException { NodeDocument doc = Utils.getRootDocument(store.getDocumentStore()); Set collisions = Sets.newHashSet(doc.getLocalMap(COLLISIONS).keySet()); Set commits = Sets.newHashSet(Iterables.transform(b.getCommits(), - new Function() { - @Override - public Revision apply(Revision input) { - return input.asTrunkRevision(); - } - })); + input -> input.asTrunkRevision())); Set conflicts = Sets.intersection(collisions, commits); if (!conflicts.isEmpty()) { throw new CommitFailedException(STATE, 2, diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java index 7d649a34c3b..a06669a4788 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java @@ -24,7 +24,6 @@ import javax.management.openmbean.CompositeData; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.api.stats.RepositoryStatistics; @@ -96,12 +95,7 @@ public String[] getInactiveClusterNodes() { public boolean apply(ClusterNodeInfoDocument input) { return !input.isActive(); } - }), new Function() { - @Override - public String apply(ClusterNodeInfoDocument input) { - return input.getClusterId() + "=" + input.getCreated(); - } - }), String.class); + }), input -> input.getClusterId() + "=" + input.getCreated()), String.class); } @Override @@ -112,12 +106,7 @@ public String[] getActiveClusterNodes() { public boolean apply(ClusterNodeInfoDocument input) { return input.isActive(); } - }), new Function() { - @Override - public String apply(ClusterNodeInfoDocument input) { - return input.getClusterId() + "=" + input.getLeaseEndTime(); - } - }), String.class); + }), input -> input.getClusterId() + "=" + input.getLeaseEndTime()), String.class); } @Override @@ -128,12 +117,7 @@ public String[] getLastKnownRevisions() { public boolean apply(Revision input) { return input.getClusterId() != getClusterId(); } - }), new Function() { - @Override - public String apply(Revision input) { - return input.getClusterId() + "=" + input.toString(); - } - }), String.class); + }), input -> input.getClusterId() + "=" + input.toString()), String.class); } @Override diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java index f0e9cd56d29..2e66eee3fde 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import org.apache.jackrabbit.oak.commons.TimeDurationFormatter; import org.apache.jackrabbit.oak.plugins.document.util.Utils; @@ -38,7 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; /** @@ -161,7 +161,7 @@ public Map.Entry apply(NodeDocument doc) { } return immutableEntry(doc.getPath(), sweepOne(doc)); } - }), new Predicate>() { + }::apply), new Predicate>() { @Override public boolean apply(Map.Entry input) { return input.getValue() != null; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java index d823247fbb0..1c76f63e122 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java @@ -33,8 +33,8 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; import org.apache.jackrabbit.guava.common.collect.ImmutableList; @@ -1351,16 +1351,12 @@ Iterable getPreviousDocs(@NotNull final String property, } // didn't find entry -> scan through remaining head ranges - return filter(transform(getPreviousRanges().headMap(revision).entrySet(), - new Function, NodeDocument>() { - @Override - public NodeDocument apply(Map.Entry input) { + return filter(transform(getPreviousRanges().headMap(revision).entrySet(), input -> { if (input.getValue().includes(revision)) { return getPreviousDoc(input.getKey(), input.getValue()); } return null; - } - }), new Predicate() { + }), new Predicate() { @Override public boolean apply(@Nullable NodeDocument input) { return input != null && input.getValueMap(property).containsKey(revision); @@ -1659,17 +1655,14 @@ private Iterable> changesFor(final List range if (ranges.isEmpty()) { return Collections.emptyList(); } - final Function>> rangeToChanges = - new Function>>() { - @Override - public Iterable> apply(Range input) { + + final Function>> rangeToChanges = input -> { NodeDocument doc = getPreviousDoc(input.high, input); if (doc == null) { return Collections.emptyList(); } return doc.getVisibleChanges(property, readRev); - } - }; + }; Iterable> changes; if (ranges.size() == 1) { @@ -1682,7 +1675,7 @@ public Iterator> iterator() { } }; } else { - changes = Iterables.concat(transform(copyOf(ranges), rangeToChanges)); + changes = Iterables.concat(transform(copyOf(ranges), rangeToChanges::apply)); } return filter(changes, new Predicate>() { @Override @@ -1786,12 +1779,7 @@ String resolveCommitValue(Revision revision) { @NotNull RevisionVector getSweepRevisions() { return new RevisionVector(transform(getLocalMap(SWEEP_REV).values(), - new Function() { - @Override - public Revision apply(String s) { - return Revision.fromString(s); - } - })); + s -> Revision.fromString(s))); } //-------------------------< UpdateOp modifiers >--------------------------- diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java index d36022c733d..16a00272278 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java @@ -20,7 +20,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.oak.commons.TimeDurationFormatter; @@ -165,18 +164,13 @@ private Revision performSweep(Iterable documents, private Iterable> sweepOperations( final Iterable docs) { - return filter(transform(docs, - new Function>() { - @Override - public Map.Entry apply(NodeDocument doc) { - return immutableEntry(doc.getPath(), sweepOne(doc)); - } - }), new Predicate>() { - @Override - public boolean apply(Map.Entry input) { - return input.getValue() != null; - } - }); + return filter(transform(docs, doc -> immutableEntry(doc.getPath(), sweepOne(doc))), + new Predicate>() { + @Override + public boolean apply(Map.Entry input) { + return input.getValue() != null; + } + }); } private UpdateOp sweepOne(NodeDocument doc) throws DocumentStoreException { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java index 37ccbb92cdd..5af7684ecdb 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.TreeMap; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -58,11 +57,7 @@ public PropertyHistory(@NotNull NodeDocument doc, @Override public Iterator iterator() { - return ensureOrder(filter(transform(doc.getPreviousRanges().entrySet(), - new Function, Map.Entry>() { - @Nullable - @Override - public Map.Entry apply(Map.Entry input) { + return ensureOrder(filter(transform(doc.getPreviousRanges().entrySet(), input -> { Revision r = input.getKey(); int h = input.getValue().height; String prevId = Utils.getPreviousIdFor(mainPath, r, h); @@ -72,8 +67,7 @@ public Map.Entry apply(Map.Entry input) return null; } return new SimpleImmutableEntry(r, prev); - } - }), Predicates.notNull())); + }), Predicates.notNull())); } /** diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java index 2644ee71bb3..d9c0e45c626 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import java.util.function.Function; import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; import org.apache.jackrabbit.oak.plugins.document.util.Utils; @@ -34,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Supplier; import org.apache.jackrabbit.guava.common.base.Suppliers; @@ -233,7 +233,7 @@ private void populateSplitRevs() { } private boolean hasBinaryPropertyForSplit(Iterable values) { - return doc.hasBinary() && any(transform(values, binarySize), BINARY_FOR_SPLIT_THRESHOLD); + return doc.hasBinary() && any(transform(values, binarySize::apply), BINARY_FOR_SPLIT_THRESHOLD); } /** diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java index be79d07f61a..8a489f697a6 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java @@ -31,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.base.Supplier; @@ -120,12 +119,7 @@ public Iterable getPaths(@NotNull final Revision start) { public boolean apply(Map.Entry input) { return start.compareRevisionTime(input.getValue()) < 1; } - }), new Function, Path>() { - @Override - public Path apply(Map.Entry input) { - return input.getKey(); - } - }); + }), input -> input.getKey()); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java index 730f46fa8b7..9dbf9c1639d 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java @@ -40,7 +40,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; @@ -51,7 +50,6 @@ import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.oak.commons.sort.StringSort; -import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType; import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key; import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation; import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation.Type; @@ -2271,22 +2269,11 @@ private Iterator previousDocIdsFor(NodeDocument doc) { // documents only. final Path path = doc.getPath(); return Iterators.transform(prevRanges.entrySet().iterator(), - new Function, String>() { - @Override - public String apply(Map.Entry input) { - int h = input.getValue().getHeight(); - return Utils.getPreviousIdFor(path, input.getKey(), h); - } - }); + input -> Utils.getPreviousIdFor(path, input.getKey(), input.getValue().getHeight())); } else { // need to fetch the previous documents to get their ids return Iterators.transform(doc.getAllPreviousDocs(), - new Function() { - @Override - public String apply(NodeDocument input) { - return input.getId(); - } - }); + input -> input.getId()); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java index f87d3e557df..531850c408c 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java @@ -93,7 +93,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Maps; import com.mongodb.BasicDBObject; import com.mongodb.MongoException; @@ -1365,20 +1364,10 @@ public List createOrUpdate(Collection collection, } } catch (MongoException e) { throw handleException(e, collection, Iterables.transform(updateOps, - new Function() { - @Override - public String apply(UpdateOp input) { - return input.getId(); - } - })); + input -> input.getId())); } finally { stats.doneCreateOrUpdate(watch.elapsed(TimeUnit.NANOSECONDS), - collection, Lists.transform(updateOps, new Function() { - @Override - public String apply(UpdateOp input) { - return input.getId(); - } - })); + collection, Lists.transform(updateOps, input -> input.getId())); } List resultList = new ArrayList(results.values()); log("createOrUpdate returns", resultList); @@ -1538,12 +1527,7 @@ private Map bulkUpdate(Collection collectio } private static Map createMap(List updateOps) { - return Maps.uniqueIndex(updateOps, new Function() { - @Override - public String apply(UpdateOp input) { - return input.getId(); - } - }); + return Maps.uniqueIndex(updateOps, input -> input.getId()); } private Map findDocuments(Collection collection, Set keys) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java index f20f51d5513..62bc6c6508c 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java @@ -16,13 +16,11 @@ * specific language governing permissions and limitations * under the License. */ - package org.apache.jackrabbit.oak.plugins.document.mongo; import static com.mongodb.client.model.Filters.eq; import static com.mongodb.client.model.Filters.exists; import static com.mongodb.client.model.Filters.gt; -import static com.mongodb.client.model.Filters.not; import static com.mongodb.client.model.Filters.or; import static com.mongodb.client.model.Projections.include; import static java.util.Optional.empty; @@ -73,7 +71,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.StandardSystemProperty; @@ -337,17 +334,13 @@ protected Iterable identifyGarbage(final Set gcTypes // queries alone (15min is still long). Iterable iterable = filter(transform(getNodeCollection().find(query) .maxTime(15, TimeUnit.MINUTES).hint(hint), - new Function() { - @Override - public NodeDocument apply(BasicDBObject input) { - return store.convertFromDBObject(NODES, input); - } - }), new Predicate() { - @Override - public boolean apply(NodeDocument input) { - return !isDefaultNoBranchSplitNewerThan(input, sweepRevs); - } - }); + input -> store.convertFromDBObject(NODES, input)), + new Predicate() { + @Override + public boolean apply(NodeDocument input) { + return !isDefaultNoBranchSplitNewerThan(input, sweepRevs); + } + }); allResults = concat(allResults, iterable); } return allResults; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java index 8639e54e0da..af0e49dd7c4 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java @@ -46,14 +46,11 @@ import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.apache.jackrabbit.oak.stats.TimerStats; import org.h2.mvstore.MVMap; -import org.h2.mvstore.WriteBuffer; import org.h2.mvstore.type.DataType; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; - class NodeCache implements Cache, GenerationCache, EvictionListener { @@ -162,10 +159,7 @@ private V asyncReadIfPresent(K key) { } private void broadcast(final K key, final V value) { - cache.broadcast(type, new Function() { - @Override - @Nullable - public Void apply(@Nullable WriteBuffer buffer) { + cache.broadcast(type, buffer -> { keyType.write(buffer, key); if (value == null) { buffer.put((byte) 0); @@ -174,8 +168,7 @@ public Void apply(@Nullable WriteBuffer buffer) { valueType.write(buffer, value); } return null; - } - }); + }); } private void write(final K key, final V value) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java index 141f81d223e..6f4a0252bdf 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java @@ -25,6 +25,7 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import org.apache.jackrabbit.guava.common.cache.Cache; import org.apache.jackrabbit.oak.cache.CacheValue; @@ -46,19 +47,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; - /** * A persistent cache for the document store. */ public class PersistentCache implements Broadcaster.Listener { - + static final Logger LOG = LoggerFactory.getLogger(PersistentCache.class); private static final String FILE_PREFIX = "cache-"; private static final String FILE_SUFFIX = ".data"; private static final AtomicInteger COUNTER = new AtomicInteger(); - + private boolean cacheNodes = true; private boolean cacheChildren = true; private boolean cacheDiff = true; @@ -70,7 +69,7 @@ public class PersistentCache implements Broadcaster.Listener { private boolean asyncDiffCache = false; private HashMap caches = new HashMap(); - + private final String directory; private MapFactory writeStore; private MapFactory readStore; @@ -87,7 +86,7 @@ public class PersistentCache implements Broadcaster.Listener { private DynamicBroadcastConfig broadcastConfig; private CacheActionDispatcher writeDispatcher; private Thread writeDispatcherThread; - + { ByteBuffer bb = ByteBuffer.wrap(new byte[16]); UUID uuid = UUID.randomUUID(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java index 82c09e213dd..c7f20564bff 100755 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java @@ -88,7 +88,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.ImmutableMap; @@ -446,12 +445,7 @@ private List internalCreateOrUpdate(Collection collec } } stats.doneCreateOrUpdate(watch.elapsed(TimeUnit.NANOSECONDS), - collection, Lists.transform(updateOps, new Function() { - @Override - public String apply(UpdateOp input) { - return input.getId(); - } - })); + collection, Lists.transform(updateOps, input -> input.getId())); return new ArrayList(results.values()); } @@ -1866,12 +1860,7 @@ public Iterator iterator() { Iterator res = db.queryAsIterator(ch, tmd, from, to, excludeKeyPatterns, conditions, limit, sortBy); returned.add(res); - Iterator tmp = Iterators.transform(res, new Function() { - @Override - public T apply(RDBRow input) { - return convertFromDBObject(collection, input); - } - }); + Iterator tmp = Iterators.transform(res, input -> convertFromDBObject(collection, input)); return CloseableIterator.wrap(tmp, (Closeable) res); } catch (SQLException ex) { throw new RuntimeException(ex); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java index 5caa65d8758..4ee8a1e41a7 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java @@ -61,7 +61,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -432,7 +431,7 @@ public Set update(Connection connection, RDBTableMe } private static void assertNoDuplicatedIds(List documents) { - if (newHashSet(transform(documents, idExtractor)).size() < documents.size()) { + if (newHashSet(transform(documents, input -> input.getId())).size() < documents.size()) { throw new IllegalArgumentException("There are duplicated ids in the document list"); } } @@ -1127,11 +1126,4 @@ public int compare(T o1, T o2) { }); return result; } - - private static final Function idExtractor = new Function() { - @Override - public String apply(Document input) { - return input.getId(); - } - }; } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java index bc8dc15d63e..84581d2f319 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java @@ -19,7 +19,7 @@ package org.apache.jackrabbit.oak.plugins.document.secondary; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.api.PropertyState; @@ -178,13 +178,8 @@ public NodeState getChildNode(@NotNull String name) throws IllegalArgumentExcept @NotNull @Override public Iterable getChildNodeEntries() { - return Iterables.transform(delegate.getChildNodeEntries(), new Function() { - @Nullable - @Override - public ChildNodeEntry apply(ChildNodeEntry input) { - return new MemoryChildNodeEntry(input.getName(), decorate(input.getName(), input.getNodeState())); - } - }); + return Iterables.transform(delegate.getChildNodeEntries(), + input -> new MemoryChildNodeEntry(input.getName(), decorate(input.getName(), input.getNodeState()))); } @NotNull diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java index 53ab10cc255..d0844c6238b 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java @@ -38,7 +38,6 @@ import java.util.TreeMap; import java.util.stream.Collectors; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; import org.apache.jackrabbit.oak.commons.OakVersion; @@ -983,12 +982,7 @@ public static boolean isHiddenPath(@NotNull String path) { */ public static Iterable asStringValueIterable( @NotNull Iterable values) { - return transform(values, new Function() { - @Override - public StringValue apply(String input) { - return new StringValue(input); - } - }); + return transform(values, input -> new StringValue(input)); } /** diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java index 52cb467a4f1..48f617e654f 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java @@ -35,6 +35,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.PathUtils; @@ -58,7 +59,6 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java index d498ab1d18d..184a4f3b520 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java @@ -18,9 +18,8 @@ import java.util.Iterator; import java.util.Map; -import java.util.NavigableMap; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Functions; import org.apache.jackrabbit.guava.common.base.Predicate; diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java index 58c1f0ef160..4b89c52d7c9 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java @@ -97,7 +97,6 @@ import static org.junit.Assume.assumeThat; import static org.junit.Assume.assumeTrue; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.cache.Cache; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; @@ -3275,12 +3274,7 @@ public void invalidateCacheOnMissingPreviousDocument() throws Exception { Long modCount = foo.getModCount(); assertNotNull(modCount); List prevIds = Lists.newArrayList(Iterators.transform( - foo.getPreviousDocLeaves(), new Function() { - @Override - public String apply(NodeDocument input) { - return input.getId(); - } - })); + foo.getPreviousDocLeaves(), input -> input.getId())); // run gc on another document node store createSecondaryStore(LeaseCheckMode.LENIENT); diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java index dcb3a92aaf3..425bd0a1da5 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java @@ -22,7 +22,6 @@ import java.util.Map.Entry; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.spi.state.AbstractChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; @@ -33,16 +32,8 @@ */ public class MemoryChildNodeEntry extends AbstractChildNodeEntry { - public static > Iterable iterable( - Iterable set) { - return Iterables.transform( - set, - new Function, ChildNodeEntry>() { - @Override - public ChildNodeEntry apply(Entry entry) { - return new MemoryChildNodeEntry(entry.getKey(), entry.getValue()); - } - }); + public static > Iterable iterable(Iterable set) { + return Iterables.transform(set, entry -> new MemoryChildNodeEntry(entry.getKey(), entry.getValue())); } private final String name; diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java index 3ae7072f62c..05900f27449 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java @@ -30,8 +30,8 @@ import java.util.Map; import java.util.Map.Entry; +import java.util.function.Function; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; @@ -39,7 +39,6 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Predicates; @@ -52,17 +51,7 @@ public class ModifiedNodeState extends AbstractNodeState { /** * Mapping from a PropertyState instance to its name. */ - private static final Function GET_NAME = - new Function() { - @Override @Nullable - public String apply(@Nullable PropertyState input) { - if (input != null) { - return input.getName(); - } else { - return null; - } - } - }; + private static final Function GET_NAME = input -> (input != null) ? input.getName() : null; /** * Unwraps the given {@code NodeState} instance into the given internals @@ -180,7 +169,7 @@ static Iterable getProperties( properties = newHashMap(properties); } Predicate predicate = Predicates.compose( - not(in(properties.keySet())), GET_NAME); + not(in(properties.keySet())), GET_NAME::apply); return concat( filter(base.getProperties(), predicate), filter(properties.values(), notNull())); @@ -361,7 +350,7 @@ public Iterable getChildNodeEntries() { return base.getChildNodeEntries(); // shortcut } else { Predicate predicate = Predicates.compose( - not(in(nodes.keySet())), ChildNodeEntry.GET_NAME); + not(in(nodes.keySet())), ChildNodeEntry.GET_NAME::apply); return concat( filter(base.getChildNodeEntries(), predicate), iterable(filterValues(nodes, NodeState.EXISTS).entrySet())); diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java index 1d286c4e772..933da8170b4 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java @@ -21,15 +21,12 @@ import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; -import java.math.BigDecimal; import java.util.List; import javax.jcr.PropertyType; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; -import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.value.Conversions.Converter; import org.jetbrains.annotations.NotNull; @@ -62,89 +59,29 @@ protected MultiPropertyState(String name, Iterable values) { private S convertTo(Type type) { switch (type.tag()) { case PropertyType.STRING: - return (S) Iterables.transform(values, new Function() { - @Override - public String apply(T value) { - return getConverter(value).toString(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toString()); case PropertyType.BINARY: - return (S) Iterables.transform(values, new Function() { - @Override - public Blob apply(T value) { - return getConverter(value).toBinary(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toBinary()); case PropertyType.LONG: - return (S) Iterables.transform(values, new Function() { - @Override - public Long apply(T value) { - return getConverter(value).toLong(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toLong()); case PropertyType.DOUBLE: - return (S) Iterables.transform(values, new Function() { - @Override - public Double apply(T value) { - return getConverter(value).toDouble(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toDouble()); case PropertyType.DATE: - return (S) Iterables.transform(values, new Function() { - @Override - public String apply(T value) { - return getConverter(value).toDate(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toDate()); case PropertyType.BOOLEAN: - return (S) Iterables.transform(values, new Function() { - @Override - public Boolean apply(T value) { - return getConverter(value).toBoolean(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toBoolean()); case PropertyType.NAME: - return (S) Iterables.transform(values, new Function() { - @Override - public String apply(T value) { - return getConverter(value).toString(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toString()); case PropertyType.PATH: - return (S) Iterables.transform(values, new Function() { - @Override - public String apply(T value) { - return getConverter(value).toString(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toString()); case PropertyType.REFERENCE: - return (S) Iterables.transform(values, new Function() { - @Override - public String apply(T value) { - return getConverter(value).toString(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toString()); case PropertyType.WEAKREFERENCE: - return (S) Iterables.transform(values, new Function() { - @Override - public String apply(T value) { - return getConverter(value).toString(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toString()); case PropertyType.URI: - return (S) Iterables.transform(values, new Function() { - @Override - public String apply(T value) { - return getConverter(value).toString(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toString()); case PropertyType.DECIMAL: - return (S) Iterables.transform(values, new Function() { - @Override - public BigDecimal apply(T value) { - return getConverter(value).toDecimal(); - } - }); + return (S) Iterables.transform(values, value -> getConverter(value).toDecimal()); default: throw new IllegalArgumentException("Unknown type:" + type); } } diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java index 393618e5da7..1c28f616963 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java @@ -21,7 +21,8 @@ import static org.apache.jackrabbit.oak.commons.PathUtils.concat; -import org.apache.jackrabbit.guava.common.base.Function; +import java.util.function.Function; + import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.NodeState; diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java index d015779ff43..d9ee3af66b0 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java @@ -33,7 +33,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; /** @@ -289,14 +288,7 @@ public long getChildNodeCount(long max) { @Override public Iterable getChildNodeNames() { - return Iterables.transform( - getChildNodeEntries(), - new Function() { - @Override - public String apply(ChildNodeEntry input) { - return input.getName(); - } - }); + return Iterables.transform(getChildNodeEntries(), input -> input.getName()); } /** diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java index c032e831044..0a59387fa01 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java @@ -16,11 +16,9 @@ */ package org.apache.jackrabbit.oak.spi.state; - -import org.apache.jackrabbit.guava.common.base.Function; +import java.util.function.Function; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; /** * A {@code ChildNodeEntry} instance represents the child node states of a @@ -54,15 +52,11 @@ public interface ChildNodeEntry { * Mapping from a ChildNodeEntry instance to its name. */ Function GET_NAME = - new Function() { - @Override @Nullable - public String apply(@Nullable ChildNodeEntry input) { + input -> { if (input != null) { return input.getName(); } else { return null; } - } - }; - + }; } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java index cf6b27da998..a55904eebc2 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java @@ -59,7 +59,7 @@ import javax.jcr.nodetype.PropertyDefinitionTemplate; import javax.jcr.security.Privilege; -import org.apache.jackrabbit.guava.common.base.Function; + import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.HashBiMap; import org.apache.jackrabbit.guava.common.collect.ImmutableList; @@ -144,7 +144,6 @@ import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermEnum; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -777,15 +776,8 @@ private void copyCustomPrivileges(PrivilegeManager pMgr) throws RepositoryExcept while (it.hasNext()) { Privilege aggrPriv = it.next(); - List aggrNames = Lists.transform( - ImmutableList.copyOf(aggrPriv.getDeclaredAggregatePrivileges()), - new Function() { - @Nullable - @Override - public String apply(@Nullable Privilege input) { - return (input == null) ? null : input.getName(); - } - }); + List aggrNames = Lists.transform(ImmutableList.copyOf(aggrPriv.getDeclaredAggregatePrivileges()), + input -> (input == null) ? null : input.getName()); if (allAggregatesRegistered(pMgr, aggrNames)) { pMgr.registerPrivilege(aggrPriv.getName(), aggrPriv.isAbstract(), aggrNames.toArray(new String[aggrNames.size()])); it.remove(); diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java index ea5e28835f7..631e261cb4a 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java @@ -44,7 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; /** @@ -172,12 +171,7 @@ private static Iterable filterChildren(NodeState parent, final Predicate public boolean apply(ChildNodeEntry input) { return predicate.apply(input.getNodeState()); } - }), new Function() { - @Override - public String apply(ChildNodeEntry input) { - return input.getName(); - } - }); + }), input -> input.getName()); } /** diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java index 1e058825c9b..1337c03fe6e 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java @@ -18,21 +18,19 @@ */ package org.apache.jackrabbit.oak.upgrade.checkpoint; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.plugins.document.DocumentCheckpointRetriever; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.segment.CheckpointAccessor; import org.apache.jackrabbit.oak.segment.SegmentNodeStore; -import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils; -import org.jetbrains.annotations.Nullable; import java.util.Collections; import java.util.List; + public final class CheckpointRetriever { public static class Checkpoint implements Comparable { @@ -88,12 +86,7 @@ public static List getCheckpoints(NodeStore nodeStore) { } private static List getCheckpoints(NodeState checkpointRoot) { - return Lists.newArrayList(Iterables.transform(checkpointRoot.getChildNodeEntries(), new Function() { - @Nullable - @Override - public Checkpoint apply(@Nullable ChildNodeEntry input) { - return Checkpoint.createFromSegmentNode(input.getName(), input.getNodeState()); - } - })); + return Lists.newArrayList(Iterables.transform(checkpointRoot.getChildNodeEntries(), + input -> Checkpoint.createFromSegmentNode(input.getName(), input.getNodeState()))); } } diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java index eef40994b99..97145a2aefe 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java @@ -37,9 +37,7 @@ import javax.jcr.RepositoryException; import javax.jcr.Session; import javax.jcr.SimpleCredentials; -import javax.jcr.Value; -import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.commons.codec.digest.DigestUtils; import org.apache.jackrabbit.oak.api.Blob; @@ -61,7 +59,6 @@ import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer; import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentNodeStoreContainer; import org.apache.jackrabbit.oak.upgrade.cli.parser.CliArgumentException; -import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -189,17 +186,13 @@ public static void verifyContent(Session session) throws RepositoryException { Node nodeType = session.getNode("/jcr:system/jcr:nodeTypes/sling:OrderedFolder"); assertEquals("rep:NodeType", nodeType.getProperty("jcr:primaryType").getString()); - List values = Lists.transform(Arrays.asList(nodeType.getProperty("rep:protectedProperties").getValues()), new Function() { - @Nullable - @Override - public String apply(@Nullable Value input) { + List values = Lists.transform(Arrays.asList(nodeType.getProperty("rep:protectedProperties").getValues()), input -> { try { return input.getString(); } catch (RepositoryException e) { return null; } - } - }); + }); assertTrue(values.contains("jcr:mixinTypes")); assertTrue(values.contains("jcr:primaryType")); assertEquals("false", nodeType.getProperty("jcr:isAbstract").getString()); From 95ed6c46a217c46c075747f2fbc1fed227c5504a Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Sat, 20 Jul 2024 18:41:42 +0200 Subject: [PATCH 17/86] OAK-10962: oak-solr-osgi: update zookeeper dependency to 3.9.2 (#1591) --- oak-solr-osgi/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-solr-osgi/pom.xml b/oak-solr-osgi/pom.xml index caad3b56f75..aadd050a0f3 100644 --- a/oak-solr-osgi/pom.xml +++ b/oak-solr-osgi/pom.xml @@ -137,7 +137,7 @@ org.apache.zookeeper zookeeper - 3.9.1 + 3.9.2 runtime From 8db9183b6f2de4d6bfc3ea12cffbdf705a868016 Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Sat, 20 Jul 2024 20:00:55 +0200 Subject: [PATCH 18/86] Update build.yml to disable Sonar for now ...because of failures, as in https://github.com/apache/jackrabbit-oak/actions/runs/10021673018 --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac020aecdc6..dcd21a9a5fc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,4 +52,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} if: ${{ env.SONAR_TOKEN != '' }} - run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pcoverage -Dsonar.projectKey=org.apache.jackrabbit:jackrabbit-oak -Dsonar.organization=apache + # run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pcoverage -Dsonar.projectKey=org.apache.jackrabbit:jackrabbit-oak -Dsonar.organization=apache + # disable Sonar for now + run: mvn -B verify -Pcoverage From 8b05caefb94252c49b62eb2499a705f3d4dd09f8 Mon Sep 17 00:00:00 2001 From: Nuno Santos Date: Mon, 22 Jul 2024 10:41:49 +0200 Subject: [PATCH 19/86] OAK-10965 - Make ConsoleIndexingReporter thread safe. (#1592) --- .../index/ConsoleIndexingReporter.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java index aa204052108..18168bfb3f6 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java @@ -26,20 +26,19 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; public class ConsoleIndexingReporter implements IndexingReporter { - // Print configuration in alphabetical order - private final Map config = new TreeMap<>(); // Print timings in the order they were added private final Map timings = new LinkedHashMap<>(); - // Print configuration in alphabetical order - private final Map metrics = new TreeMap<>(); + // Print metrics and configuration in alphabetical order. They are sorted when printed. + private final Map metrics = new HashMap<>(); + private final Map config = new HashMap<>(); private final List envVariablesToLog; private List indexes = List.of(); private final List informationStrings = new ArrayList<>(); @@ -55,24 +54,24 @@ public ConsoleIndexingReporter(@NotNull List envVariablesToLog) { this.envVariablesToLog = List.copyOf(envVariablesToLog); } - public void setIndexNames(@NotNull List indexes) { + public synchronized void setIndexNames(@NotNull List indexes) { this.indexes = List.copyOf(indexes); } - public void addConfig(String key, Object value) { + public synchronized void addConfig(String key, Object value) { config.put(key, value.toString()); } - public void addTiming(String stage, String time) { + public synchronized void addTiming(String stage, String time) { timings.put(stage, time); } - public void addMetric(String name, long value) { + public synchronized void addMetric(String name, long value) { metrics.put(name, String.valueOf(value)); } @Override - public void addInformation(String value) { + public synchronized void addInformation(String value) { informationStrings.add(value); } @@ -81,10 +80,12 @@ public void addMetricByteSize(String name, long value) { if (value >= FileUtils.ONE_KB) { v += " (" + IOUtils.humanReadableByteCountBin(value) + ")"; } - metrics.put(name, v); + synchronized (this) { + metrics.put(name, v); + } } - public String generateReport() { + public synchronized String generateReport() { return "Indexes: " + String.join(", ", indexes) + "\n" + "Date: " + DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now()) + "\n" + "OAK Version: " + OakVersion.getVersion() + "\n" + From 6c44805441cc92542c13deabb0afc4af68ec6bc3 Mon Sep 17 00:00:00 2001 From: mbaedke Date: Mon, 22 Jul 2024 12:53:24 +0200 Subject: [PATCH 20/86] OAK-6762: Convert oak-blob to OSGi R7 annotations (#1413) done --- oak-blob/pom.xml | 9 ++- .../spi/blob/osgi/FileBlobStoreService.java | 14 ++--- .../spi/blob/osgi/SplitBlobStoreService.java | 56 ++++++++++--------- 3 files changed, 45 insertions(+), 34 deletions(-) diff --git a/oak-blob/pom.xml b/oak-blob/pom.xml index fdccfc636e1..6becc735925 100644 --- a/oak-blob/pom.xml +++ b/oak-blob/pom.xml @@ -66,8 +66,13 @@ provided - org.apache.felix - org.apache.felix.scr.annotations + org.osgi + org.osgi.service.component.annotations + provided + + + org.osgi + org.osgi.service.metatype.annotations provided diff --git a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java index 305c11ff582..b7140c572a9 100644 --- a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java +++ b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java @@ -25,21 +25,21 @@ import java.util.Hashtable; import java.util.Map; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.ConfigurationPolicy; +import org.osgi.service.component.ComponentContext; +import org.osgi.framework.ServiceRegistration; import org.apache.commons.io.FilenameUtils; -import org.apache.felix.scr.annotations.Activate; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.ConfigurationPolicy; -import org.apache.felix.scr.annotations.Deactivate; import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.FileBlobStore; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; -import org.osgi.framework.ServiceRegistration; -import org.osgi.service.component.ComponentContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Component(policy = ConfigurationPolicy.REQUIRE, name = FileBlobStoreService.NAME) +@Component(configurationPolicy = ConfigurationPolicy.REQUIRE, name = FileBlobStoreService.NAME) public class FileBlobStoreService { public static final String NAME = "org.apache.jackrabbit.oak.spi.blob.FileBlobStore"; diff --git a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java index f1f82d09ce0..62e53262934 100644 --- a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java +++ b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java @@ -21,50 +21,56 @@ import java.util.Dictionary; import java.util.Hashtable; -import java.util.Map; - -import org.apache.felix.scr.annotations.Activate; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.ConfigurationPolicy; -import org.apache.felix.scr.annotations.Deactivate; -import org.apache.felix.scr.annotations.Property; -import org.apache.felix.scr.annotations.PropertyOption; -import org.apache.felix.scr.annotations.Reference; -import org.apache.felix.scr.annotations.ReferenceCardinality; -import org.apache.felix.scr.annotations.ReferencePolicy; -import org.apache.jackrabbit.oak.spi.blob.BlobStore; -import org.apache.jackrabbit.oak.spi.blob.split.DefaultSplitBlobStore; -import org.apache.jackrabbit.oak.spi.blob.split.WrappingSplitBlobStore; + +import org.osgi.service.component.ComponentContext; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.ComponentPropertyType; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.ConfigurationPolicy; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.ReferenceCardinality; +import org.osgi.service.component.annotations.ReferencePolicy; +import org.osgi.service.metatype.annotations.AttributeDefinition; +import org.osgi.service.metatype.annotations.Option; import org.osgi.framework.BundleContext; import org.osgi.framework.InvalidSyntaxException; import org.osgi.framework.ServiceRegistration; -import org.osgi.service.component.ComponentContext; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.blob.split.DefaultSplitBlobStore; +import org.apache.jackrabbit.oak.spi.blob.split.WrappingSplitBlobStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.jackrabbit.oak.spi.blob.osgi.SplitBlobStoreService.BlobStoreType.*; -@Component(policy = ConfigurationPolicy.REQUIRE) +@Component(configurationPolicy = ConfigurationPolicy.REQUIRE) public class SplitBlobStoreService { private static final Logger log = LoggerFactory.getLogger(SplitBlobStoreService.class); - @Property private static final String PROP_HOME = "repository.home"; - @Property(options = { @PropertyOption(name = "External", value = "EXTERNAL"), - @PropertyOption(name = "Internal - Segment", value = "SEGMENT"), - @PropertyOption(name = "Internal - Document", value = "DOCUMENT") }) private static final String PROP_OLD_BLOB_STORE_TYPE = "split.old.blobstore.type"; public static final String PROP_SPLIT_BLOBSTORE = "split.blobstore"; + @ComponentPropertyType + @interface Config { + @AttributeDefinition + String repository_home(); + @AttributeDefinition(options = { @Option(label = "External", value = "EXTERNAL"), + @Option(label = "Internal - Segment", value = "SEGMENT"), + @Option(label = "Internal - Document", value = "DOCUMENT") }) + String split_old_blobstore_type(); + } + public static final String ONLY_STANDALONE_TARGET = "(&(!(split.blobstore=old))(!(split.blobstore=new)))"; - @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=old)") - private BlobStore oldBlobStore; + @Reference(cardinality = ReferenceCardinality.OPTIONAL, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=old)") + private volatile BlobStore oldBlobStore; - @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=new)") - private BlobStore newBlobStore; + @Reference(cardinality = ReferenceCardinality.OPTIONAL, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=new)") + private volatile BlobStore newBlobStore; private BundleContext ctx; @@ -75,7 +81,7 @@ public class SplitBlobStoreService { private BlobStoreType oldBlobStoreType; @Activate - protected void activate(ComponentContext context, Map config) throws InvalidSyntaxException { + protected void activate(ComponentContext context, Config config) throws InvalidSyntaxException { String oldTypeName = lookup(context, PROP_OLD_BLOB_STORE_TYPE); if (oldTypeName == null) { oldBlobStoreType = BlobStoreType.EXTERNAL; From 0521c6376caa5d2c5e8b0322df3840e49aaf46ff Mon Sep 17 00:00:00 2001 From: mbaedke Date: Mon, 22 Jul 2024 15:57:24 +0200 Subject: [PATCH 21/86] OAK-6773: Convert oak-store-composite to OSGi R7 annotations (#1489) done --- oak-store-composite/pom.xml | 9 +- .../composite/CompositeNodeStoreService.java | 108 +++++++++++++----- .../CrossMountReferenceValidatorProvider.java | 60 ++++++---- .../PrivateStoreValidatorProvider.java | 44 +++++-- .../NamespacePrefixNodestoreChecker.java | 6 +- .../checks/NodeStoreChecksService.java | 43 ++++--- .../NodeTypeDefinitionNodeStoreChecker.java | 6 +- .../NodeTypeMountedNodeStoreChecker.java | 38 +++--- .../checks/UniqueIndexNodeStoreChecker.java | 6 +- .../oak/composite/checks/package-info.java | 20 ++++ .../oak/composite/package-info.java | 2 +- ...odeTypeDefinitionNodeStoreCheckerTest.java | 6 +- 12 files changed, 239 insertions(+), 109 deletions(-) create mode 100755 oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java diff --git a/oak-store-composite/pom.xml b/oak-store-composite/pom.xml index 6b708b92b53..73a55c69f30 100644 --- a/oak-store-composite/pom.xml +++ b/oak-store-composite/pom.xml @@ -93,8 +93,13 @@ provided - org.apache.felix - org.apache.felix.scr.annotations + org.osgi + org.osgi.service.component.annotations + provided + + + org.osgi + org.osgi.service.metatype.annotations provided diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java index b3fe33f2251..4fd946676d6 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java @@ -17,14 +17,15 @@ package org.apache.jackrabbit.oak.composite; import org.apache.jackrabbit.guava.common.io.Closer; -import org.apache.felix.scr.annotations.Activate; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.ConfigurationPolicy; -import org.apache.felix.scr.annotations.Deactivate; -import org.apache.felix.scr.annotations.Property; -import org.apache.felix.scr.annotations.Reference; -import org.apache.felix.scr.annotations.ReferenceCardinality; -import org.apache.felix.scr.annotations.ReferencePolicy; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.ComponentPropertyType; +import org.osgi.service.component.annotations.ConfigurationPolicy; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.ReferenceCardinality; +import org.osgi.service.component.annotations.ReferencePolicy; +import org.osgi.service.metatype.annotations.AttributeDefinition; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean; import org.apache.jackrabbit.oak.commons.PropertiesUtil; @@ -57,7 +58,7 @@ import static org.apache.jackrabbit.guava.common.collect.Sets.newIdentityHashSet; import static java.util.stream.Collectors.toSet; -@Component(policy = ConfigurationPolicy.REQUIRE) +@Component(configurationPolicy = ConfigurationPolicy.REQUIRE) public class CompositeNodeStoreService { private static final Logger LOG = LoggerFactory.getLogger(CompositeNodeStoreService.class); @@ -66,35 +67,39 @@ public class CompositeNodeStoreService { private static final String MOUNT_ROLE_PREFIX = "composite-mount-"; - @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY, policy = ReferencePolicy.STATIC) private MountInfoProvider mountInfoProvider; - @Reference(cardinality = ReferenceCardinality.MANDATORY_MULTIPLE, policy = ReferencePolicy.DYNAMIC, bind = "bindNodeStore", unbind = "unbindNodeStore", referenceInterface = NodeStoreProvider.class, target="(!(service.pid=org.apache.jackrabbit.oak.composite.CompositeNodeStore))") private List nodeStores = new ArrayList<>(); - @Reference private NodeStoreChecks checks; - @Reference private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP; - @Property(label = "Enable node store checks", - description = "Whether the composite node store constraints should be checked before start", - boolValue = true - ) private static final String ENABLE_CHECKS = "enableChecks"; - - @Property(label = "Pre-populate seed mount", - description = "Setting this parameter to a mount name will enable pre-populating the empty default store" - ) private static final String PROP_SEED_MOUNT = "seedMount"; - - @Property(label = "Gather path statistics", - description = "Whether the CompositeNodeStoreStatsMBean should gather information about the most popular paths (may be expensive)", - boolValue = false - ) private static final String PATH_STATS = "pathStats"; + @ComponentPropertyType + @interface Config { + @AttributeDefinition( + name = "Enable node store checks", + description = "Whether the composite node store constraints should be checked before start" + ) + boolean enableChecks() default true; + + @AttributeDefinition( + name = "Pre-populate seed mount", + description = "Setting this parameter to a mount name will enable pre-populating the empty default store" + ) + String seedMount(); + + @AttributeDefinition( + name = "Gather path statistics", + description = "Whether the CompositeNodeStoreStatsMBean should gather information about the most popular paths (may be expensive)" + ) + boolean pathStats() default false; + } + private ComponentContext context; private final Set nodeStoresInUse = newIdentityHashSet(); @@ -260,6 +265,12 @@ private void unregisterCompositeNodeStore() throws IOException { } @SuppressWarnings("unused") + @Reference(name = "nodeStores", + cardinality = ReferenceCardinality.AT_LEAST_ONE, + policy = ReferencePolicy.DYNAMIC, + service = NodeStoreProvider.class, + target="(!(service.pid=org.apache.jackrabbit.oak.composite.CompositeNodeStore))" + ) protected void bindNodeStore(NodeStoreProvider ns, Map config) throws IOException, CommitFailedException { NodeStoreWithProps newNs = new NodeStoreWithProps(ns, config); nodeStores.add(newNs); @@ -288,6 +299,49 @@ protected void unbindNodeStore(NodeStoreProvider ns) throws IOException { } } + @SuppressWarnings("unused") + @Reference(name = "mountInfoProvider", + cardinality = ReferenceCardinality.MANDATORY, + policy = ReferencePolicy.STATIC, + service = MountInfoProvider.class + ) + protected void bindMountInfoProvider(MountInfoProvider mip) { + this.mountInfoProvider = mip; + } + + @SuppressWarnings("unused") + protected void unbindMountInfoProvider(MountInfoProvider mip) { + if (this.mountInfoProvider == mip) { + this.mountInfoProvider = null; + } + } + + @SuppressWarnings("unused") + @Reference(name = "checks", service = NodeStoreChecks.class) + protected void bindChecks(NodeStoreChecks checks) { + this.checks = checks; + } + + @SuppressWarnings("unused") + protected void unbindChecks(NodeStoreChecks checks) { + if (this.checks == checks) { + this.checks = null; + } + } + + @SuppressWarnings("unused") + @Reference(name = "statisticsProvider", service = StatisticsProvider.class) + protected void bindStatisticsProvider(StatisticsProvider sp) { + this.statisticsProvider = sp; + } + + @SuppressWarnings("unused") + protected void unbindStatisticsProvider(StatisticsProvider sp) { + if (this.statisticsProvider == sp) { + this.statisticsProvider = null; + } + } + private static class NodeStoreWithProps { private final NodeStoreProvider nodeStore; @@ -316,4 +370,4 @@ public String getDescription() { getNodeStoreProvider().getClass().toString()); } } -} \ No newline at end of file +} diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java index 935fa0edc5b..3852b9a70cd 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java @@ -16,13 +16,15 @@ */ package org.apache.jackrabbit.oak.composite; -import org.apache.felix.scr.annotations.Activate; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.ConfigurationPolicy; -import org.apache.felix.scr.annotations.Property; -import org.apache.felix.scr.annotations.Reference; -import org.apache.felix.scr.annotations.Service; -import org.apache.jackrabbit.oak.commons.PropertiesUtil; +import org.apache.jackrabbit.oak.spi.state.NodeStoreProvider; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.ComponentPropertyType; +import org.osgi.service.component.annotations.ConfigurationPolicy; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.ReferenceCardinality; +import org.osgi.service.component.annotations.ReferencePolicy; +import org.osgi.service.metatype.annotations.AttributeDefinition; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.EditorProvider; import org.apache.jackrabbit.oak.spi.commit.Validator; @@ -30,27 +32,27 @@ import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.osgi.framework.BundleContext; - -import java.util.Map; /** * {@link Validator} which detects references crossing the mount boundaries */ -@Component(label = "Apache Jackrabbit Oak CrossMountReferenceValidatorProvider", policy = ConfigurationPolicy.REQUIRE) -@Property(name = "type", value = "crossMountRefValidator", propertyPrivate = true) -@Service({ValidatorProvider.class, EditorProvider.class}) +@Component(configurationPolicy = ConfigurationPolicy.REQUIRE, service = {ValidatorProvider.class, EditorProvider.class}) public class CrossMountReferenceValidatorProvider extends ValidatorProvider { - @Property( - boolValue = true, - label = "Fail when detecting commits cross-mount references", - description = "Commits will fail if set to true when detecting cross-mount references. If set to false the commit information is only logged." - ) - private static final String PROP_FAIL_ON_DETECTION = "failOnDetection"; + @ComponentPropertyType + @interface Config { + @AttributeDefinition + String type() default "crossMountRefValidator"; + + @AttributeDefinition( + name = "Fail when detecting commits cross-mount references", + description = "Commits will fail if set to true when detecting cross-mount references. If set to false the commit information is only logged." + ) + boolean failOnDetection() default true; + } + private boolean failOnDetection; - @Reference private MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider(); public CrossMountReferenceValidatorProvider() { @@ -62,8 +64,8 @@ public CrossMountReferenceValidatorProvider(MountInfoProvider mountInfoProvider, } @Activate - private void activate(BundleContext bundleContext, Map config) { - failOnDetection = PropertiesUtil.toBoolean(config.get(PROP_FAIL_ON_DETECTION), false); + private void activate(Config config) { + failOnDetection = config.failOnDetection(); } @Override @@ -80,4 +82,18 @@ CrossMountReferenceValidatorProvider withFailOnDetection(boolean failOnDetection this.failOnDetection = failOnDetection; return this; } + + @SuppressWarnings("unused") + @Reference(name = "mountInfoProvider") + protected void bindMountInfoProvider(MountInfoProvider mip) { + this.mountInfoProvider = mip; + } + + @SuppressWarnings("unused") + protected void unbindMountInfoProvider(MountInfoProvider mip) { + if (this.mountInfoProvider == mip) { + this.mountInfoProvider = null; + } + + } } diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java index 530def5a088..5dae99cf769 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java @@ -17,10 +17,14 @@ package org.apache.jackrabbit.oak.composite; -import org.apache.felix.scr.annotations.*; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.ComponentPropertyType; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.metatype.annotations.AttributeDefinition; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.PathUtils; -import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.spi.commit.*; import org.apache.jackrabbit.oak.spi.mount.Mount; import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; @@ -36,20 +40,22 @@ /** * {@link Validator} which detects change commits to the read only mounts. */ -@Component(label = "Apache Jackrabbit Oak PrivateStoreValidatorProvider") +@Component public class PrivateStoreValidatorProvider extends ValidatorProvider { private final Logger logger = LoggerFactory.getLogger(getClass()); private static final String ROOT_PATH = "/"; - @Property( - boolValue = true, - label = "Fail when detecting commits to the read-only stores", - description = "Commits will fail if set to true when detecting changes to any read-only store. If set to false the commit information is only logged." - ) - private static final String PROP_FAIL_ON_DETECTION = "failOnDetection"; + @ComponentPropertyType + @interface Config { + @AttributeDefinition( + name = "Fail when detecting commits to the read-only stores", + description = "Commits will fail if set to true when detecting changes to any read-only store. If set to false the commit information is only logged." + ) + boolean failOnDetection() default true; + } + private boolean failOnDetection; - @Reference private MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider(); private ServiceRegistration serviceRegistration; @@ -60,8 +66,8 @@ public Validator getRootValidator(NodeState before, NodeState after, CommitInfo } @Activate - private void activate(BundleContext bundleContext, Map config) { - failOnDetection = PropertiesUtil.toBoolean(config.get(PROP_FAIL_ON_DETECTION), true); + private void activate(BundleContext bundleContext, Config config) { + failOnDetection = config.failOnDetection(); if (mountInfoProvider.hasNonDefaultMounts()) { serviceRegistration = bundleContext.registerService(EditorProvider.class.getName(), this, null); @@ -91,6 +97,20 @@ boolean isFailOnDetection() { return failOnDetection; } + @SuppressWarnings("unused") + @Reference(name = "mountInfoProvider", service = MountInfoProvider.class) + protected void bindMountInfoProvider(MountInfoProvider mip) { + this.mountInfoProvider = mip; + } + + @SuppressWarnings("unused") + protected void unbindMountInfoProvider(MountInfoProvider mip) { + if (this.mountInfoProvider == mip) { + this.mountInfoProvider = null; + } + + } + private class PrivateStoreValidator extends DefaultValidator { private final String path; diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java index 4095c7ccaac..865022f2918 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java @@ -18,8 +18,7 @@ import java.util.Set; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.Service; +import org.osgi.service.component.annotations.Component; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.api.Tree; @@ -32,8 +31,7 @@ import org.apache.jackrabbit.guava.common.collect.Sets; -@Component -@Service(MountedNodeStoreChecker.class) +@Component(service={MountedNodeStoreChecker.class}) public class NamespacePrefixNodestoreChecker implements MountedNodeStoreChecker { @Override diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java index c1f3fbb4549..14ee6d25a82 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java @@ -19,10 +19,9 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.Reference; -import org.apache.felix.scr.annotations.ReferenceCardinality; -import org.apache.felix.scr.annotations.Service; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Reference; +import org.osgi.service.component.annotations.ReferenceCardinality; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.composite.MountedNodeStore; import org.apache.jackrabbit.oak.plugins.tree.factories.TreeFactory; @@ -32,19 +31,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Component -@Service(NodeStoreChecks.class) +@Component(service = {NodeStoreChecks.class}) public class NodeStoreChecksService implements NodeStoreChecks { private final Logger log = LoggerFactory.getLogger(getClass()); - @Reference(cardinality = ReferenceCardinality.OPTIONAL_MULTIPLE, - bind = "bindChecker", - unbind = "unbindChecker", - referenceInterface = MountedNodeStoreChecker.class) private List> checkers = new CopyOnWriteArrayList<>(); - @Reference private MountInfoProvider mip; // used by SCR @@ -102,13 +95,31 @@ private void visit(Tree tree, MountedNodeStore mountedStore, ErrorHolder err if ( ( mounted || under ) && keepGoing ) { tree.getChildren().forEach( child -> visit(child, mountedStore, errorHolder, context, c)); } - } - - protected void bindChecker(MountedNodeStoreChecker checker) { + } + + @SuppressWarnings("unused") + @Reference(name = "checkers", + cardinality = ReferenceCardinality.MULTIPLE, + service = MountedNodeStoreChecker.class) + protected void bindChecker(MountedNodeStoreChecker checker) { checkers.add(checker); } - - protected void unbindChecker(MountedNodeStoreChecker checker) { + + @SuppressWarnings("unused") + protected void unbindChecker(MountedNodeStoreChecker checker) { checkers.remove(checker); } + + @SuppressWarnings("unused") + @Reference(name = "mip", service = MountInfoProvider.class) + protected void bindMip(MountInfoProvider mip) { + this.mip = mip; + } + + @SuppressWarnings("unused") + protected void unbindMip(MountInfoProvider mip) { + if (this.mip == mip) { + this.mip = null; + } + } } diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java index 5df639fb671..76c68e74196 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java @@ -26,8 +26,7 @@ import java.util.List; import java.util.Set; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.Service; +import org.osgi.service.component.annotations.Component; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.Tree; @@ -48,8 +47,7 @@ /** * Checks that nodes present in a mount are consistent with the global node type definitions */ -@Component -@Service(MountedNodeStoreChecker.class) +@Component(service = {MountedNodeStoreChecker.class}) public class NodeTypeDefinitionNodeStoreChecker implements MountedNodeStoreChecker { @Override diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java index 7a2c6dfedd1..88ddb19b12d 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java @@ -20,10 +20,10 @@ import java.util.Set; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.ConfigurationPolicy; -import org.apache.felix.scr.annotations.Property; -import org.apache.felix.scr.annotations.Service; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.ComponentPropertyType; +import org.osgi.service.component.annotations.ConfigurationPolicy; +import org.osgi.service.metatype.annotations.AttributeDefinition; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.commons.PropertiesUtil; @@ -39,23 +39,35 @@ import org.apache.jackrabbit.guava.common.collect.ImmutableSet; -@Component(configurationFactory=true, - policy = ConfigurationPolicy.REQUIRE) -@Service(MountedNodeStoreChecker.class) +@Component(configurationPolicy = ConfigurationPolicy.REQUIRE, service = {MountedNodeStoreChecker.class}) public class NodeTypeMountedNodeStoreChecker implements MountedNodeStoreChecker { private final Logger log = LoggerFactory.getLogger(getClass()); - - @Property(label = "The name of a node type that is invalid and will be rejected when found") + private static final String INVALID_NODE_TYPE = "invalidNodeType"; - @Property(label = "The error label to use when rejecting an invalid node type") private static final String ERROR_LABEL = "errorLabel"; - - @Property(label="Node types that will cause the check to succeeed, even in the invalid node type is also found.", - cardinality = Integer.MAX_VALUE) private static final String EXCLUDED_NODE_TYPES = "excludedNodeTypes"; + @ComponentPropertyType + @interface Config { + @AttributeDefinition( + name = "The name of a node type that is invalid and will be rejected when found" + ) + String invalidNodeType(); + + @AttributeDefinition( + name = "The error label to use when rejecting an invalid node type" + ) + String errorLabel(); + + @AttributeDefinition( + name = "Node types that will cause the check to succeeed, even in the invalid node type is also found.", + cardinality = Integer.MAX_VALUE + ) + String[] excludedNodeTypes() default {}; + } + private String invalidNodeType; private String errorLabel; private Set excludedNodeTypes; diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java index 98078c33884..7f4621cc4e4 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java @@ -33,8 +33,7 @@ import java.util.Set; import java.util.TreeSet; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.Service; +import org.osgi.service.component.annotations.Component; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.composite.MountedNodeStore; import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; @@ -62,8 +61,7 @@ * on first access and skips all subsequent executions.

* */ -@Component -@Service(MountedNodeStoreChecker.class) +@Component(service = {MountedNodeStoreChecker.class}) public class UniqueIndexNodeStoreChecker implements MountedNodeStoreChecker { private static final Logger LOG = LoggerFactory.getLogger(UniqueIndexNodeStoreChecker.class); diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java new file mode 100755 index 00000000000..ecbd27d417e --- /dev/null +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@Version("0.0.1") +package org.apache.jackrabbit.oak.composite.checks; + +import org.osgi.annotation.versioning.Version; \ No newline at end of file diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java index 9599d669c2a..6efb8de2a23 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java @@ -55,7 +55,7 @@ * This is obviously correct but may be slow. * {@link org.apache.jackrabbit.oak.composite.CompositionContext#getContributingStores(java.lang.String, java.util.function.Function)} */ -@Version("0.5.0") +@Version("0.5.1") package org.apache.jackrabbit.oak.composite; import org.osgi.annotation.versioning.Version; \ No newline at end of file diff --git a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java index d710610c2c9..64489a3a4ba 100644 --- a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java +++ b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java @@ -20,8 +20,7 @@ import java.io.IOException; -import org.apache.felix.scr.annotations.Component; -import org.apache.felix.scr.annotations.Service; +import org.osgi.service.component.annotations.Component; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException; @@ -45,8 +44,7 @@ * that they are performed when needed.

* */ -@Component -@Service(MountedNodeStoreChecker.class) +@Component(service = {MountedNodeStoreChecker.class}) public class NodeTypeDefinitionNodeStoreCheckerTest { @Test From ebefe01536172e55c98fad969e30eb448408324c Mon Sep 17 00:00:00 2001 From: Nuno Santos Date: Tue, 23 Jul 2024 08:37:05 +0200 Subject: [PATCH 22/86] OAK-10951 - Add a new configuration property (#1594) - "oak.indexer.persistedLinkedList.cacheSize" - sets the cache size of the PersistedLinkedList used to traverse the FFS. This controls the number FFS entries kept in memory. --- .../indexer/document/NodeStateEntry.java | 2 +- .../flatfile/FlatFileStoreIterator.java | 38 ++++++++++++------- .../linkedList/FlatFileBufferLinkedList.java | 5 +-- .../linkedList/NodeStateEntryList.java | 14 +++---- .../linkedList/PersistedLinkedList.java | 17 +++------ 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java index 2685a60ea37..f7b5204866a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java @@ -30,7 +30,7 @@ public class NodeStateEntry { private final long lastModified; private final String id; - private NodeStateEntry(NodeState nodeState, String path, long memUsage, long lastModified, String id) { + public NodeStateEntry(NodeState nodeState, String path, long memUsage, long lastModified, String id) { this.nodeState = nodeState; this.path = path; this.memUsage = memUsage; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java index a10fe953ebb..69a7c58f3eb 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java @@ -27,10 +27,10 @@ import java.util.Set; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; -import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry.NodeStateEntryBuilder; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.NodeStateEntryList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedList; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; @@ -39,16 +39,22 @@ import org.apache.jackrabbit.guava.common.collect.AbstractIterator; class FlatFileStoreIterator extends AbstractIterator implements Iterator, Closeable { - private final Logger log = LoggerFactory.getLogger(getClass()); + private static final Logger log = LoggerFactory.getLogger(FlatFileStoreIterator.class); + + static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB"; + // by default, use the PersistedLinkedList + private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0; + static final String PERSISTED_LINKED_LIST_CACHE_SIZE = "oak.indexer.persistedLinkedList.cacheSize"; + static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000; + + private final Iterator baseItr; private final NodeStateEntryList buffer; - private NodeStateEntry current; private final Set preferredPathElements; - private int maxBufferSize; - static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB"; - // by default, use the PersistedLinkedList - private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0; + private NodeStateEntry current; + private int maxBufferSize; + private long maxBufferSizeBytes; public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator baseItr, Set preferredPathElements) { this(blobStore, fileName, baseItr, preferredPathElements, @@ -63,7 +69,8 @@ public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator maxBufferSize) { maxBufferSize = buffer.size(); log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", - maxBufferSize, buffer.estimatedMemoryUsage(), current.getPath()); + maxBufferSize, maxBufferSizeBytes, current.getPath()); + } + if (buffer.estimatedMemoryUsage() > maxBufferSizeBytes) { + maxBufferSizeBytes = buffer.estimatedMemoryUsage(); + log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", + maxBufferSize, maxBufferSizeBytes, current.getPath()); } if (!buffer.isEmpty()) { NodeStateEntry e = buffer.remove(); @@ -112,7 +124,7 @@ private NodeStateEntry computeNextEntry() { private NodeStateEntry wrap(NodeStateEntry baseEntry) { NodeState state = new LazyChildrenNodeState(baseEntry.getNodeState(), new ChildNodeStateProvider(getEntries(), baseEntry.getPath(), preferredPathElements)); - return new NodeStateEntryBuilder(state, baseEntry.getPath()).withMemUsage(baseEntry.estimatedMemUsage()).build(); + return new NodeStateEntry(state, baseEntry.getPath(), baseEntry.estimatedMemUsage(), 0, ""); } private Iterable getEntries() { @@ -121,7 +133,7 @@ private Iterable getEntries() { private Iterator queueIterator() { Iterator qitr = buffer.iterator(); - return new AbstractIterator() { + return new AbstractIterator<>() { @Override protected NodeStateEntry computeNext() { //If queue is empty try to append by getting entry from base diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java index c0effab8231..6bee80ad826 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java @@ -32,7 +32,7 @@ */ public class FlatFileBufferLinkedList implements NodeStateEntryList { - private ListNode head = new ListNode(); + private final ListNode head = new ListNode(); private ListNode tail = head; private int size = 0; @@ -52,9 +52,8 @@ public void add(@NotNull NodeStateEntry item) { long incomingSize = item.estimatedMemUsage(); long memUsage = estimatedMemoryUsage(); Preconditions.checkState(memUsage + incomingSize <= memLimit, - String.format( "Adding item (%s) estimated with %s bytes would increase mem usage beyond upper limit (%s)." + - " Current estimated mem usage is %s bytes", item.getPath(), incomingSize, memLimit, memUsage)); + " Current estimated mem usage is %s bytes", item.getPath(), incomingSize, memLimit, memUsage); tail.next = new ListNode(item); tail = tail.next; size++; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java index 60d1812c2bf..3c14d98fa91 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java @@ -28,26 +28,26 @@ public interface NodeStateEntryList { /** * Add an item at the tail of the list. */ - public void add(@NotNull NodeStateEntry item); + void add(@NotNull NodeStateEntry item); /** * Remove the first item from the list. * * @return the removed item */ - public NodeStateEntry remove(); + NodeStateEntry remove(); - public long estimatedMemoryUsage(); + long estimatedMemoryUsage(); - public int size(); + int size(); /** * Get an iterator to iterate over the whole list */ - public Iterator iterator(); + Iterator iterator(); - public boolean isEmpty(); + boolean isEmpty(); - public void close(); + void close(); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java index 8515e9d7dee..b4b8317b5f2 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java @@ -51,9 +51,7 @@ public class PersistedLinkedList implements NodeStateEntryList { private final NodeStateEntryWriter writer; private final NodeStateEntryReader reader; private final String storeFileName; - private final int compactStoreMillis = Integer.getInteger( - COMPACT_STORE_MILLIS_NAME, - 60 * 1000); + private final int compactStoreMillis = Integer.getInteger(COMPACT_STORE_MILLIS_NAME, 60 * 1000); private MVStore store; private MVMap map; @@ -66,11 +64,9 @@ public class PersistedLinkedList implements NodeStateEntryList { private long cacheHits, cacheMisses; public PersistedLinkedList(String fileName, NodeStateEntryWriter writer, NodeStateEntryReader reader, int cacheSize) { - LOG.info("Opening store " + fileName); + LOG.info("Opening store {}", fileName); this.storeFileName = fileName; - this.cache = - new LinkedHashMap(cacheSize + 1, .75F, true) { - private static final long serialVersionUID = 1L; + this.cache = new LinkedHashMap<>(cacheSize + 1, .75F, true) { @Override public boolean removeEldestEntry(Map.Entry eldest) { return size() > cacheSize; @@ -78,7 +74,7 @@ public boolean removeEldestEntry(Map.Entry eldest) { }; File oldFile = new File(fileName); if (oldFile.exists()) { - LOG.info("Deleting " + fileName); + LOG.info("Deleting {}", fileName); try { FileUtils.forceDelete(oldFile); } catch (IOException e) { @@ -107,8 +103,7 @@ public void add(@NotNull NodeStateEntry item) { long sizeBytes = store.getFileStore().size(); long now = System.currentTimeMillis(); if (now >= lastLog + 10000) { - LOG.info("Entries: " + size + " map size: " + map.sizeAsLong() + " file size: " - + sizeBytes + " bytes"); + LOG.info("Entries: {} map size: {} file size: {} bytes", size, map.sizeAsLong(), sizeBytes); lastLog = now; } boolean compactNow = now >= lastCompact + compactStoreMillis; @@ -119,7 +114,7 @@ public void add(@NotNull NodeStateEntry item) { MVStoreTool.compact(storeFileName, true); openStore(); lastCompact = System.currentTimeMillis(); - LOG.info("New size=" + store.getFileStore().size() + " bytes"); + LOG.info("New size={} bytes", store.getFileStore().size()); } } From b37db4c507b9f4efd0651db8185743c18d90477d Mon Sep 17 00:00:00 2001 From: Nuno Santos Date: Tue, 23 Jul 2024 09:55:14 +0200 Subject: [PATCH 23/86] OAK-10966 - Avoid object creation in PathUtils.isAncestor (#1596) --- .../java/org/apache/jackrabbit/oak/commons/PathUtils.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java index ed8dc09eeb6..8c94da7a76e 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java @@ -367,11 +367,13 @@ public static boolean isAncestor(String ancestor, String path) { if (denotesRoot(ancestor)) { if (denotesRoot(path)) { return false; + } else { + return path.startsWith(ancestor); } } else { - ancestor += "/"; + // Equivalent to path.startsWith(ancestor + "/") but avoids allocating a new string + return path.startsWith(ancestor) && path.length() > ancestor.length() && path.charAt(ancestor.length()) == '/'; } - return path.startsWith(ancestor); } /** From 9528fddef84449ea71f6b36dc484316afd421b0d Mon Sep 17 00:00:00 2001 From: Tushar <145645280+t-rana@users.noreply.github.com> Date: Tue, 23 Jul 2024 17:59:49 +0530 Subject: [PATCH 24/86] OAK-10964: bump nimbus-jose-jwt dependency to latest (#1593) --- oak-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml index 483409b135c..8b07e847ce7 100644 --- a/oak-parent/pom.xml +++ b/oak-parent/pom.xml @@ -770,7 +770,7 @@ com.nimbusds nimbus-jose-jwt - 9.30.2 + 9.40 com.azure From 818317c7200914cb27b7c4f791fb996e75a74dc6 Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Tue, 23 Jul 2024 13:40:46 +0100 Subject: [PATCH 25/86] OAK-6773: Convert oak-store-composite to OSGi R7 annotations - fix line ends --- .../org/apache/jackrabbit/oak/composite/checks/package-info.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java old mode 100755 new mode 100644 From 25792e7c929052a434cc0c9ee117883dc1793b79 Mon Sep 17 00:00:00 2001 From: ionutzpi Date: Wed, 24 Jul 2024 18:26:09 +0300 Subject: [PATCH 26/86] OAK-10803 -- compress/uncompress property, disabled by default (#1526) Co-authored-by: pirlogea --- .../document/DocumentPropertyState.java | 87 ++++- .../document/DocumentPropertyStateTest.java | 318 +++++++++++++++++- 2 files changed, 395 insertions(+), 10 deletions(-) diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java index 872985e5664..767e3998d6f 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java @@ -19,6 +19,12 @@ import static java.util.Collections.emptyList; import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; import javax.jcr.PropertyType; @@ -26,8 +32,10 @@ import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.commons.json.JsopReader; import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier; import org.apache.jackrabbit.oak.json.TypeCodes; import org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState; import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState; @@ -38,12 +46,16 @@ import org.apache.jackrabbit.oak.plugins.memory.StringPropertyState; import org.apache.jackrabbit.oak.plugins.value.Conversions; import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * PropertyState implementation with lazy parsing of the JSOP encoded value. */ final class DocumentPropertyState implements PropertyState { + private static final Logger LOG = LoggerFactory.getLogger(DocumentPropertyState.class); + private final DocumentNodeStore store; private final String name; @@ -51,11 +63,47 @@ final class DocumentPropertyState implements PropertyState { private final String value; private PropertyState parsed; + private final byte[] compressedValue; + private final Compression compression; + + private static int COMPRESSION_THRESHOLD = SystemPropertySupplier + .create("oak.documentMK.stringCompressionThreshold ", -1).loggingTo(LOG).get(); DocumentPropertyState(DocumentNodeStore store, String name, String value) { + this(store, name, value, Compression.GZIP); + } + + DocumentPropertyState(DocumentNodeStore store, String name, String value, Compression compression) { this.store = store; this.name = name; - this.value = value; + if (compression == null || COMPRESSION_THRESHOLD == -1) { + this.value = value; + this.compression = null; + this.compressedValue = null; + } else { + this.compression = compression; + int size = value.length(); + String localValue = value; + byte[] localCompressedValue = null; + if (size > COMPRESSION_THRESHOLD) { + try { + localCompressedValue = compress(value.getBytes(StandardCharsets.UTF_8)); + localValue = null; + } catch (IOException e) { + LOG.warn("Failed to compress property {} value: ", name, e); + } + } + this.value = localValue; + this.compressedValue = localCompressedValue; + } + } + + private byte[] compress(byte[] value) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + OutputStream compressionOutputStream = compression.getOutputStream(out); + compressionOutputStream.write(value); + compressionOutputStream.close(); + return out.toByteArray(); } @NotNull @@ -116,7 +164,20 @@ public int count() { */ @NotNull String getValue() { - return value; + return value != null ? value : decompress(this.compressedValue); + } + + private String decompress(byte[] value) { + try { + return new String(compression.getInputStream(new ByteArrayInputStream(value)).readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + LOG.error("Failed to decompress property {} value: ", getName(), e); + return "\"{}\""; + } + } + + byte[] getCompressedValue() { + return compressedValue; } //------------------------------------------------------------< Object >-- @@ -127,8 +188,18 @@ public boolean equals(Object object) { return true; } else if (object instanceof DocumentPropertyState) { DocumentPropertyState other = (DocumentPropertyState) object; - return this.name.equals(other.name) - && this.value.equals(other.value); + if (!this.name.equals(other.name) || !Arrays.equals(this.compressedValue, other.compressedValue)) { + return false; + } + if (this.compressedValue == null && other.compressedValue == null) { + return value.equals(other.value); + } else { + // Compare length and content of compressed values + if (this.compressedValue.length != other.compressedValue.length) { + return false; + } + return Arrays.equals(this.compressedValue, other.compressedValue); + } } // fall back to default equality check in AbstractPropertyState return object instanceof PropertyState @@ -145,11 +216,15 @@ public String toString() { return AbstractPropertyState.toString(this); } + static void setCompressionThreshold(int compressionThreshold) { + COMPRESSION_THRESHOLD = compressionThreshold; + } + //----------------------------< internal >---------------------------------- private PropertyState parsed() { if (parsed == null) { - JsopReader reader = new JsopTokenizer(value); + JsopReader reader = new JsopTokenizer(getValue()); if (reader.matches('[')) { parsed = readArrayProperty(name, reader); } else { @@ -277,4 +352,4 @@ static PropertyState readArrayProperty(String name, DocumentNodeStore store, Jso } return createProperty(name, values, Type.fromTag(type, true)); } -} +} \ No newline at end of file diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java index 557a0c1d1f7..a9c4ef67bc3 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java @@ -18,26 +18,46 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.lang.reflect.InvocationTargetException; import java.util.List; +import java.util.Objects; import java.util.Set; +import com.mongodb.ReadPreference; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; +import org.apache.jackrabbit.oak.plugins.document.mongo.MongoTestUtils; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; +import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.*; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; public class DocumentPropertyStateTest { private static final int BLOB_SIZE = 16 * 1024; + private static final String TEST_NODE = "test"; + private static final String STRING_HUGEVALUE = RandomStringUtils.random(10050, "dummytest"); + private static final int DEFAULT_COMPRESSION_THRESHOLD = 1024; + private static final int DISABLED_COMPRESSION = -1; @Rule public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); @@ -60,6 +80,15 @@ public void before() throws Exception { ns = builderProvider.newBuilder().setBlobStore(bs).getNodeStore(); } + @After + public void tearDown() { + try { + ns.dispose(); + } finally { + DocumentPropertyState.setCompressionThreshold(DISABLED_COMPRESSION); + } + } + // OAK-5462 @Test public void multiValuedBinarySize() throws Exception { @@ -81,4 +110,285 @@ public void multiValuedBinarySize() throws Exception { assertEquals(0, reads.size()); } -} + @Test + public void multiValuedAboveThresholdSize() throws Exception { + NodeBuilder builder = ns.getRoot().builder(); + List blobs = newArrayList(); + for (int i = 0; i < 13; i++) { + blobs.add(builder.createBlob(new RandomStream(BLOB_SIZE, i))); + } + builder.child(TEST_NODE).setProperty("p", blobs, Type.BINARIES); + TestUtils.merge(ns, builder); + + PropertyState p = ns.getRoot().getChildNode(TEST_NODE).getProperty("p"); + assertEquals(Type.BINARIES, Objects.requireNonNull(p).getType()); + assertEquals(13, p.count()); + + reads.clear(); + assertEquals(BLOB_SIZE, p.size(0)); + // must not read the blob via stream + assertEquals(0, reads.size()); + } + + @Test + public void stringBelowThresholdSize() throws Exception { + NodeBuilder builder = ns.getRoot().builder(); + builder.child(TEST_NODE).setProperty("p", "dummy", Type.STRING); + TestUtils.merge(ns, builder); + + PropertyState p = ns.getRoot().getChildNode(TEST_NODE).getProperty("p"); + assertEquals(Type.STRING, Objects.requireNonNull(p).getType()); + assertEquals(1, p.count()); + + reads.clear(); + assertEquals(5, p.size(0)); + // must not read the string via stream + assertEquals(0, reads.size()); + } + + @Test + public void stringAboveThresholdSize() throws Exception { + NodeBuilder builder = ns.getRoot().builder(); + builder.child(TEST_NODE).setProperty("p", STRING_HUGEVALUE, Type.STRING); + TestUtils.merge(ns, builder); + + PropertyState p = ns.getRoot().getChildNode(TEST_NODE).getProperty("p"); + assertEquals(Type.STRING, Objects.requireNonNull(p).getType()); + assertEquals(1, p.count()); + + reads.clear(); + assertEquals(10050, p.size(0)); + // must not read the string via streams + assertEquals(0, reads.size()); + } + + @Test + public void compressValueThrowsException() throws IOException, NoSuchFieldException, IllegalAccessException { + DocumentNodeStore mockDocumentStore = mock(DocumentNodeStore.class); + Compression mockCompression = mock(Compression.class); + when(mockCompression.getOutputStream(any(OutputStream.class))).thenThrow(new IOException("Compression failed")); + + DocumentPropertyState.setCompressionThreshold(DEFAULT_COMPRESSION_THRESHOLD); + DocumentPropertyState documentPropertyState = new DocumentPropertyState(mockDocumentStore, "p", "\"" + STRING_HUGEVALUE + "\"", mockCompression); + + assertEquals(documentPropertyState.getValue(Type.STRING), STRING_HUGEVALUE); + + verify(mockCompression, times(1)).getOutputStream(any(OutputStream.class)); + } + + @Test + public void uncompressValueThrowsException() throws IOException, NoSuchFieldException, IllegalAccessException { + + DocumentNodeStore mockDocumentStore = mock(DocumentNodeStore.class); + Compression mockCompression = mock(Compression.class); + OutputStream mockOutputStream= mock(OutputStream.class); + when(mockCompression.getOutputStream(any(OutputStream.class))).thenReturn(mockOutputStream); + when(mockCompression.getInputStream(any(InputStream.class))).thenThrow(new IOException("Compression failed")); + + DocumentPropertyState.setCompressionThreshold(DEFAULT_COMPRESSION_THRESHOLD); + DocumentPropertyState documentPropertyState = new DocumentPropertyState(mockDocumentStore, "p", STRING_HUGEVALUE, mockCompression); + + assertEquals(documentPropertyState.getValue(Type.STRING), "{}"); + + verify(mockCompression, times(1)).getInputStream(any(InputStream.class)); + } + + @Test + public void defaultValueSetToMinusOne() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException { + DocumentNodeStore store = mock(DocumentNodeStore.class); + + DocumentPropertyState.setCompressionThreshold(-1); + DocumentPropertyState state = new DocumentPropertyState(store, "propertyNameNew", "\"" + STRING_HUGEVALUE + "\"", Compression.GZIP); + + byte[] result = state.getCompressedValue(); + + assertNull(result); + assertEquals(state.getValue(Type.STRING), STRING_HUGEVALUE); + } + + @Test + public void stringAboveThresholdSizeNoCompression() { + DocumentNodeStore store = mock(DocumentNodeStore.class); + + DocumentPropertyState.setCompressionThreshold(DEFAULT_COMPRESSION_THRESHOLD); + + DocumentPropertyState state = new DocumentPropertyState(store, "propertyName", "\"" + STRING_HUGEVALUE + "\"", Compression.NONE); + + byte[] result = state.getCompressedValue(); + + assertEquals(result.length, STRING_HUGEVALUE.length() + 2 ); + + assertEquals(state.getValue(Type.STRING), STRING_HUGEVALUE); + assertEquals(STRING_HUGEVALUE, state.getValue(Type.STRING)); + } + + @Test + public void testInterestingStringsWithoutCompression() { + DocumentNodeStore store = mock(DocumentNodeStore.class); + String[] tests = new String[] { "simple:foo", "cr:a\n\b", "dquote:a\"b", "bs:a\\b", "euro:a\u201c", "gclef:\uD834\uDD1E", + "tab:a\tb", "nul:a\u0000b"}; + + for (String test : tests) { + DocumentPropertyState state = new DocumentPropertyState(store, "propertyName", test, Compression.GZIP); + assertEquals(test, state.getValue()); + } + } + + @Test + public void testInterestingStringsWithCompression() { + DocumentNodeStore store = mock(DocumentNodeStore.class); + String[] tests = new String[]{"simple:foo", "cr:a\n\b", "dquote:a\"b", "bs:a\\b", "euro:a\u201c", "gclef:\uD834\uDD1E", + "tab:a\tb", "nul:a\u0000b"}; + + DocumentPropertyState.setCompressionThreshold(1); + for (String test : tests) { + DocumentPropertyState state = new DocumentPropertyState(store, "propertyName", test, Compression.GZIP); + assertEquals(test, state.getValue()); + } + } + + @Test + public void testBrokenSurrogateWithoutCompressionForMongo() throws CommitFailedException { + getBrokenSurrogateAndInitializeDifferentStores(MONGO, false); + } + + @Test + public void testBrokenSurrogateWithoutCompressionForRDB() throws CommitFailedException { + getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, false); + } + + @Test + public void testBrokenSurrogateWithoutCompressionForInMemory() throws CommitFailedException { + getBrokenSurrogateAndInitializeDifferentStores(MEMORY, false); + } + + @Test + public void testBrokenSurrogateWithCompressionForMongo() throws CommitFailedException { + DocumentPropertyState.setCompressionThreshold(1); + getBrokenSurrogateAndInitializeDifferentStores(MONGO, true); + } + + @Test + public void testBrokenSurrogateWithCompressionForRDB() throws CommitFailedException { + DocumentPropertyState.setCompressionThreshold(1); + getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, true); + } + + @Test + public void testBrokenSurrogateWithCompressionForInMemory() throws CommitFailedException { + DocumentPropertyState.setCompressionThreshold(1); + getBrokenSurrogateAndInitializeDifferentStores(MEMORY, true); + } + + private void getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture fixture, boolean compressionEnabled) throws CommitFailedException { + String test = "brokensurrogate:dfsa\ud800"; + DocumentStore store = null; + DocumentNodeStore nodeStore = null; + + try { + store = fixture.createDocumentStore(); + + if (store instanceof MongoDocumentStore) { + // Enforce primary read preference, otherwise tests may fail on a + // replica set with a read preference configured to secondary. + // Revision GC usually runs with a modified range way in the past, + // which means changes made it to the secondary, but not in this + // test using a virtual clock + MongoTestUtils.setReadPreference(store, ReadPreference.primary()); + } + nodeStore = new DocumentMK.Builder().setDocumentStore(store).getNodeStore(); + + createPropAndCheckValue(nodeStore, test, compressionEnabled); + + } finally { + if (nodeStore != null) { + nodeStore.dispose(); + } + } + + } + + private void createPropAndCheckValue(DocumentNodeStore nodeStore, String test, boolean compressionEnabled) throws CommitFailedException { + NodeBuilder builder = nodeStore.getRoot().builder(); + if (compressionEnabled) { + DocumentPropertyState.setCompressionThreshold(1); + } + builder.child(TEST_NODE).setProperty("p", test, Type.STRING); + TestUtils.merge(nodeStore, builder); + + PropertyState p = nodeStore.getRoot().getChildNode(TEST_NODE).getProperty("p"); + assertEquals(Objects.requireNonNull(p).getValue(Type.STRING), test); + } + + @Test + public void testEqualsWithoutCompression() { + DocumentNodeStore store = mock(DocumentNodeStore.class); + String name = "propertyName"; + String value = "testValue"; + Compression compression = Compression.GZIP; + + DocumentPropertyState state1 = new DocumentPropertyState(store, name, value, compression); + DocumentPropertyState state2 = new DocumentPropertyState(store, name, value, compression); + + assertEquals(state1, state2); + + // Test for inequality + DocumentPropertyState state4 = new DocumentPropertyState(store, "differentName", value, compression); + assertNotEquals(state1, state4); + } + + @Test + public void testEqualsWithCompression() throws IOException { + DocumentNodeStore store = mock(DocumentNodeStore.class); + String name = "propertyName"; + String value = "testValue"; + Compression compression = Compression.GZIP; + + DocumentPropertyState.setCompressionThreshold(1); + // Create two DocumentPropertyState instances with the same properties + DocumentPropertyState state1 = new DocumentPropertyState(store, name, value, compression); + DocumentPropertyState state2 = new DocumentPropertyState(store, name, value, compression); + + // Check that the compressed values are not null + assertNotNull(state1.getCompressedValue()); + assertNotNull(state2.getCompressedValue()); + + // Check that the equals method returns true + assertEquals(state1, state2); + + // Decompress the values + String decompressedValue1 = state1.getValue(); + String decompressedValue2 = state2.getValue(); + + // Check that the decompressed values are equal to the original value + assertEquals(value, decompressedValue1); + assertEquals(value, decompressedValue2); + + // Check that the equals method still returns true after decompression + assertEquals(state1, state2); + } + + @Test + public void testOneCompressOtherUncompressInEquals() throws IOException { + DocumentNodeStore store = mock(DocumentNodeStore.class); + String name = "propertyName"; + String value = "testValue"; + Compression compression = Compression.GZIP; + + // Create a DocumentPropertyState instance with a compressed value + DocumentPropertyState.setCompressionThreshold(1); + DocumentPropertyState state1 = new DocumentPropertyState(store, name, value, compression); + + // Create a DocumentPropertyState instance with an uncompressed value + DocumentPropertyState.setCompressionThreshold(-1); + DocumentPropertyState state2 = new DocumentPropertyState(store, name, value, compression); + + assertNotEquals(state1, state2); + + // Decompress the value of the first instance + String decompressedValue1 = state1.getValue(); + + // Check that the decompressed value is equal to the original value + assertEquals(value, decompressedValue1); + } +} \ No newline at end of file From aefb990391f51a8ea354a68f9126d4aec959e191 Mon Sep 17 00:00:00 2001 From: Stefan Egli Date: Wed, 24 Jul 2024 19:23:50 +0200 Subject: [PATCH 27/86] OAK-10974 and OAK-10869 : temporarily disabling flaky tests --- .../oak/plugins/document/VersionGarbageCollectorIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java index 4b89c52d7c9..b6aa5efbe8b 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java @@ -1707,6 +1707,8 @@ public void testUnmergedBCRootCleanup() throws Exception { // OAK-8646 @Test public void testDeletedPropsAndUnmergedBCWithoutCollision() throws Exception { + // OAK-10974: + assumeTrue(fullGcMode != FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS); // create a node with property. NodeBuilder nb = store1.getRoot().builder(); nb.child("bar").setProperty("prop", "value"); @@ -2358,6 +2360,8 @@ public void testGCDeletedPropsWithDryRunMode() throws Exception { @Test public void testDeletedPropsAndUnmergedBCWithCollisionWithDryRunMode() throws Exception { + // OAK-10869: + assumeTrue(fullGcMode != FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS); // create a node with property. NodeBuilder nb = store1.getRoot().builder(); nb.child("bar").setProperty("prop", "value"); From c416850e3aaca0a6057aa3541531423a34f763a8 Mon Sep 17 00:00:00 2001 From: Nuno Santos Date: Thu, 25 Jul 2024 07:56:41 +0200 Subject: [PATCH 28/86] OAK-10971 - Add a method to test if a path is a direct ancestor of another: PathUtils.isDirectAncestor() (#1598) --- .../jackrabbit/oak/commons/PathUtils.java | 13 +++++++++++ .../jackrabbit/oak/commons/package-info.java | 2 +- .../jackrabbit/oak/commons/PathUtilsTest.java | 22 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java index 8c94da7a76e..d74bfd41e55 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java @@ -376,6 +376,19 @@ public static boolean isAncestor(String ancestor, String path) { } } + /** + * Check if a path is a direct ancestor of another path. + * + * @param ancestor the ancestor path + * @param path the potential direct offspring path + * @return true if the path is a direct offspring of the ancestor + */ + public static boolean isDirectAncestor(String ancestor, String path) { + int lastSlashInPath = path.lastIndexOf('/'); + return ((PathUtils.denotesRoot(ancestor) && lastSlashInPath == 0) || lastSlashInPath == ancestor.length()) + && isAncestor(ancestor, path); + } + /** * Relativize a path wrt. a parent path such that * {@code relativize(parentPath, concat(parentPath, path)) == paths} diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java index d13032236f3..417265ebd05 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Version("2.0.0") +@Version("2.1.0") package org.apache.jackrabbit.oak.commons; import org.osgi.annotation.versioning.Version; diff --git a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java index 417db72cb04..bc0e78b9f87 100644 --- a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java +++ b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java @@ -232,6 +232,28 @@ private static void test(String parent, String child) { assertFalse(PathUtils.isAncestor(parent, child)); assertFalse(PathUtils.isAncestor("/" + parent, "/" + parent + "123")); assertFalse(PathUtils.isAncestor("/" + parent, "/" + parent + "123/foo")); + assertTrue(PathUtils.isAncestor("/" + parent, "/" + parent + "/foo")); + assertFalse(PathUtils.isAncestor("/" + parent + "/foo", "/" + parent + "/foo")); + assertTrue(PathUtils.isAncestor("/" + parent, "/" + parent + "/foo/bar")); + + // isDirectAncestor + assertFalse(PathUtils.isDirectAncestor("/", "/")); + assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent)); + assertFalse(PathUtils.isDirectAncestor(parent, parent)); + assertTrue(PathUtils.isDirectAncestor("/", "/" + parent)); + assertFalse(PathUtils.isDirectAncestor("/", "/" + parent + "/foo1")); + assertFalse(PathUtils.isDirectAncestor("/", "/" + parent + "/foo1/foo2")); + assertTrue(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo1")); + assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo1/foo2")); + assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo1/foo2/foo3")); + assertTrue(PathUtils.isDirectAncestor(parent, parent + "/" + child)); + assertFalse(PathUtils.isDirectAncestor("/", parent + "/" + child)); + assertTrue(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/" + child)); + assertFalse(PathUtils.isDirectAncestor(parent, child)); + assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "123")); + assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "123/foo")); + assertTrue(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo")); + assertFalse(PathUtils.isDirectAncestor("/" + parent + "/foo", "/" + parent + "/foo")); // relativize assertEquals("", PathUtils.relativize("/", "/")); From 5dd63444fac480e104966babbae076afdb618bb0 Mon Sep 17 00:00:00 2001 From: Julian Reschke Date: Thu, 25 Jul 2024 10:45:36 +0200 Subject: [PATCH 29/86] OAK-10803: fix NPE when Mongo is unavailable, remove '*' imports (#1601) --- .../document/DocumentPropertyStateTest.java | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java index a9c4ef67bc3..02c0dda73ec 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java @@ -16,6 +16,19 @@ */ package org.apache.jackrabbit.oak.plugins.document; +import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; +import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assume.assumeTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -24,7 +37,6 @@ import java.util.Objects; import java.util.Set; -import com.mongodb.ReadPreference; import org.apache.commons.lang3.RandomStringUtils; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; @@ -36,20 +48,12 @@ import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.junit.*; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; -import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; -import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; -import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.*; -import static org.junit.Assert.assertNotEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.times; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import com.mongodb.ReadPreference; public class DocumentPropertyStateTest { @@ -249,38 +253,39 @@ public void testInterestingStringsWithCompression() { @Test public void testBrokenSurrogateWithoutCompressionForMongo() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(MONGO, false); + getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MONGO, false); } @Test public void testBrokenSurrogateWithoutCompressionForRDB() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, false); + getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.RDB_H2, false); } @Test public void testBrokenSurrogateWithoutCompressionForInMemory() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(MEMORY, false); + getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MEMORY, false); } @Test public void testBrokenSurrogateWithCompressionForMongo() throws CommitFailedException { DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(MONGO, true); + getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MONGO, true); } @Test public void testBrokenSurrogateWithCompressionForRDB() throws CommitFailedException { DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, true); + getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.RDB_H2, true); } @Test public void testBrokenSurrogateWithCompressionForInMemory() throws CommitFailedException { DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(MEMORY, true); + getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MEMORY, true); } private void getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture fixture, boolean compressionEnabled) throws CommitFailedException { + assumeTrue(fixture.isAvailable()); String test = "brokensurrogate:dfsa\ud800"; DocumentStore store = null; DocumentNodeStore nodeStore = null; From e4119605ebfe8fe7c4d2b02805291e3605af9134 Mon Sep 17 00:00:00 2001 From: Nuno Santos Date: Fri, 26 Jul 2024 08:21:35 +0200 Subject: [PATCH 30/86] OAK-10976 - Avoid unnecessary call to PathUtils.getName in IndexDefinition (#1603) --- .../jackrabbit/oak/plugins/index/search/IndexDefinition.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java index 45ad59e231b..ffe2dba7200 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java @@ -1525,10 +1525,10 @@ private NamePattern(String pattern, */ boolean matches(String propertyPath) { String parentPath = getParentPath(propertyPath); - String propertyName = PathUtils.getName(propertyPath); if (!this.parentPath.equals(parentPath)) { return false; } + String propertyName = PathUtils.getName(propertyPath); return pattern.matcher(propertyName).matches(); } From 632a15b3d0af8c98c65e4a02b40743db94729184 Mon Sep 17 00:00:00 2001 From: Tushar <145645280+t-rana@users.noreply.github.com> Date: Fri, 26 Jul 2024 13:10:35 +0530 Subject: [PATCH 31/86] OAK-10904: Close token refresh executor service after access token is no longer needed (#1545) * OAK-10904: change token refresh executor to use daemon thread * OAK-10904: add log * OAK-10904: move storage credential method to concrete class and explicitly calling close to shutdown the executor * OAK-10904: wrap code for generating storage credentials in try catch * OAK-10904: minor changes * OAK-10904: refactor log --- .../run/cli/SegmentTarFixtureProvider.java | 5 +- .../AzureSegmentStoreExplorerBackend.java | 11 +- .../oak/run/FileStoreDiffCommand.java | 29 ++-- .../oak/run/DataStoreCommandTest.java | 5 +- .../azure/AzureSegmentStoreService.java | 12 +- .../azure/AzureStorageCredentialManager.java | 147 ++++++++++++++++++ .../oak/segment/azure/AzureUtilities.java | 94 ----------- .../oak/segment/azure/package-info.java | 2 +- .../oak/segment/azure/tool/AzureCheck.java | 7 +- .../oak/segment/azure/tool/AzureCompact.java | 11 +- .../oak/segment/azure/tool/SegmentCopy.java | 13 +- .../oak/segment/azure/tool/ToolUtils.java | 107 ++++--------- ...entCopyAzureServicePrincipalToTarTest.java | 7 +- .../azure/tool/SegmentCopyTestBase.java | 2 +- .../oak/segment/azure/tool/ToolUtilsTest.java | 24 ++- .../upgrade/cli/node/SegmentAzureFactory.java | 20 ++- ...ureServicePrincipalNodeStoreContainer.java | 8 +- .../cli/node/SegmentAzureFactoryTest.java | 61 ++++---- 18 files changed, 325 insertions(+), 240 deletions(-) create mode 100644 oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java index 38746a6ee56..a2d1b05162f 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java @@ -31,6 +31,7 @@ import org.apache.jackrabbit.guava.common.io.Closer; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; @@ -53,10 +54,12 @@ static NodeStore configureSegment(Options options, BlobStore blobStore, Whiteboa FileStoreBuilder builder; if (segmentStoreType == ToolUtils.SegmentStoreType.AZURE) { + final AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager(); SegmentNodeStorePersistence segmentNodeStorePersistence = - ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri); + ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri, azureStorageCredentialManager); File tempDir = Files.createTempDirectory("azure-segment-store").toFile(); closer.register(() -> FileUtils.deleteQuietly(tempDir)); + closer.register(azureStorageCredentialManager); builder = fileStoreBuilder(tempDir).withCustomPersistence(segmentNodeStorePersistence); } else { builder = fileStoreBuilder(new File(pathOrUri)).withMaxFileSize(256); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java index f45333326a1..5f5b1580a00 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.explorer; import org.apache.jackrabbit.guava.common.io.Files; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; @@ -38,14 +39,16 @@ public class AzureSegmentStoreExplorerBackend extends AbstractSegmentTarExplorerBackend { private final String path; private SegmentNodeStorePersistence persistence; + private final AzureStorageCredentialManager azureStorageCredentialManager; public AzureSegmentStoreExplorerBackend(String path) { this.path = path; + this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } @Override public void open() throws IOException { - this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); + this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManager); try { this.store = fileStoreBuilder(Files.createTempDir()) @@ -57,6 +60,12 @@ public void open() throws IOException { this.index = store.getTarReaderIndex(); } + @Override + public void close() { + super.close(); + azureStorageCredentialManager.close(); + } + @Override protected JournalFile getJournal() { return persistence.getJournalFile(); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java index 3bc7d27d0cc..0d2ee2c6a5f 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java @@ -28,6 +28,7 @@ import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.apache.jackrabbit.oak.run.commons.Command; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; @@ -86,19 +87,21 @@ public void execute(String... args) throws Exception { } } else { if (pathOrURI.startsWith("az:")) { - SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI); - ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); - statusCode = Diff.builder() - .withPath(pathOrURI) - .withReadOnlyFileStore(store) - .withOutput(out) - .withInterval(interval) - .withIncremental(incremental) - .withFilter(path) - .withIgnoreMissingSegments(ignoreSNFEs) - .withRevisionsProcessor(ToolUtils::readRevisions) - .build() - .run(); + try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { + SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI, azureStorageCredentialManager); + ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); + statusCode = Diff.builder() + .withPath(pathOrURI) + .withReadOnlyFileStore(store) + .withOutput(out) + .withInterval(interval) + .withIncremental(incremental) + .withFilter(path) + .withIgnoreMissingSegments(ignoreSNFEs) + .withRevisionsProcessor(ToolUtils::readRevisions) + .build() + .run(); + } } else { ReadOnlyFileStore store = fileStoreBuilder(new File(pathOrURI)).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); statusCode = Diff.builder() diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java index d77b05e1cfa..6a33a27f834 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java @@ -73,6 +73,7 @@ import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions.Type; import org.apache.jackrabbit.oak.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; @@ -1142,6 +1143,7 @@ class SegmentStoreFixture implements StoreFixture { class AzureSegmentStoreFixture extends SegmentStoreFixture { private static final String AZURE_DIR = "repository"; private String container; + private final AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager(); @Override public NodeStore init(DataStoreBlobStore blobStore, File storeFile) throws Exception { Properties props = AzureDataStoreUtils.getAzureConfig(); @@ -1158,7 +1160,7 @@ class AzureSegmentStoreFixture extends SegmentStoreFixture { // initialize azure segment for test setup SegmentNodeStorePersistence segmentNodeStorePersistence = - ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath); + ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath, azureStorageCredentialManager); fileStore = fileStoreBuilder(storeFile).withBlobStore(blobStore) .withCustomPersistence(segmentNodeStorePersistence).build(); @@ -1190,6 +1192,7 @@ protected String getAzureConnectionString(String accountName, String secret, Str public void after() { try { AzureDataStoreUtils.deleteContainer(container); + azureStorageCredentialManager.close(); } catch(Exception e) { log.error("Error in cleaning the container {}", container, e); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java index 1b5640c76d5..24465ea720c 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java @@ -20,7 +20,7 @@ import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.LocationMode; -import com.microsoft.azure.storage.StorageCredentialsToken; +import com.microsoft.azure.storage.StorageCredentials; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.CloudBlobClient; @@ -43,7 +43,6 @@ import java.util.Hashtable; import java.util.Objects; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom; import static org.osgi.framework.Constants.SERVICE_PID; @Component( @@ -61,6 +60,7 @@ public class AzureSegmentStoreService { public static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net"; private ServiceRegistration registration; + private static AzureStorageCredentialManager azureStorageCredentialManager; @Activate public void activate(ComponentContext context, Configuration config) throws IOException { @@ -80,6 +80,9 @@ public void deactivate() throws IOException { registration.unregister(); registration = null; } + if (azureStorageCredentialManager != null) { + azureStorageCredentialManager.close(); + } } private static AzurePersistence createAzurePersistenceFrom(Configuration configuration) throws IOException { @@ -124,8 +127,9 @@ private static AzurePersistence createPersistenceFromConnectionURL(Configuration @NotNull private static AzurePersistence createPersistenceFromServicePrincipalCredentials(Configuration configuration) throws IOException { - StorageCredentialsToken storageCredentialsToken = storageCredentialAccessTokenFrom(configuration.accountName(), configuration.clientId(), configuration.clientSecret(), configuration.tenantId()); - + azureStorageCredentialManager = new AzureStorageCredentialManager(); + StorageCredentials storageCredentialsToken = azureStorageCredentialManager.getStorageCredentialAccessTokenFromServicePrincipals(configuration.accountName(), configuration.clientId(), configuration.clientSecret(), configuration.tenantId()); + try { CloudStorageAccount cloud = new CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX, configuration.accountName()); return createAzurePersistence(cloud, configuration, true); diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java new file mode 100644 index 00000000000..4f962f10809 --- /dev/null +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.segment.azure; + +import com.azure.core.credential.AccessToken; +import com.azure.core.credential.TokenRequestContext; +import com.azure.identity.ClientSecretCredential; +import com.azure.identity.ClientSecretCredentialBuilder; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; +import com.microsoft.azure.storage.StorageCredentialsToken; +import org.apache.commons.lang3.StringUtils; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.Closeable; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; + +public class AzureStorageCredentialManager implements Closeable { + private static final Logger log = LoggerFactory.getLogger(AzureStorageCredentialManager.class); + private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default"; + private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L; + private static final long TOKEN_REFRESHER_DELAY = 1L; + private ClientSecretCredential clientSecretCredential; + private AccessToken accessToken; + private StorageCredentialsToken storageCredentialsToken; + private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + + public StorageCredentials getStorageCredentialsFromEnvironment(@NotNull String accountName, @NotNull Environment environment) { + final String clientId = environment.getVariable(AZURE_CLIENT_ID); + final String clientSecret = environment.getVariable(AZURE_CLIENT_SECRET); + final String tenantId = environment.getVariable(AZURE_TENANT_ID); + + if (StringUtils.isNoneBlank(clientId, clientSecret, tenantId)) { + try { + return getStorageCredentialAccessTokenFromServicePrincipals(accountName, clientId, clientSecret, tenantId); + } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { + log.error("Error occurred while connecting to Azure Storage using service principals: ", e); + throw new IllegalArgumentException( + "Could not connect to the Azure Storage. Please verify if AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables are correctly set!"); + } + } + + log.warn("AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."); + + String key = environment.getVariable(AZURE_SECRET_KEY); + try { + return new StorageCredentialsAccountAndKey(accountName, key); + } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { + log.error("Error occurred while connecting to Azure Storage using secret key: ", e); + throw new IllegalArgumentException( + "Could not connect to the Azure Storage. Please verify if AZURE_SECRET_KEY environment variable is correctly set!"); + } + } + + public StorageCredentials getStorageCredentialAccessTokenFromServicePrincipals(String accountName, String clientId, String clientSecret, String tenantId) { + boolean isAccessTokenGenerated = false; + if (accessToken == null) { + clientSecretCredential = new ClientSecretCredentialBuilder() + .clientId(clientId) + .clientSecret(clientSecret) + .tenantId(tenantId) + .build(); + accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); + if (accessToken == null || StringUtils.isBlank(accessToken.getToken())) { + throw new IllegalArgumentException("Could not connect to azure storage, access token is null or empty"); + } + storageCredentialsToken = new StorageCredentialsToken(accountName, accessToken.getToken()); + isAccessTokenGenerated = true; + } + Objects.requireNonNull(storageCredentialsToken, "storageCredentialsToken cannot be null"); + + // start refresh token executor only when the access token is first generated + if (isAccessTokenGenerated) { + log.info("starting refresh token task at: {}", OffsetDateTime.now()); + TokenRefresher tokenRefresher = new TokenRefresher(); + executorService.scheduleWithFixedDelay(tokenRefresher, TOKEN_REFRESHER_INITIAL_DELAY, TOKEN_REFRESHER_DELAY, TimeUnit.MINUTES); + } + return storageCredentialsToken; + } + + /** + * This class represents a token refresher responsible for ensuring the validity of the access token used for azure AD authentication. + * The access token generated by the Azure client is valid for 1 hour only. Therefore, this class periodically checks the validity + * of the access token and refreshes it if necessary. The refresh is triggered when the current access token is about to expire, + * defined by a threshold of 5 minutes from the current time. This threshold is similar to what is being used in azure identity library to + * generate a new token + */ + private class TokenRefresher implements Runnable { + @Override + public void run() { + try { + log.debug("Checking for azure access token expiry at: {}", LocalDateTime.now()); + OffsetDateTime tokenExpiryThreshold = OffsetDateTime.now().plusMinutes(5); + if (accessToken.getExpiresAt() != null && accessToken.getExpiresAt().isBefore(tokenExpiryThreshold)) { + log.info("Access token is about to expire (5 minutes or less) at: {}. New access token will be generated", + accessToken.getExpiresAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + AccessToken newToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); + log.info("New azure access token generated at: {}", LocalDateTime.now()); + if (newToken == null || StringUtils.isBlank(newToken.getToken())) { + log.error("New access token is null or empty"); + return; + } + // update access token with newly generated token + accessToken = newToken; + storageCredentialsToken.updateToken(accessToken.getToken()); + } + } catch (Exception e) { + log.error("Error while acquiring new access token: ", e); + } + } + } + + @Override + public void close() { + new ExecutorCloser(executorService).close(); + log.info("Access token refresh executor shutdown completed"); + } +} diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java index ec5763b0cfe..5c740fac1d2 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java @@ -16,15 +16,10 @@ */ package org.apache.jackrabbit.oak.segment.azure; -import com.azure.core.credential.AccessToken; -import com.azure.core.credential.TokenRequestContext; -import com.azure.identity.ClientSecretCredential; -import com.azure.identity.ClientSecretCredentialBuilder; import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.ResultContinuation; import com.microsoft.azure.storage.ResultSegment; import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsToken; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageUri; import com.microsoft.azure.storage.blob.BlobListingDetails; @@ -33,9 +28,7 @@ import com.microsoft.azure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.LeaseStatus; import com.microsoft.azure.storage.blob.ListBlobItem; -import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.commons.Buffer; -import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; import org.apache.jackrabbit.oak.segment.spi.RepositoryNotReachableException; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -48,33 +41,19 @@ import java.net.URISyntaxException; import java.nio.file.Paths; import java.security.InvalidKeyException; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; public final class AzureUtilities { - static { - Runtime.getRuntime().addShutdownHook(new Thread(AzureUtilities::shutDown)); - } - public static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; public static final String AZURE_SECRET_KEY = "AZURE_SECRET_KEY"; public static final String AZURE_TENANT_ID = "AZURE_TENANT_ID"; public static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID"; public static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET"; - private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default"; - private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L; - private static final long TOKEN_REFRESHER_DELAY = 1L; private static final Logger log = LoggerFactory.getLogger(AzureUtilities.class); - private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); private AzureUtilities() { } @@ -145,24 +124,6 @@ public static CloudBlobDirectory cloudBlobDirectoryFrom(String connection, Strin return container.getDirectoryReference(dir); } - public static StorageCredentialsToken storageCredentialAccessTokenFrom(String accountName, String clientId, String clientSecret, String tenantId) { - ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder() - .clientId(clientId) - .clientSecret(clientSecret) - .tenantId(tenantId) - .build(); - - AccessToken accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); - if (accessToken == null || StringUtils.isBlank(accessToken.getToken())) { - log.error("Access token is null or empty"); - throw new IllegalArgumentException("Could not connect to azure storage, access token is null or empty"); - } - StorageCredentialsToken storageCredentialsToken = new StorageCredentialsToken(accountName, accessToken.getToken()); - TokenRefresher tokenRefresher = new TokenRefresher(clientSecretCredential, accessToken, storageCredentialsToken); - executorService.scheduleWithFixedDelay(tokenRefresher, TOKEN_REFRESHER_INITIAL_DELAY, TOKEN_REFRESHER_DELAY, TimeUnit.MINUTES); - return storageCredentialsToken; - } - private static ResultSegment listBlobsInSegments(CloudBlobDirectory directory, ResultContinuation token) throws IOException { ResultSegment result = null; @@ -230,61 +191,6 @@ public void write(@NotNull byte[] bytes, int offset, int length) { } } - /** - * This class represents a token refresher responsible for ensuring the validity of the access token used for azure AD authentication. - * The access token generated by the Azure client is valid for 1 hour only. Therefore, this class periodically checks the validity - * of the access token and refreshes it if necessary. The refresh is triggered when the current access token is about to expire, - * defined by a threshold of 5 minutes from the current time. This threshold is similar to what is being used in azure identity to - * generate a new token - */ - private static class TokenRefresher implements Runnable { - - private final ClientSecretCredential clientSecretCredential; - private AccessToken accessToken; - private final StorageCredentialsToken storageCredentialsToken; - - - /** - * Constructs a new TokenRefresher object with the specified parameters. - * - * @param clientSecretCredential The client secret credential used to obtain the access token. - * @param accessToken The current access token. - * @param storageCredentialsToken The storage credentials token associated with the access token. - */ - public TokenRefresher(ClientSecretCredential clientSecretCredential, - AccessToken accessToken, - StorageCredentialsToken storageCredentialsToken) { - this.clientSecretCredential = clientSecretCredential; - this.accessToken = accessToken; - this.storageCredentialsToken = storageCredentialsToken; - } - - @Override - public void run() { - try { - log.debug("Checking for azure access token expiry at: {}", LocalDateTime.now()); - OffsetDateTime tokenExpiryThreshold = OffsetDateTime.now().plusMinutes(5); - if (accessToken.getExpiresAt() != null && accessToken.getExpiresAt().isBefore(tokenExpiryThreshold)) { - log.info("Access token is about to expire (5 minutes or less) at: {}. New access token will be generated", - accessToken.getExpiresAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); - AccessToken newToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); - if (newToken == null || StringUtils.isBlank(newToken.getToken())) { - log.error("New access token is null or empty"); - return; - } - this.accessToken = newToken; - this.storageCredentialsToken.updateToken(this.accessToken.getToken()); - } - } catch (Exception e) { - log.error("Error while acquiring new access token: ", e); - } - } - } - - public static void shutDown() { - new ExecutorCloser(executorService).close(); - } - } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java index 99e2b3f1cf9..954129d77c7 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java @@ -15,7 +15,7 @@ * limitations under the License. */ @Internal(since = "1.0.0") -@Version("2.4.0") +@Version("3.0.0") package org.apache.jackrabbit.oak.segment.azure; import org.apache.jackrabbit.oak.commons.annotations.Internal; diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java index f4845155740..148d1ce30d2 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java @@ -19,6 +19,7 @@ import com.google.common.io.Files; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; @@ -347,6 +348,7 @@ public void afterSegmentRead(File file, long msb, long lsb, int length, long ela private final Integer persistentCacheSizeGb; private final CloudBlobDirectory cloudBlobDirectory; + private final AzureStorageCredentialManager azureStorageCredentialManager; private AzureCheck(Builder builder) { this.path = builder.path; @@ -365,6 +367,7 @@ private AzureCheck(Builder builder) { this.persistentCachePath = builder.persistentCachePath; this.persistentCacheSizeGb = builder.persistentCacheSizeGb; this.cloudBlobDirectory = builder.cloudBlobDirectory; + this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } private static Integer revisionsToCheckCount(Integer revisionsCount) { @@ -378,7 +381,7 @@ public int run() { if (cloudBlobDirectory != null) { persistence = new AzurePersistence(cloudBlobDirectory); } else { - persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); + persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManager); } if (persistentCachePath != null) { @@ -424,6 +427,8 @@ public int run() { } catch (Exception e) { e.printStackTrace(err); return 1; + } finally { + azureStorageCredentialManager.close(); } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java index 3e79f11837c..a5db11950b8 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java @@ -32,6 +32,7 @@ import org.apache.jackrabbit.oak.segment.SegmentCache; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; @@ -265,6 +266,7 @@ public AzureCompact build() { private final CloudBlobDirectory sourceCloudBlobDirectory; private final CloudBlobDirectory destinationCloudBlobDirectory; + private final AzureStorageCredentialManager azureStorageCredentialManager; private AzureCompact(Builder builder) { this.path = builder.path; @@ -279,6 +281,7 @@ private AzureCompact(Builder builder) { this.persistentCacheSizeGb = builder.persistentCacheSizeGb; this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory; this.destinationCloudBlobDirectory = builder.destinationCloudBlobDirectory; + this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } public int run() throws IOException, StorageException, URISyntaxException { @@ -290,8 +293,8 @@ public int run() throws IOException, StorageException, URISyntaxException { roPersistence = new AzurePersistence(sourceCloudBlobDirectory); rwPersistence = new AzurePersistence(destinationCloudBlobDirectory); } else { - roPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path); - rwPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath); + roPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path, azureStorageCredentialManager); + rwPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath, azureStorageCredentialManager); } if (persistentCachePath != null) { @@ -354,13 +357,15 @@ public int run() throws IOException, StorageException, URISyntaxException { CloudBlobContainer targetContainer = null; if (targetPath != null) { - CloudBlobDirectory targetDirectory = createCloudBlobDirectory(targetPath.substring(3)); + CloudBlobDirectory targetDirectory = createCloudBlobDirectory(targetPath.substring(3), azureStorageCredentialManager); targetContainer = targetDirectory.getContainer(); } else { targetContainer = destinationCloudBlobDirectory.getContainer(); } printTargetRepoSizeInfo(targetContainer); + // close azure storage credential manager + azureStorageCredentialManager.close(); return 0; } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java index 44bb5e3ecd1..df2452cca74 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java @@ -43,6 +43,7 @@ import java.util.concurrent.Future; import org.apache.jackrabbit.oak.commons.Buffer; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.SegmentStoreMigrator.Segment; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; import org.apache.jackrabbit.oak.segment.azure.util.Retrier; @@ -259,6 +260,7 @@ public SegmentCopy build() { private SegmentNodeStorePersistence destPersistence; private ExecutorService executor = Executors.newFixedThreadPool(READ_THREADS + 1); + private final AzureStorageCredentialManager azureStorageCredentialManager; public SegmentCopy(Builder builder) { this.source = builder.source; @@ -271,6 +273,7 @@ public SegmentCopy(Builder builder) { this.maxSizeGb = builder.maxSizeGb; this.outWriter = builder.outWriter; this.errWriter = builder.errWriter; + this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } public int run() { @@ -284,7 +287,7 @@ public int run() { if (flat && destType == SegmentStoreType.TAR) { try { - srcPersistence = newSegmentNodeStorePersistence(srcType, source); + srcPersistence = newSegmentNodeStorePersistence(srcType, source, azureStorageCredentialManager); SegmentArchiveManager sourceManager = srcPersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter()); @@ -361,12 +364,14 @@ public int run() { destination); e.printStackTrace(errWriter); return 1; + } finally { + azureStorageCredentialManager.close(); } } else { try { if (srcPersistence == null || destPersistence == null) { - srcPersistence = newSegmentNodeStorePersistence(srcType, source); - destPersistence = newSegmentNodeStorePersistence(destType, destination); + srcPersistence = newSegmentNodeStorePersistence(srcType, source, azureStorageCredentialManager); + destPersistence = newSegmentNodeStorePersistence(destType, destination, azureStorageCredentialManager); } printMessage(outWriter, "Started segment-copy transfer!"); @@ -390,6 +395,8 @@ public int run() { destination); e.printStackTrace(errWriter); return 1; + } finally { + azureStorageCredentialManager.close(); } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java index 20283ef77b8..172b7b2dccb 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java @@ -19,11 +19,6 @@ package org.apache.jackrabbit.oak.segment.azure.tool; import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE; @@ -39,11 +34,12 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.commons.Buffer; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; @@ -64,12 +60,12 @@ import org.apache.jackrabbit.guava.common.collect.Iterators; import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -126,22 +122,6 @@ public static FileStore newFileStore(SegmentNodeStorePersistence persistence, Fi .build(); } - public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, - String pathOrUri, String persistentCachePath, Integer persistentCacheSize) { - SegmentNodeStorePersistence persistence = null; - - switch (storeType) { - case AZURE: - CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); - persistence = decorateWithCache(new AzurePersistence(cloudBlobDirectory), persistentCachePath, persistentCacheSize); - break; - default: - persistence = new TarPersistence(new File(pathOrUri)); - } - - return persistence; - } - public static SegmentNodeStorePersistence decorateWithCache(SegmentNodeStorePersistence persistence, String persistentCachePath, Integer persistentCacheSize) { PersistentCache persistentCache = new PersistentDiskCache(new File(persistentCachePath), @@ -150,16 +130,18 @@ public static SegmentNodeStorePersistence decorateWithCache(SegmentNodeStorePers } public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, - String pathOrUri) { + String pathOrUri, + @Nullable AzureStorageCredentialManager azureStorageCredentialManager) { SegmentNodeStorePersistence persistence = null; switch (storeType) { - case AZURE: - CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); - persistence = new AzurePersistence(cloudBlobDirectory); - break; - default: - persistence = new TarPersistence(new File(pathOrUri)); + case AZURE: + Objects.requireNonNull(azureStorageCredentialManager, "azure storage credentials manager instance cannot be null"); + CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3), azureStorageCredentialManager); + persistence = new AzurePersistence(cloudBlobDirectory); + break; + default: + persistence = new TarPersistence(new File(pathOrUri)); } return persistence; @@ -178,11 +160,13 @@ public static SegmentArchiveManager createArchiveManager(SegmentNodeStorePersist return archiveManager; } - public static CloudBlobDirectory createCloudBlobDirectory(String path) { - return createCloudBlobDirectory(path, ENVIRONMENT); + public static CloudBlobDirectory createCloudBlobDirectory(String path, AzureStorageCredentialManager azureStorageCredentialManager) { + return createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManager); } - public static CloudBlobDirectory createCloudBlobDirectory(String path, Environment environment) { + public static CloudBlobDirectory createCloudBlobDirectory(String path, + Environment environment, + AzureStorageCredentialManager azureStorageCredentialManager) { Map config = parseAzureConfigurationFromUri(path); String accountName = config.get(KEY_ACCOUNT_NAME); @@ -191,7 +175,7 @@ public static CloudBlobDirectory createCloudBlobDirectory(String path, Environme if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) { credentials = new StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE)); } else { - credentials = getStorageCredentialsFromAccountAndEnv(accountName, environment); + credentials = azureStorageCredentialManager.getStorageCredentialsFromEnvironment(accountName, environment); } String uri = config.get(KEY_STORAGE_URI); @@ -201,58 +185,29 @@ public static CloudBlobDirectory createCloudBlobDirectory(String path, Environme return AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } catch (URISyntaxException | StorageException e) { throw new IllegalArgumentException( - "Could not connect to the Azure Storage. Please verify the path provided!"); + "Could not connect to the Azure Storage. Please verify the path provided!"); } } public static List readRevisions(String uri) { - SegmentNodeStorePersistence persistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri); - JournalFile journal = persistence.getJournalFile(); - - if (journal.exists()) { - try (JournalReader journalReader = new JournalReader(journal)) { - Iterator revisionIterator = Iterators.transform(journalReader, - entry -> entry.getRevision()); - return newArrayList(revisionIterator); - } catch (Exception e) { - e.printStackTrace(); + try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { + SegmentNodeStorePersistence persistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri, azureStorageCredentialManager); + JournalFile journal = persistence.getJournalFile(); + if (journal.exists()) { + try (JournalReader journalReader = new JournalReader(journal)) { + Iterator revisionIterator = Iterators.transform(journalReader, + entry -> entry.getRevision()); + return newArrayList(revisionIterator); + } catch (Exception e) { + log.error("Error while reading from journal file"); + e.printStackTrace(); + } } } return newArrayList(); } - @NotNull - public static StorageCredentials getStorageCredentialsFromAccountAndEnv(@NotNull String accountName) { - return getStorageCredentialsFromAccountAndEnv(accountName, ENVIRONMENT); - } - - @NotNull - private static StorageCredentials getStorageCredentialsFromAccountAndEnv(String accountName, Environment environment) { - String clientId = environment.getVariable(AZURE_CLIENT_ID); - String clientSecret = environment.getVariable(AZURE_CLIENT_SECRET); - String tenantId = environment.getVariable(AZURE_TENANT_ID); - - if (!StringUtils.isAnyBlank(clientId, clientSecret, tenantId)) { - try { - return storageCredentialAccessTokenFrom(accountName, clientId, clientSecret, tenantId); - } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { - throw new IllegalArgumentException( - "Could not connect to the Azure Storage. Please verify if AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables are correctly set!"); - } - } else { - log.warn("AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."); - } - - String key = environment.getVariable(AZURE_SECRET_KEY); - try { - return new StorageCredentialsAccountAndKey(accountName, key); - } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { - throw new IllegalArgumentException( - "Could not connect to the Azure Storage. Please verify if AZURE_SECRET_KEY environment variable is correctly set!"); - } - } - public static SegmentStoreType storeTypeFromPathOrUri(String pathOrUri) { if (pathOrUri.startsWith("az:")) { return SegmentStoreType.AZURE; diff --git a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java index 892d3801514..16c5052670e 100644 --- a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java +++ b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java @@ -20,6 +20,7 @@ import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; @@ -52,8 +53,10 @@ public void testSegmentCopy() throws Exception { protected SegmentNodeStorePersistence getSrcPersistence() { String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME); String path = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, CONTAINER_NAME, DIR); - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT); - + CloudBlobDirectory cloudBlobDirectory; + try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { + cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManager); + } return new AzurePersistence(cloudBlobDirectory); } diff --git a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java index 114161ac24a..edad03847fb 100644 --- a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java +++ b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java @@ -231,7 +231,7 @@ private void checkManifest(SegmentNodeStorePersistence srcPersistence, SegmentNo } protected SegmentNodeStorePersistence getTarPersistence() { - return newSegmentNodeStorePersistence(SegmentStoreType.TAR, folder.getRoot().getAbsolutePath()); + return newSegmentNodeStorePersistence(SegmentStoreType.TAR, folder.getRoot().getAbsolutePath(), null); } protected SegmentNodeStorePersistence getAzurePersistence() throws Exception { diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java index 6ae1a0abba1..4ce192c794e 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java @@ -37,9 +37,12 @@ import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; @@ -73,6 +76,17 @@ public class ToolUtilsTest { public static final String AZURE_SECRET_KEY_WARNING = "AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."; private final TestEnvironment environment = new TestEnvironment(); + private AzureStorageCredentialManager azureStorageCredentialManager; + + @Before + public void init() { + this.azureStorageCredentialManager = new AzureStorageCredentialManager(); + } + + @After + public void clear() { + this.azureStorageCredentialManager.close(); + } @Test public void createCloudBlobDirectoryWithAccessKey() { @@ -82,7 +96,7 @@ public void createCloudBlobDirectoryWithAccessKey() { StorageCredentialsAccountAndKey credentials = expectCredentials( StorageCredentialsAccountAndKey.class, - () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment), + () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManager), DEFAULT_CONTAINER_URL ); @@ -98,7 +112,7 @@ public void createCloudBlobDirectoryWithAccessKey() { public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() { environment.setVariable(AZURE_SECRET_KEY, null); assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment) + ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManager) ); } @@ -106,7 +120,7 @@ public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() { public void createCloudBlobDirectoryFailsWhenAccessKeyIsInvalid() { environment.setVariable(AZURE_SECRET_KEY, "invalid"); assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment) + ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManager) ); } @@ -116,7 +130,7 @@ public void createCloudBlobDirectoryWithSasUri() { StorageCredentialsSharedAccessSignature credentials = expectCredentials( StorageCredentialsSharedAccessSignature.class, - () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken), + () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken, azureStorageCredentialManager), DEFAULT_CONTAINER_URL ); @@ -135,7 +149,7 @@ public void createCloudBlobDirectoryWithServicePrincipal() throws URISyntaxExcep String containerName = "oak"; String segmentStorePath = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, containerName, DEFAULT_REPO_DIR); - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT, azureStorageCredentialManager); assertNotNull(cloudBlobDirectory); assertEquals(containerName, cloudBlobDirectory.getContainer().getName()); } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java index a48ecd6de67..ae299321978 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java @@ -25,14 +25,16 @@ import org.apache.jackrabbit.guava.common.io.Files; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; -import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; +import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.upgrade.cli.CliUtils; import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.NodeStoreWithFileStore; import java.io.File; @@ -53,6 +55,8 @@ public class SegmentAzureFactory implements NodeStoreFactory { private int segmentCacheSize; private final boolean readOnly; + private static final Environment environment = new Environment(); + private AzureStorageCredentialManager azureStorageCredentialManager; public static class Builder { private final String dir; @@ -116,7 +120,7 @@ public SegmentAzureFactory(Builder builder) { public NodeStore create(BlobStore blobStore, Closer closer) throws IOException { AzurePersistence azPersistence = null; try { - azPersistence = createAzurePersistence(); + azPersistence = createAzurePersistence(closer); } catch (StorageException | URISyntaxException | InvalidKeyException e) { throw new IllegalStateException(e); } @@ -148,7 +152,7 @@ public NodeStore create(BlobStore blobStore, Closer closer) throws IOException { } } - private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { + private AzurePersistence createAzurePersistence(Closer closer) throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; // connection string will take precedence over accountkey / sas / service principal @@ -159,7 +163,9 @@ private AzurePersistence createAzurePersistence() throws StorageException, URISy if (StringUtils.isNotBlank(sasToken)) { credentials = new StorageCredentialsSharedAccessSignature(sasToken); } else { - credentials = ToolUtils.getStorageCredentialsFromAccountAndEnv(accountName); + this.azureStorageCredentialManager = new AzureStorageCredentialManager(); + credentials = azureStorageCredentialManager.getStorageCredentialsFromEnvironment(accountName, environment); + closer.register(azureStorageCredentialManager); } cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } @@ -174,9 +180,12 @@ private AzurePersistence createAzurePersistence() throws StorageException, URISy @Override public boolean hasExternalBlobReferences() throws IOException { AzurePersistence azPersistence = null; + Closer closer = Closer.create(); + CliUtils.handleSigInt(closer); try { - azPersistence = createAzurePersistence(); + azPersistence = createAzurePersistence(closer); } catch (StorageException | URISyntaxException | InvalidKeyException e) { + closer.close(); throw new IllegalStateException(e); } @@ -192,6 +201,7 @@ public boolean hasExternalBlobReferences() throws IOException { throw new IOException(e); } finally { tmpDir.delete(); + closer.close(); } } diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java index 2347d3e210e..c0619c60419 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java @@ -20,6 +20,7 @@ import org.apache.jackrabbit.guava.common.io.Files; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; @@ -43,6 +44,7 @@ public class SegmentAzureServicePrincipalNodeStoreContainer implements NodeStore private FileStore fs; private File tmpDir; private AzurePersistence azurePersistence; + private final AzureStorageCredentialManager azureStorageCredentialManager; public SegmentAzureServicePrincipalNodeStoreContainer() { this(null); @@ -50,6 +52,7 @@ public SegmentAzureServicePrincipalNodeStoreContainer() { public SegmentAzureServicePrincipalNodeStoreContainer(BlobStore blobStore) { this.blobStore = blobStore; + this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } @@ -83,7 +86,7 @@ private AzurePersistence createAzurePersistence() { } String path = String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), CONTAINER_NAME, DIR); - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManager); return new AzurePersistence(cloudBlobDirectory); } @@ -96,6 +99,9 @@ public void close() { if (tmpDir != null) { tmpDir.delete(); } + if (azureStorageCredentialManager != null) { + azureStorageCredentialManager.close(); + } } @Override diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java index a28803f5975..00e10ddfe5f 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.guava.common.io.Closer; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; +import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; @@ -108,19 +109,21 @@ public void testConnectionWithUri_accessKey() throws IOException { String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME); Closer closer = Closer.create(); - try { - SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, - false) - .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) - .uri(uri) - .build(); - closer = Closer.create(); - CliUtils.handleSigInt(closer); - FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); - assertEquals(1, nodeStore.getFileStore().getSegmentCount()); - } finally { - closer.close(); - cleanup(uri); + try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { + try { + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, + false) + .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) + .uri(uri) + .build(); + closer = Closer.create(); + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + } finally { + closer.close(); + cleanup(uri, azureStorageCredentialManager); + } } } @@ -133,26 +136,28 @@ public void testConnectionWithUri_servicePrincipal() throws IOException, Interru String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME); Closer closer = Closer.create(); - try { - SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, - false) - .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) - .uri(uri) - .build(); - - CliUtils.handleSigInt(closer); - FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); - assertEquals(1, nodeStore.getFileStore().getSegmentCount()); - } finally { - closer.close(); - cleanup(uri); + try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { + try { + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, + false) + .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) + .uri(uri) + .build(); + + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + } finally { + closer.close(); + cleanup(uri, azureStorageCredentialManager); + } } } - private void cleanup(String uri) { + private void cleanup(String uri, AzureStorageCredentialManager azureStorageCredentialManager) { uri = uri + "/" + DIR; try { - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT, azureStorageCredentialManager); AzureUtilities.deleteAllBlobs(cloudBlobDirectory); } catch (Exception e) { throw new IllegalStateException(e); From ce5c7df8993d2319828fceaacdcfc2093d511919 Mon Sep 17 00:00:00 2001 From: Nuno Santos Date: Mon, 29 Jul 2024 11:39:57 +0200 Subject: [PATCH 32/86] OAK-10977 - Cleanup IndexDefinition class (#1604) - replace Guava usages with JDK equivalents - fix typos - improve formatting --- .../plugins/index/search/IndexDefinition.java | 430 +++++++++--------- 1 file changed, 225 insertions(+), 205 deletions(-) diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java index ffe2dba7200..503ee2e13a5 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java @@ -19,31 +19,11 @@ package org.apache.jackrabbit.oak.plugins.index.search; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import javax.jcr.PropertyType; -import javax.jcr.RepositoryException; -import javax.jcr.nodetype.NodeTypeIterator; - -import org.apache.jackrabbit.guava.common.collect.ImmutableList; +import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.guava.common.collect.ImmutableMap; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; -import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.guava.common.primitives.Ints; -import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Root; @@ -70,11 +50,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; -import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; -import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayListWithCapacity; -import static org.apache.jackrabbit.guava.common.collect.Maps.newHashMap; -import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; +import javax.jcr.PropertyType; +import javax.jcr.RepositoryException; +import javax.jcr.nodetype.NodeTypeIterator; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Stream; + import static org.apache.jackrabbit.JcrConstants.JCR_SCORE; import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM; import static org.apache.jackrabbit.JcrConstants.NT_BASE; @@ -87,7 +79,37 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_SYNC; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.BLOB_SIZE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.COMPAT_MODE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EVALUATE_PATH_RESTRICTION; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXCLUDE_PROPERTY_NAMES; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXPERIMENTAL_STORAGE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FACETS; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FIELD_BOOST; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FULL_TEXT_ENABLED; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_NAMES; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_TYPES; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_DATA_CHILD_NAME; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_SIMILARITY_BINARIES; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_SIMILARITY_STRINGS; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.ORDERED_PROP_NAMES; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS_TOP_CHILDREN; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_QUERY_FILTER_REGEX; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_RANDOM_SEED; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_INSECURE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_JVM_PARAM; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_SECURE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_STATISTICAL; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_STATISTICAL_FACET_SAMPLE_SIZE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_VALUE_REGEX; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_CONFIG; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MAPPED_TYPE; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MIME_TYPES; import static org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition.DEFAULT_BOOST; import static org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValue; import static org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValues; @@ -195,8 +217,8 @@ public class IndexDefinition implements Aggregate.AggregateMapper { public static final OrderEntry NATIVE_SORT_ORDER = new OrderEntry(JCR_SCORE, Type.UNDEFINED, OrderEntry.Order.DESCENDING); - private boolean indexSimilarityBinaries; - private boolean indexSimilarityStrings; + private final boolean indexSimilarityBinaries; + private final boolean indexSimilarityStrings; /** * Dynamic boost uses index time boosting. This requires to have a separate field for each unique term that needs to @@ -206,13 +228,13 @@ public class IndexDefinition implements Aggregate.AggregateMapper { * oak.search.dynamicBoostLite=lucene no index time boosting is used for lucene types indexes. The terms will affect * the query match clause but the scores won't be the same. In summary, in lite mode the query will have the same * recall but lower precision. - * + *

* WARNING: dynamicBoostLite needs similarityTags. In case there are no similarityTags, the query won't return the * expected results. */ private final static String DYNAMIC_BOOST_LITE_NAME = "oak.search.dynamicBoostLite"; protected final static List DYNAMIC_BOOST_LITE = - Arrays.asList(System.getProperty(DYNAMIC_BOOST_LITE_NAME, "").split(",")); + List.of(System.getProperty(DYNAMIC_BOOST_LITE_NAME, "").split(",")); protected final boolean fullTextEnabled; @@ -322,7 +344,7 @@ public boolean isTestMode() { //~--------------------------------------------------------< Builder > // TODO - this method should be removed after tests don't use it anymore - public static Builder newBuilder(NodeState root, NodeState defn, String indexPath){ + public static Builder newBuilder(NodeState root, NodeState defn, String indexPath) { return new Builder().root(root).defn(defn).indexPath(indexPath); } @@ -340,49 +362,49 @@ public static class Builder { protected IndexFormatVersion version; public Builder root(NodeState root) { - this.root = checkNotNull(root); + this.root = Objects.requireNonNull(root); return this; } public Builder defn(NodeState defn) { - this.defn = checkNotNull(defn); + this.defn = Objects.requireNonNull(defn); return this; } public Builder indexPath(String indexPath) { - this.indexPath = checkNotNull(indexPath); + this.indexPath = Objects.requireNonNull(indexPath); return this; } - public Builder uid(String uid){ + public Builder uid(String uid) { this.uid = uid; return this; } - public Builder version(IndexFormatVersion version){ + public Builder version(IndexFormatVersion version) { this.version = version; return this; } - public Builder reindex(){ + public Builder reindex() { this.reindexMode = true; return this; } - public IndexDefinition build(){ - if (version == null){ + public IndexDefinition build() { + if (version == null) { version = determineIndexFormatVersion(defn); } - if (uid == null){ + if (uid == null) { uid = determineUniqueId(defn); - if (uid == null && !IndexDefinition.hasPersistedIndex(defn)){ + if (uid == null && !IndexDefinition.hasPersistedIndex(defn)) { uid = DEFAULT_UID; } } NodeState indexDefnStateToUse = defn; - if (!reindexMode){ + if (!reindexMode) { indexDefnStateToUse = getIndexDefinitionState(defn); } return createInstance(indexDefnStateToUse); @@ -401,10 +423,10 @@ public IndexDefinition(NodeState root, NodeState defn, String indexPath) { protected IndexDefinition(NodeState root, NodeState defn, IndexFormatVersion version, String uid, String indexPath) { try { this.root = root; - this.version = checkNotNull(version); + this.version = Objects.requireNonNull(version); this.uid = uid; this.definition = defn; - this.indexPath = checkNotNull(indexPath); + this.indexPath = Objects.requireNonNull(indexPath); this.indexName = indexPath; this.indexTags = getOptionalValues(defn, IndexConstants.INDEX_TAGS, Type.STRINGS, String.class); this.indexSelectionPolicy @@ -412,17 +434,17 @@ protected IndexDefinition(NodeState root, NodeState defn, IndexFormatVersion ver this.nodeTypeIndex = getOptionalValue(defn, FulltextIndexConstants.PROP_INDEX_NODE_TYPE, false); this.blobSize = Math.max(1024, getOptionalValue(defn, BLOB_SIZE, DEFAULT_BLOB_SIZE)); - this.aggregates = nodeTypeIndex ? Collections.emptyMap() : collectAggregates(defn); + this.aggregates = nodeTypeIndex ? Map.of() : collectAggregates(defn); NodeState rulesState = defn.getChildNode(FulltextIndexConstants.INDEX_RULES); - if (!rulesState.exists()){ + if (!rulesState.exists()) { rulesState = createIndexRules(defn).getNodeState(); } this.testMode = getOptionalValue(defn, FulltextIndexConstants.TEST_MODE, false); - List definedIndexRules = newArrayList(); + List definedIndexRules = new ArrayList<>(); this.indexRules = collectIndexRules(rulesState, definedIndexRules); - this.definedRules = ImmutableList.copyOf(definedIndexRules); + this.definedRules = List.copyOf(definedIndexRules); this.fullTextEnabled = hasFulltextEnabledIndexRule(definedIndexRules); this.evaluatePathRestrictions = getOptionalValue(defn, EVALUATE_PATH_RESTRICTION, false); @@ -507,7 +529,7 @@ private boolean getSimilarityDefaultValue(NodeState defn, String propertyKey) { } private boolean isPresent(T key, Iterator iterator) { - while (iterator.hasNext()){ + while (iterator.hasNext()) { if (key.equals(iterator.next())) { return true; } @@ -555,7 +577,7 @@ public boolean isFullTextEnabled() { return fullTextEnabled; } - public String getFunctionName(){ + public String getFunctionName() { return funcName; } @@ -564,19 +586,20 @@ protected String getDefaultFunctionName() { return "fulltext";//TODO Should this be FulltextIndexConstants.FUNC_NAME } - public boolean hasFunctionDefined(){ + public boolean hasFunctionDefined() { return funcName != null; } /** * Size in bytes for the blobs created while storing the index content + * * @return size in bytes */ public int getBlobSize() { return blobSize; } - public long getReindexCount(){ + public long getReindexCount() { return reindexCount; } @@ -615,8 +638,8 @@ public double getCostPerExecution() { return costPerExecution; } - public long getFulltextEntryCount(long numOfDocs){ - if (isEntryCountDefined()){ + public long getFulltextEntryCount(long numOfDocs) { + if (isEntryCountDefined()) { return Math.min(getEntryCount(), numOfDocs); } return numOfDocs; @@ -631,7 +654,7 @@ public IndexFormatVersion getVersion() { } - public boolean isOfOldFormat(){ + public boolean isOfOldFormat() { return !hasIndexingRules(definition); } @@ -639,11 +662,11 @@ public boolean evaluatePathRestrictions() { return evaluatePathRestrictions; } - public boolean hasCustomTikaConfig(){ + public boolean hasCustomTikaConfig() { return hasCustomTikaConfig; } - public InputStream getTikaConfig(){ + public InputStream getTikaConfig() { return ConfigUtil.getBlob(getTikaConfigNode(), TIKA_CONFIG).getNewStream(); } @@ -699,7 +722,7 @@ public boolean isPureNodeTypeIndex() { /** * Check if the index definition is fresh, or (some) indexing has occurred. - * + *

* WARNING: If there is _any_ hidden node, then it is assumed that * no reindex is needed. Even if the hidden node is completely unrelated * and doesn't contain index data (for example the node ":status"). @@ -708,7 +731,7 @@ public boolean isPureNodeTypeIndex() { * @param definition nodestate for Index Definition * @return true if index has some indexed content */ - public static boolean hasPersistedIndex(NodeState definition){ + public static boolean hasPersistedIndex(NodeState definition) { for (String rm : definition.getChildNodeNames()) { if (NodeStateUtils.isHidden(rm)) { return true; @@ -725,7 +748,7 @@ public static void setDisableStoredIndexDefinition(boolean disableStoredIndexDef IndexDefinition.disableStoredIndexDefinition = disableStoredIndexDefinitionDefault; } - public Set getRelativeNodeNames(){ + public Set getRelativeNodeNames() { //Can be computed lazily as required only for oak-run indexing for now Set names = new HashSet<>(); for (IndexingRule r : definedRules) { @@ -740,7 +763,7 @@ public Set getRelativeNodeNames(){ return names; } - public boolean indexesRelativeNodes(){ + public boolean indexesRelativeNodes() { for (IndexingRule r : definedRules) { if (!r.aggregate.getIncludes().isEmpty()) { return true; @@ -758,7 +781,7 @@ public String toString() { //~---------------------------------------------------< Aggregates > @Nullable - public Aggregate getAggregate(String nodeType){ + public Aggregate getAggregate(String nodeType) { return aggregates.get(nodeType); } @@ -768,14 +791,14 @@ public Map getAggregates() { } private Map collectAggregates(NodeState defn) { - Map aggregateMap = newHashMap(); + Map aggregateMap = new HashMap<>(); for (ChildNodeEntry cne : defn.getChildNode(FulltextIndexConstants.AGGREGATES).getChildNodeEntries()) { String nodeType = cne.getName(); int recursionLimit = getOptionalValue(cne.getNodeState(), FulltextIndexConstants.AGG_RECURSIVE_LIMIT, Aggregate.RECURSIVE_AGGREGATION_LIMIT_DEFAULT); - List includes = newArrayList(); + List includes = new ArrayList<>(); for (ChildNodeEntry include : cne.getNodeState().getChildNodeEntries()) { NodeState is = include.getNodeState(); String primaryType = is.getString(FulltextIndexConstants.AGG_PRIMARY_TYPE); @@ -791,7 +814,7 @@ private Map collectAggregates(NodeState defn) { aggregateMap.put(nodeType, new Aggregate(nodeType, includes, recursionLimit)); } - return ImmutableMap.copyOf(aggregateMap); + return Map.copyOf(aggregateMap); } //~---------------------------------------------------< IndexRule > @@ -877,19 +900,19 @@ public IndexingRule getApplicableIndexingRule(NodeState state) { } private Map> collectIndexRules(NodeState indexRules, - List definedIndexRules){ + List definedIndexRules) { //TODO if a rule is defined for nt:base then this map would have entry for each //registered nodeType in the system if (!indexRules.exists()) { - return Collections.emptyMap(); + return Map.of(); } - if (!hasOrderableChildren(indexRules)){ + if (!hasOrderableChildren(indexRules)) { log.warn("IndexRule node does not have orderable children in [{}]", IndexDefinition.this); } - Map> nt2rules = newHashMap(); + Map> nt2rules = new HashMap<>(); ReadOnlyNodeTypeManager ntReg = createNodeTypeManager(RootFactory.createReadOnlyRoot(root)); //Use Tree API to read ordered child nodes @@ -903,10 +926,10 @@ private Map> collectIndexRules(NodeState indexRules, log.trace("Found rule '{}' for NodeType '{}'", rule, rule.getNodeTypeName()); List ntNames = allNames; - if (!rule.inherited){ + if (!rule.inherited) { //Trim the list to rule's nodeType so that inheritance check //is not performed for other nodeTypes - ntNames = Collections.singletonList(rule.getNodeTypeName()); + ntNames = List.of(rule.getNodeTypeName()); } for (String ntName : ntNames) { @@ -918,11 +941,11 @@ private Map> collectIndexRules(NodeState indexRules, } } - for (Map.Entry> e : nt2rules.entrySet()){ - e.setValue(ImmutableList.copyOf(e.getValue())); + for (Map.Entry> e : nt2rules.entrySet()) { + e.setValue(List.copyOf(e.getValue())); } - return ImmutableMap.copyOf(nt2rules); + return Map.copyOf(nt2rules); } private boolean evaluateSuggestionEnabled() { @@ -1004,7 +1027,7 @@ public class IndexingRule { private final String baseNodeType; private final String nodeTypeName; /** - * Case insensitive map of lower cased propertyName to propertyConfigs + * Case-insensitive map of lower cased propertyName to propertyConfigs */ private final Map propConfigs; /** @@ -1037,31 +1060,31 @@ public class IndexingRule { this.inherited = getOptionalValue(config, FulltextIndexConstants.RULE_INHERITED, true); this.propertyTypes = getSupportedTypes(config, INCLUDE_PROPERTY_TYPES, TYPES_ALLOW_ALL); - List namePatterns = newArrayList(); - List nonExistentProperties = newArrayList(); - List functionRestrictions = newArrayList(); - List existentProperties = newArrayList(); - List nodeScopeAnalyzedProps = newArrayList(); - List syncProps = newArrayList(); - List similarityProperties = newArrayList(); - List propIncludes = newArrayList(); + List namePatterns = new ArrayList<>(); + List nonExistentProperties = new ArrayList<>(); + List functionRestrictions = new ArrayList<>(); + List existentProperties = new ArrayList<>(); + List nodeScopeAnalyzedProps = new ArrayList<>(); + List syncProps = new ArrayList<>(); + List similarityProperties = new ArrayList<>(); + List propIncludes = new ArrayList<>(); this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties, existentProperties, nodeScopeAnalyzedProps, functionRestrictions, syncProps, similarityProperties); this.propAggregate = new Aggregate(nodeTypeName, propIncludes); this.aggregate = combine(propAggregate, nodeTypeName); - this.namePatterns = ImmutableList.copyOf(namePatterns); - this.nodeScopeAnalyzedProps = ImmutableList.copyOf(nodeScopeAnalyzedProps); - this.nullCheckEnabledProperties = ImmutableList.copyOf(nonExistentProperties); - this.functionRestrictions = ImmutableList.copyOf(functionRestrictions); - this.notNullCheckEnabledProperties = ImmutableList.copyOf(existentProperties); - this.similarityProperties = ImmutableList.copyOf(similarityProperties); + this.namePatterns = List.copyOf(namePatterns); + this.nodeScopeAnalyzedProps = List.copyOf(nodeScopeAnalyzedProps); + this.nullCheckEnabledProperties = List.copyOf(nonExistentProperties); + this.functionRestrictions = List.copyOf(functionRestrictions); + this.notNullCheckEnabledProperties = List.copyOf(existentProperties); + this.similarityProperties = List.copyOf(similarityProperties); this.fulltextEnabled = aggregate.hasNodeAggregates() || hasAnyFullTextEnabledProperty(); this.nodeFullTextIndexed = aggregate.hasNodeAggregates() || anyNodeScopeIndexedProperty(); this.propertyIndexEnabled = hasAnyPropertyIndexConfigured(); this.indexesAllNodesOfMatchingType = areAlMatchingNodeByTypeIndexed(); this.nodeNameIndexed = evaluateNodeNameIndexed(config); - this.syncProps = ImmutableList.copyOf(syncProps); + this.syncProps = List.copyOf(syncProps); validateRuleDefinition(); } @@ -1069,7 +1092,7 @@ public class IndexingRule { * Creates a new indexing rule base on an existing one, but for a * different node type name. * - * @param original the existing rule. + * @param original the existing rule. * @param nodeTypeName the node type name for the rule. */ IndexingRule(IndexingRule original, String nodeTypeName) { @@ -1101,7 +1124,7 @@ public class IndexingRule { * * @param propertyName the name of a property. * @return true if the property is indexed; - * false otherwise. + * false otherwise. */ public boolean isIndexed(String propertyName) { return getConfig(propertyName) != null; @@ -1126,7 +1149,7 @@ public String getBaseNodeType() { /** * Returns all the configured {@code PropertyDefinition}s for this {@code IndexRule}. - * + *

* In case of a pure nodetype index we just return primaryType and mixins. * * @return an {@code Iterable} of {@code PropertyDefinition}s. @@ -1162,8 +1185,8 @@ public Stream getNamePatternsProperties() { @Override public String toString() { - String str = "IndexRule: "+ nodeTypeName; - if (!baseNodeType.equals(nodeTypeName)){ + String str = "IndexRule: " + nodeTypeName; + if (!baseNodeType.equals(nodeTypeName)) { str += "(" + baseNodeType + ")"; } return str; @@ -1179,11 +1202,11 @@ public boolean isAggregated(String nodePath) { * * @param state the state to check. * @return true the rule applies to the given node; - * false otherwise. + * false otherwise. */ public boolean appliesTo(NodeState state) { - for (String mixinName : getMixinTypeNames(state)){ - if (nodeTypeName.equals(mixinName)){ + for (String mixinName : getMixinTypeNames(state)) { + if (nodeTypeName.equals(mixinName)) { return true; } } @@ -1219,8 +1242,8 @@ public boolean isNodeFullTextIndexed() { /** * @param propertyName name of a property. * @return the property configuration or null if this - * indexing rule does not contain a configuration for the given - * property. + * indexing rule does not contain a configuration for the given + * property. */ @Nullable public PropertyDefinition getConfig(String propertyName) { @@ -1238,7 +1261,7 @@ public PropertyDefinition getConfig(String propertyName) { return null; } - public boolean includePropertyType(int type){ + public boolean includePropertyType(int type) { return IndexDefinition.includePropertyType(propertyTypes, type); } @@ -1252,14 +1275,14 @@ public Aggregate getAggregate() { * certain property 'foo' for node type 'app:Asset' then index would only have * entries for those assets where foo is defined. Such an index cannot claim that * it has entries for all assets. - + * * @return true in case all matching node types are covered by this rule */ public boolean indexesAllNodesOfMatchingType() { return indexesAllNodesOfMatchingType; } - public boolean isBasedOnNtBase(){ + public boolean isBasedOnNtBase() { return JcrConstants.NT_BASE.equals(baseNodeType); } @@ -1272,10 +1295,10 @@ private Map collectPropConfigs(NodeState config, List functionRestrictions, List syncProps, List similarityProperties) { - Map propDefns = newHashMap(); + Map propDefns = new HashMap<>(); NodeState propNode = config.getChildNode(FulltextIndexConstants.PROP_NODE); - if (propNode.exists() && !hasOrderableChildren(propNode)){ + if (propNode.exists() && !hasOrderableChildren(propNode)) { log.warn("Properties node for [{}] does not have orderable " + "children in [{}]", this, IndexDefinition.this); } @@ -1284,8 +1307,8 @@ private Map collectPropConfigs(NodeState config, //and ignore any other property definition if (nodeTypeIndex) { boolean sync = getOptionalValue(config, FulltextIndexConstants.PROP_SYNC, false); - PropertyDefinition pdpt = createNodeTypeDefinition(this, JcrConstants.JCR_PRIMARYTYPE, sync); - PropertyDefinition pdmixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, sync); + PropertyDefinition pdpt = createNodeTypeDefinition(this, JcrConstants.JCR_PRIMARYTYPE, sync); + PropertyDefinition pdmixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, sync); propDefns.put(pdpt.name.toLowerCase(Locale.ENGLISH), pdpt); propDefns.put(pdmixin.name.toLowerCase(Locale.ENGLISH), pdmixin); @@ -1300,7 +1323,7 @@ private Map collectPropConfigs(NodeState config, "definitions", indexPath, FulltextIndexConstants.PROP_INDEX_NODE_TYPE); } - return ImmutableMap.copyOf(propDefns); + return Map.copyOf(propDefns); } //Include all immediate child nodes to 'properties' node by default @@ -1322,28 +1345,28 @@ private Map collectPropConfigs(NodeState config, // a function index has no other options continue; } - if(pd.isRegexp){ + if (pd.isRegexp) { patterns.add(new NamePattern(pd.name, pd)); } else { propDefns.put(pd.name.toLowerCase(Locale.ENGLISH), pd); } - if (pd.relative){ + if (pd.relative) { propAggregate.add(new Aggregate.PropertyInclude(pd)); } - if (pd.nullCheckEnabled){ + if (pd.nullCheckEnabled) { nonExistentProperties.add(pd); } - if (pd.notNullCheckEnabled){ + if (pd.notNullCheckEnabled) { existentProperties.add(pd); } //Include props with name, boosted and nodeScopeIndex if (pd.nodeScopeIndex && pd.analyzed - && !pd.isRegexp){ + && !pd.isRegexp) { nodeScopeAnalyzedProps.add(pd); } @@ -1356,7 +1379,7 @@ private Map collectPropConfigs(NodeState config, } } ensureNodeTypeIndexingIsConsistent(propDefns, syncProps); - return ImmutableMap.copyOf(propDefns); + return Map.copyOf(propDefns); } /** @@ -1369,21 +1392,21 @@ private void ensureNodeTypeIndexingIsConsistent(Map PropertyDefinition pd_mixin = propDefns.get(JcrConstants.JCR_MIXINTYPES.toLowerCase(Locale.ENGLISH)); if (pd_pr != null && pd_pr.propertyIndex && pd_mixin == null) { - pd_mixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, pd_pr.sync); + pd_mixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, pd_pr.sync); syncProps.add(pd_mixin); propDefns.put(JcrConstants.JCR_MIXINTYPES.toLowerCase(Locale.ENGLISH), pd_mixin); } } private boolean hasAnyFullTextEnabledProperty() { - for (PropertyDefinition pd : propConfigs.values()){ - if (pd.fulltextEnabled()){ + for (PropertyDefinition pd : propConfigs.values()) { + if (pd.fulltextEnabled()) { return true; } } - for (NamePattern np : namePatterns){ - if (np.getConfig().fulltextEnabled()){ + for (NamePattern np : namePatterns) { + if (np.getConfig().fulltextEnabled()) { return true; } } @@ -1391,31 +1414,31 @@ private boolean hasAnyFullTextEnabledProperty() { } private boolean hasAnyPropertyIndexConfigured() { - for (PropertyDefinition pd : propConfigs.values()){ - if (pd.propertyIndex){ + for (PropertyDefinition pd : propConfigs.values()) { + if (pd.propertyIndex) { return true; } } - for (NamePattern np : namePatterns){ - if (np.getConfig().propertyIndex){ + for (NamePattern np : namePatterns) { + if (np.getConfig().propertyIndex) { return true; } } return false; } - private boolean anyNodeScopeIndexedProperty(){ + private boolean anyNodeScopeIndexedProperty() { //Check if there is any nodeScope indexed property as //for such case a node would always be indexed - for (PropertyDefinition pd : propConfigs.values()){ - if (pd.nodeScopeIndex){ + for (PropertyDefinition pd : propConfigs.values()) { + if (pd.nodeScopeIndex) { return true; } } - for (NamePattern np : namePatterns){ - if (np.getConfig().nodeScopeIndex){ + for (NamePattern np : namePatterns) { + if (np.getConfig().nodeScopeIndex) { return true; } } @@ -1423,19 +1446,19 @@ private boolean anyNodeScopeIndexedProperty(){ return false; } - private boolean areAlMatchingNodeByTypeIndexed(){ + private boolean areAlMatchingNodeByTypeIndexed() { if (nodeTypeIndex) { return true; } - if (nodeFullTextIndexed){ + if (nodeFullTextIndexed) { return true; } //If there is nullCheckEnabled property which is not relative then //all nodes would be indexed. relativeProperty with nullCheckEnabled might //not ensure that (OAK-1085) - for (PropertyDefinition pd : nullCheckEnabledProperties){ + for (PropertyDefinition pd : nullCheckEnabledProperties) { if (!pd.relative) { return true; } @@ -1455,26 +1478,25 @@ private boolean evaluateNodeNameIndexed(NodeState config) { } //iterate over property definitions - for (PropertyDefinition pd : propConfigs.values()){ - if (FulltextIndexConstants.PROPDEF_PROP_NODE_NAME.equals(pd.name)){ + for (PropertyDefinition pd : propConfigs.values()) { + if (FulltextIndexConstants.PROPDEF_PROP_NODE_NAME.equals(pd.name)) { return true; } } return false; } - private Aggregate combine(Aggregate propAggregate, String nodeTypeName){ + private Aggregate combine(Aggregate propAggregate, String nodeTypeName) { Aggregate nodeTypeAgg = IndexDefinition.this.getAggregate(nodeTypeName); - List includes = newArrayList(); - includes.addAll(propAggregate.getIncludes()); - if (nodeTypeAgg != null){ + List includes = new ArrayList<>(propAggregate.getIncludes()); + if (nodeTypeAgg != null) { includes.addAll(nodeTypeAgg.getIncludes()); } return new Aggregate(nodeTypeName, includes); } private void validateRuleDefinition() { - if (!nullCheckEnabledProperties.isEmpty() && isBasedOnNtBase()){ + if (!nullCheckEnabledProperties.isEmpty() && isBasedOnNtBase()) { throw new IllegalStateException("nt:base based rule cannot have a " + "PropertyDefinition with nullCheckEnabled"); } @@ -1500,15 +1522,14 @@ private static final class NamePattern { * Creates a new name pattern. * * @param pattern the pattern as defined by the property definition - * @param config the associated configuration. + * @param config the associated configuration. */ - private NamePattern(String pattern, - PropertyDefinition config){ + private NamePattern(String pattern, PropertyDefinition config) { //Special handling for all props regex as its already being used //and use of '/' in regex would confuse the parent path calculation //logic - if (FulltextIndexConstants.REGEX_ALL_PROPS.equals(pattern)){ + if (FulltextIndexConstants.REGEX_ALL_PROPS.equals(pattern)) { this.parentPath = ""; this.pattern = Pattern.compile(pattern); } else { @@ -1521,7 +1542,7 @@ private NamePattern(String pattern, /** * @param propertyPath property name to match * @return true if property name matches this name - * pattern; false otherwise. + * pattern; false otherwise. */ boolean matches(String propertyPath) { String parentPath = getParentPath(propertyPath); @@ -1539,13 +1560,13 @@ PropertyDefinition getConfig() { //~---------------------------------------------< compatibility > - public static NodeBuilder updateDefinition(NodeBuilder indexDefn){ + public static NodeBuilder updateDefinition(NodeBuilder indexDefn) { return updateDefinition(indexDefn, "unknown"); } - public static NodeBuilder updateDefinition(NodeBuilder indexDefn, String indexPath){ + public static NodeBuilder updateDefinition(NodeBuilder indexDefn, String indexPath) { NodeState defn = indexDefn.getBaseState(); - if (!hasIndexingRules(defn)){ + if (!hasIndexingRules(defn)) { NodeState rulesState = createIndexRules(defn).getNodeState(); indexDefn.setChildNode(FulltextIndexConstants.INDEX_RULES, rulesState); indexDefn.setProperty(INDEX_VERSION, determineIndexFormatVersion(defn).getVersion()); @@ -1564,7 +1585,7 @@ public static NodeBuilder updateDefinition(NodeBuilder indexDefn, String indexPa /** * Constructs IndexingRule based on earlier format of index configuration */ - private static NodeBuilder createIndexRules(NodeState defn){ + private static NodeBuilder createIndexRules(NodeState defn) { NodeBuilder builder = EMPTY_NODE.builder(); Set declaringNodeTypes = getMultiProperty(defn, DECLARING_NODE_TYPES); Set includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES); @@ -1575,19 +1596,19 @@ private static NodeBuilder createIndexRules(NodeState defn){ NodeState propNodeState = defn.getChildNode(FulltextIndexConstants.PROP_NODE); //If no explicit nodeType defined then all config applies for nt:base - if (declaringNodeTypes.isEmpty()){ - declaringNodeTypes = Collections.singleton(NT_BASE); + if (declaringNodeTypes.isEmpty()) { + declaringNodeTypes = Set.of(NT_BASE); } - Set propNamesSet = Sets.newHashSet(); + Set propNamesSet = new HashSet<>(); propNamesSet.addAll(includes); propNamesSet.addAll(excludes); propNamesSet.addAll(orderedProps); //Also include all immediate leaf propNode names - for (ChildNodeEntry cne : propNodeState.getChildNodeEntries()){ + for (ChildNodeEntry cne : propNodeState.getChildNodeEntries()) { if (!propNamesSet.contains(cne.getName()) - && Iterables.isEmpty(cne.getNodeState().getChildNodeNames())){ + && Iterables.isEmpty(cne.getNodeState().getChildNodeNames())) { propNamesSet.add(cne.getName()); } } @@ -1595,25 +1616,24 @@ private static NodeBuilder createIndexRules(NodeState defn){ List propNames = new ArrayList<>(propNamesSet); final String includeAllProp = FulltextIndexConstants.REGEX_ALL_PROPS; - if (fullTextEnabled - && includes.isEmpty()){ + if (fullTextEnabled && includes.isEmpty()) { //Add the regEx for including all properties at the end //for fulltext index and when no explicit includes are defined propNames.add(includeAllProp); } - for (String typeName : declaringNodeTypes){ + for (String typeName : declaringNodeTypes) { NodeBuilder rule = builder.child(typeName); markAsNtUnstructured(rule); - List propNodeNames = newArrayListWithCapacity(propNamesSet.size()); + List propNodeNames = new ArrayList<>(propNamesSet.size()); NodeBuilder propNodes = rule.child(PROP_NODE); int i = 0; - for (String propName : propNames){ + for (String propName : propNames) { String propNodeName = propName; //For proper propName use the propName as childNode name - if(PropertyDefinition.isRelativeProperty(propName) - || propName.equals(includeAllProp)){ + if (PropertyDefinition.isRelativeProperty(propName) + || propName.equals(includeAllProp)) { propNodeName = "prop" + i++; } propNodeNames.add(propNodeName); @@ -1622,9 +1642,9 @@ private static NodeBuilder createIndexRules(NodeState defn){ markAsNtUnstructured(prop); prop.setProperty(FulltextIndexConstants.PROP_NAME, propName); - if (excludes.contains(propName)){ + if (excludes.contains(propName)) { prop.setProperty(FulltextIndexConstants.PROP_INDEX, false); - } else if (fullTextEnabled){ + } else if (fullTextEnabled) { prop.setProperty(FulltextIndexConstants.PROP_ANALYZED, true); prop.setProperty(FulltextIndexConstants.PROP_NODE_SCOPE_INDEX, true); prop.setProperty(FulltextIndexConstants.PROP_USE_IN_EXCERPT, storageEnabled); @@ -1632,18 +1652,18 @@ private static NodeBuilder createIndexRules(NodeState defn){ } else { prop.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX, true); - if (orderedProps.contains(propName)){ + if (orderedProps.contains(propName)) { prop.setProperty(FulltextIndexConstants.PROP_ORDERED, true); } } - if (propName.equals(includeAllProp)){ + if (propName.equals(includeAllProp)) { prop.setProperty(FulltextIndexConstants.PROP_IS_REGEX, true); } else { //Copy over the property configuration - NodeState propDefNode = getPropDefnNode(defn, propName); - if (propDefNode != null){ - for (PropertyState ps : propDefNode.getProperties()){ + NodeState propDefNode = getPropDefnNode(defn, propName); + if (propDefNode != null) { + for (PropertyState ps : propDefNode.getProperties()) { prop.setProperty(ps); } } @@ -1653,7 +1673,7 @@ private static NodeBuilder createIndexRules(NodeState defn){ //If no propertyType defined then default to UNKNOWN such that none //of the properties get indexed PropertyState supportedTypes = defn.getProperty(INCLUDE_PROPERTY_TYPES); - if (supportedTypes == null){ + if (supportedTypes == null) { supportedTypes = PropertyStates.createProperty(INCLUDE_PROPERTY_TYPES, TYPES_ALLOW_ALL_NAME); } rule.setProperty(supportedTypes); @@ -1662,16 +1682,16 @@ private static NodeBuilder createIndexRules(NodeState defn){ rule.setProperty(FulltextIndexConstants.RULE_INHERITED, false); } - propNodes.setProperty(OAK_CHILD_ORDER, propNodeNames ,NAMES); + propNodes.setProperty(OAK_CHILD_ORDER, propNodeNames, NAMES); markAsNtUnstructured(propNodes); } markAsNtUnstructured(builder); - builder.setProperty(OAK_CHILD_ORDER, declaringNodeTypes ,NAMES); + builder.setProperty(OAK_CHILD_ORDER, declaringNodeTypes, NAMES); return builder; } - private static NodeState getPropDefnNode(NodeState defn, String propName){ + private static NodeState getPropDefnNode(NodeState defn, String propName) { NodeState propNode = defn.getChildNode(FulltextIndexConstants.PROP_NODE); NodeState propDefNode; if (PropertyDefinition.isRelativeProperty(propName)) { @@ -1691,8 +1711,8 @@ private static NodeState getPropDefnNode(NodeState defn, String propName){ private int determineMaxExtractLength() { int length = getOptionalValue(definition.getChildNode(TIKA), FulltextIndexConstants.TIKA_MAX_EXTRACT_LENGTH, DEFAULT_MAX_EXTRACT_LENGTH); - if (length < 0){ - return - length * maxFieldLength; + if (length < 0) { + return -length * maxFieldLength; } return length; } @@ -1701,24 +1721,24 @@ private NodeState getTikaConfigNode() { return definition.getChildNode(TIKA).getChildNode(TIKA_CONFIG); } - private static Set getMultiProperty(NodeState definition, String propName){ + private static Set getMultiProperty(NodeState definition, String propName) { PropertyState pse = definition.getProperty(propName); - return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Collections.emptySet(); + return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Set.of(); } private static Set toLowerCase(Set values) { - Set result = newHashSet(); - for(String val : values){ + Set result = new HashSet<>(); + for (String val : values) { result.add(val.toLowerCase(Locale.ENGLISH)); } - return ImmutableSet.copyOf(result); + return Set.copyOf(result); } private static List getAllNodeTypes(ReadOnlyNodeTypeManager ntReg) { try { - List typeNames = newArrayList(); + List typeNames = new ArrayList<>(); NodeTypeIterator ntItr = ntReg.getAllNodeTypes(); - while (ntItr.hasNext()){ + while (ntItr.hasNext()) { typeNames.add(ntItr.nextNodeType().getName()); } return typeNames; @@ -1757,10 +1777,10 @@ private static String getPrimaryTypeName(NodeState state) { private static Iterable getMixinTypeNames(NodeState state) { PropertyState property = state.getProperty(JcrConstants.JCR_MIXINTYPES); - return property != null ? property.getValue(NAMES) : Collections.emptyList(); + return property != null ? property.getValue(NAMES) : List.of(); } - private static boolean hasOrderableChildren(NodeState state){ + private static boolean hasOrderableChildren(NodeState state) { return state.hasProperty(OAK_CHILD_ORDER); } @@ -1769,14 +1789,14 @@ static int getSupportedTypes(NodeState defn, String typePropertyName, int defaul if (pst != null) { int types = 0; for (String inc : pst.getValue(Type.STRINGS)) { - if (TYPES_ALLOW_ALL_NAME.equals(inc)){ + if (TYPES_ALLOW_ALL_NAME.equals(inc)) { return TYPES_ALLOW_ALL; } try { types |= 1 << PropertyType.valueFromName(inc); } catch (IllegalArgumentException e) { - log.warn("Unknown property type: " + inc); + log.warn("Unknown property type: {}", inc); } } return types; @@ -1784,12 +1804,12 @@ static int getSupportedTypes(NodeState defn, String typePropertyName, int defaul return defaultVal; } - static boolean includePropertyType(int includedPropertyTypes, int type){ - if(includedPropertyTypes == TYPES_ALLOW_ALL){ + static boolean includePropertyType(int includedPropertyTypes, int type) { + if (includedPropertyTypes == TYPES_ALLOW_ALL) { return true; } - if (includedPropertyTypes == TYPES_ALLOW_NONE){ + if (includedPropertyTypes == TYPES_ALLOW_NONE) { return false; } @@ -1797,30 +1817,30 @@ static boolean includePropertyType(int includedPropertyTypes, int type){ } private static boolean hasFulltextEnabledIndexRule(List rules) { - for (IndexingRule rule : rules){ - if (rule.isFulltextEnabled()){ + for (IndexingRule rule : rules) { + if (rule.isFulltextEnabled()) { return true; } } return false; } - private static void markAsNtUnstructured(NodeBuilder nb){ + private static void markAsNtUnstructured(NodeBuilder nb) { nb.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); } protected static IndexFormatVersion determineIndexFormatVersion(NodeState defn) { //Compat mode version if specified has highest priority - if (defn.hasProperty(COMPAT_MODE)){ + if (defn.hasProperty(COMPAT_MODE)) { return versionFrom(defn.getProperty(COMPAT_MODE)); } - if (defn.hasProperty(INDEX_VERSION)){ + if (defn.hasProperty(INDEX_VERSION)) { return versionFrom(defn.getProperty(INDEX_VERSION)); } //No existing index data i.e. reindex or fresh index - if (!defn.getChildNode(INDEX_DATA_CHILD_NAME).exists()){ + if (!defn.getChildNode(INDEX_DATA_CHILD_NAME).exists()) { return determineVersionForFreshIndex(defn); } @@ -1831,20 +1851,20 @@ protected static IndexFormatVersion determineIndexFormatVersion(NodeState defn) return fullTextEnabled ? IndexFormatVersion.V1 : IndexFormatVersion.V2; } - static IndexFormatVersion determineVersionForFreshIndex(NodeState defn){ + static IndexFormatVersion determineVersionForFreshIndex(NodeState defn) { return determineVersionForFreshIndex(defn.getProperty(FULL_TEXT_ENABLED), defn.getProperty(COMPAT_MODE), defn.getProperty(INDEX_VERSION)); } - static IndexFormatVersion determineVersionForFreshIndex(NodeBuilder defnb){ + static IndexFormatVersion determineVersionForFreshIndex(NodeBuilder defnb) { return determineVersionForFreshIndex(defnb.getProperty(FULL_TEXT_ENABLED), defnb.getProperty(COMPAT_MODE), defnb.getProperty(INDEX_VERSION)); } private static IndexFormatVersion determineVersionForFreshIndex(PropertyState fulltext, PropertyState compat, - PropertyState version){ - if (compat != null){ + PropertyState version) { + if (compat != null) { return versionFrom(compat); } @@ -1858,20 +1878,20 @@ private static IndexFormatVersion determineVersionForFreshIndex(PropertyState fu IndexFormatVersion result = defaultToUse; //If default configured is lesser than existing then prefer existing - if (existing != null){ - result = IndexFormatVersion.max(result,existing); + if (existing != null) { + result = IndexFormatVersion.max(result, existing); } - //Check if fulltext is false which indicates its a property index and + //Check if fulltext is false which indicates it's a property index and //hence confirm to V2 or above - if (fulltext != null && !fulltext.getValue(Type.BOOLEAN)){ + if (fulltext != null && !fulltext.getValue(Type.BOOLEAN)) { return IndexFormatVersion.max(result, IndexFormatVersion.V2); } return result; } - private static IndexFormatVersion versionFrom(PropertyState ps){ + private static IndexFormatVersion versionFrom(PropertyState ps) { return IndexFormatVersion.getVersion(Ints.checkedCast(ps.getValue(Type.LONG))); } @@ -1898,14 +1918,14 @@ public static boolean supportsSyncOrNRTIndexing(NodeBuilder defn) { private static boolean supportsIndexingMode(NodeBuilder defn, String mode) { PropertyState async = defn.getProperty(IndexConstants.ASYNC_PROPERTY_NAME); - if (async == null){ + if (async == null) { return false; } return Iterables.contains(async.getValue(Type.STRINGS), mode); } protected static NodeState getIndexDefinitionState(NodeState defn) { - if (isDisableStoredIndexDefinition()){ + if (isDisableStoredIndexDefinition()) { return defn; } NodeState storedState = defn.getChildNode(INDEX_DEFINITION_NODE); @@ -1994,7 +2014,7 @@ static MODE getMode(@NotNull final NodeState facetConfigRoot) { String modeString; if (securePS != null) { - if(securePS.getType() == Type.BOOLEAN) { + if (securePS.getType() == Type.BOOLEAN) { // legacy secure config boolean secure = securePS.getValue(Type.BOOLEAN); return secure ? MODE.SECURE : MODE.INSECURE; From 58146387d61e92700bf3f75a1b78d2f87e83070b Mon Sep 17 00:00:00 2001 From: Andrei Dulceanu Date: Mon, 29 Jul 2024 16:24:27 +0200 Subject: [PATCH 33/86] OAK-10978 - Skip Azure compaction when there's not enough garbage in the repository (#1606) --- .../jackrabbit/oak/run/CompactCommand.java | 18 +++++ .../oak/segment/azure/tool/AzureCompact.java | 67 ++++++++++++++++++- .../oak/segment/file/FileStore.java | 2 +- 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java index ced75fc6be6..0f8ee389fed 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java @@ -78,6 +78,16 @@ public void execute(String... args) throws Exception { .withRequiredArg() .ofType(Integer.class) .defaultsTo(50); + OptionSpec garbageThresholdGb = parser.accepts("garbage-threshold-gb", "Minimum amount of garbage in GB (defaults to 0 GB) for " + + "compaction to run") + .withRequiredArg() + .ofType(Integer.class) + .defaultsTo(0); + OptionSpec garbageThresholdPercentage = parser.accepts("garbage-threshold-percentage", "Minimum amount of garbage in percentage (defaults to 0%) for " + + "compaction to run") + .withRequiredArg() + .ofType(Integer.class) + .defaultsTo(0); OptionSet options = parser.parse(args); @@ -111,6 +121,14 @@ public void execute(String... args) throws Exception { azureBuilder.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options)); } + if (options.has(garbageThresholdGb)) { + azureBuilder.withGarbageThresholdGb(garbageThresholdGb.value(options)); + } + + if (options.has(garbageThresholdPercentage)) { + azureBuilder.withGarbageThresholdPercentage(garbageThresholdPercentage.value(options)); + } + if (options.has(tailArg)) { azureBuilder.withGCType(SegmentGCOptions.GCType.TAIL); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java index a5db11950b8..e11f30ca7b4 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java @@ -37,6 +37,8 @@ import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.GCJournal; +import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.spi.persistence.split.SplitPersistence; @@ -88,6 +90,10 @@ public static class Builder { private Integer persistentCacheSizeGb; + private int garbageThresholdGb; + + private int garbageThresholdPercentage; + private CloudBlobDirectory sourceCloudBlobDirectory; private CloudBlobDirectory destinationCloudBlobDirectory; @@ -219,6 +225,29 @@ public Builder withPersistentCacheSizeGb(Integer persistentCacheSizeGb) { return this; } + /** + * The minimum garbage size in GB for the compaction to run. + * @param garbageThresholdGb + * the minimum garbage size in GB for the compaction to run. + * + * @return this builder + */ + public Builder withGarbageThresholdGb(int garbageThresholdGb) { + this.garbageThresholdGb = garbageThresholdGb; + return this; + } + + /** + * The minimum garbage size in percentage for the compaction to run. + * @param garbageThresholdPercentage + * the minimum garbage size in percentage for the compaction to run. + * @return this builder + */ + public Builder withGarbageThresholdPercentage(int garbageThresholdPercentage) { + this.garbageThresholdPercentage = garbageThresholdPercentage; + return this; + } + public Builder withSourceCloudBlobDirectory(CloudBlobDirectory sourceCloudBlobDirectory) { this.sourceCloudBlobDirectory = checkNotNull(sourceCloudBlobDirectory); return this; @@ -243,6 +272,8 @@ public AzureCompact build() { } } + private static final long GB = 1024 * 1024 * 1024; + private final String path; private final String targetPath; @@ -263,6 +294,10 @@ public AzureCompact build() { private final Integer persistentCacheSizeGb; + private final int garbageThresholdGb; + + private final int garbageThresholdPercentage; + private final CloudBlobDirectory sourceCloudBlobDirectory; private final CloudBlobDirectory destinationCloudBlobDirectory; @@ -279,6 +314,8 @@ private AzureCompact(Builder builder) { this.concurrency = builder.concurrency; this.persistentCachePath = builder.persistentCachePath; this.persistentCacheSizeGb = builder.persistentCacheSizeGb; + this.garbageThresholdGb = builder.garbageThresholdGb; + this.garbageThresholdPercentage = builder.garbageThresholdPercentage; this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory; this.destinationCloudBlobDirectory = builder.destinationCloudBlobDirectory; this.azureStorageCredentialManager = new AzureStorageCredentialManager(); @@ -317,10 +354,19 @@ public int run() throws IOException, StorageException, URISyntaxException { } printArchives(System.out, beforeArchives); - System.out.printf(" -> compacting\n"); try (FileStore store = newFileStore(splitPersistence, Files.createTempDir(), strictVersionCheck, segmentCacheSize, gcLogInterval, compactorType, concurrency)) { + if (garbageThresholdGb > 0 && garbageThresholdPercentage > 0) { + System.out.printf(" -> minimum garbage threshold set to %d GB or %d%%\n", garbageThresholdGb, garbageThresholdPercentage); + long currentSize = store.size(); + if (!isGarbageOverMinimumThreshold(currentSize, roPersistence)) { + return 0; + } + } + + System.out.printf(" -> compacting\n"); + boolean success = false; switch (gcType) { case FULL: @@ -369,6 +415,25 @@ public int run() throws IOException, StorageException, URISyntaxException { return 0; } + private boolean isGarbageOverMinimumThreshold(long currentSize, SegmentNodeStorePersistence roPersistence) throws IOException { + long previousSize = 0; + + GCJournalFile gcJournalFile = roPersistence.getGCJournalFile(); + if (gcJournalFile != null) { + GCJournal gcJournal = new GCJournal(gcJournalFile); + GCJournal.GCJournalEntry gcJournalEntry = gcJournal.read(); + previousSize = gcJournalEntry.getRepoSize(); + } + + long potentialGarbage = currentSize - previousSize; + if (currentSize < previousSize || (potentialGarbage < garbageThresholdGb * GB && potentialGarbage < currentSize * garbageThresholdPercentage / 100)) { + System.out.printf(" -> [skipping] not enough garbage -> previous size: %d, current size: %d\n", previousSize, currentSize); + return false; + } + + return true; + } + private long printTargetRepoSizeInfo(CloudBlobContainer container) { System.out.printf("Calculating the size of container %s\n", container.getName()); long size = 0; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java index 6381e1129cf..7e72cf78247 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java @@ -298,7 +298,7 @@ public GCNodeWriteMonitor getGCNodeWriteMonitor() { /** * @return the size of this store. */ - private long size() { + public long size() { try (ShutDownCloser ignored = shutDown.keepAlive()) { return tarFiles.size(); } From 8a72ef89022e6187bb878400301aab93bd16af88 Mon Sep 17 00:00:00 2001 From: Nuno Santos Date: Mon, 29 Jul 2024 18:08:13 +0200 Subject: [PATCH 34/86] OAK-10966 - Indexing job: create optimized version of PersistedLinkedList (#1595) --- .../flatfile/ChildNodeStateProvider.java | 18 +- .../flatfile/FlatFileStoreIterator.java | 74 +++-- .../linkedList/PersistedLinkedListV2.java | 300 ++++++++++++++++++ .../FlatFileBufferLinkedListTest.java | 14 +- .../linkedList/PersistedLinkedListV2Test.java | 120 +++++++ .../spi/editor/FulltextDocumentMaker.java | 53 ++-- 6 files changed, 505 insertions(+), 74 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java index 5ad6b473a75..4123ae0fe7c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java @@ -38,7 +38,6 @@ import static org.apache.jackrabbit.guava.common.collect.Iterators.transform; import static java.util.Collections.emptyIterator; import static org.apache.jackrabbit.oak.commons.PathUtils.getName; -import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.commons.PathUtils.isAncestor; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; @@ -65,14 +64,15 @@ public NodeState getChildNode(@NotNull String name) throws IllegalArgumentExcept } public long getChildNodeCount(long max) { - if (max == 1 && children().hasNext()) { + Iterator childrenIter = children(); + if (max == 1 && childrenIter.hasNext()) { return 1; } - return size(children()); + return size(childrenIter); } public Iterable getChildNodeNames() { - return () -> transform(children(), p -> name(p)); + return () -> transform(children(), ChildNodeStateProvider::name); } @NotNull @@ -100,14 +100,14 @@ Iterator children(boolean preferred) { "Did not found path [%s] in leftover iterator. Possibly node state accessed " + "after main iterator has moved past it", path); - //Prepare an iterator to fetch all child node paths i.e. immediate and there children - return new AbstractIterator() { + //Prepare an iterator to fetch all child node paths i.e. immediate and their children + return new AbstractIterator<>() { @Override protected NodeStateEntry computeNext() { while (pitr.hasNext() && isAncestor(path, pitr.peek().getPath())) { NodeStateEntry nextEntry = pitr.next(); String nextEntryPath = nextEntry.getPath(); - if (isImmediateChild(nextEntryPath)) { + if (PathUtils.isDirectAncestor(path, nextEntryPath)) { String nextEntryName = PathUtils.getName(nextEntryPath); if (preferred && !preferredPathElements.contains(nextEntryName)) { return endOfData(); @@ -123,8 +123,4 @@ protected NodeStateEntry computeNext() { private static String name(NodeStateEntry p) { return getName(p.getPath()); } - - private boolean isImmediateChild(String childPath){ - return getParentPath(childPath).equals(path); - } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java index 69a7c58f3eb..cd80ac5e1c6 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java @@ -19,34 +19,44 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile; -import static org.apache.jackrabbit.guava.common.collect.Iterators.concat; -import static org.apache.jackrabbit.guava.common.collect.Iterators.singletonIterator; - -import java.io.Closeable; -import java.util.Iterator; -import java.util.Set; - +import org.apache.jackrabbit.guava.common.collect.AbstractIterator; +import org.apache.jackrabbit.oak.commons.IOUtils; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.NodeStateEntryList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedList; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedListV2; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.jackrabbit.guava.common.collect.AbstractIterator; +import java.io.Closeable; +import java.util.Iterator; +import java.util.Set; + +import static org.apache.jackrabbit.guava.common.collect.Iterators.concat; +import static org.apache.jackrabbit.guava.common.collect.Iterators.singletonIterator; class FlatFileStoreIterator extends AbstractIterator implements Iterator, Closeable { - private static final Logger log = LoggerFactory.getLogger(FlatFileStoreIterator.class); + private static final Logger LOG = LoggerFactory.getLogger(FlatFileStoreIterator.class); static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB"; // by default, use the PersistedLinkedList private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0; - static final String PERSISTED_LINKED_LIST_CACHE_SIZE = "oak.indexer.persistedLinkedList.cacheSize"; - static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000; + public static final String PERSISTED_LINKED_LIST_CACHE_SIZE = "oak.indexer.persistedLinkedList.cacheSize"; + public static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000; + + public static final String PERSISTED_LINKED_LIST_V2_CACHE_SIZE = "oak.indexer.persistedLinkedListV2.cacheSize"; + public static final int DEFAULT_PERSISTED_LINKED_LIST_V2_CACHE_SIZE = 10000; + + public static final String PERSISTED_LINKED_LIST_V2_MEMORY_CACHE_SIZE_MB = "oak.indexer.persistedLinkedListV2.cacheMaxSizeMB"; + public static final int DEFAULT_PERSISTED_LINKED_LIST_V2_MEMORY_CACHE_SIZE_MB = 8; + + public static final String PERSISTED_LINKED_LIST_USE_V2 = "oak.indexer.persistedLinkedList.useV2"; + public static final boolean DEFAULT_PERSISTED_LINKED_LIST_USE_V2 = false; private final Iterator baseItr; private final NodeStateEntryList buffer; @@ -66,16 +76,23 @@ public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator maxBufferSize) { - maxBufferSize = buffer.size(); - log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", - maxBufferSize, maxBufferSizeBytes, current.getPath()); - } - if (buffer.estimatedMemoryUsage() > maxBufferSizeBytes) { - maxBufferSizeBytes = buffer.estimatedMemoryUsage(); - log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", - maxBufferSize, maxBufferSizeBytes, current.getPath()); - } if (!buffer.isEmpty()) { + if (buffer.size() > maxBufferSize) { + maxBufferSize = buffer.size(); + LOG.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", + maxBufferSize, maxBufferSizeBytes, current.getPath()); + } + if (buffer.estimatedMemoryUsage() > maxBufferSizeBytes) { + maxBufferSizeBytes = buffer.estimatedMemoryUsage(); + LOG.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", + maxBufferSize, maxBufferSizeBytes, current.getPath()); + } NodeStateEntry e = buffer.remove(); return wrapIfNeeded(e); } @@ -149,7 +167,7 @@ protected NodeStateEntry computeNext() { } private NodeStateEntry wrapIfNeeded(NodeStateEntry e) { - if (buffer instanceof PersistedLinkedList) { + if (buffer instanceof PersistedLinkedList || buffer instanceof PersistedLinkedListV2) { // for the PersistedLinkedList, the entries from the iterators are // de-serialized and don't contain the LazyChildrenNodeState - // so we need to wrap them diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java new file mode 100644 index 00000000000..eeab22c89a9 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList; + +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.guava.common.base.Preconditions; +import org.apache.jackrabbit.oak.commons.IOUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import org.h2.mvstore.MVStoreTool; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; + +/** + * A persistent linked list that internally uses the MVStore. This list keeps an in-memory cache, writing to the + * persistent store the nodes only when the cache is full. The in-memory cache is limited by two parameters: + * + *

    + *
  • cacheSize: the maximum number of elements to keep in the in-memory cache
    • + *
  • cacheSizeMB: the maximum size of the in-memory cache in MB
    • + *
+ *

+ * The recommended configuration is to rely on the total memory usage to limit the cache, giving as much memory as + * available in the JVM, and setting a very high limit for the number of elements. A cache miss has a very high cost, + * so it should be avoided as much as possible. + *

+ *

+ * Each element is stored either in the cache or in the persistent store, but not in both. And elements are not moved + * between the two tiers, so even if there is a cache miss, that element will remain in the persistent store. + * For the access pattern of the indexer, this policy has a lower rate of cache misses than if we move to the cache an + * element after a miss. + *

+ * To understand why, let's assume we want to traverse the children of a node P that is at line/position 100 in the FFS. + * When we call getChildren on P, this creates an iterator that scans from position 100 for all the children. + * If we call recursively getChildren on a child C of P, this will also create a new iterator that will start also at + * position 100. Therefore, the iterators will frequently scan from 100 down. Let's assume the cache can only hold 10 + * nodes and that we use a policy of moving to the cache the last node that was accessed. Then, if an iterator scans + * from 100 to, let's say 150, when it finishes iterating, the nodes 141 to 150 will be the only ones in the cache. + * The next iterator that is scanning from 100 will have cache misses for all the nodes until 140. And this will repeat + * for every new iterator. On the other hand, if we keep in the cache the nodes from 100 to 109, every iterator starting + * from 100 will at least have 10 cache hits, which is better than having a cache miss for all elements. + */ +public class PersistedLinkedListV2 implements NodeStateEntryList { + + private final static Logger LOG = LoggerFactory.getLogger(PersistedLinkedListV2.class); + + private static final String COMPACT_STORE_MILLIS_NAME = "oak.indexer.linkedList.compactMillis"; + + private final HashMap cache = new HashMap<>(512); + private final int compactStoreMillis = Integer.getInteger(COMPACT_STORE_MILLIS_NAME, 60 * 1000); + private final NodeStateEntryWriter writer; + private final NodeStateEntryReader reader; + private final String storeFileName; + private final long cacheSizeLimitBytes; + private final long cacheSizeLimit; + + private MVStore store; + private MVMap map; + private long headIndex; + private long tailIndex; + // Total entries in the list + private long totalEntries; + private long lastLog; + private long lastCompact; + + // Estimation of the cache size + private long cacheSizeEstimationBytes; + // If the sanity check on the cache size estimation triggers, log it only once to avoid filling up the logs with + // the same message + private boolean loggedCacheSizeEstimationMismatch = false; + + // Metrics + private long cacheHits; + private long cacheMisses; + private long storeWrites; // Each cache miss is a read from the store, so no need for a storeRead counter + private long peakCacheSizeBytes; + private long peakCacheSize; + + /** + * @param cacheSize the maximum number of elements to keep in the in-memory cache + * @param cacheSizeMB the maximum size of the in-memory cache in MB + */ + public PersistedLinkedListV2(String fileName, NodeStateEntryWriter writer, NodeStateEntryReader reader, int cacheSize, int cacheSizeMB) { + this.cacheSizeLimit = cacheSize; + this.cacheSizeLimitBytes = ((long) cacheSizeMB) * 1024 * 1024; + this.storeFileName = fileName; + LOG.info("Opening store {}", fileName); + File oldFile = new File(fileName); + if (oldFile.exists()) { + LOG.info("Deleting {}", fileName); + try { + FileUtils.forceDelete(oldFile); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + openStore(); + this.writer = writer; + this.reader = reader; + lastCompact = System.currentTimeMillis(); + } + + private void openStore() { + store = MVStore.open(storeFileName); + map = store.openMap("list"); + } + + @Override + public void add(@NotNull NodeStateEntry item) { + Preconditions.checkArgument(item != null, "Can't add null to the list"); + long index = tailIndex++; + addEntryToCache(index, item); + } + + @Override + public boolean isEmpty() { + return totalEntries == 0; + } + + @Override + public Iterator iterator() { + return new NodeIterator(headIndex); + } + + @Override + public NodeStateEntry remove() { + Preconditions.checkState(!isEmpty(), "Cannot remove item from empty list"); + Long boxedHeadIndex = headIndex; + NodeStateEntry entryRemoved = cache.remove(boxedHeadIndex); + if (entryRemoved == null) { + String mapEntry = map.remove(boxedHeadIndex); + if (mapEntry == null) { + throw new IllegalStateException("Entry not found in cache or in store: " + boxedHeadIndex); + } + cacheMisses++; + entryRemoved = reader.read(mapEntry); + } else { + cacheHits++; + cacheSizeEstimationBytes -= entryRemoved.estimatedMemUsage(); + } + + headIndex++; + totalEntries--; + if (totalEntries == 0) { + if (cacheSizeEstimationBytes != 0 && !loggedCacheSizeEstimationMismatch) { + loggedCacheSizeEstimationMismatch = true; + LOG.warn("Total entries is 0, but cache size estimation is not zero: {}. Metrics: {}", cacheSizeEstimationBytes, formatMetrics()); + } + map.clear(); + cache.clear(); + } + return entryRemoved; + } + + private NodeStateEntry get(Long index) { + NodeStateEntry result = cache.get(index); + if (result == null) { + cacheMisses++; + String s = map.get(index); + result = reader.read(s); + LOG.trace("Cache miss: {}={}", index, result.getPath()); + } else { + cacheHits++; + } + return result; + } + + private void addEntryToCache(long index, NodeStateEntry entry) { + long now = System.currentTimeMillis(); + long newCacheSizeBytes = cacheSizeEstimationBytes + entry.estimatedMemUsage(); + if (cache.size() >= cacheSizeLimit || newCacheSizeBytes > cacheSizeLimitBytes) { + if (LOG.isTraceEnabled()) { + LOG.trace("Mem cache size {}/{} or byte size {}/{} would exceed maximum. Writing to persistent map: {} = {}, entry size: {}", + newCacheSizeBytes, cacheSizeLimitBytes, cache.size() + 1, cacheSizeLimit, index, entry.getPath(), entry.estimatedMemUsage()); + } + storeWrites++; + map.put(index, writer.toString(entry)); + long storeSizeBytes = store.getFileStore().size(); + boolean compactNow = now >= lastCompact + compactStoreMillis; + if (compactNow && storeSizeBytes > 10L * 1000 * 1000) { + // compact once a minute, if larger than 10 MB + LOG.info("Compacting. Current size: {}", storeSizeBytes); + store.close(); + MVStoreTool.compact(storeFileName, true); + openStore(); + lastCompact = System.currentTimeMillis(); + LOG.info("Finished compaction. Previous size: {}, new size={} bytes", storeSizeBytes, store.getFileStore().size()); + } + } else { + // new element fits in the in-memory cache. Do not write it to the persistent store. + cache.put(index, entry); + cacheSizeEstimationBytes = newCacheSizeBytes; + if (cacheSizeEstimationBytes > peakCacheSizeBytes) { + peakCacheSizeBytes = cacheSizeEstimationBytes; + } + if (cache.size() > peakCacheSize) { + peakCacheSize = cache.size(); + } + } + totalEntries++; + if (index % (1024 * 1000) == 0 || now >= lastLog + 10000) { // Print every million entries or every 10 seconds + LOG.info("Metrics: {}", formatMetrics()); + lastLog = now; + } + } + + @Override + public int size() { + return (int) totalEntries; + } + + @Override + public void close() { + store.close(); + LOG.info("Closing. Metrics: {}", formatMetrics()); + } + + public String formatMetrics() { + return "totalEntriesAdded: " + headIndex + ", totalEntriesInList: " + totalEntries + ", mapSize: " + map.sizeAsLong() + + ", mapSizeBytes: " + store.getFileStore().size() + " (" + IOUtils.humanReadableByteCountBin(store.getFileStore().size()) + ")" + + ", cacheHits: " + cacheHits + ", cacheMisses: " + cacheMisses + ", storeWrites: " + storeWrites + + ", peakCacheSize: " + peakCacheSize + + ", peakCacheSizeBytes: " + peakCacheSizeBytes + " (" + IOUtils.humanReadableByteCountBin(peakCacheSizeBytes) + ")"; + } + + @Override + public long estimatedMemoryUsage() { + return cacheSizeEstimationBytes; + } + + /** + * A node iterator over this list. + */ + final class NodeIterator implements Iterator { + + private long index; + + NodeIterator(long index) { + this.index = index; + } + + @Override + public boolean hasNext() { + return index < tailIndex; + } + + @Override + public NodeStateEntry next() { + if (index < headIndex || index >= tailIndex) { + throw new IllegalStateException(); + } + return get(index++); + } + } + + public long getCacheHits() { + return cacheHits; + } + + public long getCacheMisses() { + return cacheMisses; + } + + public long getStoreWrites() { + return storeWrites; + } + + public long getPeakCacheSizeBytes() { + return peakCacheSizeBytes; + } + + public long getPeakCacheSize() { + return peakCacheSize; + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java index 8fa844d736d..0c2abaab4ea 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java @@ -144,10 +144,10 @@ public void isEmpty() { public void memUsage() { Assert.assertEquals("Empty list must be estimate 0", 0, list.estimatedMemoryUsage()); - list.add(new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(20).build()); + list.add(testNode("/", 20)); Assert.assertEquals(20, list.estimatedMemoryUsage()); - list.add(new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(30).build()); + list.add(testNode("/", 30)); Assert.assertEquals(50, list.estimatedMemoryUsage()); list.remove(); @@ -157,8 +157,8 @@ public void memUsage() { @Test public void memLimit() { list = new FlatFileBufferLinkedList(10); - NodeStateEntry e10Bytes = new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(10).build(); - NodeStateEntry e1Byte = new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(1).build(); + NodeStateEntry e10Bytes = testNode("/", 10); + NodeStateEntry e1Byte = testNode("/", 1); list.add(e10Bytes); //this should succeed @@ -181,7 +181,11 @@ public void basics() { Assert.assertTrue("Adding an item should be available", list.iterator().hasNext()); } - private NodeStateEntry testNode(String n) { + protected NodeStateEntry testNode(String n) { return new NodeStateEntryBuilder(EMPTY_NODE, n).build(); } + + protected NodeStateEntry testNode(String n, long memoryUsage) { + return new NodeStateEntryBuilder(EMPTY_NODE, n).withMemUsage(memoryUsage).build(); + } } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java new file mode 100644 index 00000000000..37ea219eed4 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList; + +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +public class PersistedLinkedListV2Test extends FlatFileBufferLinkedListTest { + + @Rule + public final TemporaryFolder folder = new TemporaryFolder(new File("target")); + + @Before + public void setup() throws IOException { + list = newList(2, 1); + } + + private PersistedLinkedListV2 newList(int cacheSize, int cacheSizeMB) throws IOException { + String fileName = folder.newFile().getAbsolutePath(); + BlobStore blobStore = new MemoryBlobStore(); + NodeStateEntryReader reader = new NodeStateEntryReader(blobStore); + NodeStateEntryWriter writer = new NodeStateEntryWriter(blobStore); + return new PersistedLinkedListV2(fileName, writer, reader, cacheSize, cacheSizeMB); + } + + @After + public void tearDown() { + list.close(); + } + + @Test + public void testCacheHitsMissNumberOfEntries() throws IOException { + PersistedLinkedListV2 persistedLinkedList = newList(2, 1); + persistedLinkedList.add(testNode("/")); + persistedLinkedList.add(testNode("/a")); + persistedLinkedList.add(testNode("/a/b")); + + Assert.assertEquals(3, persistedLinkedList.size()); + ArrayList names1 = extracted(persistedLinkedList.iterator()); + Assert.assertEquals(List.of("/", "/a", "/a/b"), names1); + Assert.assertEquals(2, persistedLinkedList.getCacheHits()); + Assert.assertEquals(1, persistedLinkedList.getCacheMisses()); + + ArrayList names2 = extracted(persistedLinkedList.iterator()); + Assert.assertEquals(List.of("/", "/a", "/a/b"), names2); + Assert.assertEquals(4, persistedLinkedList.getCacheHits()); + Assert.assertEquals(2, persistedLinkedList.getCacheMisses()); + + // Does not count as a cache miss/hit + Assert.assertEquals("/", persistedLinkedList.remove().getPath()); + + var names3 = extracted(persistedLinkedList.iterator()); + Assert.assertEquals(List.of("/a", "/a/b"), names3); + Assert.assertEquals(6, persistedLinkedList.getCacheHits()); + Assert.assertEquals(3, persistedLinkedList.getCacheMisses()); + + Assert.assertEquals("/a", persistedLinkedList.remove().getPath()); + Assert.assertEquals("/a/b", persistedLinkedList.remove().getPath()); + + var names4 = extracted(persistedLinkedList.iterator()); + Assert.assertTrue(names4.isEmpty()); + Assert.assertEquals(7, persistedLinkedList.getCacheHits()); + Assert.assertEquals(4, persistedLinkedList.getCacheMisses()); + } + + @Test + public void testCacheHitsMissSizeOfEntries() throws IOException { + PersistedLinkedListV2 persistedLinkedList = newList(100, 1); + persistedLinkedList.add(testNode("/", 500 * 1024)); + persistedLinkedList.add(testNode("/a", 500 * 1024)); + persistedLinkedList.add(testNode("/a/b", 500 * 1024)); + + // The first two nodes should be kept in the cache. The last node exceeds the cache size and should be stored in + // the persistent map. So there will be 2 cache hits and one miss. + Assert.assertEquals(3, persistedLinkedList.size()); + ArrayList names1 = extracted(persistedLinkedList.iterator()); + Assert.assertEquals(List.of("/", "/a", "/a/b"), names1); + Assert.assertEquals(2, persistedLinkedList.getCacheHits()); + Assert.assertEquals(1, persistedLinkedList.getCacheMisses()); + } + + private static ArrayList extracted(Iterator iter) { + ArrayList names = new ArrayList<>(); + while (iter.hasNext()) { + names.add(iter.next().getPath()); + } + return names; + } +} diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java index 864ace265e1..8516194a63d 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java @@ -236,11 +236,10 @@ private boolean indexFacets(D doc, PropertyState property, String pname, Propert int idxDefinedTag = pd.getType(); // Try converting type to the defined type in the index definition if (tag != idxDefinedTag) { - log.debug("[{}] Facet property defined with type {} differs from property {} with type {} in " - + "path {}", - getIndexName(), - Type.fromTag(idxDefinedTag, false), property, - Type.fromTag(tag, false), path); + if (log.isDebugEnabled()) { + log.debug("[{}] Facet property defined with type {} differs from property {} with type {} in path {}", + getIndexName(), Type.fromTag(idxDefinedTag, false), property, Type.fromTag(tag, false), path); + } tag = idxDefinedTag; } return indexFacetProperty(doc, tag, property, pname); @@ -263,9 +262,8 @@ private boolean indexProperty(String path, dirty = true; } } catch (Exception e) { - log.error( - "could not index similarity field for property {} and definition {} for path {}", - property.getName(), pd, path); + log.error("could not index similarity field for property {} and definition {} for path {}", + property.getName(), pd, path); } } else if (Type.BINARY.tag() == property.getType().tag() && includeTypeForFullText) { @@ -318,9 +316,8 @@ private boolean indexProperty(String path, indexSimilarityStrings(doc, pd, value); } } catch (Exception e) { - log.error( - "could not index similarity field for property {} and definition {} for path {}", - property.getName(), pd, path); + log.error("could not index similarity field for property {} and definition {} for path {}", + property.getName(), pd, path); } } } @@ -389,10 +386,8 @@ private boolean addTypedOrderedFields(D doc, // Also there is no handling for different indexes having the same property since those are usually different versions of the same index. if (!throttleWarnLogs || MV_ORDERED_PROPERTY_SET.add(pname) || WARN_LOG_COUNTER_MV_ORDERED_PROPERTY.incrementAndGet() % throttleWarnLogThreshold == 0) { - log.warn( - "[{}] Ignoring ordered property {} of type {} for path {} as multivalued ordered property not supported", - getIndexName(), pname, - Type.fromTag(property.getType().tag(), true), path); + log.warn("[{}] Ignoring ordered property {} of type {} for path {} as multivalued ordered property not supported", + getIndexName(), pname, Type.fromTag(property.getType().tag(), true), path); } return false; } @@ -401,12 +396,10 @@ private boolean addTypedOrderedFields(D doc, int idxDefinedTag = pd.getType(); // Try converting type to the defined type in the index definition if (tag != idxDefinedTag) { - log.debug( - "[{}] Ordered property defined with type {} differs from property {} with type {} in " - + "path {}", - getIndexName(), - Type.fromTag(idxDefinedTag, false), property, - Type.fromTag(tag, false), path); + if (log.isDebugEnabled()) { + log.debug("[{}] Ordered property defined with type {} differs from property {} with type {} in path {}", + getIndexName(), Type.fromTag(idxDefinedTag, false), property, Type.fromTag(tag, false), path); + } tag = idxDefinedTag; } return indexTypeOrderedFields(doc, pname, tag, property, pd); @@ -606,7 +599,7 @@ public void onResult(Aggregate.PropertyIncludeResult result) { */ private boolean indexAggregatedNode(String path, D doc, Aggregate.NodeIncludeResult result) { //rule for node being aggregated might be null if such nodes - //are not indexed on there own. In such cases we rely in current + //are not indexed on their own. In such cases we rely on current //rule for some checks IndexDefinition.IndexingRule ruleAggNode = definition .getApplicableIndexingRule(getPrimaryTypeName(result.nodeState)); @@ -633,19 +626,19 @@ private boolean indexAggregatedNode(String path, D doc, Aggregate.NodeIncludeRes //Check if any explicit property defn is defined via relative path // and is marked to exclude this property from being indexed. We exclude //it from aggregation if - // 1. Its not to be indexed i.e. index=false - // 2. Its explicitly excluded from aggregation i.e. excludeFromAggregation=true + // 1. It's not to be indexed i.e. index=false + // 2. It's explicitly excluded from aggregation i.e. excludeFromAggregation=true PropertyDefinition pdForRootNode = indexingRule.getConfig(propertyPath); if (pdForRootNode != null && (!pdForRootNode.index || pdForRootNode.excludeFromAggregate)) { continue; } if (Type.BINARY == property.getType()) { - String aggreagtedNodePath = PathUtils.concat(path, result.nodePath); + String aggregatedNodePath = PathUtils.concat(path, result.nodePath); //Here the fulltext is being created for aggregate root hence nodePath passed //should be null String nodePath = result.isRelativeNode() ? result.rootIncludePath : null; - List binaryValues = newBinary(property, result.nodeState, aggreagtedNodePath + "@" + pname); + List binaryValues = newBinary(property, result.nodeState, aggregatedNodePath + "@" + pname); addBinary(doc, nodePath, binaryValues); dirty = true; } else { @@ -682,7 +675,7 @@ protected boolean indexDynamicBoost(D doc, String propertyName, String nodeName, continue; } if (p.isArray()) { - log.warn(p.getName() + " is an array: {}", parentName); + log.warn("{} is an array: {}", p.getName(), parentName); continue; } String dynaTagValue = p.getValue(Type.STRING); @@ -692,18 +685,18 @@ protected boolean indexDynamicBoost(D doc, String propertyName, String nodeName, continue; } if (p.isArray()) { - log.warn(p.getName() + " is an array: {}", parentName); + log.warn("{} is an array: {}", p.getName(), parentName); continue; } double dynaTagConfidence; try { dynaTagConfidence = p.getValue(Type.DOUBLE); } catch (NumberFormatException e) { - log.warn(p.getName() + " parsing failed: {}", parentName, e); + log.warn("{} parsing failed: {}", p.getName(), parentName, e); continue; } if (!Double.isFinite(dynaTagConfidence)) { - log.warn(p.getName() + " is not finite: {}", parentName); + log.warn("{} is not finite: {}", p.getName(), parentName); continue; } From 560cc483a9fc176a933c13f0fe334a578870bc15 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 10:34:45 +0200 Subject: [PATCH 35/86] Revert "OAK-10966 - Indexing job: create optimized version of PersistedLinkedList (#1595)" This reverts commit 8a72ef89022e6187bb878400301aab93bd16af88. --- .../flatfile/ChildNodeStateProvider.java | 18 +- .../flatfile/FlatFileStoreIterator.java | 74 ++--- .../linkedList/PersistedLinkedListV2.java | 300 ------------------ .../FlatFileBufferLinkedListTest.java | 14 +- .../linkedList/PersistedLinkedListV2Test.java | 120 ------- .../spi/editor/FulltextDocumentMaker.java | 53 ++-- 6 files changed, 74 insertions(+), 505 deletions(-) delete mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java delete mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java index 4123ae0fe7c..5ad6b473a75 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java @@ -38,6 +38,7 @@ import static org.apache.jackrabbit.guava.common.collect.Iterators.transform; import static java.util.Collections.emptyIterator; import static org.apache.jackrabbit.oak.commons.PathUtils.getName; +import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.commons.PathUtils.isAncestor; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; @@ -64,15 +65,14 @@ public NodeState getChildNode(@NotNull String name) throws IllegalArgumentExcept } public long getChildNodeCount(long max) { - Iterator childrenIter = children(); - if (max == 1 && childrenIter.hasNext()) { + if (max == 1 && children().hasNext()) { return 1; } - return size(childrenIter); + return size(children()); } public Iterable getChildNodeNames() { - return () -> transform(children(), ChildNodeStateProvider::name); + return () -> transform(children(), p -> name(p)); } @NotNull @@ -100,14 +100,14 @@ Iterator children(boolean preferred) { "Did not found path [%s] in leftover iterator. Possibly node state accessed " + "after main iterator has moved past it", path); - //Prepare an iterator to fetch all child node paths i.e. immediate and their children - return new AbstractIterator<>() { + //Prepare an iterator to fetch all child node paths i.e. immediate and there children + return new AbstractIterator() { @Override protected NodeStateEntry computeNext() { while (pitr.hasNext() && isAncestor(path, pitr.peek().getPath())) { NodeStateEntry nextEntry = pitr.next(); String nextEntryPath = nextEntry.getPath(); - if (PathUtils.isDirectAncestor(path, nextEntryPath)) { + if (isImmediateChild(nextEntryPath)) { String nextEntryName = PathUtils.getName(nextEntryPath); if (preferred && !preferredPathElements.contains(nextEntryName)) { return endOfData(); @@ -123,4 +123,8 @@ protected NodeStateEntry computeNext() { private static String name(NodeStateEntry p) { return getName(p.getPath()); } + + private boolean isImmediateChild(String childPath){ + return getParentPath(childPath).equals(path); + } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java index cd80ac5e1c6..69a7c58f3eb 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java @@ -19,44 +19,34 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile; -import org.apache.jackrabbit.guava.common.collect.AbstractIterator; -import org.apache.jackrabbit.oak.commons.IOUtils; +import static org.apache.jackrabbit.guava.common.collect.Iterators.concat; +import static org.apache.jackrabbit.guava.common.collect.Iterators.singletonIterator; + +import java.io.Closeable; +import java.util.Iterator; +import java.util.Set; + import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.NodeStateEntryList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedList; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedListV2; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Closeable; -import java.util.Iterator; -import java.util.Set; - -import static org.apache.jackrabbit.guava.common.collect.Iterators.concat; -import static org.apache.jackrabbit.guava.common.collect.Iterators.singletonIterator; +import org.apache.jackrabbit.guava.common.collect.AbstractIterator; class FlatFileStoreIterator extends AbstractIterator implements Iterator, Closeable { - private static final Logger LOG = LoggerFactory.getLogger(FlatFileStoreIterator.class); + private static final Logger log = LoggerFactory.getLogger(FlatFileStoreIterator.class); static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB"; // by default, use the PersistedLinkedList private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0; + static final String PERSISTED_LINKED_LIST_CACHE_SIZE = "oak.indexer.persistedLinkedList.cacheSize"; + static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000; - public static final String PERSISTED_LINKED_LIST_CACHE_SIZE = "oak.indexer.persistedLinkedList.cacheSize"; - public static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000; - - public static final String PERSISTED_LINKED_LIST_V2_CACHE_SIZE = "oak.indexer.persistedLinkedListV2.cacheSize"; - public static final int DEFAULT_PERSISTED_LINKED_LIST_V2_CACHE_SIZE = 10000; - - public static final String PERSISTED_LINKED_LIST_V2_MEMORY_CACHE_SIZE_MB = "oak.indexer.persistedLinkedListV2.cacheMaxSizeMB"; - public static final int DEFAULT_PERSISTED_LINKED_LIST_V2_MEMORY_CACHE_SIZE_MB = 8; - - public static final String PERSISTED_LINKED_LIST_USE_V2 = "oak.indexer.persistedLinkedList.useV2"; - public static final boolean DEFAULT_PERSISTED_LINKED_LIST_USE_V2 = false; private final Iterator baseItr; private final NodeStateEntryList buffer; @@ -76,23 +66,16 @@ public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator maxBufferSize) { + maxBufferSize = buffer.size(); + log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", + maxBufferSize, maxBufferSizeBytes, current.getPath()); + } + if (buffer.estimatedMemoryUsage() > maxBufferSizeBytes) { + maxBufferSizeBytes = buffer.estimatedMemoryUsage(); + log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", + maxBufferSize, maxBufferSizeBytes, current.getPath()); + } if (!buffer.isEmpty()) { - if (buffer.size() > maxBufferSize) { - maxBufferSize = buffer.size(); - LOG.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", - maxBufferSize, maxBufferSizeBytes, current.getPath()); - } - if (buffer.estimatedMemoryUsage() > maxBufferSizeBytes) { - maxBufferSizeBytes = buffer.estimatedMemoryUsage(); - LOG.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", - maxBufferSize, maxBufferSizeBytes, current.getPath()); - } NodeStateEntry e = buffer.remove(); return wrapIfNeeded(e); } @@ -167,7 +149,7 @@ protected NodeStateEntry computeNext() { } private NodeStateEntry wrapIfNeeded(NodeStateEntry e) { - if (buffer instanceof PersistedLinkedList || buffer instanceof PersistedLinkedListV2) { + if (buffer instanceof PersistedLinkedList) { // for the PersistedLinkedList, the entries from the iterators are // de-serialized and don't contain the LazyChildrenNodeState - // so we need to wrap them diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java deleted file mode 100644 index eeab22c89a9..00000000000 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList; - -import org.apache.commons.io.FileUtils; -import org.apache.jackrabbit.guava.common.base.Preconditions; -import org.apache.jackrabbit.oak.commons.IOUtils; -import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter; -import org.h2.mvstore.MVMap; -import org.h2.mvstore.MVStore; -import org.h2.mvstore.MVStoreTool; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; - -/** - * A persistent linked list that internally uses the MVStore. This list keeps an in-memory cache, writing to the - * persistent store the nodes only when the cache is full. The in-memory cache is limited by two parameters: - * - *

    - *
  • cacheSize: the maximum number of elements to keep in the in-memory cache
    • - *
  • cacheSizeMB: the maximum size of the in-memory cache in MB
    • - *
- *

- * The recommended configuration is to rely on the total memory usage to limit the cache, giving as much memory as - * available in the JVM, and setting a very high limit for the number of elements. A cache miss has a very high cost, - * so it should be avoided as much as possible. - *

- *

- * Each element is stored either in the cache or in the persistent store, but not in both. And elements are not moved - * between the two tiers, so even if there is a cache miss, that element will remain in the persistent store. - * For the access pattern of the indexer, this policy has a lower rate of cache misses than if we move to the cache an - * element after a miss. - *

- * To understand why, let's assume we want to traverse the children of a node P that is at line/position 100 in the FFS. - * When we call getChildren on P, this creates an iterator that scans from position 100 for all the children. - * If we call recursively getChildren on a child C of P, this will also create a new iterator that will start also at - * position 100. Therefore, the iterators will frequently scan from 100 down. Let's assume the cache can only hold 10 - * nodes and that we use a policy of moving to the cache the last node that was accessed. Then, if an iterator scans - * from 100 to, let's say 150, when it finishes iterating, the nodes 141 to 150 will be the only ones in the cache. - * The next iterator that is scanning from 100 will have cache misses for all the nodes until 140. And this will repeat - * for every new iterator. On the other hand, if we keep in the cache the nodes from 100 to 109, every iterator starting - * from 100 will at least have 10 cache hits, which is better than having a cache miss for all elements. - */ -public class PersistedLinkedListV2 implements NodeStateEntryList { - - private final static Logger LOG = LoggerFactory.getLogger(PersistedLinkedListV2.class); - - private static final String COMPACT_STORE_MILLIS_NAME = "oak.indexer.linkedList.compactMillis"; - - private final HashMap cache = new HashMap<>(512); - private final int compactStoreMillis = Integer.getInteger(COMPACT_STORE_MILLIS_NAME, 60 * 1000); - private final NodeStateEntryWriter writer; - private final NodeStateEntryReader reader; - private final String storeFileName; - private final long cacheSizeLimitBytes; - private final long cacheSizeLimit; - - private MVStore store; - private MVMap map; - private long headIndex; - private long tailIndex; - // Total entries in the list - private long totalEntries; - private long lastLog; - private long lastCompact; - - // Estimation of the cache size - private long cacheSizeEstimationBytes; - // If the sanity check on the cache size estimation triggers, log it only once to avoid filling up the logs with - // the same message - private boolean loggedCacheSizeEstimationMismatch = false; - - // Metrics - private long cacheHits; - private long cacheMisses; - private long storeWrites; // Each cache miss is a read from the store, so no need for a storeRead counter - private long peakCacheSizeBytes; - private long peakCacheSize; - - /** - * @param cacheSize the maximum number of elements to keep in the in-memory cache - * @param cacheSizeMB the maximum size of the in-memory cache in MB - */ - public PersistedLinkedListV2(String fileName, NodeStateEntryWriter writer, NodeStateEntryReader reader, int cacheSize, int cacheSizeMB) { - this.cacheSizeLimit = cacheSize; - this.cacheSizeLimitBytes = ((long) cacheSizeMB) * 1024 * 1024; - this.storeFileName = fileName; - LOG.info("Opening store {}", fileName); - File oldFile = new File(fileName); - if (oldFile.exists()) { - LOG.info("Deleting {}", fileName); - try { - FileUtils.forceDelete(oldFile); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - openStore(); - this.writer = writer; - this.reader = reader; - lastCompact = System.currentTimeMillis(); - } - - private void openStore() { - store = MVStore.open(storeFileName); - map = store.openMap("list"); - } - - @Override - public void add(@NotNull NodeStateEntry item) { - Preconditions.checkArgument(item != null, "Can't add null to the list"); - long index = tailIndex++; - addEntryToCache(index, item); - } - - @Override - public boolean isEmpty() { - return totalEntries == 0; - } - - @Override - public Iterator iterator() { - return new NodeIterator(headIndex); - } - - @Override - public NodeStateEntry remove() { - Preconditions.checkState(!isEmpty(), "Cannot remove item from empty list"); - Long boxedHeadIndex = headIndex; - NodeStateEntry entryRemoved = cache.remove(boxedHeadIndex); - if (entryRemoved == null) { - String mapEntry = map.remove(boxedHeadIndex); - if (mapEntry == null) { - throw new IllegalStateException("Entry not found in cache or in store: " + boxedHeadIndex); - } - cacheMisses++; - entryRemoved = reader.read(mapEntry); - } else { - cacheHits++; - cacheSizeEstimationBytes -= entryRemoved.estimatedMemUsage(); - } - - headIndex++; - totalEntries--; - if (totalEntries == 0) { - if (cacheSizeEstimationBytes != 0 && !loggedCacheSizeEstimationMismatch) { - loggedCacheSizeEstimationMismatch = true; - LOG.warn("Total entries is 0, but cache size estimation is not zero: {}. Metrics: {}", cacheSizeEstimationBytes, formatMetrics()); - } - map.clear(); - cache.clear(); - } - return entryRemoved; - } - - private NodeStateEntry get(Long index) { - NodeStateEntry result = cache.get(index); - if (result == null) { - cacheMisses++; - String s = map.get(index); - result = reader.read(s); - LOG.trace("Cache miss: {}={}", index, result.getPath()); - } else { - cacheHits++; - } - return result; - } - - private void addEntryToCache(long index, NodeStateEntry entry) { - long now = System.currentTimeMillis(); - long newCacheSizeBytes = cacheSizeEstimationBytes + entry.estimatedMemUsage(); - if (cache.size() >= cacheSizeLimit || newCacheSizeBytes > cacheSizeLimitBytes) { - if (LOG.isTraceEnabled()) { - LOG.trace("Mem cache size {}/{} or byte size {}/{} would exceed maximum. Writing to persistent map: {} = {}, entry size: {}", - newCacheSizeBytes, cacheSizeLimitBytes, cache.size() + 1, cacheSizeLimit, index, entry.getPath(), entry.estimatedMemUsage()); - } - storeWrites++; - map.put(index, writer.toString(entry)); - long storeSizeBytes = store.getFileStore().size(); - boolean compactNow = now >= lastCompact + compactStoreMillis; - if (compactNow && storeSizeBytes > 10L * 1000 * 1000) { - // compact once a minute, if larger than 10 MB - LOG.info("Compacting. Current size: {}", storeSizeBytes); - store.close(); - MVStoreTool.compact(storeFileName, true); - openStore(); - lastCompact = System.currentTimeMillis(); - LOG.info("Finished compaction. Previous size: {}, new size={} bytes", storeSizeBytes, store.getFileStore().size()); - } - } else { - // new element fits in the in-memory cache. Do not write it to the persistent store. - cache.put(index, entry); - cacheSizeEstimationBytes = newCacheSizeBytes; - if (cacheSizeEstimationBytes > peakCacheSizeBytes) { - peakCacheSizeBytes = cacheSizeEstimationBytes; - } - if (cache.size() > peakCacheSize) { - peakCacheSize = cache.size(); - } - } - totalEntries++; - if (index % (1024 * 1000) == 0 || now >= lastLog + 10000) { // Print every million entries or every 10 seconds - LOG.info("Metrics: {}", formatMetrics()); - lastLog = now; - } - } - - @Override - public int size() { - return (int) totalEntries; - } - - @Override - public void close() { - store.close(); - LOG.info("Closing. Metrics: {}", formatMetrics()); - } - - public String formatMetrics() { - return "totalEntriesAdded: " + headIndex + ", totalEntriesInList: " + totalEntries + ", mapSize: " + map.sizeAsLong() + - ", mapSizeBytes: " + store.getFileStore().size() + " (" + IOUtils.humanReadableByteCountBin(store.getFileStore().size()) + ")" + - ", cacheHits: " + cacheHits + ", cacheMisses: " + cacheMisses + ", storeWrites: " + storeWrites + - ", peakCacheSize: " + peakCacheSize + - ", peakCacheSizeBytes: " + peakCacheSizeBytes + " (" + IOUtils.humanReadableByteCountBin(peakCacheSizeBytes) + ")"; - } - - @Override - public long estimatedMemoryUsage() { - return cacheSizeEstimationBytes; - } - - /** - * A node iterator over this list. - */ - final class NodeIterator implements Iterator { - - private long index; - - NodeIterator(long index) { - this.index = index; - } - - @Override - public boolean hasNext() { - return index < tailIndex; - } - - @Override - public NodeStateEntry next() { - if (index < headIndex || index >= tailIndex) { - throw new IllegalStateException(); - } - return get(index++); - } - } - - public long getCacheHits() { - return cacheHits; - } - - public long getCacheMisses() { - return cacheMisses; - } - - public long getStoreWrites() { - return storeWrites; - } - - public long getPeakCacheSizeBytes() { - return peakCacheSizeBytes; - } - - public long getPeakCacheSize() { - return peakCacheSize; - } -} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java index 0c2abaab4ea..8fa844d736d 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java @@ -144,10 +144,10 @@ public void isEmpty() { public void memUsage() { Assert.assertEquals("Empty list must be estimate 0", 0, list.estimatedMemoryUsage()); - list.add(testNode("/", 20)); + list.add(new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(20).build()); Assert.assertEquals(20, list.estimatedMemoryUsage()); - list.add(testNode("/", 30)); + list.add(new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(30).build()); Assert.assertEquals(50, list.estimatedMemoryUsage()); list.remove(); @@ -157,8 +157,8 @@ public void memUsage() { @Test public void memLimit() { list = new FlatFileBufferLinkedList(10); - NodeStateEntry e10Bytes = testNode("/", 10); - NodeStateEntry e1Byte = testNode("/", 1); + NodeStateEntry e10Bytes = new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(10).build(); + NodeStateEntry e1Byte = new NodeStateEntryBuilder(EMPTY_NODE, "/").withMemUsage(1).build(); list.add(e10Bytes); //this should succeed @@ -181,11 +181,7 @@ public void basics() { Assert.assertTrue("Adding an item should be available", list.iterator().hasNext()); } - protected NodeStateEntry testNode(String n) { + private NodeStateEntry testNode(String n) { return new NodeStateEntryBuilder(EMPTY_NODE, n).build(); } - - protected NodeStateEntry testNode(String n, long memoryUsage) { - return new NodeStateEntryBuilder(EMPTY_NODE, n).withMemUsage(memoryUsage).build(); - } } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java deleted file mode 100644 index 37ea219eed4..00000000000 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListV2Test.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList; - -import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter; -import org.apache.jackrabbit.oak.spi.blob.BlobStore; -import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -public class PersistedLinkedListV2Test extends FlatFileBufferLinkedListTest { - - @Rule - public final TemporaryFolder folder = new TemporaryFolder(new File("target")); - - @Before - public void setup() throws IOException { - list = newList(2, 1); - } - - private PersistedLinkedListV2 newList(int cacheSize, int cacheSizeMB) throws IOException { - String fileName = folder.newFile().getAbsolutePath(); - BlobStore blobStore = new MemoryBlobStore(); - NodeStateEntryReader reader = new NodeStateEntryReader(blobStore); - NodeStateEntryWriter writer = new NodeStateEntryWriter(blobStore); - return new PersistedLinkedListV2(fileName, writer, reader, cacheSize, cacheSizeMB); - } - - @After - public void tearDown() { - list.close(); - } - - @Test - public void testCacheHitsMissNumberOfEntries() throws IOException { - PersistedLinkedListV2 persistedLinkedList = newList(2, 1); - persistedLinkedList.add(testNode("/")); - persistedLinkedList.add(testNode("/a")); - persistedLinkedList.add(testNode("/a/b")); - - Assert.assertEquals(3, persistedLinkedList.size()); - ArrayList names1 = extracted(persistedLinkedList.iterator()); - Assert.assertEquals(List.of("/", "/a", "/a/b"), names1); - Assert.assertEquals(2, persistedLinkedList.getCacheHits()); - Assert.assertEquals(1, persistedLinkedList.getCacheMisses()); - - ArrayList names2 = extracted(persistedLinkedList.iterator()); - Assert.assertEquals(List.of("/", "/a", "/a/b"), names2); - Assert.assertEquals(4, persistedLinkedList.getCacheHits()); - Assert.assertEquals(2, persistedLinkedList.getCacheMisses()); - - // Does not count as a cache miss/hit - Assert.assertEquals("/", persistedLinkedList.remove().getPath()); - - var names3 = extracted(persistedLinkedList.iterator()); - Assert.assertEquals(List.of("/a", "/a/b"), names3); - Assert.assertEquals(6, persistedLinkedList.getCacheHits()); - Assert.assertEquals(3, persistedLinkedList.getCacheMisses()); - - Assert.assertEquals("/a", persistedLinkedList.remove().getPath()); - Assert.assertEquals("/a/b", persistedLinkedList.remove().getPath()); - - var names4 = extracted(persistedLinkedList.iterator()); - Assert.assertTrue(names4.isEmpty()); - Assert.assertEquals(7, persistedLinkedList.getCacheHits()); - Assert.assertEquals(4, persistedLinkedList.getCacheMisses()); - } - - @Test - public void testCacheHitsMissSizeOfEntries() throws IOException { - PersistedLinkedListV2 persistedLinkedList = newList(100, 1); - persistedLinkedList.add(testNode("/", 500 * 1024)); - persistedLinkedList.add(testNode("/a", 500 * 1024)); - persistedLinkedList.add(testNode("/a/b", 500 * 1024)); - - // The first two nodes should be kept in the cache. The last node exceeds the cache size and should be stored in - // the persistent map. So there will be 2 cache hits and one miss. - Assert.assertEquals(3, persistedLinkedList.size()); - ArrayList names1 = extracted(persistedLinkedList.iterator()); - Assert.assertEquals(List.of("/", "/a", "/a/b"), names1); - Assert.assertEquals(2, persistedLinkedList.getCacheHits()); - Assert.assertEquals(1, persistedLinkedList.getCacheMisses()); - } - - private static ArrayList extracted(Iterator iter) { - ArrayList names = new ArrayList<>(); - while (iter.hasNext()) { - names.add(iter.next().getPath()); - } - return names; - } -} diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java index 8516194a63d..864ace265e1 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/editor/FulltextDocumentMaker.java @@ -236,10 +236,11 @@ private boolean indexFacets(D doc, PropertyState property, String pname, Propert int idxDefinedTag = pd.getType(); // Try converting type to the defined type in the index definition if (tag != idxDefinedTag) { - if (log.isDebugEnabled()) { - log.debug("[{}] Facet property defined with type {} differs from property {} with type {} in path {}", - getIndexName(), Type.fromTag(idxDefinedTag, false), property, Type.fromTag(tag, false), path); - } + log.debug("[{}] Facet property defined with type {} differs from property {} with type {} in " + + "path {}", + getIndexName(), + Type.fromTag(idxDefinedTag, false), property, + Type.fromTag(tag, false), path); tag = idxDefinedTag; } return indexFacetProperty(doc, tag, property, pname); @@ -262,8 +263,9 @@ private boolean indexProperty(String path, dirty = true; } } catch (Exception e) { - log.error("could not index similarity field for property {} and definition {} for path {}", - property.getName(), pd, path); + log.error( + "could not index similarity field for property {} and definition {} for path {}", + property.getName(), pd, path); } } else if (Type.BINARY.tag() == property.getType().tag() && includeTypeForFullText) { @@ -316,8 +318,9 @@ private boolean indexProperty(String path, indexSimilarityStrings(doc, pd, value); } } catch (Exception e) { - log.error("could not index similarity field for property {} and definition {} for path {}", - property.getName(), pd, path); + log.error( + "could not index similarity field for property {} and definition {} for path {}", + property.getName(), pd, path); } } } @@ -386,8 +389,10 @@ private boolean addTypedOrderedFields(D doc, // Also there is no handling for different indexes having the same property since those are usually different versions of the same index. if (!throttleWarnLogs || MV_ORDERED_PROPERTY_SET.add(pname) || WARN_LOG_COUNTER_MV_ORDERED_PROPERTY.incrementAndGet() % throttleWarnLogThreshold == 0) { - log.warn("[{}] Ignoring ordered property {} of type {} for path {} as multivalued ordered property not supported", - getIndexName(), pname, Type.fromTag(property.getType().tag(), true), path); + log.warn( + "[{}] Ignoring ordered property {} of type {} for path {} as multivalued ordered property not supported", + getIndexName(), pname, + Type.fromTag(property.getType().tag(), true), path); } return false; } @@ -396,10 +401,12 @@ private boolean addTypedOrderedFields(D doc, int idxDefinedTag = pd.getType(); // Try converting type to the defined type in the index definition if (tag != idxDefinedTag) { - if (log.isDebugEnabled()) { - log.debug("[{}] Ordered property defined with type {} differs from property {} with type {} in path {}", - getIndexName(), Type.fromTag(idxDefinedTag, false), property, Type.fromTag(tag, false), path); - } + log.debug( + "[{}] Ordered property defined with type {} differs from property {} with type {} in " + + "path {}", + getIndexName(), + Type.fromTag(idxDefinedTag, false), property, + Type.fromTag(tag, false), path); tag = idxDefinedTag; } return indexTypeOrderedFields(doc, pname, tag, property, pd); @@ -599,7 +606,7 @@ public void onResult(Aggregate.PropertyIncludeResult result) { */ private boolean indexAggregatedNode(String path, D doc, Aggregate.NodeIncludeResult result) { //rule for node being aggregated might be null if such nodes - //are not indexed on their own. In such cases we rely on current + //are not indexed on there own. In such cases we rely in current //rule for some checks IndexDefinition.IndexingRule ruleAggNode = definition .getApplicableIndexingRule(getPrimaryTypeName(result.nodeState)); @@ -626,19 +633,19 @@ private boolean indexAggregatedNode(String path, D doc, Aggregate.NodeIncludeRes //Check if any explicit property defn is defined via relative path // and is marked to exclude this property from being indexed. We exclude //it from aggregation if - // 1. It's not to be indexed i.e. index=false - // 2. It's explicitly excluded from aggregation i.e. excludeFromAggregation=true + // 1. Its not to be indexed i.e. index=false + // 2. Its explicitly excluded from aggregation i.e. excludeFromAggregation=true PropertyDefinition pdForRootNode = indexingRule.getConfig(propertyPath); if (pdForRootNode != null && (!pdForRootNode.index || pdForRootNode.excludeFromAggregate)) { continue; } if (Type.BINARY == property.getType()) { - String aggregatedNodePath = PathUtils.concat(path, result.nodePath); + String aggreagtedNodePath = PathUtils.concat(path, result.nodePath); //Here the fulltext is being created for aggregate root hence nodePath passed //should be null String nodePath = result.isRelativeNode() ? result.rootIncludePath : null; - List binaryValues = newBinary(property, result.nodeState, aggregatedNodePath + "@" + pname); + List binaryValues = newBinary(property, result.nodeState, aggreagtedNodePath + "@" + pname); addBinary(doc, nodePath, binaryValues); dirty = true; } else { @@ -675,7 +682,7 @@ protected boolean indexDynamicBoost(D doc, String propertyName, String nodeName, continue; } if (p.isArray()) { - log.warn("{} is an array: {}", p.getName(), parentName); + log.warn(p.getName() + " is an array: {}", parentName); continue; } String dynaTagValue = p.getValue(Type.STRING); @@ -685,18 +692,18 @@ protected boolean indexDynamicBoost(D doc, String propertyName, String nodeName, continue; } if (p.isArray()) { - log.warn("{} is an array: {}", p.getName(), parentName); + log.warn(p.getName() + " is an array: {}", parentName); continue; } double dynaTagConfidence; try { dynaTagConfidence = p.getValue(Type.DOUBLE); } catch (NumberFormatException e) { - log.warn("{} parsing failed: {}", p.getName(), parentName, e); + log.warn(p.getName() + " parsing failed: {}", parentName, e); continue; } if (!Double.isFinite(dynaTagConfidence)) { - log.warn("{} is not finite: {}", p.getName(), parentName); + log.warn(p.getName() + " is not finite: {}", parentName); continue; } From 992e532027ea381da1a7a30f583042de7ab81226 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 10:34:49 +0200 Subject: [PATCH 36/86] Revert "OAK-10978 - Skip Azure compaction when there's not enough garbage in the repository (#1606)" This reverts commit 58146387d61e92700bf3f75a1b78d2f87e83070b. --- .../jackrabbit/oak/run/CompactCommand.java | 18 ----- .../oak/segment/azure/tool/AzureCompact.java | 67 +------------------ .../oak/segment/file/FileStore.java | 2 +- 3 files changed, 2 insertions(+), 85 deletions(-) diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java index 0f8ee389fed..ced75fc6be6 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java @@ -78,16 +78,6 @@ public void execute(String... args) throws Exception { .withRequiredArg() .ofType(Integer.class) .defaultsTo(50); - OptionSpec garbageThresholdGb = parser.accepts("garbage-threshold-gb", "Minimum amount of garbage in GB (defaults to 0 GB) for " - + "compaction to run") - .withRequiredArg() - .ofType(Integer.class) - .defaultsTo(0); - OptionSpec garbageThresholdPercentage = parser.accepts("garbage-threshold-percentage", "Minimum amount of garbage in percentage (defaults to 0%) for " - + "compaction to run") - .withRequiredArg() - .ofType(Integer.class) - .defaultsTo(0); OptionSet options = parser.parse(args); @@ -121,14 +111,6 @@ public void execute(String... args) throws Exception { azureBuilder.withPersistentCacheSizeGb(persistentCacheSizeGb.value(options)); } - if (options.has(garbageThresholdGb)) { - azureBuilder.withGarbageThresholdGb(garbageThresholdGb.value(options)); - } - - if (options.has(garbageThresholdPercentage)) { - azureBuilder.withGarbageThresholdPercentage(garbageThresholdPercentage.value(options)); - } - if (options.has(tailArg)) { azureBuilder.withGCType(SegmentGCOptions.GCType.TAIL); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java index e11f30ca7b4..a5db11950b8 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java @@ -37,8 +37,6 @@ import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; import org.apache.jackrabbit.oak.segment.file.FileStore; -import org.apache.jackrabbit.oak.segment.file.GCJournal; -import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.spi.persistence.split.SplitPersistence; @@ -90,10 +88,6 @@ public static class Builder { private Integer persistentCacheSizeGb; - private int garbageThresholdGb; - - private int garbageThresholdPercentage; - private CloudBlobDirectory sourceCloudBlobDirectory; private CloudBlobDirectory destinationCloudBlobDirectory; @@ -225,29 +219,6 @@ public Builder withPersistentCacheSizeGb(Integer persistentCacheSizeGb) { return this; } - /** - * The minimum garbage size in GB for the compaction to run. - * @param garbageThresholdGb - * the minimum garbage size in GB for the compaction to run. - * - * @return this builder - */ - public Builder withGarbageThresholdGb(int garbageThresholdGb) { - this.garbageThresholdGb = garbageThresholdGb; - return this; - } - - /** - * The minimum garbage size in percentage for the compaction to run. - * @param garbageThresholdPercentage - * the minimum garbage size in percentage for the compaction to run. - * @return this builder - */ - public Builder withGarbageThresholdPercentage(int garbageThresholdPercentage) { - this.garbageThresholdPercentage = garbageThresholdPercentage; - return this; - } - public Builder withSourceCloudBlobDirectory(CloudBlobDirectory sourceCloudBlobDirectory) { this.sourceCloudBlobDirectory = checkNotNull(sourceCloudBlobDirectory); return this; @@ -272,8 +243,6 @@ public AzureCompact build() { } } - private static final long GB = 1024 * 1024 * 1024; - private final String path; private final String targetPath; @@ -294,10 +263,6 @@ public AzureCompact build() { private final Integer persistentCacheSizeGb; - private final int garbageThresholdGb; - - private final int garbageThresholdPercentage; - private final CloudBlobDirectory sourceCloudBlobDirectory; private final CloudBlobDirectory destinationCloudBlobDirectory; @@ -314,8 +279,6 @@ private AzureCompact(Builder builder) { this.concurrency = builder.concurrency; this.persistentCachePath = builder.persistentCachePath; this.persistentCacheSizeGb = builder.persistentCacheSizeGb; - this.garbageThresholdGb = builder.garbageThresholdGb; - this.garbageThresholdPercentage = builder.garbageThresholdPercentage; this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory; this.destinationCloudBlobDirectory = builder.destinationCloudBlobDirectory; this.azureStorageCredentialManager = new AzureStorageCredentialManager(); @@ -354,19 +317,10 @@ public int run() throws IOException, StorageException, URISyntaxException { } printArchives(System.out, beforeArchives); + System.out.printf(" -> compacting\n"); try (FileStore store = newFileStore(splitPersistence, Files.createTempDir(), strictVersionCheck, segmentCacheSize, gcLogInterval, compactorType, concurrency)) { - if (garbageThresholdGb > 0 && garbageThresholdPercentage > 0) { - System.out.printf(" -> minimum garbage threshold set to %d GB or %d%%\n", garbageThresholdGb, garbageThresholdPercentage); - long currentSize = store.size(); - if (!isGarbageOverMinimumThreshold(currentSize, roPersistence)) { - return 0; - } - } - - System.out.printf(" -> compacting\n"); - boolean success = false; switch (gcType) { case FULL: @@ -415,25 +369,6 @@ public int run() throws IOException, StorageException, URISyntaxException { return 0; } - private boolean isGarbageOverMinimumThreshold(long currentSize, SegmentNodeStorePersistence roPersistence) throws IOException { - long previousSize = 0; - - GCJournalFile gcJournalFile = roPersistence.getGCJournalFile(); - if (gcJournalFile != null) { - GCJournal gcJournal = new GCJournal(gcJournalFile); - GCJournal.GCJournalEntry gcJournalEntry = gcJournal.read(); - previousSize = gcJournalEntry.getRepoSize(); - } - - long potentialGarbage = currentSize - previousSize; - if (currentSize < previousSize || (potentialGarbage < garbageThresholdGb * GB && potentialGarbage < currentSize * garbageThresholdPercentage / 100)) { - System.out.printf(" -> [skipping] not enough garbage -> previous size: %d, current size: %d\n", previousSize, currentSize); - return false; - } - - return true; - } - private long printTargetRepoSizeInfo(CloudBlobContainer container) { System.out.printf("Calculating the size of container %s\n", container.getName()); long size = 0; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java index 7e72cf78247..6381e1129cf 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java @@ -298,7 +298,7 @@ public GCNodeWriteMonitor getGCNodeWriteMonitor() { /** * @return the size of this store. */ - public long size() { + private long size() { try (ShutDownCloser ignored = shutDown.keepAlive()) { return tarFiles.size(); } From 5af130638ea97e95d37d907db06f78c2d48c3d9a Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:27:42 +0200 Subject: [PATCH 37/86] Revert "OAK-10977 - Cleanup IndexDefinition class (#1604)" This reverts commit ce5c7df8993d2319828fceaacdcfc2093d511919. --- .../plugins/index/search/IndexDefinition.java | 430 +++++++++--------- 1 file changed, 205 insertions(+), 225 deletions(-) diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java index 503ee2e13a5..ffe2dba7200 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java @@ -19,11 +19,31 @@ package org.apache.jackrabbit.oak.plugins.index.search; -import org.apache.jackrabbit.JcrConstants; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import javax.jcr.PropertyType; +import javax.jcr.RepositoryException; +import javax.jcr.nodetype.NodeTypeIterator; + +import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.ImmutableMap; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; +import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.guava.common.primitives.Ints; +import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Root; @@ -50,23 +70,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.jcr.PropertyType; -import javax.jcr.RepositoryException; -import javax.jcr.nodetype.NodeTypeIterator; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; -import java.util.regex.Pattern; -import java.util.stream.Stream; - +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; +import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayListWithCapacity; +import static org.apache.jackrabbit.guava.common.collect.Maps.newHashMap; +import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; import static org.apache.jackrabbit.JcrConstants.JCR_SCORE; import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM; import static org.apache.jackrabbit.JcrConstants.NT_BASE; @@ -79,37 +87,7 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEXING_MODE_SYNC; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.BLOB_SIZE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.COMPAT_MODE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EVALUATE_PATH_RESTRICTION; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXCLUDE_PROPERTY_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.EXPERIMENTAL_STORAGE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FACETS; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FIELD_BOOST; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.FULL_TEXT_ENABLED; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INCLUDE_PROPERTY_TYPES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_DATA_CHILD_NAME; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_SIMILARITY_BINARIES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.INDEX_SIMILARITY_STRINGS; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.ORDERED_PROP_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS_TOP_CHILDREN; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_NODE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_QUERY_FILTER_REGEX; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_RANDOM_SEED; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_INSECURE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_JVM_PARAM; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_SECURE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_SECURE_FACETS_VALUE_STATISTICAL; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_STATISTICAL_FACET_SAMPLE_SIZE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_VALUE_REGEX; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_JVM_PARAM; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_CONFIG; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MAPPED_TYPE; -import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.TIKA_MIME_TYPES; +import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.*; import static org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition.DEFAULT_BOOST; import static org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValue; import static org.apache.jackrabbit.oak.plugins.index.search.util.ConfigUtil.getOptionalValues; @@ -217,8 +195,8 @@ public class IndexDefinition implements Aggregate.AggregateMapper { public static final OrderEntry NATIVE_SORT_ORDER = new OrderEntry(JCR_SCORE, Type.UNDEFINED, OrderEntry.Order.DESCENDING); - private final boolean indexSimilarityBinaries; - private final boolean indexSimilarityStrings; + private boolean indexSimilarityBinaries; + private boolean indexSimilarityStrings; /** * Dynamic boost uses index time boosting. This requires to have a separate field for each unique term that needs to @@ -228,13 +206,13 @@ public class IndexDefinition implements Aggregate.AggregateMapper { * oak.search.dynamicBoostLite=lucene no index time boosting is used for lucene types indexes. The terms will affect * the query match clause but the scores won't be the same. In summary, in lite mode the query will have the same * recall but lower precision. - *

+ * * WARNING: dynamicBoostLite needs similarityTags. In case there are no similarityTags, the query won't return the * expected results. */ private final static String DYNAMIC_BOOST_LITE_NAME = "oak.search.dynamicBoostLite"; protected final static List DYNAMIC_BOOST_LITE = - List.of(System.getProperty(DYNAMIC_BOOST_LITE_NAME, "").split(",")); + Arrays.asList(System.getProperty(DYNAMIC_BOOST_LITE_NAME, "").split(",")); protected final boolean fullTextEnabled; @@ -344,7 +322,7 @@ public boolean isTestMode() { //~--------------------------------------------------------< Builder > // TODO - this method should be removed after tests don't use it anymore - public static Builder newBuilder(NodeState root, NodeState defn, String indexPath) { + public static Builder newBuilder(NodeState root, NodeState defn, String indexPath){ return new Builder().root(root).defn(defn).indexPath(indexPath); } @@ -362,49 +340,49 @@ public static class Builder { protected IndexFormatVersion version; public Builder root(NodeState root) { - this.root = Objects.requireNonNull(root); + this.root = checkNotNull(root); return this; } public Builder defn(NodeState defn) { - this.defn = Objects.requireNonNull(defn); + this.defn = checkNotNull(defn); return this; } public Builder indexPath(String indexPath) { - this.indexPath = Objects.requireNonNull(indexPath); + this.indexPath = checkNotNull(indexPath); return this; } - public Builder uid(String uid) { + public Builder uid(String uid){ this.uid = uid; return this; } - public Builder version(IndexFormatVersion version) { + public Builder version(IndexFormatVersion version){ this.version = version; return this; } - public Builder reindex() { + public Builder reindex(){ this.reindexMode = true; return this; } - public IndexDefinition build() { - if (version == null) { + public IndexDefinition build(){ + if (version == null){ version = determineIndexFormatVersion(defn); } - if (uid == null) { + if (uid == null){ uid = determineUniqueId(defn); - if (uid == null && !IndexDefinition.hasPersistedIndex(defn)) { + if (uid == null && !IndexDefinition.hasPersistedIndex(defn)){ uid = DEFAULT_UID; } } NodeState indexDefnStateToUse = defn; - if (!reindexMode) { + if (!reindexMode){ indexDefnStateToUse = getIndexDefinitionState(defn); } return createInstance(indexDefnStateToUse); @@ -423,10 +401,10 @@ public IndexDefinition(NodeState root, NodeState defn, String indexPath) { protected IndexDefinition(NodeState root, NodeState defn, IndexFormatVersion version, String uid, String indexPath) { try { this.root = root; - this.version = Objects.requireNonNull(version); + this.version = checkNotNull(version); this.uid = uid; this.definition = defn; - this.indexPath = Objects.requireNonNull(indexPath); + this.indexPath = checkNotNull(indexPath); this.indexName = indexPath; this.indexTags = getOptionalValues(defn, IndexConstants.INDEX_TAGS, Type.STRINGS, String.class); this.indexSelectionPolicy @@ -434,17 +412,17 @@ protected IndexDefinition(NodeState root, NodeState defn, IndexFormatVersion ver this.nodeTypeIndex = getOptionalValue(defn, FulltextIndexConstants.PROP_INDEX_NODE_TYPE, false); this.blobSize = Math.max(1024, getOptionalValue(defn, BLOB_SIZE, DEFAULT_BLOB_SIZE)); - this.aggregates = nodeTypeIndex ? Map.of() : collectAggregates(defn); + this.aggregates = nodeTypeIndex ? Collections.emptyMap() : collectAggregates(defn); NodeState rulesState = defn.getChildNode(FulltextIndexConstants.INDEX_RULES); - if (!rulesState.exists()) { + if (!rulesState.exists()){ rulesState = createIndexRules(defn).getNodeState(); } this.testMode = getOptionalValue(defn, FulltextIndexConstants.TEST_MODE, false); - List definedIndexRules = new ArrayList<>(); + List definedIndexRules = newArrayList(); this.indexRules = collectIndexRules(rulesState, definedIndexRules); - this.definedRules = List.copyOf(definedIndexRules); + this.definedRules = ImmutableList.copyOf(definedIndexRules); this.fullTextEnabled = hasFulltextEnabledIndexRule(definedIndexRules); this.evaluatePathRestrictions = getOptionalValue(defn, EVALUATE_PATH_RESTRICTION, false); @@ -529,7 +507,7 @@ private boolean getSimilarityDefaultValue(NodeState defn, String propertyKey) { } private boolean isPresent(T key, Iterator iterator) { - while (iterator.hasNext()) { + while (iterator.hasNext()){ if (key.equals(iterator.next())) { return true; } @@ -577,7 +555,7 @@ public boolean isFullTextEnabled() { return fullTextEnabled; } - public String getFunctionName() { + public String getFunctionName(){ return funcName; } @@ -586,20 +564,19 @@ protected String getDefaultFunctionName() { return "fulltext";//TODO Should this be FulltextIndexConstants.FUNC_NAME } - public boolean hasFunctionDefined() { + public boolean hasFunctionDefined(){ return funcName != null; } /** * Size in bytes for the blobs created while storing the index content - * * @return size in bytes */ public int getBlobSize() { return blobSize; } - public long getReindexCount() { + public long getReindexCount(){ return reindexCount; } @@ -638,8 +615,8 @@ public double getCostPerExecution() { return costPerExecution; } - public long getFulltextEntryCount(long numOfDocs) { - if (isEntryCountDefined()) { + public long getFulltextEntryCount(long numOfDocs){ + if (isEntryCountDefined()){ return Math.min(getEntryCount(), numOfDocs); } return numOfDocs; @@ -654,7 +631,7 @@ public IndexFormatVersion getVersion() { } - public boolean isOfOldFormat() { + public boolean isOfOldFormat(){ return !hasIndexingRules(definition); } @@ -662,11 +639,11 @@ public boolean evaluatePathRestrictions() { return evaluatePathRestrictions; } - public boolean hasCustomTikaConfig() { + public boolean hasCustomTikaConfig(){ return hasCustomTikaConfig; } - public InputStream getTikaConfig() { + public InputStream getTikaConfig(){ return ConfigUtil.getBlob(getTikaConfigNode(), TIKA_CONFIG).getNewStream(); } @@ -722,7 +699,7 @@ public boolean isPureNodeTypeIndex() { /** * Check if the index definition is fresh, or (some) indexing has occurred. - *

+ * * WARNING: If there is _any_ hidden node, then it is assumed that * no reindex is needed. Even if the hidden node is completely unrelated * and doesn't contain index data (for example the node ":status"). @@ -731,7 +708,7 @@ public boolean isPureNodeTypeIndex() { * @param definition nodestate for Index Definition * @return true if index has some indexed content */ - public static boolean hasPersistedIndex(NodeState definition) { + public static boolean hasPersistedIndex(NodeState definition){ for (String rm : definition.getChildNodeNames()) { if (NodeStateUtils.isHidden(rm)) { return true; @@ -748,7 +725,7 @@ public static void setDisableStoredIndexDefinition(boolean disableStoredIndexDef IndexDefinition.disableStoredIndexDefinition = disableStoredIndexDefinitionDefault; } - public Set getRelativeNodeNames() { + public Set getRelativeNodeNames(){ //Can be computed lazily as required only for oak-run indexing for now Set names = new HashSet<>(); for (IndexingRule r : definedRules) { @@ -763,7 +740,7 @@ public Set getRelativeNodeNames() { return names; } - public boolean indexesRelativeNodes() { + public boolean indexesRelativeNodes(){ for (IndexingRule r : definedRules) { if (!r.aggregate.getIncludes().isEmpty()) { return true; @@ -781,7 +758,7 @@ public String toString() { //~---------------------------------------------------< Aggregates > @Nullable - public Aggregate getAggregate(String nodeType) { + public Aggregate getAggregate(String nodeType){ return aggregates.get(nodeType); } @@ -791,14 +768,14 @@ public Map getAggregates() { } private Map collectAggregates(NodeState defn) { - Map aggregateMap = new HashMap<>(); + Map aggregateMap = newHashMap(); for (ChildNodeEntry cne : defn.getChildNode(FulltextIndexConstants.AGGREGATES).getChildNodeEntries()) { String nodeType = cne.getName(); int recursionLimit = getOptionalValue(cne.getNodeState(), FulltextIndexConstants.AGG_RECURSIVE_LIMIT, Aggregate.RECURSIVE_AGGREGATION_LIMIT_DEFAULT); - List includes = new ArrayList<>(); + List includes = newArrayList(); for (ChildNodeEntry include : cne.getNodeState().getChildNodeEntries()) { NodeState is = include.getNodeState(); String primaryType = is.getString(FulltextIndexConstants.AGG_PRIMARY_TYPE); @@ -814,7 +791,7 @@ private Map collectAggregates(NodeState defn) { aggregateMap.put(nodeType, new Aggregate(nodeType, includes, recursionLimit)); } - return Map.copyOf(aggregateMap); + return ImmutableMap.copyOf(aggregateMap); } //~---------------------------------------------------< IndexRule > @@ -900,19 +877,19 @@ public IndexingRule getApplicableIndexingRule(NodeState state) { } private Map> collectIndexRules(NodeState indexRules, - List definedIndexRules) { + List definedIndexRules){ //TODO if a rule is defined for nt:base then this map would have entry for each //registered nodeType in the system if (!indexRules.exists()) { - return Map.of(); + return Collections.emptyMap(); } - if (!hasOrderableChildren(indexRules)) { + if (!hasOrderableChildren(indexRules)){ log.warn("IndexRule node does not have orderable children in [{}]", IndexDefinition.this); } - Map> nt2rules = new HashMap<>(); + Map> nt2rules = newHashMap(); ReadOnlyNodeTypeManager ntReg = createNodeTypeManager(RootFactory.createReadOnlyRoot(root)); //Use Tree API to read ordered child nodes @@ -926,10 +903,10 @@ private Map> collectIndexRules(NodeState indexRules, log.trace("Found rule '{}' for NodeType '{}'", rule, rule.getNodeTypeName()); List ntNames = allNames; - if (!rule.inherited) { + if (!rule.inherited){ //Trim the list to rule's nodeType so that inheritance check //is not performed for other nodeTypes - ntNames = List.of(rule.getNodeTypeName()); + ntNames = Collections.singletonList(rule.getNodeTypeName()); } for (String ntName : ntNames) { @@ -941,11 +918,11 @@ private Map> collectIndexRules(NodeState indexRules, } } - for (Map.Entry> e : nt2rules.entrySet()) { - e.setValue(List.copyOf(e.getValue())); + for (Map.Entry> e : nt2rules.entrySet()){ + e.setValue(ImmutableList.copyOf(e.getValue())); } - return Map.copyOf(nt2rules); + return ImmutableMap.copyOf(nt2rules); } private boolean evaluateSuggestionEnabled() { @@ -1027,7 +1004,7 @@ public class IndexingRule { private final String baseNodeType; private final String nodeTypeName; /** - * Case-insensitive map of lower cased propertyName to propertyConfigs + * Case insensitive map of lower cased propertyName to propertyConfigs */ private final Map propConfigs; /** @@ -1060,31 +1037,31 @@ public class IndexingRule { this.inherited = getOptionalValue(config, FulltextIndexConstants.RULE_INHERITED, true); this.propertyTypes = getSupportedTypes(config, INCLUDE_PROPERTY_TYPES, TYPES_ALLOW_ALL); - List namePatterns = new ArrayList<>(); - List nonExistentProperties = new ArrayList<>(); - List functionRestrictions = new ArrayList<>(); - List existentProperties = new ArrayList<>(); - List nodeScopeAnalyzedProps = new ArrayList<>(); - List syncProps = new ArrayList<>(); - List similarityProperties = new ArrayList<>(); - List propIncludes = new ArrayList<>(); + List namePatterns = newArrayList(); + List nonExistentProperties = newArrayList(); + List functionRestrictions = newArrayList(); + List existentProperties = newArrayList(); + List nodeScopeAnalyzedProps = newArrayList(); + List syncProps = newArrayList(); + List similarityProperties = newArrayList(); + List propIncludes = newArrayList(); this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties, existentProperties, nodeScopeAnalyzedProps, functionRestrictions, syncProps, similarityProperties); this.propAggregate = new Aggregate(nodeTypeName, propIncludes); this.aggregate = combine(propAggregate, nodeTypeName); - this.namePatterns = List.copyOf(namePatterns); - this.nodeScopeAnalyzedProps = List.copyOf(nodeScopeAnalyzedProps); - this.nullCheckEnabledProperties = List.copyOf(nonExistentProperties); - this.functionRestrictions = List.copyOf(functionRestrictions); - this.notNullCheckEnabledProperties = List.copyOf(existentProperties); - this.similarityProperties = List.copyOf(similarityProperties); + this.namePatterns = ImmutableList.copyOf(namePatterns); + this.nodeScopeAnalyzedProps = ImmutableList.copyOf(nodeScopeAnalyzedProps); + this.nullCheckEnabledProperties = ImmutableList.copyOf(nonExistentProperties); + this.functionRestrictions = ImmutableList.copyOf(functionRestrictions); + this.notNullCheckEnabledProperties = ImmutableList.copyOf(existentProperties); + this.similarityProperties = ImmutableList.copyOf(similarityProperties); this.fulltextEnabled = aggregate.hasNodeAggregates() || hasAnyFullTextEnabledProperty(); this.nodeFullTextIndexed = aggregate.hasNodeAggregates() || anyNodeScopeIndexedProperty(); this.propertyIndexEnabled = hasAnyPropertyIndexConfigured(); this.indexesAllNodesOfMatchingType = areAlMatchingNodeByTypeIndexed(); this.nodeNameIndexed = evaluateNodeNameIndexed(config); - this.syncProps = List.copyOf(syncProps); + this.syncProps = ImmutableList.copyOf(syncProps); validateRuleDefinition(); } @@ -1092,7 +1069,7 @@ public class IndexingRule { * Creates a new indexing rule base on an existing one, but for a * different node type name. * - * @param original the existing rule. + * @param original the existing rule. * @param nodeTypeName the node type name for the rule. */ IndexingRule(IndexingRule original, String nodeTypeName) { @@ -1124,7 +1101,7 @@ public class IndexingRule { * * @param propertyName the name of a property. * @return true if the property is indexed; - * false otherwise. + * false otherwise. */ public boolean isIndexed(String propertyName) { return getConfig(propertyName) != null; @@ -1149,7 +1126,7 @@ public String getBaseNodeType() { /** * Returns all the configured {@code PropertyDefinition}s for this {@code IndexRule}. - *

+ * * In case of a pure nodetype index we just return primaryType and mixins. * * @return an {@code Iterable} of {@code PropertyDefinition}s. @@ -1185,8 +1162,8 @@ public Stream getNamePatternsProperties() { @Override public String toString() { - String str = "IndexRule: " + nodeTypeName; - if (!baseNodeType.equals(nodeTypeName)) { + String str = "IndexRule: "+ nodeTypeName; + if (!baseNodeType.equals(nodeTypeName)){ str += "(" + baseNodeType + ")"; } return str; @@ -1202,11 +1179,11 @@ public boolean isAggregated(String nodePath) { * * @param state the state to check. * @return true the rule applies to the given node; - * false otherwise. + * false otherwise. */ public boolean appliesTo(NodeState state) { - for (String mixinName : getMixinTypeNames(state)) { - if (nodeTypeName.equals(mixinName)) { + for (String mixinName : getMixinTypeNames(state)){ + if (nodeTypeName.equals(mixinName)){ return true; } } @@ -1242,8 +1219,8 @@ public boolean isNodeFullTextIndexed() { /** * @param propertyName name of a property. * @return the property configuration or null if this - * indexing rule does not contain a configuration for the given - * property. + * indexing rule does not contain a configuration for the given + * property. */ @Nullable public PropertyDefinition getConfig(String propertyName) { @@ -1261,7 +1238,7 @@ public PropertyDefinition getConfig(String propertyName) { return null; } - public boolean includePropertyType(int type) { + public boolean includePropertyType(int type){ return IndexDefinition.includePropertyType(propertyTypes, type); } @@ -1275,14 +1252,14 @@ public Aggregate getAggregate() { * certain property 'foo' for node type 'app:Asset' then index would only have * entries for those assets where foo is defined. Such an index cannot claim that * it has entries for all assets. - * + * @return true in case all matching node types are covered by this rule */ public boolean indexesAllNodesOfMatchingType() { return indexesAllNodesOfMatchingType; } - public boolean isBasedOnNtBase() { + public boolean isBasedOnNtBase(){ return JcrConstants.NT_BASE.equals(baseNodeType); } @@ -1295,10 +1272,10 @@ private Map collectPropConfigs(NodeState config, List functionRestrictions, List syncProps, List similarityProperties) { - Map propDefns = new HashMap<>(); + Map propDefns = newHashMap(); NodeState propNode = config.getChildNode(FulltextIndexConstants.PROP_NODE); - if (propNode.exists() && !hasOrderableChildren(propNode)) { + if (propNode.exists() && !hasOrderableChildren(propNode)){ log.warn("Properties node for [{}] does not have orderable " + "children in [{}]", this, IndexDefinition.this); } @@ -1307,8 +1284,8 @@ private Map collectPropConfigs(NodeState config, //and ignore any other property definition if (nodeTypeIndex) { boolean sync = getOptionalValue(config, FulltextIndexConstants.PROP_SYNC, false); - PropertyDefinition pdpt = createNodeTypeDefinition(this, JcrConstants.JCR_PRIMARYTYPE, sync); - PropertyDefinition pdmixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, sync); + PropertyDefinition pdpt = createNodeTypeDefinition(this, JcrConstants.JCR_PRIMARYTYPE, sync); + PropertyDefinition pdmixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, sync); propDefns.put(pdpt.name.toLowerCase(Locale.ENGLISH), pdpt); propDefns.put(pdmixin.name.toLowerCase(Locale.ENGLISH), pdmixin); @@ -1323,7 +1300,7 @@ private Map collectPropConfigs(NodeState config, "definitions", indexPath, FulltextIndexConstants.PROP_INDEX_NODE_TYPE); } - return Map.copyOf(propDefns); + return ImmutableMap.copyOf(propDefns); } //Include all immediate child nodes to 'properties' node by default @@ -1345,28 +1322,28 @@ private Map collectPropConfigs(NodeState config, // a function index has no other options continue; } - if (pd.isRegexp) { + if(pd.isRegexp){ patterns.add(new NamePattern(pd.name, pd)); } else { propDefns.put(pd.name.toLowerCase(Locale.ENGLISH), pd); } - if (pd.relative) { + if (pd.relative){ propAggregate.add(new Aggregate.PropertyInclude(pd)); } - if (pd.nullCheckEnabled) { + if (pd.nullCheckEnabled){ nonExistentProperties.add(pd); } - if (pd.notNullCheckEnabled) { + if (pd.notNullCheckEnabled){ existentProperties.add(pd); } //Include props with name, boosted and nodeScopeIndex if (pd.nodeScopeIndex && pd.analyzed - && !pd.isRegexp) { + && !pd.isRegexp){ nodeScopeAnalyzedProps.add(pd); } @@ -1379,7 +1356,7 @@ private Map collectPropConfigs(NodeState config, } } ensureNodeTypeIndexingIsConsistent(propDefns, syncProps); - return Map.copyOf(propDefns); + return ImmutableMap.copyOf(propDefns); } /** @@ -1392,21 +1369,21 @@ private void ensureNodeTypeIndexingIsConsistent(Map PropertyDefinition pd_mixin = propDefns.get(JcrConstants.JCR_MIXINTYPES.toLowerCase(Locale.ENGLISH)); if (pd_pr != null && pd_pr.propertyIndex && pd_mixin == null) { - pd_mixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, pd_pr.sync); + pd_mixin = createNodeTypeDefinition(this, JcrConstants.JCR_MIXINTYPES, pd_pr.sync); syncProps.add(pd_mixin); propDefns.put(JcrConstants.JCR_MIXINTYPES.toLowerCase(Locale.ENGLISH), pd_mixin); } } private boolean hasAnyFullTextEnabledProperty() { - for (PropertyDefinition pd : propConfigs.values()) { - if (pd.fulltextEnabled()) { + for (PropertyDefinition pd : propConfigs.values()){ + if (pd.fulltextEnabled()){ return true; } } - for (NamePattern np : namePatterns) { - if (np.getConfig().fulltextEnabled()) { + for (NamePattern np : namePatterns){ + if (np.getConfig().fulltextEnabled()){ return true; } } @@ -1414,31 +1391,31 @@ private boolean hasAnyFullTextEnabledProperty() { } private boolean hasAnyPropertyIndexConfigured() { - for (PropertyDefinition pd : propConfigs.values()) { - if (pd.propertyIndex) { + for (PropertyDefinition pd : propConfigs.values()){ + if (pd.propertyIndex){ return true; } } - for (NamePattern np : namePatterns) { - if (np.getConfig().propertyIndex) { + for (NamePattern np : namePatterns){ + if (np.getConfig().propertyIndex){ return true; } } return false; } - private boolean anyNodeScopeIndexedProperty() { + private boolean anyNodeScopeIndexedProperty(){ //Check if there is any nodeScope indexed property as //for such case a node would always be indexed - for (PropertyDefinition pd : propConfigs.values()) { - if (pd.nodeScopeIndex) { + for (PropertyDefinition pd : propConfigs.values()){ + if (pd.nodeScopeIndex){ return true; } } - for (NamePattern np : namePatterns) { - if (np.getConfig().nodeScopeIndex) { + for (NamePattern np : namePatterns){ + if (np.getConfig().nodeScopeIndex){ return true; } } @@ -1446,19 +1423,19 @@ private boolean anyNodeScopeIndexedProperty() { return false; } - private boolean areAlMatchingNodeByTypeIndexed() { + private boolean areAlMatchingNodeByTypeIndexed(){ if (nodeTypeIndex) { return true; } - if (nodeFullTextIndexed) { + if (nodeFullTextIndexed){ return true; } //If there is nullCheckEnabled property which is not relative then //all nodes would be indexed. relativeProperty with nullCheckEnabled might //not ensure that (OAK-1085) - for (PropertyDefinition pd : nullCheckEnabledProperties) { + for (PropertyDefinition pd : nullCheckEnabledProperties){ if (!pd.relative) { return true; } @@ -1478,25 +1455,26 @@ private boolean evaluateNodeNameIndexed(NodeState config) { } //iterate over property definitions - for (PropertyDefinition pd : propConfigs.values()) { - if (FulltextIndexConstants.PROPDEF_PROP_NODE_NAME.equals(pd.name)) { + for (PropertyDefinition pd : propConfigs.values()){ + if (FulltextIndexConstants.PROPDEF_PROP_NODE_NAME.equals(pd.name)){ return true; } } return false; } - private Aggregate combine(Aggregate propAggregate, String nodeTypeName) { + private Aggregate combine(Aggregate propAggregate, String nodeTypeName){ Aggregate nodeTypeAgg = IndexDefinition.this.getAggregate(nodeTypeName); - List includes = new ArrayList<>(propAggregate.getIncludes()); - if (nodeTypeAgg != null) { + List includes = newArrayList(); + includes.addAll(propAggregate.getIncludes()); + if (nodeTypeAgg != null){ includes.addAll(nodeTypeAgg.getIncludes()); } return new Aggregate(nodeTypeName, includes); } private void validateRuleDefinition() { - if (!nullCheckEnabledProperties.isEmpty() && isBasedOnNtBase()) { + if (!nullCheckEnabledProperties.isEmpty() && isBasedOnNtBase()){ throw new IllegalStateException("nt:base based rule cannot have a " + "PropertyDefinition with nullCheckEnabled"); } @@ -1522,14 +1500,15 @@ private static final class NamePattern { * Creates a new name pattern. * * @param pattern the pattern as defined by the property definition - * @param config the associated configuration. + * @param config the associated configuration. */ - private NamePattern(String pattern, PropertyDefinition config) { + private NamePattern(String pattern, + PropertyDefinition config){ //Special handling for all props regex as its already being used //and use of '/' in regex would confuse the parent path calculation //logic - if (FulltextIndexConstants.REGEX_ALL_PROPS.equals(pattern)) { + if (FulltextIndexConstants.REGEX_ALL_PROPS.equals(pattern)){ this.parentPath = ""; this.pattern = Pattern.compile(pattern); } else { @@ -1542,7 +1521,7 @@ private NamePattern(String pattern, PropertyDefinition config) { /** * @param propertyPath property name to match * @return true if property name matches this name - * pattern; false otherwise. + * pattern; false otherwise. */ boolean matches(String propertyPath) { String parentPath = getParentPath(propertyPath); @@ -1560,13 +1539,13 @@ PropertyDefinition getConfig() { //~---------------------------------------------< compatibility > - public static NodeBuilder updateDefinition(NodeBuilder indexDefn) { + public static NodeBuilder updateDefinition(NodeBuilder indexDefn){ return updateDefinition(indexDefn, "unknown"); } - public static NodeBuilder updateDefinition(NodeBuilder indexDefn, String indexPath) { + public static NodeBuilder updateDefinition(NodeBuilder indexDefn, String indexPath){ NodeState defn = indexDefn.getBaseState(); - if (!hasIndexingRules(defn)) { + if (!hasIndexingRules(defn)){ NodeState rulesState = createIndexRules(defn).getNodeState(); indexDefn.setChildNode(FulltextIndexConstants.INDEX_RULES, rulesState); indexDefn.setProperty(INDEX_VERSION, determineIndexFormatVersion(defn).getVersion()); @@ -1585,7 +1564,7 @@ public static NodeBuilder updateDefinition(NodeBuilder indexDefn, String indexPa /** * Constructs IndexingRule based on earlier format of index configuration */ - private static NodeBuilder createIndexRules(NodeState defn) { + private static NodeBuilder createIndexRules(NodeState defn){ NodeBuilder builder = EMPTY_NODE.builder(); Set declaringNodeTypes = getMultiProperty(defn, DECLARING_NODE_TYPES); Set includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES); @@ -1596,19 +1575,19 @@ private static NodeBuilder createIndexRules(NodeState defn) { NodeState propNodeState = defn.getChildNode(FulltextIndexConstants.PROP_NODE); //If no explicit nodeType defined then all config applies for nt:base - if (declaringNodeTypes.isEmpty()) { - declaringNodeTypes = Set.of(NT_BASE); + if (declaringNodeTypes.isEmpty()){ + declaringNodeTypes = Collections.singleton(NT_BASE); } - Set propNamesSet = new HashSet<>(); + Set propNamesSet = Sets.newHashSet(); propNamesSet.addAll(includes); propNamesSet.addAll(excludes); propNamesSet.addAll(orderedProps); //Also include all immediate leaf propNode names - for (ChildNodeEntry cne : propNodeState.getChildNodeEntries()) { + for (ChildNodeEntry cne : propNodeState.getChildNodeEntries()){ if (!propNamesSet.contains(cne.getName()) - && Iterables.isEmpty(cne.getNodeState().getChildNodeNames())) { + && Iterables.isEmpty(cne.getNodeState().getChildNodeNames())){ propNamesSet.add(cne.getName()); } } @@ -1616,24 +1595,25 @@ private static NodeBuilder createIndexRules(NodeState defn) { List propNames = new ArrayList<>(propNamesSet); final String includeAllProp = FulltextIndexConstants.REGEX_ALL_PROPS; - if (fullTextEnabled && includes.isEmpty()) { + if (fullTextEnabled + && includes.isEmpty()){ //Add the regEx for including all properties at the end //for fulltext index and when no explicit includes are defined propNames.add(includeAllProp); } - for (String typeName : declaringNodeTypes) { + for (String typeName : declaringNodeTypes){ NodeBuilder rule = builder.child(typeName); markAsNtUnstructured(rule); - List propNodeNames = new ArrayList<>(propNamesSet.size()); + List propNodeNames = newArrayListWithCapacity(propNamesSet.size()); NodeBuilder propNodes = rule.child(PROP_NODE); int i = 0; - for (String propName : propNames) { + for (String propName : propNames){ String propNodeName = propName; //For proper propName use the propName as childNode name - if (PropertyDefinition.isRelativeProperty(propName) - || propName.equals(includeAllProp)) { + if(PropertyDefinition.isRelativeProperty(propName) + || propName.equals(includeAllProp)){ propNodeName = "prop" + i++; } propNodeNames.add(propNodeName); @@ -1642,9 +1622,9 @@ private static NodeBuilder createIndexRules(NodeState defn) { markAsNtUnstructured(prop); prop.setProperty(FulltextIndexConstants.PROP_NAME, propName); - if (excludes.contains(propName)) { + if (excludes.contains(propName)){ prop.setProperty(FulltextIndexConstants.PROP_INDEX, false); - } else if (fullTextEnabled) { + } else if (fullTextEnabled){ prop.setProperty(FulltextIndexConstants.PROP_ANALYZED, true); prop.setProperty(FulltextIndexConstants.PROP_NODE_SCOPE_INDEX, true); prop.setProperty(FulltextIndexConstants.PROP_USE_IN_EXCERPT, storageEnabled); @@ -1652,18 +1632,18 @@ private static NodeBuilder createIndexRules(NodeState defn) { } else { prop.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX, true); - if (orderedProps.contains(propName)) { + if (orderedProps.contains(propName)){ prop.setProperty(FulltextIndexConstants.PROP_ORDERED, true); } } - if (propName.equals(includeAllProp)) { + if (propName.equals(includeAllProp)){ prop.setProperty(FulltextIndexConstants.PROP_IS_REGEX, true); } else { //Copy over the property configuration - NodeState propDefNode = getPropDefnNode(defn, propName); - if (propDefNode != null) { - for (PropertyState ps : propDefNode.getProperties()) { + NodeState propDefNode = getPropDefnNode(defn, propName); + if (propDefNode != null){ + for (PropertyState ps : propDefNode.getProperties()){ prop.setProperty(ps); } } @@ -1673,7 +1653,7 @@ private static NodeBuilder createIndexRules(NodeState defn) { //If no propertyType defined then default to UNKNOWN such that none //of the properties get indexed PropertyState supportedTypes = defn.getProperty(INCLUDE_PROPERTY_TYPES); - if (supportedTypes == null) { + if (supportedTypes == null){ supportedTypes = PropertyStates.createProperty(INCLUDE_PROPERTY_TYPES, TYPES_ALLOW_ALL_NAME); } rule.setProperty(supportedTypes); @@ -1682,16 +1662,16 @@ private static NodeBuilder createIndexRules(NodeState defn) { rule.setProperty(FulltextIndexConstants.RULE_INHERITED, false); } - propNodes.setProperty(OAK_CHILD_ORDER, propNodeNames, NAMES); + propNodes.setProperty(OAK_CHILD_ORDER, propNodeNames ,NAMES); markAsNtUnstructured(propNodes); } markAsNtUnstructured(builder); - builder.setProperty(OAK_CHILD_ORDER, declaringNodeTypes, NAMES); + builder.setProperty(OAK_CHILD_ORDER, declaringNodeTypes ,NAMES); return builder; } - private static NodeState getPropDefnNode(NodeState defn, String propName) { + private static NodeState getPropDefnNode(NodeState defn, String propName){ NodeState propNode = defn.getChildNode(FulltextIndexConstants.PROP_NODE); NodeState propDefNode; if (PropertyDefinition.isRelativeProperty(propName)) { @@ -1711,8 +1691,8 @@ private static NodeState getPropDefnNode(NodeState defn, String propName) { private int determineMaxExtractLength() { int length = getOptionalValue(definition.getChildNode(TIKA), FulltextIndexConstants.TIKA_MAX_EXTRACT_LENGTH, DEFAULT_MAX_EXTRACT_LENGTH); - if (length < 0) { - return -length * maxFieldLength; + if (length < 0){ + return - length * maxFieldLength; } return length; } @@ -1721,24 +1701,24 @@ private NodeState getTikaConfigNode() { return definition.getChildNode(TIKA).getChildNode(TIKA_CONFIG); } - private static Set getMultiProperty(NodeState definition, String propName) { + private static Set getMultiProperty(NodeState definition, String propName){ PropertyState pse = definition.getProperty(propName); - return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Set.of(); + return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Collections.emptySet(); } private static Set toLowerCase(Set values) { - Set result = new HashSet<>(); - for (String val : values) { + Set result = newHashSet(); + for(String val : values){ result.add(val.toLowerCase(Locale.ENGLISH)); } - return Set.copyOf(result); + return ImmutableSet.copyOf(result); } private static List getAllNodeTypes(ReadOnlyNodeTypeManager ntReg) { try { - List typeNames = new ArrayList<>(); + List typeNames = newArrayList(); NodeTypeIterator ntItr = ntReg.getAllNodeTypes(); - while (ntItr.hasNext()) { + while (ntItr.hasNext()){ typeNames.add(ntItr.nextNodeType().getName()); } return typeNames; @@ -1777,10 +1757,10 @@ private static String getPrimaryTypeName(NodeState state) { private static Iterable getMixinTypeNames(NodeState state) { PropertyState property = state.getProperty(JcrConstants.JCR_MIXINTYPES); - return property != null ? property.getValue(NAMES) : List.of(); + return property != null ? property.getValue(NAMES) : Collections.emptyList(); } - private static boolean hasOrderableChildren(NodeState state) { + private static boolean hasOrderableChildren(NodeState state){ return state.hasProperty(OAK_CHILD_ORDER); } @@ -1789,14 +1769,14 @@ static int getSupportedTypes(NodeState defn, String typePropertyName, int defaul if (pst != null) { int types = 0; for (String inc : pst.getValue(Type.STRINGS)) { - if (TYPES_ALLOW_ALL_NAME.equals(inc)) { + if (TYPES_ALLOW_ALL_NAME.equals(inc)){ return TYPES_ALLOW_ALL; } try { types |= 1 << PropertyType.valueFromName(inc); } catch (IllegalArgumentException e) { - log.warn("Unknown property type: {}", inc); + log.warn("Unknown property type: " + inc); } } return types; @@ -1804,12 +1784,12 @@ static int getSupportedTypes(NodeState defn, String typePropertyName, int defaul return defaultVal; } - static boolean includePropertyType(int includedPropertyTypes, int type) { - if (includedPropertyTypes == TYPES_ALLOW_ALL) { + static boolean includePropertyType(int includedPropertyTypes, int type){ + if(includedPropertyTypes == TYPES_ALLOW_ALL){ return true; } - if (includedPropertyTypes == TYPES_ALLOW_NONE) { + if (includedPropertyTypes == TYPES_ALLOW_NONE){ return false; } @@ -1817,30 +1797,30 @@ static boolean includePropertyType(int includedPropertyTypes, int type) { } private static boolean hasFulltextEnabledIndexRule(List rules) { - for (IndexingRule rule : rules) { - if (rule.isFulltextEnabled()) { + for (IndexingRule rule : rules){ + if (rule.isFulltextEnabled()){ return true; } } return false; } - private static void markAsNtUnstructured(NodeBuilder nb) { + private static void markAsNtUnstructured(NodeBuilder nb){ nb.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME); } protected static IndexFormatVersion determineIndexFormatVersion(NodeState defn) { //Compat mode version if specified has highest priority - if (defn.hasProperty(COMPAT_MODE)) { + if (defn.hasProperty(COMPAT_MODE)){ return versionFrom(defn.getProperty(COMPAT_MODE)); } - if (defn.hasProperty(INDEX_VERSION)) { + if (defn.hasProperty(INDEX_VERSION)){ return versionFrom(defn.getProperty(INDEX_VERSION)); } //No existing index data i.e. reindex or fresh index - if (!defn.getChildNode(INDEX_DATA_CHILD_NAME).exists()) { + if (!defn.getChildNode(INDEX_DATA_CHILD_NAME).exists()){ return determineVersionForFreshIndex(defn); } @@ -1851,20 +1831,20 @@ protected static IndexFormatVersion determineIndexFormatVersion(NodeState defn) return fullTextEnabled ? IndexFormatVersion.V1 : IndexFormatVersion.V2; } - static IndexFormatVersion determineVersionForFreshIndex(NodeState defn) { + static IndexFormatVersion determineVersionForFreshIndex(NodeState defn){ return determineVersionForFreshIndex(defn.getProperty(FULL_TEXT_ENABLED), defn.getProperty(COMPAT_MODE), defn.getProperty(INDEX_VERSION)); } - static IndexFormatVersion determineVersionForFreshIndex(NodeBuilder defnb) { + static IndexFormatVersion determineVersionForFreshIndex(NodeBuilder defnb){ return determineVersionForFreshIndex(defnb.getProperty(FULL_TEXT_ENABLED), defnb.getProperty(COMPAT_MODE), defnb.getProperty(INDEX_VERSION)); } private static IndexFormatVersion determineVersionForFreshIndex(PropertyState fulltext, PropertyState compat, - PropertyState version) { - if (compat != null) { + PropertyState version){ + if (compat != null){ return versionFrom(compat); } @@ -1878,20 +1858,20 @@ private static IndexFormatVersion determineVersionForFreshIndex(PropertyState fu IndexFormatVersion result = defaultToUse; //If default configured is lesser than existing then prefer existing - if (existing != null) { - result = IndexFormatVersion.max(result, existing); + if (existing != null){ + result = IndexFormatVersion.max(result,existing); } - //Check if fulltext is false which indicates it's a property index and + //Check if fulltext is false which indicates its a property index and //hence confirm to V2 or above - if (fulltext != null && !fulltext.getValue(Type.BOOLEAN)) { + if (fulltext != null && !fulltext.getValue(Type.BOOLEAN)){ return IndexFormatVersion.max(result, IndexFormatVersion.V2); } return result; } - private static IndexFormatVersion versionFrom(PropertyState ps) { + private static IndexFormatVersion versionFrom(PropertyState ps){ return IndexFormatVersion.getVersion(Ints.checkedCast(ps.getValue(Type.LONG))); } @@ -1918,14 +1898,14 @@ public static boolean supportsSyncOrNRTIndexing(NodeBuilder defn) { private static boolean supportsIndexingMode(NodeBuilder defn, String mode) { PropertyState async = defn.getProperty(IndexConstants.ASYNC_PROPERTY_NAME); - if (async == null) { + if (async == null){ return false; } return Iterables.contains(async.getValue(Type.STRINGS), mode); } protected static NodeState getIndexDefinitionState(NodeState defn) { - if (isDisableStoredIndexDefinition()) { + if (isDisableStoredIndexDefinition()){ return defn; } NodeState storedState = defn.getChildNode(INDEX_DEFINITION_NODE); @@ -2014,7 +1994,7 @@ static MODE getMode(@NotNull final NodeState facetConfigRoot) { String modeString; if (securePS != null) { - if (securePS.getType() == Type.BOOLEAN) { + if(securePS.getType() == Type.BOOLEAN) { // legacy secure config boolean secure = securePS.getValue(Type.BOOLEAN); return secure ? MODE.SECURE : MODE.INSECURE; From c57579fa7fb50a9780bb44d252e6bbcd21963e6c Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:27:47 +0200 Subject: [PATCH 38/86] Revert "OAK-10904: Close token refresh executor service after access token is no longer needed (#1545)" This reverts commit 632a15b3d0af8c98c65e4a02b40743db94729184. --- .../run/cli/SegmentTarFixtureProvider.java | 5 +- .../AzureSegmentStoreExplorerBackend.java | 11 +- .../oak/run/FileStoreDiffCommand.java | 29 ++-- .../oak/run/DataStoreCommandTest.java | 5 +- .../azure/AzureSegmentStoreService.java | 12 +- .../azure/AzureStorageCredentialManager.java | 147 ------------------ .../oak/segment/azure/AzureUtilities.java | 94 +++++++++++ .../oak/segment/azure/package-info.java | 2 +- .../oak/segment/azure/tool/AzureCheck.java | 7 +- .../oak/segment/azure/tool/AzureCompact.java | 11 +- .../oak/segment/azure/tool/SegmentCopy.java | 13 +- .../oak/segment/azure/tool/ToolUtils.java | 107 +++++++++---- ...entCopyAzureServicePrincipalToTarTest.java | 7 +- .../azure/tool/SegmentCopyTestBase.java | 2 +- .../oak/segment/azure/tool/ToolUtilsTest.java | 24 +-- .../upgrade/cli/node/SegmentAzureFactory.java | 20 +-- ...ureServicePrincipalNodeStoreContainer.java | 8 +- .../cli/node/SegmentAzureFactoryTest.java | 61 ++++---- 18 files changed, 240 insertions(+), 325 deletions(-) delete mode 100644 oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java index a2d1b05162f..38746a6ee56 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java @@ -31,7 +31,6 @@ import org.apache.jackrabbit.guava.common.io.Closer; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; @@ -54,12 +53,10 @@ static NodeStore configureSegment(Options options, BlobStore blobStore, Whiteboa FileStoreBuilder builder; if (segmentStoreType == ToolUtils.SegmentStoreType.AZURE) { - final AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager(); SegmentNodeStorePersistence segmentNodeStorePersistence = - ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri, azureStorageCredentialManager); + ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri); File tempDir = Files.createTempDirectory("azure-segment-store").toFile(); closer.register(() -> FileUtils.deleteQuietly(tempDir)); - closer.register(azureStorageCredentialManager); builder = fileStoreBuilder(tempDir).withCustomPersistence(segmentNodeStorePersistence); } else { builder = fileStoreBuilder(new File(pathOrUri)).withMaxFileSize(256); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java index 5f5b1580a00..f45333326a1 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java @@ -19,7 +19,6 @@ package org.apache.jackrabbit.oak.explorer; import org.apache.jackrabbit.guava.common.io.Files; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; @@ -39,16 +38,14 @@ public class AzureSegmentStoreExplorerBackend extends AbstractSegmentTarExplorerBackend { private final String path; private SegmentNodeStorePersistence persistence; - private final AzureStorageCredentialManager azureStorageCredentialManager; public AzureSegmentStoreExplorerBackend(String path) { this.path = path; - this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } @Override public void open() throws IOException { - this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManager); + this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); try { this.store = fileStoreBuilder(Files.createTempDir()) @@ -60,12 +57,6 @@ public void open() throws IOException { this.index = store.getTarReaderIndex(); } - @Override - public void close() { - super.close(); - azureStorageCredentialManager.close(); - } - @Override protected JournalFile getJournal() { return persistence.getJournalFile(); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java index 0d2ee2c6a5f..3bc7d27d0cc 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java @@ -28,7 +28,6 @@ import joptsimple.OptionSet; import joptsimple.OptionSpec; import org.apache.jackrabbit.oak.run.commons.Command; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; @@ -87,21 +86,19 @@ public void execute(String... args) throws Exception { } } else { if (pathOrURI.startsWith("az:")) { - try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { - SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI, azureStorageCredentialManager); - ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); - statusCode = Diff.builder() - .withPath(pathOrURI) - .withReadOnlyFileStore(store) - .withOutput(out) - .withInterval(interval) - .withIncremental(incremental) - .withFilter(path) - .withIgnoreMissingSegments(ignoreSNFEs) - .withRevisionsProcessor(ToolUtils::readRevisions) - .build() - .run(); - } + SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI); + ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); + statusCode = Diff.builder() + .withPath(pathOrURI) + .withReadOnlyFileStore(store) + .withOutput(out) + .withInterval(interval) + .withIncremental(incremental) + .withFilter(path) + .withIgnoreMissingSegments(ignoreSNFEs) + .withRevisionsProcessor(ToolUtils::readRevisions) + .build() + .run(); } else { ReadOnlyFileStore store = fileStoreBuilder(new File(pathOrURI)).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly(); statusCode = Diff.builder() diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java index 6a33a27f834..d77b05e1cfa 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java @@ -73,7 +73,6 @@ import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions.Type; import org.apache.jackrabbit.oak.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; @@ -1143,7 +1142,6 @@ class SegmentStoreFixture implements StoreFixture { class AzureSegmentStoreFixture extends SegmentStoreFixture { private static final String AZURE_DIR = "repository"; private String container; - private final AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager(); @Override public NodeStore init(DataStoreBlobStore blobStore, File storeFile) throws Exception { Properties props = AzureDataStoreUtils.getAzureConfig(); @@ -1160,7 +1158,7 @@ class AzureSegmentStoreFixture extends SegmentStoreFixture { // initialize azure segment for test setup SegmentNodeStorePersistence segmentNodeStorePersistence = - ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath, azureStorageCredentialManager); + ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath); fileStore = fileStoreBuilder(storeFile).withBlobStore(blobStore) .withCustomPersistence(segmentNodeStorePersistence).build(); @@ -1192,7 +1190,6 @@ protected String getAzureConnectionString(String accountName, String secret, Str public void after() { try { AzureDataStoreUtils.deleteContainer(container); - azureStorageCredentialManager.close(); } catch(Exception e) { log.error("Error in cleaning the container {}", container, e); } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java index 24465ea720c..1b5640c76d5 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureSegmentStoreService.java @@ -20,7 +20,7 @@ import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.LocationMode; -import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsToken; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.BlobRequestOptions; import com.microsoft.azure.storage.blob.CloudBlobClient; @@ -43,6 +43,7 @@ import java.util.Hashtable; import java.util.Objects; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom; import static org.osgi.framework.Constants.SERVICE_PID; @Component( @@ -60,7 +61,6 @@ public class AzureSegmentStoreService { public static final String DEFAULT_ENDPOINT_SUFFIX = "core.windows.net"; private ServiceRegistration registration; - private static AzureStorageCredentialManager azureStorageCredentialManager; @Activate public void activate(ComponentContext context, Configuration config) throws IOException { @@ -80,9 +80,6 @@ public void deactivate() throws IOException { registration.unregister(); registration = null; } - if (azureStorageCredentialManager != null) { - azureStorageCredentialManager.close(); - } } private static AzurePersistence createAzurePersistenceFrom(Configuration configuration) throws IOException { @@ -127,9 +124,8 @@ private static AzurePersistence createPersistenceFromConnectionURL(Configuration @NotNull private static AzurePersistence createPersistenceFromServicePrincipalCredentials(Configuration configuration) throws IOException { - azureStorageCredentialManager = new AzureStorageCredentialManager(); - StorageCredentials storageCredentialsToken = azureStorageCredentialManager.getStorageCredentialAccessTokenFromServicePrincipals(configuration.accountName(), configuration.clientId(), configuration.clientSecret(), configuration.tenantId()); - + StorageCredentialsToken storageCredentialsToken = storageCredentialAccessTokenFrom(configuration.accountName(), configuration.clientId(), configuration.clientSecret(), configuration.tenantId()); + try { CloudStorageAccount cloud = new CloudStorageAccount(storageCredentialsToken, true, DEFAULT_ENDPOINT_SUFFIX, configuration.accountName()); return createAzurePersistence(cloud, configuration, true); diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java deleted file mode 100644 index 4f962f10809..00000000000 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureStorageCredentialManager.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.segment.azure; - -import com.azure.core.credential.AccessToken; -import com.azure.core.credential.TokenRequestContext; -import com.azure.identity.ClientSecretCredential; -import com.azure.identity.ClientSecretCredentialBuilder; -import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; -import com.microsoft.azure.storage.StorageCredentialsToken; -import org.apache.commons.lang3.StringUtils; -import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; -import org.apache.jackrabbit.oak.segment.azure.util.Environment; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Objects; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; -import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; - -public class AzureStorageCredentialManager implements Closeable { - private static final Logger log = LoggerFactory.getLogger(AzureStorageCredentialManager.class); - private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default"; - private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L; - private static final long TOKEN_REFRESHER_DELAY = 1L; - private ClientSecretCredential clientSecretCredential; - private AccessToken accessToken; - private StorageCredentialsToken storageCredentialsToken; - private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); - - public StorageCredentials getStorageCredentialsFromEnvironment(@NotNull String accountName, @NotNull Environment environment) { - final String clientId = environment.getVariable(AZURE_CLIENT_ID); - final String clientSecret = environment.getVariable(AZURE_CLIENT_SECRET); - final String tenantId = environment.getVariable(AZURE_TENANT_ID); - - if (StringUtils.isNoneBlank(clientId, clientSecret, tenantId)) { - try { - return getStorageCredentialAccessTokenFromServicePrincipals(accountName, clientId, clientSecret, tenantId); - } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { - log.error("Error occurred while connecting to Azure Storage using service principals: ", e); - throw new IllegalArgumentException( - "Could not connect to the Azure Storage. Please verify if AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables are correctly set!"); - } - } - - log.warn("AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."); - - String key = environment.getVariable(AZURE_SECRET_KEY); - try { - return new StorageCredentialsAccountAndKey(accountName, key); - } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { - log.error("Error occurred while connecting to Azure Storage using secret key: ", e); - throw new IllegalArgumentException( - "Could not connect to the Azure Storage. Please verify if AZURE_SECRET_KEY environment variable is correctly set!"); - } - } - - public StorageCredentials getStorageCredentialAccessTokenFromServicePrincipals(String accountName, String clientId, String clientSecret, String tenantId) { - boolean isAccessTokenGenerated = false; - if (accessToken == null) { - clientSecretCredential = new ClientSecretCredentialBuilder() - .clientId(clientId) - .clientSecret(clientSecret) - .tenantId(tenantId) - .build(); - accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); - if (accessToken == null || StringUtils.isBlank(accessToken.getToken())) { - throw new IllegalArgumentException("Could not connect to azure storage, access token is null or empty"); - } - storageCredentialsToken = new StorageCredentialsToken(accountName, accessToken.getToken()); - isAccessTokenGenerated = true; - } - Objects.requireNonNull(storageCredentialsToken, "storageCredentialsToken cannot be null"); - - // start refresh token executor only when the access token is first generated - if (isAccessTokenGenerated) { - log.info("starting refresh token task at: {}", OffsetDateTime.now()); - TokenRefresher tokenRefresher = new TokenRefresher(); - executorService.scheduleWithFixedDelay(tokenRefresher, TOKEN_REFRESHER_INITIAL_DELAY, TOKEN_REFRESHER_DELAY, TimeUnit.MINUTES); - } - return storageCredentialsToken; - } - - /** - * This class represents a token refresher responsible for ensuring the validity of the access token used for azure AD authentication. - * The access token generated by the Azure client is valid for 1 hour only. Therefore, this class periodically checks the validity - * of the access token and refreshes it if necessary. The refresh is triggered when the current access token is about to expire, - * defined by a threshold of 5 minutes from the current time. This threshold is similar to what is being used in azure identity library to - * generate a new token - */ - private class TokenRefresher implements Runnable { - @Override - public void run() { - try { - log.debug("Checking for azure access token expiry at: {}", LocalDateTime.now()); - OffsetDateTime tokenExpiryThreshold = OffsetDateTime.now().plusMinutes(5); - if (accessToken.getExpiresAt() != null && accessToken.getExpiresAt().isBefore(tokenExpiryThreshold)) { - log.info("Access token is about to expire (5 minutes or less) at: {}. New access token will be generated", - accessToken.getExpiresAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); - AccessToken newToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); - log.info("New azure access token generated at: {}", LocalDateTime.now()); - if (newToken == null || StringUtils.isBlank(newToken.getToken())) { - log.error("New access token is null or empty"); - return; - } - // update access token with newly generated token - accessToken = newToken; - storageCredentialsToken.updateToken(accessToken.getToken()); - } - } catch (Exception e) { - log.error("Error while acquiring new access token: ", e); - } - } - } - - @Override - public void close() { - new ExecutorCloser(executorService).close(); - log.info("Access token refresh executor shutdown completed"); - } -} diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java index 5c740fac1d2..ec5763b0cfe 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzureUtilities.java @@ -16,10 +16,15 @@ */ package org.apache.jackrabbit.oak.segment.azure; +import com.azure.core.credential.AccessToken; +import com.azure.core.credential.TokenRequestContext; +import com.azure.identity.ClientSecretCredential; +import com.azure.identity.ClientSecretCredentialBuilder; import com.microsoft.azure.storage.CloudStorageAccount; import com.microsoft.azure.storage.ResultContinuation; import com.microsoft.azure.storage.ResultSegment; import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsToken; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.StorageUri; import com.microsoft.azure.storage.blob.BlobListingDetails; @@ -28,7 +33,9 @@ import com.microsoft.azure.storage.blob.CloudBlobDirectory; import com.microsoft.azure.storage.blob.LeaseStatus; import com.microsoft.azure.storage.blob.ListBlobItem; +import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.commons.Buffer; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; import org.apache.jackrabbit.oak.segment.spi.RepositoryNotReachableException; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -41,19 +48,33 @@ import java.net.URISyntaxException; import java.nio.file.Paths; import java.security.InvalidKeyException; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; public final class AzureUtilities { + static { + Runtime.getRuntime().addShutdownHook(new Thread(AzureUtilities::shutDown)); + } + public static final String AZURE_ACCOUNT_NAME = "AZURE_ACCOUNT_NAME"; public static final String AZURE_SECRET_KEY = "AZURE_SECRET_KEY"; public static final String AZURE_TENANT_ID = "AZURE_TENANT_ID"; public static final String AZURE_CLIENT_ID = "AZURE_CLIENT_ID"; public static final String AZURE_CLIENT_SECRET = "AZURE_CLIENT_SECRET"; + private static final String AZURE_DEFAULT_SCOPE = "https://storage.azure.com/.default"; + private static final long TOKEN_REFRESHER_INITIAL_DELAY = 45L; + private static final long TOKEN_REFRESHER_DELAY = 1L; private static final Logger log = LoggerFactory.getLogger(AzureUtilities.class); + private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); private AzureUtilities() { } @@ -124,6 +145,24 @@ public static CloudBlobDirectory cloudBlobDirectoryFrom(String connection, Strin return container.getDirectoryReference(dir); } + public static StorageCredentialsToken storageCredentialAccessTokenFrom(String accountName, String clientId, String clientSecret, String tenantId) { + ClientSecretCredential clientSecretCredential = new ClientSecretCredentialBuilder() + .clientId(clientId) + .clientSecret(clientSecret) + .tenantId(tenantId) + .build(); + + AccessToken accessToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); + if (accessToken == null || StringUtils.isBlank(accessToken.getToken())) { + log.error("Access token is null or empty"); + throw new IllegalArgumentException("Could not connect to azure storage, access token is null or empty"); + } + StorageCredentialsToken storageCredentialsToken = new StorageCredentialsToken(accountName, accessToken.getToken()); + TokenRefresher tokenRefresher = new TokenRefresher(clientSecretCredential, accessToken, storageCredentialsToken); + executorService.scheduleWithFixedDelay(tokenRefresher, TOKEN_REFRESHER_INITIAL_DELAY, TOKEN_REFRESHER_DELAY, TimeUnit.MINUTES); + return storageCredentialsToken; + } + private static ResultSegment listBlobsInSegments(CloudBlobDirectory directory, ResultContinuation token) throws IOException { ResultSegment result = null; @@ -191,6 +230,61 @@ public void write(@NotNull byte[] bytes, int offset, int length) { } } + /** + * This class represents a token refresher responsible for ensuring the validity of the access token used for azure AD authentication. + * The access token generated by the Azure client is valid for 1 hour only. Therefore, this class periodically checks the validity + * of the access token and refreshes it if necessary. The refresh is triggered when the current access token is about to expire, + * defined by a threshold of 5 minutes from the current time. This threshold is similar to what is being used in azure identity to + * generate a new token + */ + private static class TokenRefresher implements Runnable { + + private final ClientSecretCredential clientSecretCredential; + private AccessToken accessToken; + private final StorageCredentialsToken storageCredentialsToken; + + + /** + * Constructs a new TokenRefresher object with the specified parameters. + * + * @param clientSecretCredential The client secret credential used to obtain the access token. + * @param accessToken The current access token. + * @param storageCredentialsToken The storage credentials token associated with the access token. + */ + public TokenRefresher(ClientSecretCredential clientSecretCredential, + AccessToken accessToken, + StorageCredentialsToken storageCredentialsToken) { + this.clientSecretCredential = clientSecretCredential; + this.accessToken = accessToken; + this.storageCredentialsToken = storageCredentialsToken; + } + + @Override + public void run() { + try { + log.debug("Checking for azure access token expiry at: {}", LocalDateTime.now()); + OffsetDateTime tokenExpiryThreshold = OffsetDateTime.now().plusMinutes(5); + if (accessToken.getExpiresAt() != null && accessToken.getExpiresAt().isBefore(tokenExpiryThreshold)) { + log.info("Access token is about to expire (5 minutes or less) at: {}. New access token will be generated", + accessToken.getExpiresAt().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + AccessToken newToken = clientSecretCredential.getTokenSync(new TokenRequestContext().addScopes(AZURE_DEFAULT_SCOPE)); + if (newToken == null || StringUtils.isBlank(newToken.getToken())) { + log.error("New access token is null or empty"); + return; + } + this.accessToken = newToken; + this.storageCredentialsToken.updateToken(this.accessToken.getToken()); + } + } catch (Exception e) { + log.error("Error while acquiring new access token: ", e); + } + } + } + + public static void shutDown() { + new ExecutorCloser(executorService).close(); + } + } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java index 954129d77c7..99e2b3f1cf9 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/package-info.java @@ -15,7 +15,7 @@ * limitations under the License. */ @Internal(since = "1.0.0") -@Version("3.0.0") +@Version("2.4.0") package org.apache.jackrabbit.oak.segment.azure; import org.apache.jackrabbit.oak.commons.annotations.Internal; diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java index 148d1ce30d2..f4845155740 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCheck.java @@ -19,7 +19,6 @@ import com.google.common.io.Files; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; @@ -348,7 +347,6 @@ public void afterSegmentRead(File file, long msb, long lsb, int length, long ela private final Integer persistentCacheSizeGb; private final CloudBlobDirectory cloudBlobDirectory; - private final AzureStorageCredentialManager azureStorageCredentialManager; private AzureCheck(Builder builder) { this.path = builder.path; @@ -367,7 +365,6 @@ private AzureCheck(Builder builder) { this.persistentCachePath = builder.persistentCachePath; this.persistentCacheSizeGb = builder.persistentCacheSizeGb; this.cloudBlobDirectory = builder.cloudBlobDirectory; - this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } private static Integer revisionsToCheckCount(Integer revisionsCount) { @@ -381,7 +378,7 @@ public int run() { if (cloudBlobDirectory != null) { persistence = new AzurePersistence(cloudBlobDirectory); } else { - persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManager); + persistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); } if (persistentCachePath != null) { @@ -427,8 +424,6 @@ public int run() { } catch (Exception e) { e.printStackTrace(err); return 1; - } finally { - azureStorageCredentialManager.close(); } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java index a5db11950b8..3e79f11837c 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/AzureCompact.java @@ -32,7 +32,6 @@ import org.apache.jackrabbit.oak.segment.SegmentCache; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GCType; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; @@ -266,7 +265,6 @@ public AzureCompact build() { private final CloudBlobDirectory sourceCloudBlobDirectory; private final CloudBlobDirectory destinationCloudBlobDirectory; - private final AzureStorageCredentialManager azureStorageCredentialManager; private AzureCompact(Builder builder) { this.path = builder.path; @@ -281,7 +279,6 @@ private AzureCompact(Builder builder) { this.persistentCacheSizeGb = builder.persistentCacheSizeGb; this.sourceCloudBlobDirectory = builder.sourceCloudBlobDirectory; this.destinationCloudBlobDirectory = builder.destinationCloudBlobDirectory; - this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } public int run() throws IOException, StorageException, URISyntaxException { @@ -293,8 +290,8 @@ public int run() throws IOException, StorageException, URISyntaxException { roPersistence = new AzurePersistence(sourceCloudBlobDirectory); rwPersistence = new AzurePersistence(destinationCloudBlobDirectory); } else { - roPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path, azureStorageCredentialManager); - rwPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath, azureStorageCredentialManager); + roPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, path); + rwPersistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, targetPath); } if (persistentCachePath != null) { @@ -357,15 +354,13 @@ public int run() throws IOException, StorageException, URISyntaxException { CloudBlobContainer targetContainer = null; if (targetPath != null) { - CloudBlobDirectory targetDirectory = createCloudBlobDirectory(targetPath.substring(3), azureStorageCredentialManager); + CloudBlobDirectory targetDirectory = createCloudBlobDirectory(targetPath.substring(3)); targetContainer = targetDirectory.getContainer(); } else { targetContainer = destinationCloudBlobDirectory.getContainer(); } printTargetRepoSizeInfo(targetContainer); - // close azure storage credential manager - azureStorageCredentialManager.close(); return 0; } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java index df2452cca74..44bb5e3ecd1 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java @@ -43,7 +43,6 @@ import java.util.concurrent.Future; import org.apache.jackrabbit.oak.commons.Buffer; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.SegmentStoreMigrator.Segment; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.SegmentStoreType; import org.apache.jackrabbit.oak.segment.azure.util.Retrier; @@ -260,7 +259,6 @@ public SegmentCopy build() { private SegmentNodeStorePersistence destPersistence; private ExecutorService executor = Executors.newFixedThreadPool(READ_THREADS + 1); - private final AzureStorageCredentialManager azureStorageCredentialManager; public SegmentCopy(Builder builder) { this.source = builder.source; @@ -273,7 +271,6 @@ public SegmentCopy(Builder builder) { this.maxSizeGb = builder.maxSizeGb; this.outWriter = builder.outWriter; this.errWriter = builder.errWriter; - this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } public int run() { @@ -287,7 +284,7 @@ public int run() { if (flat && destType == SegmentStoreType.TAR) { try { - srcPersistence = newSegmentNodeStorePersistence(srcType, source, azureStorageCredentialManager); + srcPersistence = newSegmentNodeStorePersistence(srcType, source); SegmentArchiveManager sourceManager = srcPersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter(), new RemoteStoreMonitorAdapter()); @@ -364,14 +361,12 @@ public int run() { destination); e.printStackTrace(errWriter); return 1; - } finally { - azureStorageCredentialManager.close(); } } else { try { if (srcPersistence == null || destPersistence == null) { - srcPersistence = newSegmentNodeStorePersistence(srcType, source, azureStorageCredentialManager); - destPersistence = newSegmentNodeStorePersistence(destType, destination, azureStorageCredentialManager); + srcPersistence = newSegmentNodeStorePersistence(srcType, source); + destPersistence = newSegmentNodeStorePersistence(destType, destination); } printMessage(outWriter, "Started segment-copy transfer!"); @@ -395,8 +390,6 @@ public int run() { destination); e.printStackTrace(errWriter); return 1; - } finally { - azureStorageCredentialManager.close(); } } diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java index 172b7b2dccb..20283ef77b8 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java @@ -19,6 +19,11 @@ package org.apache.jackrabbit.oak.segment.azure.tool; import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_CLIENT_SECRET; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_SECRET_KEY; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.AZURE_TENANT_ID; +import static org.apache.jackrabbit.oak.segment.azure.AzureUtilities.storageCredentialAccessTokenFrom; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_ACCOUNT_NAME; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_DIR; import static org.apache.jackrabbit.oak.segment.azure.util.AzureConfigurationParserUtils.KEY_SHARED_ACCESS_SIGNATURE; @@ -34,12 +39,11 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.oak.commons.Buffer; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.CompactorType; @@ -60,12 +64,12 @@ import org.apache.jackrabbit.guava.common.collect.Iterators; import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -122,6 +126,22 @@ public static FileStore newFileStore(SegmentNodeStorePersistence persistence, Fi .build(); } + public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, + String pathOrUri, String persistentCachePath, Integer persistentCacheSize) { + SegmentNodeStorePersistence persistence = null; + + switch (storeType) { + case AZURE: + CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); + persistence = decorateWithCache(new AzurePersistence(cloudBlobDirectory), persistentCachePath, persistentCacheSize); + break; + default: + persistence = new TarPersistence(new File(pathOrUri)); + } + + return persistence; + } + public static SegmentNodeStorePersistence decorateWithCache(SegmentNodeStorePersistence persistence, String persistentCachePath, Integer persistentCacheSize) { PersistentCache persistentCache = new PersistentDiskCache(new File(persistentCachePath), @@ -130,18 +150,16 @@ public static SegmentNodeStorePersistence decorateWithCache(SegmentNodeStorePers } public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, - String pathOrUri, - @Nullable AzureStorageCredentialManager azureStorageCredentialManager) { + String pathOrUri) { SegmentNodeStorePersistence persistence = null; switch (storeType) { - case AZURE: - Objects.requireNonNull(azureStorageCredentialManager, "azure storage credentials manager instance cannot be null"); - CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3), azureStorageCredentialManager); - persistence = new AzurePersistence(cloudBlobDirectory); - break; - default: - persistence = new TarPersistence(new File(pathOrUri)); + case AZURE: + CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); + persistence = new AzurePersistence(cloudBlobDirectory); + break; + default: + persistence = new TarPersistence(new File(pathOrUri)); } return persistence; @@ -160,13 +178,11 @@ public static SegmentArchiveManager createArchiveManager(SegmentNodeStorePersist return archiveManager; } - public static CloudBlobDirectory createCloudBlobDirectory(String path, AzureStorageCredentialManager azureStorageCredentialManager) { - return createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManager); + public static CloudBlobDirectory createCloudBlobDirectory(String path) { + return createCloudBlobDirectory(path, ENVIRONMENT); } - public static CloudBlobDirectory createCloudBlobDirectory(String path, - Environment environment, - AzureStorageCredentialManager azureStorageCredentialManager) { + public static CloudBlobDirectory createCloudBlobDirectory(String path, Environment environment) { Map config = parseAzureConfigurationFromUri(path); String accountName = config.get(KEY_ACCOUNT_NAME); @@ -175,7 +191,7 @@ public static CloudBlobDirectory createCloudBlobDirectory(String path, if (config.containsKey(KEY_SHARED_ACCESS_SIGNATURE)) { credentials = new StorageCredentialsSharedAccessSignature(config.get(KEY_SHARED_ACCESS_SIGNATURE)); } else { - credentials = azureStorageCredentialManager.getStorageCredentialsFromEnvironment(accountName, environment); + credentials = getStorageCredentialsFromAccountAndEnv(accountName, environment); } String uri = config.get(KEY_STORAGE_URI); @@ -185,29 +201,58 @@ public static CloudBlobDirectory createCloudBlobDirectory(String path, return AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } catch (URISyntaxException | StorageException e) { throw new IllegalArgumentException( - "Could not connect to the Azure Storage. Please verify the path provided!"); + "Could not connect to the Azure Storage. Please verify the path provided!"); } } public static List readRevisions(String uri) { - try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { - SegmentNodeStorePersistence persistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri, azureStorageCredentialManager); - JournalFile journal = persistence.getJournalFile(); - if (journal.exists()) { - try (JournalReader journalReader = new JournalReader(journal)) { - Iterator revisionIterator = Iterators.transform(journalReader, - entry -> entry.getRevision()); - return newArrayList(revisionIterator); - } catch (Exception e) { - log.error("Error while reading from journal file"); - e.printStackTrace(); - } + SegmentNodeStorePersistence persistence = newSegmentNodeStorePersistence(SegmentStoreType.AZURE, uri); + JournalFile journal = persistence.getJournalFile(); + + if (journal.exists()) { + try (JournalReader journalReader = new JournalReader(journal)) { + Iterator revisionIterator = Iterators.transform(journalReader, + entry -> entry.getRevision()); + return newArrayList(revisionIterator); + } catch (Exception e) { + e.printStackTrace(); } } return newArrayList(); } + @NotNull + public static StorageCredentials getStorageCredentialsFromAccountAndEnv(@NotNull String accountName) { + return getStorageCredentialsFromAccountAndEnv(accountName, ENVIRONMENT); + } + + @NotNull + private static StorageCredentials getStorageCredentialsFromAccountAndEnv(String accountName, Environment environment) { + String clientId = environment.getVariable(AZURE_CLIENT_ID); + String clientSecret = environment.getVariable(AZURE_CLIENT_SECRET); + String tenantId = environment.getVariable(AZURE_TENANT_ID); + + if (!StringUtils.isAnyBlank(clientId, clientSecret, tenantId)) { + try { + return storageCredentialAccessTokenFrom(accountName, clientId, clientSecret, tenantId); + } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { + throw new IllegalArgumentException( + "Could not connect to the Azure Storage. Please verify if AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables are correctly set!"); + } + } else { + log.warn("AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."); + } + + String key = environment.getVariable(AZURE_SECRET_KEY); + try { + return new StorageCredentialsAccountAndKey(accountName, key); + } catch (IllegalArgumentException | StringIndexOutOfBoundsException e) { + throw new IllegalArgumentException( + "Could not connect to the Azure Storage. Please verify if AZURE_SECRET_KEY environment variable is correctly set!"); + } + } + public static SegmentStoreType storeTypeFromPathOrUri(String pathOrUri) { if (pathOrUri.startsWith("az:")) { return SegmentStoreType.AZURE; diff --git a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java index 16c5052670e..892d3801514 100644 --- a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java +++ b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyAzureServicePrincipalToTarTest.java @@ -20,7 +20,6 @@ import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; @@ -53,10 +52,8 @@ public void testSegmentCopy() throws Exception { protected SegmentNodeStorePersistence getSrcPersistence() { String accountName = ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME); String path = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, CONTAINER_NAME, DIR); - CloudBlobDirectory cloudBlobDirectory; - try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { - cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManager); - } + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT); + return new AzurePersistence(cloudBlobDirectory); } diff --git a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java index edad03847fb..114161ac24a 100644 --- a/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java +++ b/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java @@ -231,7 +231,7 @@ private void checkManifest(SegmentNodeStorePersistence srcPersistence, SegmentNo } protected SegmentNodeStorePersistence getTarPersistence() { - return newSegmentNodeStorePersistence(SegmentStoreType.TAR, folder.getRoot().getAbsolutePath(), null); + return newSegmentNodeStorePersistence(SegmentStoreType.TAR, folder.getRoot().getAbsolutePath()); } protected SegmentNodeStorePersistence getAzurePersistence() throws Exception { diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java index 4ce192c794e..6ae1a0abba1 100644 --- a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtilsTest.java @@ -37,12 +37,9 @@ import com.microsoft.azure.storage.StorageException; import com.microsoft.azure.storage.blob.CloudBlobDirectory; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.util.Environment; import org.jetbrains.annotations.NotNull; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.MockedStatic; @@ -76,17 +73,6 @@ public class ToolUtilsTest { public static final String AZURE_SECRET_KEY_WARNING = "AZURE_CLIENT_ID, AZURE_CLIENT_SECRET and AZURE_TENANT_ID environment variables empty or missing. Switching to authentication with AZURE_SECRET_KEY."; private final TestEnvironment environment = new TestEnvironment(); - private AzureStorageCredentialManager azureStorageCredentialManager; - - @Before - public void init() { - this.azureStorageCredentialManager = new AzureStorageCredentialManager(); - } - - @After - public void clear() { - this.azureStorageCredentialManager.close(); - } @Test public void createCloudBlobDirectoryWithAccessKey() { @@ -96,7 +82,7 @@ public void createCloudBlobDirectoryWithAccessKey() { StorageCredentialsAccountAndKey credentials = expectCredentials( StorageCredentialsAccountAndKey.class, - () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManager), + () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment), DEFAULT_CONTAINER_URL ); @@ -112,7 +98,7 @@ public void createCloudBlobDirectoryWithAccessKey() { public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() { environment.setVariable(AZURE_SECRET_KEY, null); assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManager) + ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment) ); } @@ -120,7 +106,7 @@ public void createCloudBlobDirectoryFailsWhenAccessKeyNotPresent() { public void createCloudBlobDirectoryFailsWhenAccessKeyIsInvalid() { environment.setVariable(AZURE_SECRET_KEY, "invalid"); assertThrows(IllegalArgumentException.class, () -> - ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment, azureStorageCredentialManager) + ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH, environment) ); } @@ -130,7 +116,7 @@ public void createCloudBlobDirectoryWithSasUri() { StorageCredentialsSharedAccessSignature credentials = expectCredentials( StorageCredentialsSharedAccessSignature.class, - () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken, azureStorageCredentialManager), + () -> ToolUtils.createCloudBlobDirectory(DEFAULT_SEGMENT_STORE_PATH + '?' + sasToken), DEFAULT_CONTAINER_URL ); @@ -149,7 +135,7 @@ public void createCloudBlobDirectoryWithServicePrincipal() throws URISyntaxExcep String containerName = "oak"; String segmentStorePath = String.format(SEGMENT_STORE_PATH_FORMAT, accountName, containerName, DEFAULT_REPO_DIR); - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT, azureStorageCredentialManager); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(segmentStorePath, ENVIRONMENT); assertNotNull(cloudBlobDirectory); assertEquals(containerName, cloudBlobDirectory.getContainer().getName()); } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java index ae299321978..a48ecd6de67 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactory.java @@ -25,16 +25,14 @@ import org.apache.jackrabbit.guava.common.io.Files; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; -import org.apache.jackrabbit.oak.segment.azure.util.Environment; +import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.file.FileStore; import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.apache.jackrabbit.oak.upgrade.cli.CliUtils; import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils.NodeStoreWithFileStore; import java.io.File; @@ -55,8 +53,6 @@ public class SegmentAzureFactory implements NodeStoreFactory { private int segmentCacheSize; private final boolean readOnly; - private static final Environment environment = new Environment(); - private AzureStorageCredentialManager azureStorageCredentialManager; public static class Builder { private final String dir; @@ -120,7 +116,7 @@ public SegmentAzureFactory(Builder builder) { public NodeStore create(BlobStore blobStore, Closer closer) throws IOException { AzurePersistence azPersistence = null; try { - azPersistence = createAzurePersistence(closer); + azPersistence = createAzurePersistence(); } catch (StorageException | URISyntaxException | InvalidKeyException e) { throw new IllegalStateException(e); } @@ -152,7 +148,7 @@ public NodeStore create(BlobStore blobStore, Closer closer) throws IOException { } } - private AzurePersistence createAzurePersistence(Closer closer) throws StorageException, URISyntaxException, InvalidKeyException { + private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; // connection string will take precedence over accountkey / sas / service principal @@ -163,9 +159,7 @@ private AzurePersistence createAzurePersistence(Closer closer) throws StorageExc if (StringUtils.isNotBlank(sasToken)) { credentials = new StorageCredentialsSharedAccessSignature(sasToken); } else { - this.azureStorageCredentialManager = new AzureStorageCredentialManager(); - credentials = azureStorageCredentialManager.getStorageCredentialsFromEnvironment(accountName, environment); - closer.register(azureStorageCredentialManager); + credentials = ToolUtils.getStorageCredentialsFromAccountAndEnv(accountName); } cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } @@ -180,12 +174,9 @@ private AzurePersistence createAzurePersistence(Closer closer) throws StorageExc @Override public boolean hasExternalBlobReferences() throws IOException { AzurePersistence azPersistence = null; - Closer closer = Closer.create(); - CliUtils.handleSigInt(closer); try { - azPersistence = createAzurePersistence(closer); + azPersistence = createAzurePersistence(); } catch (StorageException | URISyntaxException | InvalidKeyException e) { - closer.close(); throw new IllegalStateException(e); } @@ -201,7 +192,6 @@ public boolean hasExternalBlobReferences() throws IOException { throw new IOException(e); } finally { tmpDir.delete(); - closer.close(); } } diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java index c0619c60419..2347d3e210e 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/container/SegmentAzureServicePrincipalNodeStoreContainer.java @@ -20,7 +20,6 @@ import org.apache.jackrabbit.guava.common.io.Files; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.azure.AzurePersistence; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; @@ -44,7 +43,6 @@ public class SegmentAzureServicePrincipalNodeStoreContainer implements NodeStore private FileStore fs; private File tmpDir; private AzurePersistence azurePersistence; - private final AzureStorageCredentialManager azureStorageCredentialManager; public SegmentAzureServicePrincipalNodeStoreContainer() { this(null); @@ -52,7 +50,6 @@ public SegmentAzureServicePrincipalNodeStoreContainer() { public SegmentAzureServicePrincipalNodeStoreContainer(BlobStore blobStore) { this.blobStore = blobStore; - this.azureStorageCredentialManager = new AzureStorageCredentialManager(); } @@ -86,7 +83,7 @@ private AzurePersistence createAzurePersistence() { } String path = String.format(AZURE_SEGMENT_STORE_PATH, ENVIRONMENT.getVariable(AzureUtilities.AZURE_ACCOUNT_NAME), CONTAINER_NAME, DIR); - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT, azureStorageCredentialManager); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(path, ENVIRONMENT); return new AzurePersistence(cloudBlobDirectory); } @@ -99,9 +96,6 @@ public void close() { if (tmpDir != null) { tmpDir.delete(); } - if (azureStorageCredentialManager != null) { - azureStorageCredentialManager.close(); - } } @Override diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java index 00e10ddfe5f..a28803f5975 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentAzureFactoryTest.java @@ -25,7 +25,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.guava.common.io.Closer; import org.apache.jackrabbit.oak.blob.cloud.azure.blobstorage.AzuriteDockerRule; -import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager; import org.apache.jackrabbit.oak.segment.azure.AzureUtilities; import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; import org.apache.jackrabbit.oak.segment.azure.util.Environment; @@ -109,21 +108,19 @@ public void testConnectionWithUri_accessKey() throws IOException { String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME); Closer closer = Closer.create(); - try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { - try { - SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, - false) - .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) - .uri(uri) - .build(); - closer = Closer.create(); - CliUtils.handleSigInt(closer); - FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); - assertEquals(1, nodeStore.getFileStore().getSegmentCount()); - } finally { - closer.close(); - cleanup(uri, azureStorageCredentialManager); - } + try { + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, + false) + .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) + .uri(uri) + .build(); + closer = Closer.create(); + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + } finally { + closer.close(); + cleanup(uri); } } @@ -136,28 +133,26 @@ public void testConnectionWithUri_servicePrincipal() throws IOException, Interru String uri = String.format(CONNECTION_URI, ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME), CONTAINER_NAME); Closer closer = Closer.create(); - try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) { - try { - SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, - false) - .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) - .uri(uri) - .build(); - - CliUtils.handleSigInt(closer); - FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); - assertEquals(1, nodeStore.getFileStore().getSegmentCount()); - } finally { - closer.close(); - cleanup(uri, azureStorageCredentialManager); - } + try { + SegmentAzureFactory segmentAzureFactory = new SegmentAzureFactory.Builder(DIR, 256, + false) + .accountName(ENVIRONMENT.getVariable(AZURE_ACCOUNT_NAME)) + .uri(uri) + .build(); + + CliUtils.handleSigInt(closer); + FileStoreUtils.NodeStoreWithFileStore nodeStore = (FileStoreUtils.NodeStoreWithFileStore) segmentAzureFactory.create(null, closer); + assertEquals(1, nodeStore.getFileStore().getSegmentCount()); + } finally { + closer.close(); + cleanup(uri); } } - private void cleanup(String uri, AzureStorageCredentialManager azureStorageCredentialManager) { + private void cleanup(String uri) { uri = uri + "/" + DIR; try { - CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT, azureStorageCredentialManager); + CloudBlobDirectory cloudBlobDirectory = ToolUtils.createCloudBlobDirectory(uri, ENVIRONMENT); AzureUtilities.deleteAllBlobs(cloudBlobDirectory); } catch (Exception e) { throw new IllegalStateException(e); From c2be5ba765dce00a1c459cb5530c3353fb3f16eb Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:27:49 +0200 Subject: [PATCH 39/86] Revert "OAK-10976 - Avoid unnecessary call to PathUtils.getName in IndexDefinition (#1603)" This reverts commit e4119605ebfe8fe7c4d2b02805291e3605af9134. --- .../jackrabbit/oak/plugins/index/search/IndexDefinition.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java index ffe2dba7200..45ad59e231b 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexDefinition.java @@ -1525,10 +1525,10 @@ private NamePattern(String pattern, */ boolean matches(String propertyPath) { String parentPath = getParentPath(propertyPath); + String propertyName = PathUtils.getName(propertyPath); if (!this.parentPath.equals(parentPath)) { return false; } - String propertyName = PathUtils.getName(propertyPath); return pattern.matcher(propertyName).matches(); } From 329b7518124ccddd6946845deaf985d4bd738337 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:27:50 +0200 Subject: [PATCH 40/86] Revert "OAK-10803: fix NPE when Mongo is unavailable, remove '*' imports (#1601)" This reverts commit 5dd63444fac480e104966babbae076afdb618bb0. --- .../document/DocumentPropertyStateTest.java | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java index 02c0dda73ec..a9c4ef67bc3 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java @@ -16,19 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.document; -import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; -import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assume.assumeTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -37,6 +24,7 @@ import java.util.Objects; import java.util.Set; +import com.mongodb.ReadPreference; import org.apache.commons.lang3.RandomStringUtils; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; @@ -48,12 +36,20 @@ import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; -import com.mongodb.ReadPreference; +import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; +import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; +import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.*; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; public class DocumentPropertyStateTest { @@ -253,39 +249,38 @@ public void testInterestingStringsWithCompression() { @Test public void testBrokenSurrogateWithoutCompressionForMongo() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MONGO, false); + getBrokenSurrogateAndInitializeDifferentStores(MONGO, false); } @Test public void testBrokenSurrogateWithoutCompressionForRDB() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.RDB_H2, false); + getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, false); } @Test public void testBrokenSurrogateWithoutCompressionForInMemory() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MEMORY, false); + getBrokenSurrogateAndInitializeDifferentStores(MEMORY, false); } @Test public void testBrokenSurrogateWithCompressionForMongo() throws CommitFailedException { DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MONGO, true); + getBrokenSurrogateAndInitializeDifferentStores(MONGO, true); } @Test public void testBrokenSurrogateWithCompressionForRDB() throws CommitFailedException { DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.RDB_H2, true); + getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, true); } @Test public void testBrokenSurrogateWithCompressionForInMemory() throws CommitFailedException { DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture.MEMORY, true); + getBrokenSurrogateAndInitializeDifferentStores(MEMORY, true); } private void getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture fixture, boolean compressionEnabled) throws CommitFailedException { - assumeTrue(fixture.isAvailable()); String test = "brokensurrogate:dfsa\ud800"; DocumentStore store = null; DocumentNodeStore nodeStore = null; From 46b91a69977c3306786ef256bac49993b13fddeb Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:02 +0200 Subject: [PATCH 41/86] Revert "OAK-10971 - Add a method to test if a path is a direct ancestor of another: PathUtils.isDirectAncestor() (#1598)" This reverts commit c416850e3aaca0a6057aa3541531423a34f763a8. --- .../jackrabbit/oak/commons/PathUtils.java | 13 ----------- .../jackrabbit/oak/commons/package-info.java | 2 +- .../jackrabbit/oak/commons/PathUtilsTest.java | 22 ------------------- 3 files changed, 1 insertion(+), 36 deletions(-) diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java index d74bfd41e55..8c94da7a76e 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java @@ -376,19 +376,6 @@ public static boolean isAncestor(String ancestor, String path) { } } - /** - * Check if a path is a direct ancestor of another path. - * - * @param ancestor the ancestor path - * @param path the potential direct offspring path - * @return true if the path is a direct offspring of the ancestor - */ - public static boolean isDirectAncestor(String ancestor, String path) { - int lastSlashInPath = path.lastIndexOf('/'); - return ((PathUtils.denotesRoot(ancestor) && lastSlashInPath == 0) || lastSlashInPath == ancestor.length()) - && isAncestor(ancestor, path); - } - /** * Relativize a path wrt. a parent path such that * {@code relativize(parentPath, concat(parentPath, path)) == paths} diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java index 417265ebd05..d13032236f3 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/package-info.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Version("2.1.0") +@Version("2.0.0") package org.apache.jackrabbit.oak.commons; import org.osgi.annotation.versioning.Version; diff --git a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java index bc0e78b9f87..417db72cb04 100644 --- a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java +++ b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/PathUtilsTest.java @@ -232,28 +232,6 @@ private static void test(String parent, String child) { assertFalse(PathUtils.isAncestor(parent, child)); assertFalse(PathUtils.isAncestor("/" + parent, "/" + parent + "123")); assertFalse(PathUtils.isAncestor("/" + parent, "/" + parent + "123/foo")); - assertTrue(PathUtils.isAncestor("/" + parent, "/" + parent + "/foo")); - assertFalse(PathUtils.isAncestor("/" + parent + "/foo", "/" + parent + "/foo")); - assertTrue(PathUtils.isAncestor("/" + parent, "/" + parent + "/foo/bar")); - - // isDirectAncestor - assertFalse(PathUtils.isDirectAncestor("/", "/")); - assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent)); - assertFalse(PathUtils.isDirectAncestor(parent, parent)); - assertTrue(PathUtils.isDirectAncestor("/", "/" + parent)); - assertFalse(PathUtils.isDirectAncestor("/", "/" + parent + "/foo1")); - assertFalse(PathUtils.isDirectAncestor("/", "/" + parent + "/foo1/foo2")); - assertTrue(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo1")); - assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo1/foo2")); - assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo1/foo2/foo3")); - assertTrue(PathUtils.isDirectAncestor(parent, parent + "/" + child)); - assertFalse(PathUtils.isDirectAncestor("/", parent + "/" + child)); - assertTrue(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/" + child)); - assertFalse(PathUtils.isDirectAncestor(parent, child)); - assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "123")); - assertFalse(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "123/foo")); - assertTrue(PathUtils.isDirectAncestor("/" + parent, "/" + parent + "/foo")); - assertFalse(PathUtils.isDirectAncestor("/" + parent + "/foo", "/" + parent + "/foo")); // relativize assertEquals("", PathUtils.relativize("/", "/")); From aea743cf3a63b513e3b9047df3adce70cf61d055 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:05 +0200 Subject: [PATCH 42/86] Revert "OAK-10974 and OAK-10869 : temporarily disabling flaky tests" This reverts commit aefb990391f51a8ea354a68f9126d4aec959e191. --- .../oak/plugins/document/VersionGarbageCollectorIT.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java index b6aa5efbe8b..4b89c52d7c9 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java @@ -1707,8 +1707,6 @@ public void testUnmergedBCRootCleanup() throws Exception { // OAK-8646 @Test public void testDeletedPropsAndUnmergedBCWithoutCollision() throws Exception { - // OAK-10974: - assumeTrue(fullGcMode != FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS); // create a node with property. NodeBuilder nb = store1.getRoot().builder(); nb.child("bar").setProperty("prop", "value"); @@ -2360,8 +2358,6 @@ public void testGCDeletedPropsWithDryRunMode() throws Exception { @Test public void testDeletedPropsAndUnmergedBCWithCollisionWithDryRunMode() throws Exception { - // OAK-10869: - assumeTrue(fullGcMode != FullGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS); // create a node with property. NodeBuilder nb = store1.getRoot().builder(); nb.child("bar").setProperty("prop", "value"); From 1985a5f490552728a23a48c0daf9000fd9a2329d Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:06 +0200 Subject: [PATCH 43/86] Revert "OAK-10803 -- compress/uncompress property, disabled by default (#1526)" This reverts commit 25792e7c929052a434cc0c9ee117883dc1793b79. --- .../document/DocumentPropertyState.java | 87 +---- .../document/DocumentPropertyStateTest.java | 318 +----------------- 2 files changed, 10 insertions(+), 395 deletions(-) diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java index 767e3998d6f..872985e5664 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyState.java @@ -19,12 +19,6 @@ import static java.util.Collections.emptyList; import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.List; import javax.jcr.PropertyType; @@ -32,10 +26,8 @@ import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; -import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.commons.json.JsopReader; import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; -import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier; import org.apache.jackrabbit.oak.json.TypeCodes; import org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState; import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState; @@ -46,16 +38,12 @@ import org.apache.jackrabbit.oak.plugins.memory.StringPropertyState; import org.apache.jackrabbit.oak.plugins.value.Conversions; import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * PropertyState implementation with lazy parsing of the JSOP encoded value. */ final class DocumentPropertyState implements PropertyState { - private static final Logger LOG = LoggerFactory.getLogger(DocumentPropertyState.class); - private final DocumentNodeStore store; private final String name; @@ -63,47 +51,11 @@ final class DocumentPropertyState implements PropertyState { private final String value; private PropertyState parsed; - private final byte[] compressedValue; - private final Compression compression; - - private static int COMPRESSION_THRESHOLD = SystemPropertySupplier - .create("oak.documentMK.stringCompressionThreshold ", -1).loggingTo(LOG).get(); DocumentPropertyState(DocumentNodeStore store, String name, String value) { - this(store, name, value, Compression.GZIP); - } - - DocumentPropertyState(DocumentNodeStore store, String name, String value, Compression compression) { this.store = store; this.name = name; - if (compression == null || COMPRESSION_THRESHOLD == -1) { - this.value = value; - this.compression = null; - this.compressedValue = null; - } else { - this.compression = compression; - int size = value.length(); - String localValue = value; - byte[] localCompressedValue = null; - if (size > COMPRESSION_THRESHOLD) { - try { - localCompressedValue = compress(value.getBytes(StandardCharsets.UTF_8)); - localValue = null; - } catch (IOException e) { - LOG.warn("Failed to compress property {} value: ", name, e); - } - } - this.value = localValue; - this.compressedValue = localCompressedValue; - } - } - - private byte[] compress(byte[] value) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - OutputStream compressionOutputStream = compression.getOutputStream(out); - compressionOutputStream.write(value); - compressionOutputStream.close(); - return out.toByteArray(); + this.value = value; } @NotNull @@ -164,20 +116,7 @@ public int count() { */ @NotNull String getValue() { - return value != null ? value : decompress(this.compressedValue); - } - - private String decompress(byte[] value) { - try { - return new String(compression.getInputStream(new ByteArrayInputStream(value)).readAllBytes(), StandardCharsets.UTF_8); - } catch (IOException e) { - LOG.error("Failed to decompress property {} value: ", getName(), e); - return "\"{}\""; - } - } - - byte[] getCompressedValue() { - return compressedValue; + return value; } //------------------------------------------------------------< Object >-- @@ -188,18 +127,8 @@ public boolean equals(Object object) { return true; } else if (object instanceof DocumentPropertyState) { DocumentPropertyState other = (DocumentPropertyState) object; - if (!this.name.equals(other.name) || !Arrays.equals(this.compressedValue, other.compressedValue)) { - return false; - } - if (this.compressedValue == null && other.compressedValue == null) { - return value.equals(other.value); - } else { - // Compare length and content of compressed values - if (this.compressedValue.length != other.compressedValue.length) { - return false; - } - return Arrays.equals(this.compressedValue, other.compressedValue); - } + return this.name.equals(other.name) + && this.value.equals(other.value); } // fall back to default equality check in AbstractPropertyState return object instanceof PropertyState @@ -216,15 +145,11 @@ public String toString() { return AbstractPropertyState.toString(this); } - static void setCompressionThreshold(int compressionThreshold) { - COMPRESSION_THRESHOLD = compressionThreshold; - } - //----------------------------< internal >---------------------------------- private PropertyState parsed() { if (parsed == null) { - JsopReader reader = new JsopTokenizer(getValue()); + JsopReader reader = new JsopTokenizer(value); if (reader.matches('[')) { parsed = readArrayProperty(name, reader); } else { @@ -352,4 +277,4 @@ static PropertyState readArrayProperty(String name, DocumentNodeStore store, Jso } return createProperty(name, values, Type.fromTag(type, true)); } -} \ No newline at end of file +} diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java index a9c4ef67bc3..557a0c1d1f7 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentPropertyStateTest.java @@ -18,46 +18,26 @@ import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; -import java.lang.reflect.InvocationTargetException; import java.util.List; -import java.util.Objects; import java.util.Set; -import com.mongodb.ReadPreference; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; -import org.apache.jackrabbit.oak.commons.Compression; -import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; -import org.apache.jackrabbit.oak.plugins.document.mongo.MongoTestUtils; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.junit.*; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; -import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.*; -import static org.junit.Assert.assertNotEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import static org.mockito.Mockito.times; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; public class DocumentPropertyStateTest { private static final int BLOB_SIZE = 16 * 1024; - private static final String TEST_NODE = "test"; - private static final String STRING_HUGEVALUE = RandomStringUtils.random(10050, "dummytest"); - private static final int DEFAULT_COMPRESSION_THRESHOLD = 1024; - private static final int DISABLED_COMPRESSION = -1; @Rule public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); @@ -80,15 +60,6 @@ public void before() throws Exception { ns = builderProvider.newBuilder().setBlobStore(bs).getNodeStore(); } - @After - public void tearDown() { - try { - ns.dispose(); - } finally { - DocumentPropertyState.setCompressionThreshold(DISABLED_COMPRESSION); - } - } - // OAK-5462 @Test public void multiValuedBinarySize() throws Exception { @@ -110,285 +81,4 @@ public void multiValuedBinarySize() throws Exception { assertEquals(0, reads.size()); } - @Test - public void multiValuedAboveThresholdSize() throws Exception { - NodeBuilder builder = ns.getRoot().builder(); - List blobs = newArrayList(); - for (int i = 0; i < 13; i++) { - blobs.add(builder.createBlob(new RandomStream(BLOB_SIZE, i))); - } - builder.child(TEST_NODE).setProperty("p", blobs, Type.BINARIES); - TestUtils.merge(ns, builder); - - PropertyState p = ns.getRoot().getChildNode(TEST_NODE).getProperty("p"); - assertEquals(Type.BINARIES, Objects.requireNonNull(p).getType()); - assertEquals(13, p.count()); - - reads.clear(); - assertEquals(BLOB_SIZE, p.size(0)); - // must not read the blob via stream - assertEquals(0, reads.size()); - } - - @Test - public void stringBelowThresholdSize() throws Exception { - NodeBuilder builder = ns.getRoot().builder(); - builder.child(TEST_NODE).setProperty("p", "dummy", Type.STRING); - TestUtils.merge(ns, builder); - - PropertyState p = ns.getRoot().getChildNode(TEST_NODE).getProperty("p"); - assertEquals(Type.STRING, Objects.requireNonNull(p).getType()); - assertEquals(1, p.count()); - - reads.clear(); - assertEquals(5, p.size(0)); - // must not read the string via stream - assertEquals(0, reads.size()); - } - - @Test - public void stringAboveThresholdSize() throws Exception { - NodeBuilder builder = ns.getRoot().builder(); - builder.child(TEST_NODE).setProperty("p", STRING_HUGEVALUE, Type.STRING); - TestUtils.merge(ns, builder); - - PropertyState p = ns.getRoot().getChildNode(TEST_NODE).getProperty("p"); - assertEquals(Type.STRING, Objects.requireNonNull(p).getType()); - assertEquals(1, p.count()); - - reads.clear(); - assertEquals(10050, p.size(0)); - // must not read the string via streams - assertEquals(0, reads.size()); - } - - @Test - public void compressValueThrowsException() throws IOException, NoSuchFieldException, IllegalAccessException { - DocumentNodeStore mockDocumentStore = mock(DocumentNodeStore.class); - Compression mockCompression = mock(Compression.class); - when(mockCompression.getOutputStream(any(OutputStream.class))).thenThrow(new IOException("Compression failed")); - - DocumentPropertyState.setCompressionThreshold(DEFAULT_COMPRESSION_THRESHOLD); - DocumentPropertyState documentPropertyState = new DocumentPropertyState(mockDocumentStore, "p", "\"" + STRING_HUGEVALUE + "\"", mockCompression); - - assertEquals(documentPropertyState.getValue(Type.STRING), STRING_HUGEVALUE); - - verify(mockCompression, times(1)).getOutputStream(any(OutputStream.class)); - } - - @Test - public void uncompressValueThrowsException() throws IOException, NoSuchFieldException, IllegalAccessException { - - DocumentNodeStore mockDocumentStore = mock(DocumentNodeStore.class); - Compression mockCompression = mock(Compression.class); - OutputStream mockOutputStream= mock(OutputStream.class); - when(mockCompression.getOutputStream(any(OutputStream.class))).thenReturn(mockOutputStream); - when(mockCompression.getInputStream(any(InputStream.class))).thenThrow(new IOException("Compression failed")); - - DocumentPropertyState.setCompressionThreshold(DEFAULT_COMPRESSION_THRESHOLD); - DocumentPropertyState documentPropertyState = new DocumentPropertyState(mockDocumentStore, "p", STRING_HUGEVALUE, mockCompression); - - assertEquals(documentPropertyState.getValue(Type.STRING), "{}"); - - verify(mockCompression, times(1)).getInputStream(any(InputStream.class)); - } - - @Test - public void defaultValueSetToMinusOne() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, NoSuchFieldException { - DocumentNodeStore store = mock(DocumentNodeStore.class); - - DocumentPropertyState.setCompressionThreshold(-1); - DocumentPropertyState state = new DocumentPropertyState(store, "propertyNameNew", "\"" + STRING_HUGEVALUE + "\"", Compression.GZIP); - - byte[] result = state.getCompressedValue(); - - assertNull(result); - assertEquals(state.getValue(Type.STRING), STRING_HUGEVALUE); - } - - @Test - public void stringAboveThresholdSizeNoCompression() { - DocumentNodeStore store = mock(DocumentNodeStore.class); - - DocumentPropertyState.setCompressionThreshold(DEFAULT_COMPRESSION_THRESHOLD); - - DocumentPropertyState state = new DocumentPropertyState(store, "propertyName", "\"" + STRING_HUGEVALUE + "\"", Compression.NONE); - - byte[] result = state.getCompressedValue(); - - assertEquals(result.length, STRING_HUGEVALUE.length() + 2 ); - - assertEquals(state.getValue(Type.STRING), STRING_HUGEVALUE); - assertEquals(STRING_HUGEVALUE, state.getValue(Type.STRING)); - } - - @Test - public void testInterestingStringsWithoutCompression() { - DocumentNodeStore store = mock(DocumentNodeStore.class); - String[] tests = new String[] { "simple:foo", "cr:a\n\b", "dquote:a\"b", "bs:a\\b", "euro:a\u201c", "gclef:\uD834\uDD1E", - "tab:a\tb", "nul:a\u0000b"}; - - for (String test : tests) { - DocumentPropertyState state = new DocumentPropertyState(store, "propertyName", test, Compression.GZIP); - assertEquals(test, state.getValue()); - } - } - - @Test - public void testInterestingStringsWithCompression() { - DocumentNodeStore store = mock(DocumentNodeStore.class); - String[] tests = new String[]{"simple:foo", "cr:a\n\b", "dquote:a\"b", "bs:a\\b", "euro:a\u201c", "gclef:\uD834\uDD1E", - "tab:a\tb", "nul:a\u0000b"}; - - DocumentPropertyState.setCompressionThreshold(1); - for (String test : tests) { - DocumentPropertyState state = new DocumentPropertyState(store, "propertyName", test, Compression.GZIP); - assertEquals(test, state.getValue()); - } - } - - @Test - public void testBrokenSurrogateWithoutCompressionForMongo() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(MONGO, false); - } - - @Test - public void testBrokenSurrogateWithoutCompressionForRDB() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, false); - } - - @Test - public void testBrokenSurrogateWithoutCompressionForInMemory() throws CommitFailedException { - getBrokenSurrogateAndInitializeDifferentStores(MEMORY, false); - } - - @Test - public void testBrokenSurrogateWithCompressionForMongo() throws CommitFailedException { - DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(MONGO, true); - } - - @Test - public void testBrokenSurrogateWithCompressionForRDB() throws CommitFailedException { - DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(RDB_H2, true); - } - - @Test - public void testBrokenSurrogateWithCompressionForInMemory() throws CommitFailedException { - DocumentPropertyState.setCompressionThreshold(1); - getBrokenSurrogateAndInitializeDifferentStores(MEMORY, true); - } - - private void getBrokenSurrogateAndInitializeDifferentStores(DocumentStoreFixture fixture, boolean compressionEnabled) throws CommitFailedException { - String test = "brokensurrogate:dfsa\ud800"; - DocumentStore store = null; - DocumentNodeStore nodeStore = null; - - try { - store = fixture.createDocumentStore(); - - if (store instanceof MongoDocumentStore) { - // Enforce primary read preference, otherwise tests may fail on a - // replica set with a read preference configured to secondary. - // Revision GC usually runs with a modified range way in the past, - // which means changes made it to the secondary, but not in this - // test using a virtual clock - MongoTestUtils.setReadPreference(store, ReadPreference.primary()); - } - nodeStore = new DocumentMK.Builder().setDocumentStore(store).getNodeStore(); - - createPropAndCheckValue(nodeStore, test, compressionEnabled); - - } finally { - if (nodeStore != null) { - nodeStore.dispose(); - } - } - - } - - private void createPropAndCheckValue(DocumentNodeStore nodeStore, String test, boolean compressionEnabled) throws CommitFailedException { - NodeBuilder builder = nodeStore.getRoot().builder(); - if (compressionEnabled) { - DocumentPropertyState.setCompressionThreshold(1); - } - builder.child(TEST_NODE).setProperty("p", test, Type.STRING); - TestUtils.merge(nodeStore, builder); - - PropertyState p = nodeStore.getRoot().getChildNode(TEST_NODE).getProperty("p"); - assertEquals(Objects.requireNonNull(p).getValue(Type.STRING), test); - } - - @Test - public void testEqualsWithoutCompression() { - DocumentNodeStore store = mock(DocumentNodeStore.class); - String name = "propertyName"; - String value = "testValue"; - Compression compression = Compression.GZIP; - - DocumentPropertyState state1 = new DocumentPropertyState(store, name, value, compression); - DocumentPropertyState state2 = new DocumentPropertyState(store, name, value, compression); - - assertEquals(state1, state2); - - // Test for inequality - DocumentPropertyState state4 = new DocumentPropertyState(store, "differentName", value, compression); - assertNotEquals(state1, state4); - } - - @Test - public void testEqualsWithCompression() throws IOException { - DocumentNodeStore store = mock(DocumentNodeStore.class); - String name = "propertyName"; - String value = "testValue"; - Compression compression = Compression.GZIP; - - DocumentPropertyState.setCompressionThreshold(1); - // Create two DocumentPropertyState instances with the same properties - DocumentPropertyState state1 = new DocumentPropertyState(store, name, value, compression); - DocumentPropertyState state2 = new DocumentPropertyState(store, name, value, compression); - - // Check that the compressed values are not null - assertNotNull(state1.getCompressedValue()); - assertNotNull(state2.getCompressedValue()); - - // Check that the equals method returns true - assertEquals(state1, state2); - - // Decompress the values - String decompressedValue1 = state1.getValue(); - String decompressedValue2 = state2.getValue(); - - // Check that the decompressed values are equal to the original value - assertEquals(value, decompressedValue1); - assertEquals(value, decompressedValue2); - - // Check that the equals method still returns true after decompression - assertEquals(state1, state2); - } - - @Test - public void testOneCompressOtherUncompressInEquals() throws IOException { - DocumentNodeStore store = mock(DocumentNodeStore.class); - String name = "propertyName"; - String value = "testValue"; - Compression compression = Compression.GZIP; - - // Create a DocumentPropertyState instance with a compressed value - DocumentPropertyState.setCompressionThreshold(1); - DocumentPropertyState state1 = new DocumentPropertyState(store, name, value, compression); - - // Create a DocumentPropertyState instance with an uncompressed value - DocumentPropertyState.setCompressionThreshold(-1); - DocumentPropertyState state2 = new DocumentPropertyState(store, name, value, compression); - - assertNotEquals(state1, state2); - - // Decompress the value of the first instance - String decompressedValue1 = state1.getValue(); - - // Check that the decompressed value is equal to the original value - assertEquals(value, decompressedValue1); - } -} \ No newline at end of file +} From de1d34cb4f1e39ed5d965e8f0ab4e3f90221c733 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:07 +0200 Subject: [PATCH 44/86] Revert "OAK-6773: Convert oak-store-composite to OSGi R7 annotations - fix line ends" This reverts commit 818317c7200914cb27b7c4f791fb996e75a74dc6. --- .../org/apache/jackrabbit/oak/composite/checks/package-info.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java old mode 100644 new mode 100755 From b0df159d3ecd803d0b9e5551f49f4fd1cd1fc962 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:08 +0200 Subject: [PATCH 45/86] Revert "OAK-10964: bump nimbus-jose-jwt dependency to latest (#1593)" This reverts commit 9528fddef84449ea71f6b36dc484316afd421b0d. --- oak-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml index 8b07e847ce7..483409b135c 100644 --- a/oak-parent/pom.xml +++ b/oak-parent/pom.xml @@ -770,7 +770,7 @@ com.nimbusds nimbus-jose-jwt - 9.40 + 9.30.2 com.azure From bddb5cc98b532c6a7c2c5e3bc4acc1e69f89eb54 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:09 +0200 Subject: [PATCH 46/86] Revert "OAK-10966 - Avoid object creation in PathUtils.isAncestor (#1596)" This reverts commit b37db4c507b9f4efd0651db8185743c18d90477d. --- .../java/org/apache/jackrabbit/oak/commons/PathUtils.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java index 8c94da7a76e..ed8dc09eeb6 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/PathUtils.java @@ -367,13 +367,11 @@ public static boolean isAncestor(String ancestor, String path) { if (denotesRoot(ancestor)) { if (denotesRoot(path)) { return false; - } else { - return path.startsWith(ancestor); } } else { - // Equivalent to path.startsWith(ancestor + "/") but avoids allocating a new string - return path.startsWith(ancestor) && path.length() > ancestor.length() && path.charAt(ancestor.length()) == '/'; + ancestor += "/"; } + return path.startsWith(ancestor); } /** From 44c72d8fab01f88d8d138adcfdaf2e5230c62a53 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:10 +0200 Subject: [PATCH 47/86] Revert "OAK-10951 - Add a new configuration property (#1594)" This reverts commit ebefe01536172e55c98fad969e30eb448408324c. --- .../indexer/document/NodeStateEntry.java | 2 +- .../flatfile/FlatFileStoreIterator.java | 38 +++++++------------ .../linkedList/FlatFileBufferLinkedList.java | 5 ++- .../linkedList/NodeStateEntryList.java | 14 +++---- .../linkedList/PersistedLinkedList.java | 17 ++++++--- 5 files changed, 35 insertions(+), 41 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java index f7b5204866a..2685a60ea37 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntry.java @@ -30,7 +30,7 @@ public class NodeStateEntry { private final long lastModified; private final String id; - public NodeStateEntry(NodeState nodeState, String path, long memUsage, long lastModified, String id) { + private NodeStateEntry(NodeState nodeState, String path, long memUsage, long lastModified, String id) { this.nodeState = nodeState; this.path = path; this.memUsage = memUsage; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java index 69a7c58f3eb..a10fe953ebb 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java @@ -27,10 +27,10 @@ import java.util.Set; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry.NodeStateEntryBuilder; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.NodeStateEntryList; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedList; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; @@ -39,22 +39,16 @@ import org.apache.jackrabbit.guava.common.collect.AbstractIterator; class FlatFileStoreIterator extends AbstractIterator implements Iterator, Closeable { - private static final Logger log = LoggerFactory.getLogger(FlatFileStoreIterator.class); - - static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB"; - // by default, use the PersistedLinkedList - private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0; - static final String PERSISTED_LINKED_LIST_CACHE_SIZE = "oak.indexer.persistedLinkedList.cacheSize"; - static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000; - - + private final Logger log = LoggerFactory.getLogger(getClass()); private final Iterator baseItr; private final NodeStateEntryList buffer; - private final Set preferredPathElements; - private NodeStateEntry current; + private final Set preferredPathElements; private int maxBufferSize; - private long maxBufferSizeBytes; + static final String BUFFER_MEM_LIMIT_CONFIG_NAME = "oak.indexer.memLimitInMB"; + + // by default, use the PersistedLinkedList + private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0; public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator baseItr, Set preferredPathElements) { this(blobStore, fileName, baseItr, preferredPathElements, @@ -69,8 +63,7 @@ public FlatFileStoreIterator(BlobStore blobStore, String fileName, Iterator maxBufferSize) { maxBufferSize = buffer.size(); log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", - maxBufferSize, maxBufferSizeBytes, current.getPath()); - } - if (buffer.estimatedMemoryUsage() > maxBufferSizeBytes) { - maxBufferSizeBytes = buffer.estimatedMemoryUsage(); - log.info("Max buffer size changed {} (estimated memory usage: {} bytes) for path {}", - maxBufferSize, maxBufferSizeBytes, current.getPath()); + maxBufferSize, buffer.estimatedMemoryUsage(), current.getPath()); } if (!buffer.isEmpty()) { NodeStateEntry e = buffer.remove(); @@ -124,7 +112,7 @@ private NodeStateEntry computeNextEntry() { private NodeStateEntry wrap(NodeStateEntry baseEntry) { NodeState state = new LazyChildrenNodeState(baseEntry.getNodeState(), new ChildNodeStateProvider(getEntries(), baseEntry.getPath(), preferredPathElements)); - return new NodeStateEntry(state, baseEntry.getPath(), baseEntry.estimatedMemUsage(), 0, ""); + return new NodeStateEntryBuilder(state, baseEntry.getPath()).withMemUsage(baseEntry.estimatedMemUsage()).build(); } private Iterable getEntries() { @@ -133,7 +121,7 @@ private Iterable getEntries() { private Iterator queueIterator() { Iterator qitr = buffer.iterator(); - return new AbstractIterator<>() { + return new AbstractIterator() { @Override protected NodeStateEntry computeNext() { //If queue is empty try to append by getting entry from base diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java index 6bee80ad826..c0effab8231 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java @@ -32,7 +32,7 @@ */ public class FlatFileBufferLinkedList implements NodeStateEntryList { - private final ListNode head = new ListNode(); + private ListNode head = new ListNode(); private ListNode tail = head; private int size = 0; @@ -52,8 +52,9 @@ public void add(@NotNull NodeStateEntry item) { long incomingSize = item.estimatedMemUsage(); long memUsage = estimatedMemoryUsage(); Preconditions.checkState(memUsage + incomingSize <= memLimit, + String.format( "Adding item (%s) estimated with %s bytes would increase mem usage beyond upper limit (%s)." + - " Current estimated mem usage is %s bytes", item.getPath(), incomingSize, memLimit, memUsage); + " Current estimated mem usage is %s bytes", item.getPath(), incomingSize, memLimit, memUsage)); tail.next = new ListNode(item); tail = tail.next; size++; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java index 3c14d98fa91..60d1812c2bf 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java @@ -28,26 +28,26 @@ public interface NodeStateEntryList { /** * Add an item at the tail of the list. */ - void add(@NotNull NodeStateEntry item); + public void add(@NotNull NodeStateEntry item); /** * Remove the first item from the list. * * @return the removed item */ - NodeStateEntry remove(); + public NodeStateEntry remove(); - long estimatedMemoryUsage(); + public long estimatedMemoryUsage(); - int size(); + public int size(); /** * Get an iterator to iterate over the whole list */ - Iterator iterator(); + public Iterator iterator(); - boolean isEmpty(); + public boolean isEmpty(); - void close(); + public void close(); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java index b4b8317b5f2..8515e9d7dee 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java @@ -51,7 +51,9 @@ public class PersistedLinkedList implements NodeStateEntryList { private final NodeStateEntryWriter writer; private final NodeStateEntryReader reader; private final String storeFileName; - private final int compactStoreMillis = Integer.getInteger(COMPACT_STORE_MILLIS_NAME, 60 * 1000); + private final int compactStoreMillis = Integer.getInteger( + COMPACT_STORE_MILLIS_NAME, + 60 * 1000); private MVStore store; private MVMap map; @@ -64,9 +66,11 @@ public class PersistedLinkedList implements NodeStateEntryList { private long cacheHits, cacheMisses; public PersistedLinkedList(String fileName, NodeStateEntryWriter writer, NodeStateEntryReader reader, int cacheSize) { - LOG.info("Opening store {}", fileName); + LOG.info("Opening store " + fileName); this.storeFileName = fileName; - this.cache = new LinkedHashMap<>(cacheSize + 1, .75F, true) { + this.cache = + new LinkedHashMap(cacheSize + 1, .75F, true) { + private static final long serialVersionUID = 1L; @Override public boolean removeEldestEntry(Map.Entry eldest) { return size() > cacheSize; @@ -74,7 +78,7 @@ public boolean removeEldestEntry(Map.Entry eldest) { }; File oldFile = new File(fileName); if (oldFile.exists()) { - LOG.info("Deleting {}", fileName); + LOG.info("Deleting " + fileName); try { FileUtils.forceDelete(oldFile); } catch (IOException e) { @@ -103,7 +107,8 @@ public void add(@NotNull NodeStateEntry item) { long sizeBytes = store.getFileStore().size(); long now = System.currentTimeMillis(); if (now >= lastLog + 10000) { - LOG.info("Entries: {} map size: {} file size: {} bytes", size, map.sizeAsLong(), sizeBytes); + LOG.info("Entries: " + size + " map size: " + map.sizeAsLong() + " file size: " + + sizeBytes + " bytes"); lastLog = now; } boolean compactNow = now >= lastCompact + compactStoreMillis; @@ -114,7 +119,7 @@ public void add(@NotNull NodeStateEntry item) { MVStoreTool.compact(storeFileName, true); openStore(); lastCompact = System.currentTimeMillis(); - LOG.info("New size={} bytes", store.getFileStore().size()); + LOG.info("New size=" + store.getFileStore().size() + " bytes"); } } From d74446544e82a0962df249caff5269e4fb7e9cce Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:11 +0200 Subject: [PATCH 48/86] Revert "OAK-6773: Convert oak-store-composite to OSGi R7 annotations (#1489)" This reverts commit 0521c6376caa5d2c5e8b0322df3840e49aaf46ff. --- oak-store-composite/pom.xml | 9 +- .../composite/CompositeNodeStoreService.java | 108 +++++------------- .../CrossMountReferenceValidatorProvider.java | 60 ++++------ .../PrivateStoreValidatorProvider.java | 44 ++----- .../NamespacePrefixNodestoreChecker.java | 6 +- .../checks/NodeStoreChecksService.java | 43 +++---- .../NodeTypeDefinitionNodeStoreChecker.java | 6 +- .../NodeTypeMountedNodeStoreChecker.java | 38 +++--- .../checks/UniqueIndexNodeStoreChecker.java | 6 +- .../oak/composite/checks/package-info.java | 20 ---- .../oak/composite/package-info.java | 2 +- ...odeTypeDefinitionNodeStoreCheckerTest.java | 6 +- 12 files changed, 109 insertions(+), 239 deletions(-) delete mode 100755 oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java diff --git a/oak-store-composite/pom.xml b/oak-store-composite/pom.xml index 73a55c69f30..6b708b92b53 100644 --- a/oak-store-composite/pom.xml +++ b/oak-store-composite/pom.xml @@ -93,13 +93,8 @@ provided - org.osgi - org.osgi.service.component.annotations - provided - - - org.osgi - org.osgi.service.metatype.annotations + org.apache.felix + org.apache.felix.scr.annotations provided diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java index 4fd946676d6..b3fe33f2251 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreService.java @@ -17,15 +17,14 @@ package org.apache.jackrabbit.oak.composite; import org.apache.jackrabbit.guava.common.io.Closer; -import org.osgi.service.component.annotations.Activate; -import org.osgi.service.component.annotations.Deactivate; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.ComponentPropertyType; -import org.osgi.service.component.annotations.ConfigurationPolicy; -import org.osgi.service.component.annotations.Reference; -import org.osgi.service.component.annotations.ReferenceCardinality; -import org.osgi.service.component.annotations.ReferencePolicy; -import org.osgi.service.metatype.annotations.AttributeDefinition; +import org.apache.felix.scr.annotations.Activate; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.ConfigurationPolicy; +import org.apache.felix.scr.annotations.Deactivate; +import org.apache.felix.scr.annotations.Property; +import org.apache.felix.scr.annotations.Reference; +import org.apache.felix.scr.annotations.ReferenceCardinality; +import org.apache.felix.scr.annotations.ReferencePolicy; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean; import org.apache.jackrabbit.oak.commons.PropertiesUtil; @@ -58,7 +57,7 @@ import static org.apache.jackrabbit.guava.common.collect.Sets.newIdentityHashSet; import static java.util.stream.Collectors.toSet; -@Component(configurationPolicy = ConfigurationPolicy.REQUIRE) +@Component(policy = ConfigurationPolicy.REQUIRE) public class CompositeNodeStoreService { private static final Logger LOG = LoggerFactory.getLogger(CompositeNodeStoreService.class); @@ -67,38 +66,34 @@ public class CompositeNodeStoreService { private static final String MOUNT_ROLE_PREFIX = "composite-mount-"; + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY, policy = ReferencePolicy.STATIC) private MountInfoProvider mountInfoProvider; + @Reference(cardinality = ReferenceCardinality.MANDATORY_MULTIPLE, policy = ReferencePolicy.DYNAMIC, bind = "bindNodeStore", unbind = "unbindNodeStore", referenceInterface = NodeStoreProvider.class, target="(!(service.pid=org.apache.jackrabbit.oak.composite.CompositeNodeStore))") private List nodeStores = new ArrayList<>(); + @Reference private NodeStoreChecks checks; + @Reference private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP; + @Property(label = "Enable node store checks", + description = "Whether the composite node store constraints should be checked before start", + boolValue = true + ) private static final String ENABLE_CHECKS = "enableChecks"; + + @Property(label = "Pre-populate seed mount", + description = "Setting this parameter to a mount name will enable pre-populating the empty default store" + ) private static final String PROP_SEED_MOUNT = "seedMount"; - private static final String PATH_STATS = "pathStats"; - @ComponentPropertyType - @interface Config { - @AttributeDefinition( - name = "Enable node store checks", - description = "Whether the composite node store constraints should be checked before start" - ) - boolean enableChecks() default true; - - @AttributeDefinition( - name = "Pre-populate seed mount", - description = "Setting this parameter to a mount name will enable pre-populating the empty default store" - ) - String seedMount(); - - @AttributeDefinition( - name = "Gather path statistics", - description = "Whether the CompositeNodeStoreStatsMBean should gather information about the most popular paths (may be expensive)" - ) - boolean pathStats() default false; - } + @Property(label = "Gather path statistics", + description = "Whether the CompositeNodeStoreStatsMBean should gather information about the most popular paths (may be expensive)", + boolValue = false + ) + private static final String PATH_STATS = "pathStats"; private ComponentContext context; @@ -265,12 +260,6 @@ private void unregisterCompositeNodeStore() throws IOException { } @SuppressWarnings("unused") - @Reference(name = "nodeStores", - cardinality = ReferenceCardinality.AT_LEAST_ONE, - policy = ReferencePolicy.DYNAMIC, - service = NodeStoreProvider.class, - target="(!(service.pid=org.apache.jackrabbit.oak.composite.CompositeNodeStore))" - ) protected void bindNodeStore(NodeStoreProvider ns, Map config) throws IOException, CommitFailedException { NodeStoreWithProps newNs = new NodeStoreWithProps(ns, config); nodeStores.add(newNs); @@ -299,49 +288,6 @@ protected void unbindNodeStore(NodeStoreProvider ns) throws IOException { } } - @SuppressWarnings("unused") - @Reference(name = "mountInfoProvider", - cardinality = ReferenceCardinality.MANDATORY, - policy = ReferencePolicy.STATIC, - service = MountInfoProvider.class - ) - protected void bindMountInfoProvider(MountInfoProvider mip) { - this.mountInfoProvider = mip; - } - - @SuppressWarnings("unused") - protected void unbindMountInfoProvider(MountInfoProvider mip) { - if (this.mountInfoProvider == mip) { - this.mountInfoProvider = null; - } - } - - @SuppressWarnings("unused") - @Reference(name = "checks", service = NodeStoreChecks.class) - protected void bindChecks(NodeStoreChecks checks) { - this.checks = checks; - } - - @SuppressWarnings("unused") - protected void unbindChecks(NodeStoreChecks checks) { - if (this.checks == checks) { - this.checks = null; - } - } - - @SuppressWarnings("unused") - @Reference(name = "statisticsProvider", service = StatisticsProvider.class) - protected void bindStatisticsProvider(StatisticsProvider sp) { - this.statisticsProvider = sp; - } - - @SuppressWarnings("unused") - protected void unbindStatisticsProvider(StatisticsProvider sp) { - if (this.statisticsProvider == sp) { - this.statisticsProvider = null; - } - } - private static class NodeStoreWithProps { private final NodeStoreProvider nodeStore; @@ -370,4 +316,4 @@ public String getDescription() { getNodeStoreProvider().getClass().toString()); } } -} +} \ No newline at end of file diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java index 3852b9a70cd..935fa0edc5b 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/CrossMountReferenceValidatorProvider.java @@ -16,15 +16,13 @@ */ package org.apache.jackrabbit.oak.composite; -import org.apache.jackrabbit.oak.spi.state.NodeStoreProvider; -import org.osgi.service.component.annotations.Activate; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.ComponentPropertyType; -import org.osgi.service.component.annotations.ConfigurationPolicy; -import org.osgi.service.component.annotations.Reference; -import org.osgi.service.component.annotations.ReferenceCardinality; -import org.osgi.service.component.annotations.ReferencePolicy; -import org.osgi.service.metatype.annotations.AttributeDefinition; +import org.apache.felix.scr.annotations.Activate; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.ConfigurationPolicy; +import org.apache.felix.scr.annotations.Property; +import org.apache.felix.scr.annotations.Reference; +import org.apache.felix.scr.annotations.Service; +import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.EditorProvider; import org.apache.jackrabbit.oak.spi.commit.Validator; @@ -32,27 +30,27 @@ import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.osgi.framework.BundleContext; + +import java.util.Map; /** * {@link Validator} which detects references crossing the mount boundaries */ -@Component(configurationPolicy = ConfigurationPolicy.REQUIRE, service = {ValidatorProvider.class, EditorProvider.class}) +@Component(label = "Apache Jackrabbit Oak CrossMountReferenceValidatorProvider", policy = ConfigurationPolicy.REQUIRE) +@Property(name = "type", value = "crossMountRefValidator", propertyPrivate = true) +@Service({ValidatorProvider.class, EditorProvider.class}) public class CrossMountReferenceValidatorProvider extends ValidatorProvider { - @ComponentPropertyType - @interface Config { - @AttributeDefinition - String type() default "crossMountRefValidator"; - - @AttributeDefinition( - name = "Fail when detecting commits cross-mount references", - description = "Commits will fail if set to true when detecting cross-mount references. If set to false the commit information is only logged." - ) - boolean failOnDetection() default true; - } - + @Property( + boolValue = true, + label = "Fail when detecting commits cross-mount references", + description = "Commits will fail if set to true when detecting cross-mount references. If set to false the commit information is only logged." + ) + private static final String PROP_FAIL_ON_DETECTION = "failOnDetection"; private boolean failOnDetection; + @Reference private MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider(); public CrossMountReferenceValidatorProvider() { @@ -64,8 +62,8 @@ public CrossMountReferenceValidatorProvider(MountInfoProvider mountInfoProvider, } @Activate - private void activate(Config config) { - failOnDetection = config.failOnDetection(); + private void activate(BundleContext bundleContext, Map config) { + failOnDetection = PropertiesUtil.toBoolean(config.get(PROP_FAIL_ON_DETECTION), false); } @Override @@ -82,18 +80,4 @@ CrossMountReferenceValidatorProvider withFailOnDetection(boolean failOnDetection this.failOnDetection = failOnDetection; return this; } - - @SuppressWarnings("unused") - @Reference(name = "mountInfoProvider") - protected void bindMountInfoProvider(MountInfoProvider mip) { - this.mountInfoProvider = mip; - } - - @SuppressWarnings("unused") - protected void unbindMountInfoProvider(MountInfoProvider mip) { - if (this.mountInfoProvider == mip) { - this.mountInfoProvider = null; - } - - } } diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java index 5dae99cf769..530def5a088 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/PrivateStoreValidatorProvider.java @@ -17,14 +17,10 @@ package org.apache.jackrabbit.oak.composite; -import org.osgi.service.component.annotations.Activate; -import org.osgi.service.component.annotations.Deactivate; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.ComponentPropertyType; -import org.osgi.service.component.annotations.Reference; -import org.osgi.service.metatype.annotations.AttributeDefinition; +import org.apache.felix.scr.annotations.*; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.spi.commit.*; import org.apache.jackrabbit.oak.spi.mount.Mount; import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; @@ -40,22 +36,20 @@ /** * {@link Validator} which detects change commits to the read only mounts. */ -@Component +@Component(label = "Apache Jackrabbit Oak PrivateStoreValidatorProvider") public class PrivateStoreValidatorProvider extends ValidatorProvider { private final Logger logger = LoggerFactory.getLogger(getClass()); private static final String ROOT_PATH = "/"; - @ComponentPropertyType - @interface Config { - @AttributeDefinition( - name = "Fail when detecting commits to the read-only stores", - description = "Commits will fail if set to true when detecting changes to any read-only store. If set to false the commit information is only logged." - ) - boolean failOnDetection() default true; - } - + @Property( + boolValue = true, + label = "Fail when detecting commits to the read-only stores", + description = "Commits will fail if set to true when detecting changes to any read-only store. If set to false the commit information is only logged." + ) + private static final String PROP_FAIL_ON_DETECTION = "failOnDetection"; private boolean failOnDetection; + @Reference private MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider(); private ServiceRegistration serviceRegistration; @@ -66,8 +60,8 @@ public Validator getRootValidator(NodeState before, NodeState after, CommitInfo } @Activate - private void activate(BundleContext bundleContext, Config config) { - failOnDetection = config.failOnDetection(); + private void activate(BundleContext bundleContext, Map config) { + failOnDetection = PropertiesUtil.toBoolean(config.get(PROP_FAIL_ON_DETECTION), true); if (mountInfoProvider.hasNonDefaultMounts()) { serviceRegistration = bundleContext.registerService(EditorProvider.class.getName(), this, null); @@ -97,20 +91,6 @@ boolean isFailOnDetection() { return failOnDetection; } - @SuppressWarnings("unused") - @Reference(name = "mountInfoProvider", service = MountInfoProvider.class) - protected void bindMountInfoProvider(MountInfoProvider mip) { - this.mountInfoProvider = mip; - } - - @SuppressWarnings("unused") - protected void unbindMountInfoProvider(MountInfoProvider mip) { - if (this.mountInfoProvider == mip) { - this.mountInfoProvider = null; - } - - } - private class PrivateStoreValidator extends DefaultValidator { private final String path; diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java index 865022f2918..4095c7ccaac 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NamespacePrefixNodestoreChecker.java @@ -18,7 +18,8 @@ import java.util.Set; -import org.osgi.service.component.annotations.Component; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.api.Tree; @@ -31,7 +32,8 @@ import org.apache.jackrabbit.guava.common.collect.Sets; -@Component(service={MountedNodeStoreChecker.class}) +@Component +@Service(MountedNodeStoreChecker.class) public class NamespacePrefixNodestoreChecker implements MountedNodeStoreChecker { @Override diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java index 14ee6d25a82..c1f3fbb4549 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeStoreChecksService.java @@ -19,9 +19,10 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.Reference; -import org.osgi.service.component.annotations.ReferenceCardinality; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Reference; +import org.apache.felix.scr.annotations.ReferenceCardinality; +import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.composite.MountedNodeStore; import org.apache.jackrabbit.oak.plugins.tree.factories.TreeFactory; @@ -31,13 +32,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Component(service = {NodeStoreChecks.class}) +@Component +@Service(NodeStoreChecks.class) public class NodeStoreChecksService implements NodeStoreChecks { private final Logger log = LoggerFactory.getLogger(getClass()); + @Reference(cardinality = ReferenceCardinality.OPTIONAL_MULTIPLE, + bind = "bindChecker", + unbind = "unbindChecker", + referenceInterface = MountedNodeStoreChecker.class) private List> checkers = new CopyOnWriteArrayList<>(); + @Reference private MountInfoProvider mip; // used by SCR @@ -95,31 +102,13 @@ private void visit(Tree tree, MountedNodeStore mountedStore, ErrorHolder err if ( ( mounted || under ) && keepGoing ) { tree.getChildren().forEach( child -> visit(child, mountedStore, errorHolder, context, c)); } - } - - @SuppressWarnings("unused") - @Reference(name = "checkers", - cardinality = ReferenceCardinality.MULTIPLE, - service = MountedNodeStoreChecker.class) - protected void bindChecker(MountedNodeStoreChecker checker) { + } + + protected void bindChecker(MountedNodeStoreChecker checker) { checkers.add(checker); } - - @SuppressWarnings("unused") - protected void unbindChecker(MountedNodeStoreChecker checker) { + + protected void unbindChecker(MountedNodeStoreChecker checker) { checkers.remove(checker); } - - @SuppressWarnings("unused") - @Reference(name = "mip", service = MountInfoProvider.class) - protected void bindMip(MountInfoProvider mip) { - this.mip = mip; - } - - @SuppressWarnings("unused") - protected void unbindMip(MountInfoProvider mip) { - if (this.mip == mip) { - this.mip = null; - } - } } diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java index 76c68e74196..5df639fb671 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreChecker.java @@ -26,7 +26,8 @@ import java.util.List; import java.util.Set; -import org.osgi.service.component.annotations.Component; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.Tree; @@ -47,7 +48,8 @@ /** * Checks that nodes present in a mount are consistent with the global node type definitions */ -@Component(service = {MountedNodeStoreChecker.class}) +@Component +@Service(MountedNodeStoreChecker.class) public class NodeTypeDefinitionNodeStoreChecker implements MountedNodeStoreChecker { @Override diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java index 88ddb19b12d..7a2c6dfedd1 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeMountedNodeStoreChecker.java @@ -20,10 +20,10 @@ import java.util.Set; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.ComponentPropertyType; -import org.osgi.service.component.annotations.ConfigurationPolicy; -import org.osgi.service.metatype.annotations.AttributeDefinition; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.ConfigurationPolicy; +import org.apache.felix.scr.annotations.Property; +import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.commons.PropertiesUtil; @@ -39,35 +39,23 @@ import org.apache.jackrabbit.guava.common.collect.ImmutableSet; -@Component(configurationPolicy = ConfigurationPolicy.REQUIRE, service = {MountedNodeStoreChecker.class}) +@Component(configurationFactory=true, + policy = ConfigurationPolicy.REQUIRE) +@Service(MountedNodeStoreChecker.class) public class NodeTypeMountedNodeStoreChecker implements MountedNodeStoreChecker { private final Logger log = LoggerFactory.getLogger(getClass()); - + + @Property(label = "The name of a node type that is invalid and will be rejected when found") private static final String INVALID_NODE_TYPE = "invalidNodeType"; + @Property(label = "The error label to use when rejecting an invalid node type") private static final String ERROR_LABEL = "errorLabel"; + + @Property(label="Node types that will cause the check to succeeed, even in the invalid node type is also found.", + cardinality = Integer.MAX_VALUE) private static final String EXCLUDED_NODE_TYPES = "excludedNodeTypes"; - @ComponentPropertyType - @interface Config { - @AttributeDefinition( - name = "The name of a node type that is invalid and will be rejected when found" - ) - String invalidNodeType(); - - @AttributeDefinition( - name = "The error label to use when rejecting an invalid node type" - ) - String errorLabel(); - - @AttributeDefinition( - name = "Node types that will cause the check to succeeed, even in the invalid node type is also found.", - cardinality = Integer.MAX_VALUE - ) - String[] excludedNodeTypes() default {}; - } - private String invalidNodeType; private String errorLabel; private Set excludedNodeTypes; diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java index 7f4621cc4e4..98078c33884 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/UniqueIndexNodeStoreChecker.java @@ -33,7 +33,8 @@ import java.util.Set; import java.util.TreeSet; -import org.osgi.service.component.annotations.Component; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.composite.MountedNodeStore; import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; @@ -61,7 +62,8 @@ * on first access and skips all subsequent executions.

* */ -@Component(service = {MountedNodeStoreChecker.class}) +@Component +@Service(MountedNodeStoreChecker.class) public class UniqueIndexNodeStoreChecker implements MountedNodeStoreChecker { private static final Logger LOG = LoggerFactory.getLogger(UniqueIndexNodeStoreChecker.class); diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java deleted file mode 100755 index ecbd27d417e..00000000000 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/checks/package-info.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -@Version("0.0.1") -package org.apache.jackrabbit.oak.composite.checks; - -import org.osgi.annotation.versioning.Version; \ No newline at end of file diff --git a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java index 6efb8de2a23..9599d669c2a 100644 --- a/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java +++ b/oak-store-composite/src/main/java/org/apache/jackrabbit/oak/composite/package-info.java @@ -55,7 +55,7 @@ * This is obviously correct but may be slow. * {@link org.apache.jackrabbit.oak.composite.CompositionContext#getContributingStores(java.lang.String, java.util.function.Function)} */ -@Version("0.5.1") +@Version("0.5.0") package org.apache.jackrabbit.oak.composite; import org.osgi.annotation.versioning.Version; \ No newline at end of file diff --git a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java index 64489a3a4ba..d710610c2c9 100644 --- a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java +++ b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/checks/NodeTypeDefinitionNodeStoreCheckerTest.java @@ -20,7 +20,8 @@ import java.io.IOException; -import org.osgi.service.component.annotations.Component; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException; @@ -44,7 +45,8 @@ * that they are performed when needed.

* */ -@Component(service = {MountedNodeStoreChecker.class}) +@Component +@Service(MountedNodeStoreChecker.class) public class NodeTypeDefinitionNodeStoreCheckerTest { @Test From 87d4d92e0328d4b044577bd5cb07ef4bc22fc1be Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:13 +0200 Subject: [PATCH 49/86] Revert "OAK-6762: Convert oak-blob to OSGi R7 annotations (#1413)" This reverts commit 6c44805441cc92542c13deabb0afc4af68ec6bc3. --- oak-blob/pom.xml | 9 +-- .../spi/blob/osgi/FileBlobStoreService.java | 14 ++--- .../spi/blob/osgi/SplitBlobStoreService.java | 56 +++++++++---------- 3 files changed, 34 insertions(+), 45 deletions(-) diff --git a/oak-blob/pom.xml b/oak-blob/pom.xml index 6becc735925..fdccfc636e1 100644 --- a/oak-blob/pom.xml +++ b/oak-blob/pom.xml @@ -66,13 +66,8 @@ provided - org.osgi - org.osgi.service.component.annotations - provided - - - org.osgi - org.osgi.service.metatype.annotations + org.apache.felix + org.apache.felix.scr.annotations provided diff --git a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java index b7140c572a9..305c11ff582 100644 --- a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java +++ b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/FileBlobStoreService.java @@ -25,21 +25,21 @@ import java.util.Hashtable; import java.util.Map; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.Activate; -import org.osgi.service.component.annotations.Deactivate; -import org.osgi.service.component.annotations.ConfigurationPolicy; -import org.osgi.service.component.ComponentContext; -import org.osgi.framework.ServiceRegistration; import org.apache.commons.io.FilenameUtils; +import org.apache.felix.scr.annotations.Activate; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.ConfigurationPolicy; +import org.apache.felix.scr.annotations.Deactivate; import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.FileBlobStore; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; +import org.osgi.framework.ServiceRegistration; +import org.osgi.service.component.ComponentContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Component(configurationPolicy = ConfigurationPolicy.REQUIRE, name = FileBlobStoreService.NAME) +@Component(policy = ConfigurationPolicy.REQUIRE, name = FileBlobStoreService.NAME) public class FileBlobStoreService { public static final String NAME = "org.apache.jackrabbit.oak.spi.blob.FileBlobStore"; diff --git a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java index 62e53262934..f1f82d09ce0 100644 --- a/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java +++ b/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/osgi/SplitBlobStoreService.java @@ -21,56 +21,50 @@ import java.util.Dictionary; import java.util.Hashtable; - -import org.osgi.service.component.ComponentContext; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.Activate; -import org.osgi.service.component.annotations.ComponentPropertyType; -import org.osgi.service.component.annotations.Deactivate; -import org.osgi.service.component.annotations.ConfigurationPolicy; -import org.osgi.service.component.annotations.Reference; -import org.osgi.service.component.annotations.ReferenceCardinality; -import org.osgi.service.component.annotations.ReferencePolicy; -import org.osgi.service.metatype.annotations.AttributeDefinition; -import org.osgi.service.metatype.annotations.Option; -import org.osgi.framework.BundleContext; -import org.osgi.framework.InvalidSyntaxException; -import org.osgi.framework.ServiceRegistration; +import java.util.Map; + +import org.apache.felix.scr.annotations.Activate; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.ConfigurationPolicy; +import org.apache.felix.scr.annotations.Deactivate; +import org.apache.felix.scr.annotations.Property; +import org.apache.felix.scr.annotations.PropertyOption; +import org.apache.felix.scr.annotations.Reference; +import org.apache.felix.scr.annotations.ReferenceCardinality; +import org.apache.felix.scr.annotations.ReferencePolicy; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.split.DefaultSplitBlobStore; import org.apache.jackrabbit.oak.spi.blob.split.WrappingSplitBlobStore; +import org.osgi.framework.BundleContext; +import org.osgi.framework.InvalidSyntaxException; +import org.osgi.framework.ServiceRegistration; +import org.osgi.service.component.ComponentContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.jackrabbit.oak.spi.blob.osgi.SplitBlobStoreService.BlobStoreType.*; -@Component(configurationPolicy = ConfigurationPolicy.REQUIRE) +@Component(policy = ConfigurationPolicy.REQUIRE) public class SplitBlobStoreService { private static final Logger log = LoggerFactory.getLogger(SplitBlobStoreService.class); + @Property private static final String PROP_HOME = "repository.home"; + @Property(options = { @PropertyOption(name = "External", value = "EXTERNAL"), + @PropertyOption(name = "Internal - Segment", value = "SEGMENT"), + @PropertyOption(name = "Internal - Document", value = "DOCUMENT") }) private static final String PROP_OLD_BLOB_STORE_TYPE = "split.old.blobstore.type"; public static final String PROP_SPLIT_BLOBSTORE = "split.blobstore"; - @ComponentPropertyType - @interface Config { - @AttributeDefinition - String repository_home(); - @AttributeDefinition(options = { @Option(label = "External", value = "EXTERNAL"), - @Option(label = "Internal - Segment", value = "SEGMENT"), - @Option(label = "Internal - Document", value = "DOCUMENT") }) - String split_old_blobstore_type(); - } - public static final String ONLY_STANDALONE_TARGET = "(&(!(split.blobstore=old))(!(split.blobstore=new)))"; - @Reference(cardinality = ReferenceCardinality.OPTIONAL, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=old)") - private volatile BlobStore oldBlobStore; + @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=old)") + private BlobStore oldBlobStore; - @Reference(cardinality = ReferenceCardinality.OPTIONAL, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=new)") - private volatile BlobStore newBlobStore; + @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY, policy = ReferencePolicy.DYNAMIC, target = "(split.blobstore=new)") + private BlobStore newBlobStore; private BundleContext ctx; @@ -81,7 +75,7 @@ public class SplitBlobStoreService { private BlobStoreType oldBlobStoreType; @Activate - protected void activate(ComponentContext context, Config config) throws InvalidSyntaxException { + protected void activate(ComponentContext context, Map config) throws InvalidSyntaxException { String oldTypeName = lookup(context, PROP_OLD_BLOB_STORE_TYPE); if (oldTypeName == null) { oldBlobStoreType = BlobStoreType.EXTERNAL; From d49014d51868068552ab515eb229094cc29c4645 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:14 +0200 Subject: [PATCH 50/86] Revert "OAK-10965 - Make ConsoleIndexingReporter thread safe. (#1592)" This reverts commit 8b05caefb94252c49b62eb2499a705f3d4dd09f8. --- .../index/ConsoleIndexingReporter.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java index 18168bfb3f6..aa204052108 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/ConsoleIndexingReporter.java @@ -26,19 +26,20 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; public class ConsoleIndexingReporter implements IndexingReporter { + // Print configuration in alphabetical order + private final Map config = new TreeMap<>(); // Print timings in the order they were added private final Map timings = new LinkedHashMap<>(); - // Print metrics and configuration in alphabetical order. They are sorted when printed. - private final Map metrics = new HashMap<>(); - private final Map config = new HashMap<>(); + // Print configuration in alphabetical order + private final Map metrics = new TreeMap<>(); private final List envVariablesToLog; private List indexes = List.of(); private final List informationStrings = new ArrayList<>(); @@ -54,24 +55,24 @@ public ConsoleIndexingReporter(@NotNull List envVariablesToLog) { this.envVariablesToLog = List.copyOf(envVariablesToLog); } - public synchronized void setIndexNames(@NotNull List indexes) { + public void setIndexNames(@NotNull List indexes) { this.indexes = List.copyOf(indexes); } - public synchronized void addConfig(String key, Object value) { + public void addConfig(String key, Object value) { config.put(key, value.toString()); } - public synchronized void addTiming(String stage, String time) { + public void addTiming(String stage, String time) { timings.put(stage, time); } - public synchronized void addMetric(String name, long value) { + public void addMetric(String name, long value) { metrics.put(name, String.valueOf(value)); } @Override - public synchronized void addInformation(String value) { + public void addInformation(String value) { informationStrings.add(value); } @@ -80,12 +81,10 @@ public void addMetricByteSize(String name, long value) { if (value >= FileUtils.ONE_KB) { v += " (" + IOUtils.humanReadableByteCountBin(value) + ")"; } - synchronized (this) { - metrics.put(name, v); - } + metrics.put(name, v); } - public synchronized String generateReport() { + public String generateReport() { return "Indexes: " + String.join(", ", indexes) + "\n" + "Date: " + DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now()) + "\n" + "OAK Version: " + OakVersion.getVersion() + "\n" + From 501780ac64392cd5fbf1fa36d6091931c52ba178 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:15 +0200 Subject: [PATCH 51/86] Revert "Update build.yml to disable Sonar for now" This reverts commit 8db9183b6f2de4d6bfc3ea12cffbdf705a868016. --- .github/workflows/build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dcd21a9a5fc..ac020aecdc6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,6 +52,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} if: ${{ env.SONAR_TOKEN != '' }} - # run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pcoverage -Dsonar.projectKey=org.apache.jackrabbit:jackrabbit-oak -Dsonar.organization=apache - # disable Sonar for now - run: mvn -B verify -Pcoverage + run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Pcoverage -Dsonar.projectKey=org.apache.jackrabbit:jackrabbit-oak -Dsonar.organization=apache From 474745f849dfc51797b876aba15c1c856525b46b Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:16 +0200 Subject: [PATCH 52/86] Revert "OAK-10962: oak-solr-osgi: update zookeeper dependency to 3.9.2 (#1591)" This reverts commit 95ed6c46a217c46c075747f2fbc1fed227c5504a. --- oak-solr-osgi/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-solr-osgi/pom.xml b/oak-solr-osgi/pom.xml index aadd050a0f3..caad3b56f75 100644 --- a/oak-solr-osgi/pom.xml +++ b/oak-solr-osgi/pom.xml @@ -137,7 +137,7 @@ org.apache.zookeeper zookeeper - 3.9.2 + 3.9.1 runtime From 9585fd920281dbca909541c7d3a7bb75cce6a8a1 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:18 +0200 Subject: [PATCH 53/86] Revert "OAK-10945: Remove usage of Guava Function interface (#1578)" This reverts commit 5d56db1224f91992db8fb2978b2a4a482a7ca705. --- .../blobstorage/AzureBlobStoreBackend.java | 10 ++- .../oak/blob/cloud/s3/S3Backend.java | 20 ++++- .../blob/AbstractSharedCachingDataStore.java | 7 +- .../blob/MarkSweepGarbageCollector.java | 27 ++++-- .../plugins/blob/datastore/BlobIdTracker.java | 9 +- .../blob/datastore/DataStoreBlobStore.java | 17 ++-- .../blob/datastore/SharedDataStoreUtils.java | 24 +++-- .../blob/SharedDataStoreUtilsTest.java | 14 ++- .../datastore/DataStoreBlobStoreTest.java | 10 ++- .../blob/datastore/OakFileDataStoreTest.java | 9 +- .../blob/datastore/SharedDataStoreTest.java | 9 +- .../oak/commons/FileIOUtilsTest.java | 7 +- .../jackrabbit/oak/spi/mount/MountInfo.java | 5 +- .../java/org/apache/jackrabbit/oak/Oak.java | 8 +- .../jackrabbit/oak/core/SecureNodeState.java | 6 +- .../oak/management/RepositoryManager.java | 3 +- .../plugins/identifier/IdentifierManager.java | 9 +- .../plugins/index/IndexInfoServiceImpl.java | 23 +++-- .../index/aggregate/AggregateIndex.java | 10 ++- .../plugins/index/cursor/AncestorCursor.java | 17 +++- .../index/reference/ReferenceIndex.java | 8 +- .../migration/AbstractDecoratedNodeState.java | 36 +++++--- .../nodetype/EffectiveNodeTypeImpl.java | 11 ++- .../oak/plugins/tree/impl/AbstractTree.java | 8 +- .../oak/plugins/version/VersionHook.java | 11 ++- .../oak/query/ast/LowerCaseImpl.java | 13 ++- .../principal/PrincipalProviderImpl.java | 4 +- .../security/user/AuthorizableIterator.java | 5 +- .../security/user/UserPrincipalProvider.java | 4 +- .../user/autosave/AuthorizableWrapper.java | 6 +- .../user/query/ResultRowToAuthorizable.java | 5 +- .../security/user/query/UserQueryManager.java | 3 +- .../plugins/tree/impl/ImmutableTreeTest.java | 11 ++- .../evaluation/ChildOrderPropertyTest.java | 10 ++- .../oak/security/privilege/JcrAllTest.java | 9 +- .../security/privilege/PrivilegeImplTest.java | 10 ++- .../apache/jackrabbit/oak/util/NodeUtil.java | 8 +- .../CustomExternalIdentityProvider.java | 11 ++- .../privilege/L4_CustomPrivilegeTest.java | 11 ++- .../oak/composite/CompositeNodeStoreTest.java | 2 +- .../oak/jcr/delegate/NodeDelegate.java | 16 +++- .../jcr/delegate/VersionHistoryDelegate.java | 9 +- .../jackrabbit/oak/jcr/session/NodeImpl.java | 23 +++-- .../oak/jcr/version/VersionHistoryImpl.java | 15 +++- .../jackrabbit/oak/jcr/xml/ImporterImpl.java | 17 +++- .../oak/jcr/version/VersionableTest.java | 18 ++-- .../oak/plugins/index/lucene/IndexCopier.java | 16 +++- .../index/lucene/hybrid/IndexedPaths.java | 31 ++++--- .../lucene/hybrid/LuceneDocumentHolder.java | 9 +- .../lucene/ResultCountingIndexProvider.java | 17 ++-- .../document/DocumentNodeStoreHelper.java | 12 ++- .../AbstractSegmentTarExplorerBackend.java | 9 +- .../tika/CSVFileBinaryResourceProvider.java | 5 +- .../tika/NodeStoreBinaryResourceProvider.java | 5 +- .../oak/run/DataStoreCheckCommand.java | 2 +- .../jackrabbit/oak/run/PrintingDiff.java | 5 +- .../oak/plugins/tika/BinarySourceMapper.java | 3 +- .../CSVFileBinaryResourceProviderTest.java | 5 +- .../oak/run/DataStoreCheckTest.java | 18 ++-- .../oak/run/DataStoreCommandTest.java | 16 +++- .../plugins/index/search/AggregateTest.java | 3 +- .../spi/security/CompositeConfiguration.java | 36 +++++--- .../credentials/SimpleCredentialsSupport.java | 10 ++- .../token/CompositeTokenConfiguration.java | 10 ++- .../privilege/PrivilegeBitsProvider.java | 5 +- .../oak/segment/azure/tool/ToolUtils.java | 10 ++- .../oak/segment/CachingSegmentReader.java | 19 +++- .../jackrabbit/oak/segment/ReaderCache.java | 3 +- .../jackrabbit/oak/segment/Revisions.java | 3 +- .../oak/segment/WriterCacheManager.java | 9 +- .../oak/segment/file/ReadOnlyRevisions.java | 2 +- .../oak/segment/file/TarRevisions.java | 3 +- .../segment/file/tooling/RevisionHistory.java | 10 ++- .../segment/memory/MemoryStoreRevisions.java | 2 +- .../oak/segment/tool/PrintingDiff.java | 4 +- .../jackrabbit/oak/segment/tool/Utils.java | 12 ++- .../oak/segment/ReaderCacheTest.java | 29 +++++-- .../oak/segment/file/TarRevisionsTest.java | 2 +- .../composite/CompositeChildrenCountTest.java | 18 +++- .../oak/plugins/document/Branch.java | 15 +++- .../oak/plugins/document/Commit.java | 11 ++- .../plugins/document/DocumentNodeState.java | 18 +++- .../plugins/document/DocumentNodeStore.java | 27 ++++-- .../document/DocumentNodeStoreBranch.java | 8 +- .../document/DocumentNodeStoreMBeanImpl.java | 22 ++++- .../plugins/document/MissingBcSweeper2.java | 4 +- .../oak/plugins/document/NodeDocument.java | 28 ++++-- .../plugins/document/NodeDocumentSweeper.java | 20 +++-- .../oak/plugins/document/PropertyHistory.java | 10 ++- .../oak/plugins/document/SplitOperations.java | 4 +- .../document/UnsavedModifications.java | 8 +- .../document/VersionGarbageCollector.java | 17 +++- .../document/mongo/MongoDocumentStore.java | 22 ++++- .../document/mongo/MongoVersionGCSupport.java | 21 +++-- .../document/persistentCache/NodeCache.java | 11 ++- .../persistentCache/PersistentCache.java | 11 +-- .../document/rdb/RDBDocumentStore.java | 15 +++- .../document/rdb/RDBDocumentStoreJDBC.java | 10 ++- .../DelegatingDocumentNodeState.java | 11 ++- .../oak/plugins/document/util/Utils.java | 8 +- ...goVersionGCSupportDefaultNoBranchTest.java | 2 +- .../oak/plugins/document/TestUtils.java | 3 +- .../document/VersionGarbageCollectorIT.java | 8 +- .../plugins/memory/MemoryChildNodeEntry.java | 13 ++- .../oak/plugins/memory/ModifiedNodeState.java | 19 +++- .../plugins/memory/MultiPropertyState.java | 87 ++++++++++++++++--- .../commit/ProgressNotificationEditor.java | 3 +- .../oak/spi/state/AbstractNodeState.java | 10 ++- .../oak/spi/state/ChildNodeEntry.java | 12 ++- .../oak/upgrade/RepositoryUpgrade.java | 14 ++- .../oak/upgrade/SameNameSiblingsEditor.java | 8 +- .../checkpoint/CheckpointRetriever.java | 13 ++- .../oak/upgrade/cli/AbstractOak2OakTest.java | 11 ++- 113 files changed, 1056 insertions(+), 319 deletions(-) diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java index 044301fb33c..33add8dafa4 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java @@ -45,8 +45,8 @@ import java.util.Queue; import java.util.UUID; import java.util.concurrent.TimeUnit; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.cache.Cache; import org.apache.jackrabbit.guava.common.cache.CacheBuilder; @@ -462,7 +462,13 @@ public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException @Override public Iterator getAllIdentifiers() throws DataStoreException { return new RecordsIterator( - input -> new DataIdentifier(getIdentifierName(input.getName()))); + new Function() { + @Override + public DataIdentifier apply(AzureBlobInfo input) { + return new DataIdentifier(getIdentifierName(input.getName())); + } + } + ); } diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java index 81f910ab1a9..37f28d36ced 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java @@ -38,7 +38,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.function.Function; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; @@ -90,6 +89,7 @@ import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.Upload; import com.amazonaws.util.StringUtils; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.cache.Cache; @@ -441,7 +441,13 @@ public InputStream read(DataIdentifier identifier) @Override public Iterator getAllIdentifiers() throws DataStoreException { - return new RecordsIterator(input -> new DataIdentifier(getIdentifierName(input.getKey()))); + return new RecordsIterator( + new Function() { + @Override + public DataIdentifier apply(S3ObjectSummary input) { + return new DataIdentifier(getIdentifierName(input.getKey())); + } + }); } @Override @@ -636,8 +642,14 @@ public void deleteAllMetadataRecords(String prefix) { public Iterator getAllRecords() { final AbstractSharedBackend backend = this; return new RecordsIterator( - input -> new S3DataRecord(backend, s3service, bucket, new DataIdentifier(getIdentifierName(input.getKey())), - input.getLastModified().getTime(), input.getSize(), s3ReqDecorator)); + new Function() { + @Override + public DataRecord apply(S3ObjectSummary input) { + return new S3DataRecord(backend, s3service, bucket, + new DataIdentifier(getIdentifierName(input.getKey())), + input.getLastModified().getTime(), input.getSize(), s3ReqDecorator); + } + }); } @Override diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java index 70fb3e0bfc7..6f81f95064d 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/AbstractSharedCachingDataStore.java @@ -56,6 +56,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -276,7 +277,11 @@ public DataRecord addRecord(InputStream inputStream, BlobOptions blobOptions) @Override public Iterator getAllIdentifiers() throws DataStoreException { return Iterators.concat(Iterators.transform(cache.getStagingCache().getAllIdentifiers(), - id -> new DataIdentifier(id)), backend.getAllIdentifiers()); + new Function() { + @Nullable @Override public DataIdentifier apply(@Nullable String id) { + return new DataIdentifier(id); + } + }), backend.getAllIdentifiers()); } @Override diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java index 45b3edb16ad..9f016f161fb 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java @@ -51,6 +51,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.StandardSystemProperty; import org.apache.jackrabbit.guava.common.base.Stopwatch; @@ -263,14 +264,22 @@ public List getStats() throws Exception { List refFiles = ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ImmutableListMultimap references = - FluentIterable.from(refFiles).index( - input -> SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString())); + FluentIterable.from(refFiles).index(new Function() { + @Override public String apply(DataRecord input) { + return SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString()); + + } + }); // Get all the markers available List markerFiles = ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType()); - Map markers = Maps.uniqueIndex(markerFiles, - input -> input.getIdentifier().toString().substring(SharedStoreRecordType.MARKED_START_MARKER.getType().length() + 1)); + Map markers = Maps.uniqueIndex(markerFiles, new Function() { + @Override + public String apply(DataRecord input) { + return input.getIdentifier().toString().substring(SharedStoreRecordType.MARKED_START_MARKER.getType().length() + 1); + } + }); // Get all the repositories registered List repoFiles = @@ -636,12 +645,16 @@ public void addReference(String blobId, final String nodeId) { final Joiner delimJoiner = Joiner.on(DELIM).skipNulls(); Iterator> partitions = Iterators.partition(idIter, getBatchCount()); while (partitions.hasNext()) { - List idBatch = Lists.transform(partitions.next(), id -> { + List idBatch = Lists.transform(partitions.next(), new Function() { + @Nullable @Override + public String apply(@Nullable String id) { if (logPath) { return delimJoiner.join(id, nodeId); } return id; - }); + } + }); if (debugMode) { LOG.trace("chunkIds : {}", idBatch); } @@ -878,7 +891,7 @@ long mergeAllMarkedReferences(GarbageCollectableBlobStore blobStore, GarbageColl List repoFiles = ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); LOG.info("Repositories registered {}", repoFiles); - + // Retrieve repos for which reference files have not been created Set unAvailRepos = SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles); diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java index 402d573d5f1..ad4f653c577 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java @@ -24,12 +24,14 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.Iterator; import java.util.List; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -75,6 +77,7 @@ import static org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker.BlobIdStore.Type.IN_PROCESS; import static org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker.BlobIdStore.Type.REFS; + /** * Tracks the blob ids available or added in the blob store using the {@link BlobIdStore} . * @@ -268,7 +271,8 @@ private void globalMerge() throws IOException { Iterable refRecords = datastore.getAllMetadataRecords(fileNamePrefix); // Download all the corresponding files for the records - List refFiles = newArrayList(transform(refRecords, input -> { + List refFiles = newArrayList(transform(refRecords, new Function() { + @Override public File apply(DataRecord input) { InputStream inputStream = null; try { inputStream = input.getStream(); @@ -279,7 +283,8 @@ private void globalMerge() throws IOException { closeQuietly(inputStream); } return null; - })); + } + })); LOG.info("Retrieved all blob id files in [{}]", watch.elapsed(TimeUnit.MILLISECONDS)); // Merge all the downloaded files in to the local store diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java index 5536e2dac7b..d057007bc57 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java @@ -78,7 +78,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -525,12 +525,15 @@ public boolean apply(@Nullable DataRecord input) { } return false; } - }), input -> { + }), new Function() { + @Override + public String apply(DataRecord input) { if (encodeLengthInId) { return BlobId.of(input).encodedValue(); } return input.getIdentifier().toString(); - }); + } + }); } @Override @@ -772,14 +775,18 @@ public Iterator getAllRecords() throws DataStoreException { Iterator result = delegate instanceof SharedDataStore ? ((SharedDataStore) delegate).getAllRecords() : Iterators.transform(delegate.getAllIdentifiers(), - input -> { + new Function() { + @Nullable + @Override + public DataRecord apply(@Nullable DataIdentifier input) { try { return delegate.getRecord(input); } catch (DataStoreException e) { log.warn("Error occurred while fetching DataRecord for identifier {}", input, e); } return null; - }); + } + }); if (stats instanceof ExtendedBlobStatsCollector) { ((ExtendedBlobStatsCollector) stats).getAllRecordsCalled(System.nanoTime() - start, TimeUnit.NANOSECONDS); diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java index 294040b0bdf..8c0a076b17f 100644 --- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java +++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreUtils.java @@ -18,9 +18,9 @@ import java.util.List; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.collect.FluentIterable; @@ -61,7 +61,7 @@ public static DataRecord getEarliestRecord(List recs) { public Long apply(@NotNull DataRecord input) { return input.getLastModified(); } - }::apply).min(recs); + }).min(recs); } /** @@ -73,10 +73,22 @@ public Long apply(@NotNull DataRecord input) { */ public static Set refsNotAvailableFromRepos(List repos, List refs) { - return Sets.difference(FluentIterable.from(repos) - .uniqueIndex(input -> SharedStoreRecordType.REPOSITORY.getIdFromName(input.getIdentifier().toString())).keySet(), - FluentIterable.from(refs) - .index(input -> SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString())).keySet()); + return Sets.difference(FluentIterable.from(repos).uniqueIndex( + new Function() { + @Override + @Nullable + public String apply(@NotNull DataRecord input) { + return SharedStoreRecordType.REPOSITORY.getIdFromName(input.getIdentifier().toString()); + } + }).keySet(), + FluentIterable.from(refs).index( + new Function() { + @Override + @Nullable + public String apply(@NotNull DataRecord input) { + return SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString()); + } + }).keySet()); } /** diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java index 4a125a0d34a..48eafa75ba4 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreUtilsTest.java @@ -33,6 +33,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.guava.common.collect.Maps; @@ -48,6 +49,7 @@ import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils; import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType; import org.apache.jackrabbit.oak.stats.Clock; +import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -479,7 +481,11 @@ public void testGetAllRecords() throws Exception { } Set retrieved = newHashSet(Iterables.transform(newHashSet(dataStore.getAllRecords()), - input -> input.getIdentifier().toString())); + new Function() { + @Nullable @Override public String apply(@Nullable DataRecord input) { + return input.getIdentifier().toString(); + } + })); assertEquals(added, retrieved); } @@ -509,7 +515,11 @@ public void testGetAllRecordsWithMeta() throws Exception { } Set retrieved = newHashSet(Iterables.transform(newHashSet(dataStore.getAllRecords()), - input -> input.getIdentifier().toString())); + new Function() { + @Nullable @Override public String apply(@Nullable DataRecord input) { + return input.getIdentifier().toString(); + } + })); assertEquals(added, retrieved); } diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java index 8893928c20a..42a896dacb9 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStoreTest.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.UUID; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -43,6 +44,7 @@ import org.apache.jackrabbit.oak.spi.blob.AbstractBlobStoreTest; import org.apache.jackrabbit.oak.spi.blob.BlobStoreInputStream; import org.apache.jackrabbit.oak.spi.blob.stats.BlobStatsCollector; +import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -172,7 +174,13 @@ public void testGetAllChunks() throws Exception{ DataIdentifier d30 = new DataIdentifier("d-30"); List dis = ImmutableList.of(d10, d20, d30); List recs = Lists.newArrayList( - Iterables.transform(dis, input -> new TimeDataRecord(input))); + Iterables.transform(dis, new Function() { + @Nullable + @Override + public DataRecord apply(@Nullable DataIdentifier input) { + return new TimeDataRecord(input); + } + })); OakFileDataStore mockedDS = mock(OakFileDataStore.class); when(mockedDS.getAllRecords()).thenReturn(recs.iterator()); when(mockedDS.getRecord(new DataIdentifier("d-10"))).thenReturn(new TimeDataRecord(d10)); diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java index e9e533f536f..8f58a64e89f 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/OakFileDataStoreTest.java @@ -24,12 +24,14 @@ import java.util.Map; import java.util.Set; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.jackrabbit.core.data.DataIdentifier; import org.apache.jackrabbit.core.data.FileDataStore; +import org.jetbrains.annotations.Nullable; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -75,7 +77,12 @@ private void testGetAllIdentifiers(String path, String unnormalizedPath) throws fds.init(null); Iterator dis = fds.getAllIdentifiers(); - Set fileNames = Sets.newHashSet(Iterators.transform(dis, input -> input.toString())); + Set fileNames = Sets.newHashSet(Iterators.transform(dis, new Function() { + @Override + public String apply(@Nullable DataIdentifier input) { + return input.toString(); + } + })); Set expectedNames = Sets.newHashSet("abcdef","bcdefg","cdefgh"); assertEquals(expectedNames, fileNames); diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java index 9f0514bc464..87804e28c12 100644 --- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java +++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/SharedDataStoreTest.java @@ -31,6 +31,7 @@ import java.util.Properties; import java.util.Set; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -46,6 +47,7 @@ import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore; import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreTest.FixtureHelper.DATA_STORE; +import org.jetbrains.annotations.Nullable; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -144,7 +146,12 @@ private void testGetAllIdentifiers(String path, String unnormalizedPath) throws fds.init(null); Iterator dis = fds.getAllIdentifiers(); - Set fileNames = Sets.newHashSet(Iterators.transform(dis, input-> input.toString())); + Set fileNames = Sets.newHashSet(Iterators.transform(dis, new Function() { + @Override + public String apply(@Nullable DataIdentifier input) { + return input.toString(); + } + })); Set expectedNames = Sets.newHashSet("abcdef","bcdefg","cdefgh"); assertEquals(expectedNames, fileNames); diff --git a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java index 3b52f24b3ff..c172dd429fb 100644 --- a/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java +++ b/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java @@ -60,6 +60,7 @@ import java.util.Set; import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -191,7 +192,11 @@ public void sortLargeFileWithCustomComparatorTest() throws IOException { } Iterator boxedEntries = Longs.asList(entries).iterator(); - Iterator hexEntries = Iterators.transform(boxedEntries, input -> Long.toHexString(input)); + Iterator hexEntries = Iterators.transform(boxedEntries, new Function() { + @Nullable @Override public String apply(@Nullable Long input) { + return Long.toHexString(input); + } + }); File f = assertWrite(hexEntries, false, numEntries); Comparator prefixComparator = new Comparator() { diff --git a/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java b/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java index c7fe29eabe4..8f927824033 100644 --- a/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java +++ b/oak-core-spi/src/main/java/org/apache/jackrabbit/oak/spi/mount/MountInfo.java @@ -26,12 +26,13 @@ import java.util.NavigableSet; import java.util.Set; import java.util.TreeSet; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; +import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; import static org.apache.jackrabbit.guava.common.collect.Sets.newTreeSet; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.commons.PathUtils.isAncestor; @@ -140,7 +141,7 @@ public String getPathFragmentName() { private static TreeSet cleanCopy(Collection includedPaths) { // ensure that paths don't have trailing slashes - this triggers an assertion in PathUtils isAncestor - return newTreeSet(transform(includedPaths, SANITIZE_PATH::apply)); + return newTreeSet(transform(includedPaths, SANITIZE_PATH)); } public Set getPathsSupportingFragments() { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java index 8d9309f37b8..4f2beb318de 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java @@ -50,6 +50,7 @@ import javax.management.StandardMBean; import javax.security.auth.login.LoginException; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -700,13 +701,16 @@ private void initialContent(IndexEditorProvider indexEditors, QueryIndexProvider // FIXME: OAK-810 move to proper workspace initialization // initialize default workspace Iterable workspaceInitializers = Iterables.transform(securityProvider.getConfigurations(), - sc -> { + new Function() { + @Override + public WorkspaceInitializer apply(SecurityConfiguration sc) { WorkspaceInitializer wi = sc.getWorkspaceInitializer(); if (wi instanceof QueryIndexProviderAware) { ((QueryIndexProviderAware) wi).setQueryIndexProvider(indexProvider); } return wi; - }); + } + }); OakInitializer.initialize(workspaceInitializers, store, defaultWorkspaceName, initHook); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java index db9dbca1397..51a772fc108 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/core/SecureNodeState.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.core; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry; @@ -31,9 +32,6 @@ import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.guava.common.collect.Iterables.filter; import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; - -import java.util.function.Function; - import static java.util.Collections.emptyList; class SecureNodeState extends AbstractNodeState { @@ -146,7 +144,7 @@ public Iterable getChildNodeEntries() { } else if (treePermission.canRead()) { Iterable readable = transform( state.getChildNodeEntries(), - new WrapChildEntryFunction()::apply); + new WrapChildEntryFunction()); return filter(readable, new IterableNodePredicate()); } else { return emptyList(); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java index 9e256f6e5a2..4ecf4c6740d 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/management/RepositoryManager.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.management; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; @@ -31,11 +32,11 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Function; import javax.management.openmbean.CompositeData; import javax.management.openmbean.TabularData; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.jmx.FileStoreBackupRestoreMBean; import org.apache.jackrabbit.oak.api.jmx.RepositoryManagementMBean; import org.apache.jackrabbit.oak.api.jmx.SessionMBean; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java index f9fe4b19524..02f119feaab 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/identifier/IdentifierManager.java @@ -20,11 +20,10 @@ import java.util.Collections; import java.util.Iterator; import java.util.Map; -import java.util.function.Function; - import javax.jcr.PropertyType; import javax.jcr.query.Query; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.JcrConstants; @@ -238,7 +237,7 @@ private Iterable findPaths(@NotNull final Result result, @NotNull final return new Iterable() { @Override public Iterator iterator() { - return Iterators.concat(transform(result.getRows().iterator(), new RowToPaths()::apply)); + return Iterators.concat(transform(result.getRows().iterator(), new RowToPaths())); } class RowToPaths implements Function> { @@ -272,7 +271,7 @@ public String apply(PropertyState pState) { if (!rowPath.startsWith(VersionConstants.VERSION_STORE_PATH)) { if (propertyName == null) { return filter( - transform(root.getTree(rowPath).getProperties().iterator(), new PropertyToPath()::apply), + transform(root.getTree(rowPath).getProperties().iterator(), new PropertyToPath()), notNull()); } else { // for a fixed property name, we don't need to look for it, but just assume that @@ -319,7 +318,7 @@ public Iterable getReferences(@NotNull Tree tree, @NotNull final String pr QueryEngine.INTERNAL_SQL2_QUERY, Query.JCR_SQL2, bindings, NO_MAPPINGS); - Iterable resultTrees = Iterables.transform(result.getRows(), row -> row.getTree(null)); + Iterable resultTrees = Iterables.transform(result.getRows(), (Function) row -> row.getTree(null)); return Iterables.filter(resultTrees, tree1 -> !tree1.getPath().startsWith(VersionConstants.VERSION_STORE_PATH) ); } catch (ParseException e) { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java index 756bef928c2..c263bc767a3 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexInfoServiceImpl.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.index; import java.io.IOException; @@ -23,6 +24,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; @@ -69,16 +71,19 @@ public Iterable getAllIndexInfo() { } else { activeIndexes.addAll(allIndexes); } - return Iterables.filter(Iterables.transform(indexPathService.getIndexPaths(), indexPath -> { - try { - IndexInfo info = getInfo(indexPath); - if (info != null) { - info.setActive(activeIndexes.contains(indexPath)); + return Iterables.filter(Iterables.transform(indexPathService.getIndexPaths(), new Function() { + @Override + public IndexInfo apply(String indexPath) { + try { + IndexInfo info = getInfo(indexPath); + if (info != null) { + info.setActive(activeIndexes.contains(indexPath)); + } + return info; + } catch (Exception e) { + log.warn("Error occurred while capturing IndexInfo for path {}", indexPath, e); + return null; } - return info; - } catch (Exception e) { - log.warn("Error occurred while capturing IndexInfo for path {}", indexPath, e); - return null; } }), notNull()); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java index da5fe35b5d0..b22cfbe2189 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java @@ -35,6 +35,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvanceFulltextQueryIndex; @@ -207,8 +208,13 @@ public boolean visit(FullTextAnd and) { public boolean visit(FullTextOr or) { final int[] index = new int[1]; List cursors = Lists.transform(or.list, - input -> flatten(input, plan, filter, state, - path + " or(" + index[0]++ + ")")); + new Function() { + @Override + public Cursor apply(FullTextExpression input) { + return flatten(input, plan, filter, state, + path + " or(" + index[0]++ + ")"); + } + }); result.set(Cursors.newConcatCursor(cursors, filter.getQueryLimits())); return true; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java index 2842e8202ea..c24a8de67d4 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/cursor/AncestorCursor.java @@ -20,9 +20,11 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.spi.query.Cursor; +import org.apache.jackrabbit.oak.spi.query.IndexRow; import org.apache.jackrabbit.oak.spi.query.QueryLimits; import org.jetbrains.annotations.Nullable; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -37,7 +39,12 @@ public AncestorCursor(Cursor cursor, int level, QueryLimits settings) { private static Iterator transform(Cursor cursor, final int level) { Iterator unfiltered = Iterators.transform(cursor, - input -> input != null ? input.getPath() : null); + new Function() { + @Override + public String apply(@Nullable IndexRow input) { + return input != null ? input.getPath() : null; + } + }); Iterator filtered = Iterators.filter(unfiltered, new Predicate() { @Override @@ -45,7 +52,11 @@ public boolean apply(@Nullable String input) { return input != null && PathUtils.getDepth(input) >= level; } }); - return Iterators.transform(filtered, - input -> PathUtils.getAncestorPath(input, level)); + return Iterators.transform(filtered, new Function() { + @Override + public String apply(String input) { + return PathUtils.getAncestorPath(input, level); + } + }); } } \ No newline at end of file diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java index 089ed8ff6d7..438ede13c03 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java @@ -45,6 +45,7 @@ import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -146,7 +147,12 @@ public boolean apply(String path) { } }); } - paths = transform(paths, path -> getParentPath(path)); + paths = transform(paths, new Function() { + @Override + public String apply(String path) { + return getParentPath(path); + } + }); return newPathCursor(paths, filter.getQueryLimits()); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java index 9b513bf141d..374d29a464b 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/migration/AbstractDecoratedNodeState.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.plugins.migration; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.api.PropertyState; @@ -37,7 +38,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; - import static org.apache.jackrabbit.guava.common.base.Predicates.notNull; import static org.apache.jackrabbit.oak.plugins.tree.TreeConstants.OAK_CHILD_ORDER; @@ -139,16 +139,23 @@ public NodeState getChildNode(@NotNull final String name) throws IllegalArgument @Override @NotNull public Iterable getChildNodeEntries() { - final Iterable transformed = Iterables.transform(delegate.getChildNodeEntries(), childNodeEntry -> { - if (childNodeEntry != null) { - final String name = childNodeEntry.getName(); - final NodeState nodeState = decorate(name, childNodeEntry.getNodeState()); - if (nodeState.exists()) { - return new MemoryChildNodeEntry(name, nodeState); + final Iterable transformed = Iterables.transform( + delegate.getChildNodeEntries(), + new Function() { + @Nullable + @Override + public ChildNodeEntry apply(@Nullable final ChildNodeEntry childNodeEntry) { + if (childNodeEntry != null) { + final String name = childNodeEntry.getName(); + final NodeState nodeState = decorate(name, childNodeEntry.getNodeState()); + if (nodeState.exists()) { + return new MemoryChildNodeEntry(name, nodeState); + } + } + return null; + } } - } - return null; - }); + ); return Iterables.filter(transformed, notNull()); } @@ -172,7 +179,14 @@ public PropertyState getProperty(@NotNull String name) { public Iterable getProperties() { final Iterable propertyStates = Iterables.transform( delegate.getProperties(), - propertyState -> decorate(propertyState)); + new Function() { + @Override + @Nullable + public PropertyState apply(@Nullable final PropertyState propertyState) { + return decorate(propertyState); + } + } + ); return Iterables.filter(Iterables.concat(propertyStates, getNewPropertyStates()), notNull()); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java index 1b34ec9608c..61dbab78e31 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/nodetype/EffectiveNodeTypeImpl.java @@ -44,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Maps; @@ -226,8 +227,14 @@ public boolean apply(PropertyDefinition propertyDefinition) { @NotNull @Override public Iterable getNamedNodeDefinitions(@NotNull final String oakName) { - return Iterables.concat(Iterables.transform(nodeTypes.values(), - input -> input.getDeclaredNamedNodeDefinitions(oakName))); + return Iterables.concat(Iterables.transform( + nodeTypes.values(), + new Function>() { + @Override + public Iterable apply(NodeTypeImpl input) { + return input.getDeclaredNamedNodeDefinitions(oakName); + } + })); } /** diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java index 62eced8d187..4ba4f6734d0 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/tree/impl/AbstractTree.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Set; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.oak.api.PropertyState; @@ -319,10 +320,13 @@ public long getChildrenCount(long max) { @NotNull public Iterable getChildren() { Iterable children = transform(getChildNames(), - name -> { + new Function() { + @Override + public Tree apply(String name) { AbstractTree child = createChild(name); return child.exists() ? child : null; - }); + } + }); return filter(children, notNull()); } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java index 5c46f93a4a2..2bce99afae6 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/version/VersionHook.java @@ -18,6 +18,7 @@ */ package org.apache.jackrabbit.oak.plugins.version; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.spi.commit.CommitHook; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; @@ -25,11 +26,11 @@ import org.apache.jackrabbit.oak.spi.commit.EditorProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.osgi.service.component.annotations.Component; import java.util.List; import java.util.Set; - import static org.apache.jackrabbit.guava.common.collect.Collections2.transform; import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet; @@ -76,6 +77,12 @@ public NodeState processCommit(NodeState before, NodeState after, CommitInfo inf providers.add(new VersionableCollector.Provider(existingVersionables)); providers.add(new OrphanedVersionCleaner.Provider(existingVersionables)); - return compose(transform(providers, input -> new EditorHook(input))).processCommit(before, after, info); + return compose(transform(providers, new Function() { + @Nullable + @Override + public CommitHook apply(@Nullable EditorProvider input) { + return new EditorHook(input); + } + })).processCommit(before, after, info); } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java index df46dd82735..d306eac10e5 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/LowerCaseImpl.java @@ -30,6 +30,8 @@ import org.apache.jackrabbit.oak.spi.query.QueryConstants; import org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry; +import org.apache.jackrabbit.guava.common.base.Function; + import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.api.Type.STRINGS; @@ -58,12 +60,12 @@ boolean accept(AstVisitor v) { public String toString() { return "lower(" + operand + ')'; } - + @Override public PropertyExistenceImpl getPropertyExistence() { return operand.getPropertyExistence(); } - + @Override public Set getSelectors() { return operand.getSelectors(); @@ -78,7 +80,12 @@ public PropertyValue currentProperty() { // TODO toLowerCase(): document the Turkish locale problem if (p.getType().isArray()) { Iterable lowerCase = transform(p.getValue(STRINGS), - input -> input.toLowerCase()); + new Function() { + @Override + public String apply(String input) { + return input.toLowerCase(); + } + }); return PropertyValues.newString(lowerCase); } else { String value = p.getValue(STRING); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java index 0fd19da1cd5..598f7702eb3 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/principal/PrincipalProviderImpl.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.security.principal; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.principal.ItemBasedPrincipal; import org.apache.jackrabbit.api.security.user.Authorizable; @@ -44,7 +45,6 @@ import java.util.Iterator; import java.util.Objects; import java.util.Set; -import java.util.function.Function; /** * The {@code PrincipalProviderImpl} is a principal provider implementation @@ -144,7 +144,7 @@ public Iterator findPrincipals(@Nullable final String nameH } try { Iterator authorizables = findAuthorizables(nameHint, searchType, offset, limit); - Iterator principals = Iterators.filter(Iterators.transform(authorizables, new AuthorizableToPrincipal()::apply), Objects::nonNull); + Iterator principals = Iterators.filter(Iterators.transform(authorizables, new AuthorizableToPrincipal()), Objects::nonNull); return EveryoneFilter.filter(principals, nameHint, searchType, offset, limit); } catch (RepositoryException e) { log.debug(e.getMessage()); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java index 9179bcfd7da..632bdbd3be1 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/AuthorizableIterator.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.security.user; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.user.Authorizable; import org.apache.jackrabbit.oak.api.Tree; @@ -32,7 +33,6 @@ import java.util.Iterator; import java.util.Objects; import java.util.Set; -import java.util.function.Function; import java.util.function.Predicate; /** @@ -50,8 +50,7 @@ final class AuthorizableIterator implements Iterator { static AuthorizableIterator create(@NotNull Iterator authorizableTrees, @NotNull UserManagerImpl userManager, @NotNull AuthorizableType authorizableType) { - Iterator it = Iterators.transform(authorizableTrees, - new TreeToAuthorizable(userManager, authorizableType)::apply); + Iterator it = Iterators.transform(authorizableTrees, new TreeToAuthorizable(userManager, authorizableType)); long size = getSize(authorizableTrees); return new AuthorizableIterator(it, size, false); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java index 3b65a8e3886..2d48d311928 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/UserPrincipalProvider.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.security.user; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.principal.ItemBasedPrincipal; @@ -50,7 +51,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.Set; -import java.util.function.Function; import static org.apache.jackrabbit.oak.api.QueryEngine.NO_BINDINGS; import static org.apache.jackrabbit.oak.api.Type.STRING; @@ -184,7 +184,7 @@ public Iterator findPrincipals(@Nullable final String nameH limit, offset, NO_BINDINGS, namePathMapper.getSessionLocalMappings()); Iterator principals = Iterators.filter( - Iterators.transform(result.getRows().iterator(), new ResultRowToPrincipal()::apply), + Iterators.transform(result.getRows().iterator(), new ResultRowToPrincipal()), Predicates.notNull()); // everyone is injected only in complete set, not on pages diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java index 9ae0dd6db4e..378297fc48b 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/autosave/AuthorizableWrapper.java @@ -17,8 +17,8 @@ package org.apache.jackrabbit.oak.security.user.autosave; import java.util.Iterator; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.api.security.user.Authorizable; import org.apache.jackrabbit.api.security.user.Group; @@ -41,10 +41,10 @@ public T apply(T authorizable) { } static Iterator createIterator(Iterator dlgs, AutoSaveEnabledManager mgr) { - return Iterators.transform(dlgs, new AuthorizableWrapper(mgr)::apply); + return Iterators.transform(dlgs, new AuthorizableWrapper<>(mgr)); } static Iterator createGroupIterator(Iterator dlgs, AutoSaveEnabledManager mgr) { - return Iterators.transform(dlgs, new AuthorizableWrapper(mgr)::apply); + return Iterators.transform(dlgs, new AuthorizableWrapper<>(mgr)); } } \ No newline at end of file diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java index b9a5dbcadf3..aeab5feca05 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/ResultRowToAuthorizable.java @@ -16,11 +16,14 @@ */ package org.apache.jackrabbit.oak.security.user.query; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.api.security.user.Authorizable; import org.apache.jackrabbit.oak.api.ResultRow; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.security.user.UserManagerImpl; +import org.apache.jackrabbit.oak.spi.query.QueryConstants; import org.apache.jackrabbit.oak.spi.security.user.AuthorizableType; import org.apache.jackrabbit.oak.spi.security.user.util.UserUtil; import org.jetbrains.annotations.NotNull; @@ -28,8 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.function.Function; - import javax.jcr.RepositoryException; /** diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java index 5a76e75d0f1..c14982565f8 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/query/UserQueryManager.java @@ -273,8 +273,7 @@ private Iterator findAuthorizables(@NotNull String statement, statement, javax.jcr.query.Query.XPATH, limit, offset, NO_BINDINGS, namePathMapper.getSessionLocalMappings()); Iterable resultRows = query.getRows(); - Iterator authorizables = Iterators.transform(resultRows.iterator(), - new ResultRowToAuthorizable(userManager, root, type, query.getSelectorNames())::apply); + Iterator authorizables = Iterators.transform(resultRows.iterator(), new ResultRowToAuthorizable(userManager, root, type, query.getSelectorNames())); return Iterators.filter(authorizables, new UniqueResultPredicate()); } catch (ParseException e) { throw new RepositoryException("Invalid user query "+statement, e); diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java index 2ff4ce0ffbe..25082d51dcb 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/tree/impl/ImmutableTreeTest.java @@ -19,7 +19,7 @@ package org.apache.jackrabbit.oak.plugins.tree.impl; import java.util.List; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.JcrConstants; @@ -34,6 +34,7 @@ import org.apache.jackrabbit.oak.util.NodeUtil; import org.apache.jackrabbit.util.Text; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -263,7 +264,13 @@ private static ImmutableTree getHiddenTree(@NotNull ImmutableTree immutable) { } private static void assertSequence(Iterable trees, String... names) { - List actual = Lists.newArrayList(Iterables.transform(trees, input -> input.getName())); + List actual = Lists.newArrayList(Iterables.transform(trees, new Function() { + @Nullable + @Override + public String apply(Tree input) { + return input.getName(); + } + })); assertEquals(Lists.newArrayList(names), actual); } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java index 9b79e8686a3..f01bbf8bb4d 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/authorization/evaluation/ChildOrderPropertyTest.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Set; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -33,6 +34,7 @@ import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; +import org.jetbrains.annotations.Nullable; import org.junit.Before; import org.junit.Test; @@ -99,7 +101,13 @@ public void testChildOrderWithoutPropertyReadAccess() throws Exception { assertFalse(aTree.hasProperty(JcrConstants.JCR_PRIMARYTYPE)); List expected = ImmutableList.of("/a/bb", "/a/b"); - Iterable childPaths = Iterables.transform(aTree.getChildren(), input -> input.getPath()); + Iterable childPaths = Iterables.transform(aTree.getChildren(), new Function() { + @Nullable + @Override + public String apply(Tree input) { + return input.getPath(); + } + }); assertTrue(childPaths.toString(), Iterables.elementsEqual(expected, childPaths)); } } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java index ff42518973d..f053db0eea9 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/JcrAllTest.java @@ -20,6 +20,7 @@ import java.util.Collections; import javax.jcr.security.Privilege; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.api.security.authorization.PrivilegeManager; import org.apache.jackrabbit.oak.AbstractSecurityTest; @@ -29,6 +30,7 @@ import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeBitsProvider; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConfiguration; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; +import org.jetbrains.annotations.Nullable; import org.junit.Test; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkNotNull; @@ -69,7 +71,12 @@ public void testAllAggregation() throws Exception { Iterable declaredAggr = Arrays.asList(pMgr.getPrivilege(JCR_ALL).getDeclaredAggregatePrivileges()); String[] allAggregates = Iterables.toArray(Iterables.transform( declaredAggr, - privilege -> checkNotNull(privilege).getName()), String.class); + new Function() { + @Override + public String apply(@Nullable Privilege privilege) { + return checkNotNull(privilege).getName(); + } + }), String.class); PrivilegeBits all2 = bitsProvider.getBits(allAggregates); assertEquals(all, all2); diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java index 52a641b537a..bf71f9e2c1b 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/privilege/PrivilegeImplTest.java @@ -19,6 +19,7 @@ import java.util.Set; import javax.jcr.security.Privilege; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -31,6 +32,7 @@ import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeDefinition; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -67,7 +69,13 @@ private static void assertAggregation(@NotNull Privilege[] aggr, @NotNull String assertEquals(expectedNames.length, aggr.length); Set expected = Sets.newHashSet(expectedNames); - Set result = Sets.newHashSet(Iterables.transform(ImmutableSet.copyOf(aggr), input -> input.getName())); + Set result = Sets.newHashSet(Iterables.transform(ImmutableSet.copyOf(aggr), new Function() { + @Nullable + @Override + public String apply(Privilege input) { + return input.getName(); + } + })); assertEquals(expected, result); } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java index e5ab83aba58..809043ccbb5 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/util/NodeUtil.java @@ -22,6 +22,7 @@ import javax.jcr.RepositoryException; import javax.jcr.Value; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.PropertyState; @@ -220,7 +221,12 @@ public void setName(String propertyName, String value) { } public void setNames(String propertyName, String... values) { - tree.setProperty(propertyName, Lists.transform(Arrays.asList(values), jcrName -> getOakName(jcrName)), NAMES); + tree.setProperty(propertyName, Lists.transform(Arrays.asList(values), new Function() { + @Override + public String apply(String jcrName) { + return getOakName(jcrName); + } + }), NAMES); } public void setDate(String name, long time) { diff --git a/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java b/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java index 2bacc21c2c4..6b52903668a 100644 --- a/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java +++ b/oak-exercise/src/main/java/org/apache/jackrabbit/oak/exercise/security/authentication/external/CustomExternalIdentityProvider.java @@ -16,6 +16,7 @@ */ package org.apache.jackrabbit.oak.exercise.security.authentication.external; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableMap; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -27,6 +28,7 @@ import org.apache.jackrabbit.oak.spi.security.authentication.external.ExternalUser; import org.apache.jackrabbit.util.Text; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.osgi.service.component.annotations.Activate; import org.osgi.service.component.annotations.Component; import org.osgi.service.component.annotations.ConfigurationPolicy; @@ -153,8 +155,13 @@ public Iterable getDeclaredGroups() { if (groupIds == null || groupIds.isEmpty()) { return ImmutableSet.of(); } else { - return Iterables.transform(groupIds, - input -> new ExternalIdentityRef(input, getName())); + return Iterables.transform(groupIds, new Function() { + @Nullable + @Override + public ExternalIdentityRef apply(String input) { + return new ExternalIdentityRef(input, getName()); + } + }); } } diff --git a/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java b/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java index 3287dba151b..5f1cac95442 100644 --- a/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java +++ b/oak-exercise/src/test/java/org/apache/jackrabbit/oak/exercise/security/privilege/L4_CustomPrivilegeTest.java @@ -21,6 +21,7 @@ import java.util.UUID; import javax.jcr.security.Privilege; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -28,6 +29,7 @@ import org.apache.jackrabbit.oak.AbstractSecurityTest; import org.apache.jackrabbit.oak.spi.security.principal.EveryonePrincipal; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; +import org.jetbrains.annotations.Nullable; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -116,8 +118,13 @@ private static void assertEqualPrivileges(Set expectedNames, Privilege[] fail(); } - Iterable resultNames = Iterables.transform(Sets.newHashSet(result), - input -> input.toString()); + Iterable resultNames = Iterables.transform(Sets.newHashSet(result), new Function() { + @Nullable + @Override + public String apply(Privilege input) { + return input.toString(); + } + }); Iterables.removeAll(resultNames, expectedNames); assertFalse(resultNames.iterator().hasNext()); diff --git a/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java b/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java index 9d0fd3654ad..52494fd01ae 100644 --- a/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java +++ b/oak-it/src/test/java/org/apache/jackrabbit/oak/composite/CompositeNodeStoreTest.java @@ -658,7 +658,7 @@ public void duplicatedChildren() throws CommitFailedException { deepMountBuilder.child("new").setProperty("store", "deepMounted", Type.STRING); deepMountedStore.merge(deepMountBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); - List children = newArrayList(filter(store.getRoot().getChildNodeEntries(), compose(Predicates.equalTo("new"), GET_NAME::apply))); + List children = newArrayList(filter(store.getRoot().getChildNodeEntries(), compose(Predicates.equalTo("new"), GET_NAME))); assertEquals(1, children.size()); assertEquals("global", children.get(0).getNodeState().getString("store")); diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java index 9316bf10a73..4bed1fb2464 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/NodeDelegate.java @@ -61,7 +61,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.function.Function; import javax.jcr.InvalidItemStateException; import javax.jcr.ItemNotFoundException; @@ -72,6 +71,7 @@ import javax.jcr.nodetype.NoSuchNodeTypeException; import javax.jcr.security.AccessControlException; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.JcrConstants; @@ -299,7 +299,12 @@ public PropertyDelegate getProperty(String relPath) throws RepositoryException { @NotNull public Iterator getProperties() throws InvalidItemStateException { return transform(getTree().getProperties().iterator(), - propertyState -> new PropertyDelegate(sessionDelegate, tree, propertyState.getName())); + new Function() { + @Override + public PropertyDelegate apply(PropertyState propertyState) { + return new PropertyDelegate(sessionDelegate, tree, propertyState.getName()); + } + }); } /** @@ -350,7 +355,12 @@ public boolean apply(Tree tree) { return tree.exists(); } }), - tree -> new NodeDelegate(sessionDelegate, tree)); + new Function() { + @Override + public NodeDelegate apply(Tree tree) { + return new NodeDelegate(sessionDelegate, tree); + } + }); } public void orderBefore(String source, String target) diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java index ab67996295b..95e05e13b3f 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/delegate/VersionHistoryDelegate.java @@ -29,6 +29,7 @@ import javax.jcr.version.LabelExistsVersionException; import javax.jcr.version.VersionException; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.JcrConstants; @@ -161,8 +162,12 @@ public int compare(NodeDelegate n1, NodeDelegate n2) { } }); final Tree thisTree = getTree(); - return Iterators.transform(versions.iterator(), - nd -> VersionDelegate.create(sessionDelegate, thisTree.getChild(nd.getName()))); + return Iterators.transform(versions.iterator(), new Function() { + @Override + public VersionDelegate apply(NodeDelegate nd) { + return VersionDelegate.create(sessionDelegate, thisTree.getChild(nd.getName())); + } + }); } @NotNull diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java index ead1b743023..b9b992c6368 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/session/NodeImpl.java @@ -64,7 +64,7 @@ import javax.jcr.version.VersionException; import javax.jcr.version.VersionHistory; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -823,11 +823,14 @@ public PropertyIterator perform() throws InvalidItemStateException { Iterable propertyOakPaths = idManager.getReferences(weak, node.getTree(), name); // TODO: oak name? Iterable properties = Iterables.transform( propertyOakPaths, - oakPath -> { + new Function() { + @Override + public Property apply(String oakPath) { PropertyDelegate pd = sessionDelegate.getProperty(oakPath); return pd == null ? null : new PropertyImpl(pd, sessionContext); } - ); + } + ); return new PropertyIteratorAdapter(sessionDelegate.sync(properties.iterator())); } @@ -1371,13 +1374,23 @@ private EffectiveNodeType getEffectiveNodeType() throws RepositoryException { private Iterator nodeIterator(Iterator childNodes) { return sessionDelegate.sync(transform( childNodes, - nodeDelegate -> new NodeImpl(nodeDelegate, sessionContext))); + new Function() { + @Override + public Node apply(NodeDelegate nodeDelegate) { + return new NodeImpl(nodeDelegate, sessionContext); + } + })); } private Iterator propertyIterator(Iterator properties) { return sessionDelegate.sync(transform( properties, - propertyDelegate -> new PropertyImpl(propertyDelegate, sessionContext))); + new Function() { + @Override + public Property apply(PropertyDelegate propertyDelegate) { + return new PropertyImpl(propertyDelegate, sessionContext); + } + })); } private void checkValidWorkspace(String workspaceName) diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java index e1160a78a5b..e5bd7a3f0cf 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/version/VersionHistoryImpl.java @@ -34,6 +34,7 @@ import javax.jcr.version.VersionHistory; import javax.jcr.version.VersionIterator; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.commons.iterator.FrozenNodeIteratorAdapter; import org.apache.jackrabbit.commons.iterator.VersionIteratorAdapter; import org.apache.jackrabbit.oak.jcr.delegate.VersionDelegate; @@ -87,7 +88,12 @@ public VersionIterator getAllLinearVersions() throws RepositoryException { @Override public VersionIterator perform() throws RepositoryException { Iterator versions = transform(dlg.getAllLinearVersions(), - input -> new VersionImpl(input, sessionContext)); + new Function() { + @Override + public Version apply(VersionDelegate input) { + return new VersionImpl(input, sessionContext); + } + }); return new VersionIteratorAdapter(sessionDelegate.sync(versions)); } }); @@ -100,7 +106,12 @@ public VersionIterator getAllVersions() throws RepositoryException { @Override public VersionIterator perform() throws RepositoryException { Iterator versions = transform(dlg.getAllVersions(), - input -> new VersionImpl(input, sessionContext)); + new Function() { + @Override + public Version apply(VersionDelegate input) { + return new VersionImpl(input, sessionContext); + } + }); return new VersionIteratorAdapter(sessionDelegate.sync(versions)); } }); diff --git a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java index 30beec363cf..d53ac86d11a 100644 --- a/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java +++ b/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/xml/ImporterImpl.java @@ -36,6 +36,7 @@ import javax.jcr.nodetype.PropertyDefinition; import javax.jcr.version.VersionException; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -298,23 +299,31 @@ private void importProperties(@NotNull Tree tree, } private Iterable getPropertyImporters() { - return Iterables.filter(Iterables.transform(pItemImporters, importer -> { + return Iterables.filter(Iterables.transform(pItemImporters, new Function() { + @Nullable + @Override + public ProtectedPropertyImporter apply(@Nullable ProtectedItemImporter importer) { if (importer instanceof ProtectedPropertyImporter) { return (ProtectedPropertyImporter) importer; } else { return null; } - }), Predicates.notNull()); + } + }), Predicates.notNull()); } private Iterable getNodeImporters() { - return Iterables.filter(Iterables.transform(pItemImporters, importer -> { + return Iterables.filter(Iterables.transform(pItemImporters, new Function() { + @Nullable + @Override + public ProtectedNodeImporter apply(@Nullable ProtectedItemImporter importer) { if (importer instanceof ProtectedNodeImporter) { return (ProtectedNodeImporter) importer; } else { return null; } - }), Predicates.notNull()); + } + }), Predicates.notNull()); } //-----------------------------------------------------------< Importer >--- diff --git a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java index 822cf62f87f..2ffc9328157 100644 --- a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java +++ b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/version/VersionableTest.java @@ -28,12 +28,15 @@ import javax.jcr.version.VersionHistory; import javax.jcr.version.VersionManager; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.commons.jackrabbit.authorization.AccessControlUtils; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.spi.security.principal.EveryonePrincipal; import org.apache.jackrabbit.oak.spi.security.privilege.PrivilegeConstants; import org.apache.jackrabbit.test.AbstractJCRTest; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; import java.util.Set; @@ -439,12 +442,17 @@ private static void assertSuccessors(VersionHistory history, Set expecte } private static Set getNames(Version[] versions) { - return newHashSet(transform(asList(versions), input -> { - try { - return input.getName(); - } catch (RepositoryException e) { - return null; + return newHashSet(transform(asList(versions), new Function() { + @Nullable + @Override + public String apply(@Nullable Version input) { + try { + return input.getName(); + } catch (RepositoryException e) { + return null; + } } })); } + } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java index a18f560a0ef..7d9d7094fa9 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.index.lucene; import java.io.Closeable; @@ -40,6 +41,7 @@ import javax.management.openmbean.TabularDataSupport; import javax.management.openmbean.TabularType; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.guava.common.util.concurrent.Monitor; @@ -610,7 +612,12 @@ public long getLocalIndexDirSize() { @Override public String[] getGarbageDetails() { return toArray(transform(failedToDeleteFiles.values(), - input -> input.deleteLog()), String.class); + new Function() { + @Override + public String apply(LocalIndexFile input) { + return input.deleteLog(); + } + }), String.class); } @Override @@ -654,7 +661,12 @@ public String getSkippedFromUploadSize() { @Override public String[] getCopyInProgressDetails() { return toArray(transform(copyInProgressFiles, - input -> input.copyLog()), String.class); + new Function() { + @Override + public String apply(LocalIndexFile input) { + return input.copyLog(); + } + }), String.class); } @Override diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java index 374afe09432..5d0f373f581 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/IndexedPaths.java @@ -16,10 +16,14 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid; +import java.util.Collection; import java.util.Iterator; +import java.util.Map; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Multimap; import org.apache.jackrabbit.oak.plugins.document.spi.JournalProperty; @@ -35,18 +39,23 @@ public IndexedPaths(Multimap indexedPaths) { @Override public Iterator iterator() { - return Iterators.transform(indexedPaths.asMap().entrySet().iterator(), input -> - new IndexedPathInfo() { - @Override - public String getPath() { - return input.getKey(); - } + return Iterators.transform(indexedPaths.asMap().entrySet().iterator(), + new Function>, IndexedPathInfo>() { + @Override + public IndexedPathInfo apply(final Map.Entry> input) { + return new IndexedPathInfo() { + @Override + public String getPath() { + return input.getKey(); + } - @Override - public Iterable getIndexPaths() { - return input.getValue(); - } - }); + @Override + public Iterable getIndexPaths() { + return input.getValue(); + } + }; + } + }); } @Override diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java index 462bc28f4ec..83375129d86 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/LuceneDocumentHolder.java @@ -16,11 +16,13 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid; import java.util.Collection; import java.util.Map; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ArrayListMultimap; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.ListMultimap; @@ -115,7 +117,9 @@ private boolean queueSizeWithinLimits(){ } private static Iterable asLuceneDocInfo(ListMultimap docs) { - return Iterables.transform(docs.entries(), input -> { + return Iterables.transform(docs.entries(), new Function, LuceneDocInfo>() { + @Override + public LuceneDocInfo apply(final Map.Entry input) { return new LuceneDocInfo() { @Override public String getIndexPath() { @@ -127,6 +131,7 @@ public String getDocPath() { return input.getValue(); } }; - }); + } + }); } } diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java index ac7a4d5459c..6f0946ccf5c 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ResultCountingIndexProvider.java @@ -16,8 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.index.lucene; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.api.Result; import org.apache.jackrabbit.oak.spi.query.Cursor; @@ -30,7 +32,6 @@ import org.jetbrains.annotations.NotNull; import java.util.List; - class ResultCountingIndexProvider implements QueryIndexProvider { private final QueryIndexProvider delegate; private final CountingCursorFactory cursorFactory; @@ -64,11 +65,15 @@ void incrementCount() { @Override public List getQueryIndexes(NodeState nodeState) { if (shouldCount) { - return Lists.transform(delegate.getQueryIndexes(nodeState), input -> { - if (input instanceof AdvanceFulltextQueryIndex) { - return new CountingIndex((AdvanceFulltextQueryIndex) input, cursorFactory); - } else { - return input; + return Lists.transform(delegate.getQueryIndexes(nodeState), new Function() { + @NotNull + @Override + public QueryIndex apply(@NotNull QueryIndex input) { + if (input instanceof AdvanceFulltextQueryIndex) { + return new CountingIndex((AdvanceFulltextQueryIndex)input, cursorFactory); + } else { + return input; + } } }); } else { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java index 448a0d644dc..9cb26f2e0b6 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java @@ -35,12 +35,15 @@ import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStoreHelper; import org.apache.jackrabbit.oak.plugins.document.util.Utils; import org.bson.conversions.Bson; +import org.jetbrains.annotations.Nullable; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.guava.common.primitives.Longs; import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; import com.mongodb.client.FindIterable; import com.mongodb.client.MongoCollection; import com.mongodb.client.model.Filters; @@ -153,8 +156,13 @@ private static Iterable getDocuments(DocumentStore store) { mds, Collection.NODES); Bson query = Filters.eq(NodeDocument.HAS_BINARY_FLAG, NodeDocument.HAS_BINARY_VAL); FindIterable cursor = dbCol.find(query); - return Iterables.transform(cursor, - input -> MongoDocumentStoreHelper.convertFromDBObject(mds, Collection.NODES, input)); + return Iterables.transform(cursor, new Function() { + @Nullable + @Override + public NodeDocument apply(DBObject input) { + return MongoDocumentStoreHelper.convertFromDBObject(mds, Collection.NODES, input); + } + }); } else { return Utils.getSelectedDocuments(store, NodeDocument.HAS_BINARY_FLAG, NodeDocument.HAS_BINARY_VAL); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java index 55696e8f809..2c929dfd4ca 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java @@ -18,6 +18,7 @@ */ package org.apache.jackrabbit.oak.explorer; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.guava.common.collect.Maps; import org.apache.jackrabbit.oak.api.Blob; @@ -28,6 +29,7 @@ import org.apache.jackrabbit.oak.segment.SegmentNodeState; import org.apache.jackrabbit.oak.segment.SegmentNodeStateHelper; import org.apache.jackrabbit.oak.segment.SegmentPropertyState; +import org.apache.jackrabbit.oak.segment.file.JournalEntry; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; @@ -82,7 +84,12 @@ public List readRevisions() { try { journalReader = new JournalReader(journal); Iterator revisionIterator = Iterators.transform(journalReader, - entry -> entry.getRevision()); + new Function() { + @Override + public String apply(JournalEntry entry) { + return entry.getRevision(); + } + }); try { revs = newArrayList(revisionIterator); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java index 2066a69d35b..5b46b310936 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProvider.java @@ -16,14 +16,15 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.tika; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.FluentIterable; import org.apache.jackrabbit.guava.common.io.Closer; @@ -74,7 +75,7 @@ public FluentIterable getBinaries(final String path) throws IOEx CSVParser parser = CSVParser.parse(dataFile, StandardCharsets.UTF_8, FORMAT); closer.register(parser); return FluentIterable.from(parser) - .transform(new RecordTransformer()::apply) + .transform(new RecordTransformer()) .filter(notNull()) .filter(new Predicate() { @Override diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java index 658787a73be..53669591820 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/tika/NodeStoreBinaryResourceProvider.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugins.tika; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.FluentIterable; import org.apache.jackrabbit.guava.common.collect.TreeTraverser; import org.apache.jackrabbit.JcrConstants; @@ -36,8 +37,6 @@ import static org.apache.jackrabbit.oak.plugins.tree.factories.TreeFactory.createReadOnlyTree; import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode; -import java.util.function.Function; - class NodeStoreBinaryResourceProvider implements BinaryResourceProvider { private static final Logger log = LoggerFactory.getLogger(NodeStoreBinaryResourceProvider.class); private final NodeStore nodeStore; @@ -51,7 +50,7 @@ public NodeStoreBinaryResourceProvider(NodeStore nodeStore, BlobStore blobStore) public FluentIterable getBinaries(String path) { return new OakTreeTraverser() .preOrderTraversal(createReadOnlyTree(getNode(nodeStore.getRoot(), path))) - .transform(new TreeToBinarySource()::apply) + .transform(new TreeToBinarySource()) .filter(notNull()); } diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java index 6107f2d9608..7d3e4c53cc8 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DataStoreCheckCommand.java @@ -45,7 +45,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.base.Stopwatch; diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java index 1bda8da88e1..1e400b19ebe 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/PrintingDiff.java @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.jackrabbit.oak.run; import static org.apache.jackrabbit.guava.common.collect.Iterables.transform; @@ -27,8 +28,8 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; import java.io.PrintWriter; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -126,7 +127,7 @@ private static String toString(PropertyState ps) { String v = BLOB_LENGTH.apply(ps.getValue(BINARY)); val.append(" = {").append(v).append("}"); } else if (ps.getType() == BINARIES) { - String v = transform(ps.getValue(BINARIES), BLOB_LENGTH::apply).toString(); + String v = transform(ps.getValue(BINARIES), BLOB_LENGTH).toString(); val.append("[").append(ps.count()).append("] = ").append(v); } else if (ps.isArray()) { val.append("[").append(ps.count()).append("] = ").append(ps.getValue(STRINGS)); diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java index 1a1d526b88f..25fa44927d7 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/BinarySourceMapper.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.tika; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; public enum BinarySourceMapper implements Function { BY_BLOBID { diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java index 62e4c676c5f..ebd3a7077e8 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/plugins/tika/CSVFileBinaryResourceProviderTest.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.tika; import java.io.File; @@ -51,12 +52,12 @@ public void testGetBinaries() throws Exception { CSVFileBinaryResourceProvider provider = new CSVFileBinaryResourceProvider(dataFile, new MemoryBlobStore()); - Map binaries = provider.getBinaries("/").uniqueIndex(BinarySourceMapper.BY_BLOBID::apply); + Map binaries = provider.getBinaries("/").uniqueIndex(BinarySourceMapper.BY_BLOBID); assertEquals(3, binaries.size()); assertEquals("a", binaries.get("a").getBlobId()); assertEquals("/a", binaries.get("a").getPath()); - binaries = provider.getBinaries("/a").uniqueIndex(BinarySourceMapper.BY_BLOBID::apply); + binaries = provider.getBinaries("/a").uniqueIndex(BinarySourceMapper.BY_BLOBID); assertEquals(1, binaries.size()); provider.close(); diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java index 17f454e6cb3..60e720cf311 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCheckTest.java @@ -39,6 +39,7 @@ import java.util.Random; import java.util.Set; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; @@ -66,6 +67,7 @@ import org.apache.jackrabbit.oak.spi.commit.EmptyHook; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -442,14 +444,20 @@ private static String createTempConfig(File cfgFile, Properties props) throws IO } private static Set encodedIds(Set ids, String dsOption) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), - input -> DataStoreCheckCommand.encodeId(input, "--" + dsOption))); + return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { + @Nullable @Override public String apply(@Nullable String input) { + return DataStoreCheckCommand.encodeId(input, "--"+dsOption); + } + })); } private static Set encodedIdsAndPath(Set ids, String dsOption, Map blobsAddedWithNodes) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), - input -> Joiner.on(",").join( + return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { + @Nullable @Override public String apply(@Nullable String input) { + return Joiner.on(",").join( DataStoreCheckCommand.encodeId(input, "--"+dsOption), - blobsAddedWithNodes.get(input)))); + blobsAddedWithNodes.get(input)); + } + })); } } diff --git a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java index d77b05e1cfa..63def135d06 100644 --- a/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java +++ b/oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java @@ -38,6 +38,7 @@ import java.util.stream.StreamSupport; import ch.qos.logback.classic.Level; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Splitter; import org.apache.jackrabbit.guava.common.base.Strings; @@ -86,6 +87,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.stats.Clock; +import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Assert; import org.junit.Assume; @@ -934,13 +936,19 @@ private static String createTempConfig(File cfgFile, Properties props) throws IO private static Set encodedIdsAndPath(Set ids, Type dsOption, Map idToNodes, boolean encodeId) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), - input -> Joiner.on(",").join(encodeId ? encodeId(input, dsOption) : input, idToNodes.get(input)))); + return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { + @Nullable @Override public String apply(@Nullable String input) { + return Joiner.on(",").join(encodeId ? encodeId(input, dsOption) : input, idToNodes.get(input)); + } + })); } private static Set encodeIds(Set ids, Type dsOption) { - return Sets.newHashSet(Iterators.transform(ids.iterator(), - input -> encodeId(input, dsOption))); + return Sets.newHashSet(Iterators.transform(ids.iterator(), new Function() { + @Nullable @Override public String apply(@Nullable String input) { + return encodeId(input, dsOption); + } + })); } diff --git a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java index 38ac728f152..bd369b42529 100644 --- a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java +++ b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/AggregateTest.java @@ -28,6 +28,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ArrayListMultimap; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.ListMultimap; @@ -105,7 +106,7 @@ public void noOfChildNodeRead() { NodeState state = nb.getNodeState(); final AtomicInteger counter = new AtomicInteger(); Iterable countingIterator = Iterables.transform(state.getChildNodeEntries(), - input -> { + (Function) input -> { counter.incrementAndGet(); return input; }); diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java index 8a131765e68..092275b7d02 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/CompositeConfiguration.java @@ -18,6 +18,7 @@ */ package org.apache.jackrabbit.oak.spi.security; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableList; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -50,7 +51,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Function; /** * Abstract base implementation for {@link SecurityConfiguration}s that can @@ -205,22 +205,34 @@ public ConfigurationParameters getParameters() { @NotNull @Override public WorkspaceInitializer getWorkspaceInitializer() { - return new CompositeWorkspaceInitializer(Lists.transform(getConfigurations(), - securityConfiguration -> securityConfiguration.getWorkspaceInitializer())); + return new CompositeWorkspaceInitializer(Lists.transform(getConfigurations(), new Function() { + @Override + public WorkspaceInitializer apply(T securityConfiguration) { + return securityConfiguration.getWorkspaceInitializer(); + } + })); } @NotNull @Override public RepositoryInitializer getRepositoryInitializer() { - return new CompositeInitializer(Lists.transform(getConfigurations(), - securityConfiguration -> securityConfiguration.getRepositoryInitializer())); + return new CompositeInitializer(Lists.transform(getConfigurations(), new Function() { + @Override + public RepositoryInitializer apply(T securityConfiguration) { + return securityConfiguration.getRepositoryInitializer(); + } + })); } @NotNull @Override public List getCommitHooks(@NotNull final String workspaceName) { - Iterable t = Iterables.concat(Lists.transform(getConfigurations(), - securityConfiguration -> securityConfiguration.getCommitHooks(workspaceName))); + Iterable t = Iterables.concat(Lists.transform(getConfigurations(), new Function>() { + @Override + public List apply(T securityConfiguration) { + return securityConfiguration.getCommitHooks(workspaceName); + } + })); return ImmutableList.copyOf(t); } @@ -240,8 +252,12 @@ public List getConflictHandlers() { @NotNull @Override public List getProtectedItemImporters() { - Iterable t = Iterables.concat(Lists.transform(getConfigurations(), - securityConfiguration -> securityConfiguration.getProtectedItemImporters())); + Iterable t = Iterables.concat(Lists.transform(getConfigurations(), new Function>() { + @Override + public List apply(T securityConfiguration) { + return securityConfiguration.getProtectedItemImporters(); + } + })); return ImmutableList.copyOf(t); } @@ -291,7 +307,7 @@ private static final class CompositeContext implements Context { private void refresh(@NotNull List configurations) { Set s = Sets.newLinkedHashSetWithExpectedSize(configurations.size()); - for (Context c : Iterables.transform(configurations, ContextFunction.INSTANCE::apply)) { + for (Context c : Iterables.transform(configurations, ContextFunction.INSTANCE)) { if (DEFAULT != c) { s.add(c); } diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java index 47011218563..b5ea558753a 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/credentials/SimpleCredentialsSupport.java @@ -23,6 +23,7 @@ import javax.jcr.Credentials; import javax.jcr.SimpleCredentials; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; import org.apache.jackrabbit.guava.common.collect.Maps; @@ -65,8 +66,13 @@ public String getUserId(@NotNull Credentials credentials) { public Map getAttributes(@NotNull Credentials credentials) { if (credentials instanceof SimpleCredentials) { final SimpleCredentials sc = (SimpleCredentials) credentials; - return Maps.asMap(ImmutableSet.copyOf(sc.getAttributeNames()), - input -> sc.getAttribute(input)); + return Maps.asMap(ImmutableSet.copyOf(sc.getAttributeNames()), new Function() { + @Nullable + @Override + public Object apply(String input) { + return sc.getAttribute(input); + } + }); } else { return Collections.emptyMap(); } diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java index fffe95cfb71..0af35372189 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/authentication/token/CompositeTokenConfiguration.java @@ -17,7 +17,7 @@ package org.apache.jackrabbit.oak.spi.security.authentication.token; import java.util.List; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.api.Root; import org.apache.jackrabbit.oak.spi.security.CompositeConfiguration; @@ -40,8 +40,12 @@ public CompositeTokenConfiguration(@NotNull SecurityProvider securityProvider) { @NotNull @Override public TokenProvider getTokenProvider(@NotNull final Root root) { - List providers = Lists.transform(getConfigurations(), - tokenConfiguration -> tokenConfiguration.getTokenProvider(root)); + List providers = Lists.transform(getConfigurations(), new Function() { + @Override + public TokenProvider apply(TokenConfiguration tokenConfiguration) { + return tokenConfiguration.getTokenProvider(root); + } + }); return CompositeTokenProvider.newInstance(providers); } } diff --git a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java index 1e0bb2def92..b8cfb4cabdd 100644 --- a/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java +++ b/oak-security-spi/src/main/java/org/apache/jackrabbit/oak/spi/security/privilege/PrivilegeBitsProvider.java @@ -22,11 +22,10 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Function; - import javax.jcr.security.AccessControlException; import javax.jcr.security.Privilege; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.FluentIterable; import org.apache.jackrabbit.guava.common.collect.ImmutableSet; @@ -262,7 +261,7 @@ public Iterable getAggregatedPrivilegeNames(@NotNull String... privilege @NotNull private Iterable extractAggregatedPrivileges(@NotNull Iterable privilegeNames) { - return FluentIterable.from(privilegeNames).transformAndConcat(new ExtractAggregatedPrivileges()::apply); + return FluentIterable.from(privilegeNames).transformAndConcat(new ExtractAggregatedPrivileges()); } @NotNull diff --git a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java index 20283ef77b8..f1845bc0a64 100644 --- a/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java +++ b/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java @@ -61,6 +61,7 @@ import org.apache.jackrabbit.oak.segment.spi.persistence.persistentcache.PersistentCache; import org.apache.jackrabbit.guava.common.base.Stopwatch; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import com.microsoft.azure.storage.StorageCredentials; @@ -211,8 +212,13 @@ public static List readRevisions(String uri) { if (journal.exists()) { try (JournalReader journalReader = new JournalReader(journal)) { - Iterator revisionIterator = Iterators.transform(journalReader, - entry -> entry.getRevision()); + Iterator revisionIterator = Iterators.transform(journalReader, new Function() { + @NotNull + @Override + public String apply(JournalEntry entry) { + return entry.getRevision(); + } + }); return newArrayList(revisionIterator); } catch (Exception e) { e.printStackTrace(); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java index c69bb62b531..9f6fa1d81ab 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/CachingSegmentReader.java @@ -23,6 +23,7 @@ import java.io.UnsupportedEncodingException; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Supplier; import org.apache.jackrabbit.oak.cache.CacheStats; import org.apache.jackrabbit.oak.segment.util.SafeEncode; @@ -97,8 +98,13 @@ public String readString(@NotNull RecordId id) { final SegmentId segmentId = id.getSegmentId(); long msb = segmentId.getMostSignificantBits(); long lsb = segmentId.getLeastSignificantBits(); - return stringCache.get(msb, lsb, id.getRecordNumber(), - offset -> segmentId.getSegment().readString(offset)); + return stringCache.get(msb, lsb, id.getRecordNumber(), new Function() { + @NotNull + @Override + public String apply(Integer offset) { + return segmentId.getSegment().readString(offset); + } + }); } @NotNull @@ -116,8 +122,13 @@ public Template readTemplate(@NotNull RecordId id) { final SegmentId segmentId = id.getSegmentId(); long msb = segmentId.getMostSignificantBits(); long lsb = segmentId.getLeastSignificantBits(); - return templateCache.get(msb, lsb, id.getRecordNumber(), - offset -> segmentId.getSegment().readTemplate(offset)); + return templateCache.get(msb, lsb, id.getRecordNumber(), new Function() { + @NotNull + @Override + public Template apply(Integer offset) { + return segmentId.getSegment().readTemplate(offset); + } + }); } private static String safeEncode(String value) { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java index a6e41a4bca1..0d4e489c1ae 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/ReaderCache.java @@ -23,7 +23,6 @@ import static org.apache.jackrabbit.oak.segment.CacheWeights.OBJECT_HEADER_SIZE; import java.util.Arrays; -import java.util.function.Function; import org.apache.jackrabbit.guava.common.cache.Weigher; import org.apache.jackrabbit.oak.cache.CacheLIRS; @@ -31,7 +30,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; - +import org.apache.jackrabbit.guava.common.base.Function; /** * A cache consisting of a fast and slow component. The fast cache for small items is based diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java index 92b88b7ce04..d17d0f0f60b 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Revisions.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.segment; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.jetbrains.annotations.NotNull; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java index 799c716e93a..0db45bc6e5a 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java @@ -36,6 +36,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Supplier; @@ -280,8 +281,12 @@ T getGeneration(final int generation) { @NotNull @Override public Iterator iterator() { - return transform(generations.values().iterator(), - cacheFactory -> cacheFactory.get()); + return transform(generations.values().iterator(), new Function, T>() { + @Nullable @Override + public T apply(Supplier cacheFactory) { + return cacheFactory.get(); + } + }); } void evictGenerations(@NotNull Predicate evict) { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java index 32e229daf01..985853de231 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java @@ -25,8 +25,8 @@ import java.io.Closeable; import java.io.IOException; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; import org.apache.jackrabbit.oak.segment.SegmentIdProvider; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java index 485b4384e4a..b484629e0a0 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.segment.file; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; @@ -31,8 +32,8 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Supplier; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java index 3618761da04..1241ffb214c 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java @@ -28,12 +28,14 @@ import java.io.IOException; import java.util.Iterator; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.oak.json.BlobSerializer; import org.apache.jackrabbit.oak.json.JsonSerializer; import org.apache.jackrabbit.oak.segment.SegmentNodeState; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.segment.file.JournalEntry; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -78,11 +80,15 @@ public Iterator getHistory(@NotNull JournalFile journal, @NotNul checkNotNull(path); try (JournalReader journalReader = new JournalReader(checkNotNull(journal))) { - return Iterators.transform(journalReader, entry -> { + return Iterators.transform(journalReader, + new Function() { + @NotNull @Override + public HistoryElement apply(JournalEntry entry) { store.setRevision(entry.getRevision()); NodeState node = getNode(store.getHead(), path); return new HistoryElement(entry.getRevision(), node); - }); + } + }); } } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java index dab99d8c072..53c3b92d7d2 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStoreRevisions.java @@ -23,8 +23,8 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import java.io.IOException; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Revisions; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java index 3cea0f777d8..7920bc36a49 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/PrintingDiff.java @@ -28,8 +28,8 @@ import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.MISSING_NODE; import java.io.PrintWriter; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -112,7 +112,7 @@ private static String toString(PropertyState ps) { String v = BLOB_LENGTH.apply(ps.getValue(BINARY)); val.append(" = {").append(v).append("}"); } else if (ps.getType() == BINARIES) { - String v = transform(ps.getValue(BINARIES), BLOB_LENGTH::apply).toString(); + String v = transform(ps.getValue(BINARIES), BLOB_LENGTH).toString(); val.append("[").append(ps.count()).append("] = ").append(v); } else if (ps.isArray()) { val.append("[").append(ps.count()).append("] = ").append(ps.getValue(STRINGS)); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java index 4c73ac90669..06ea7b87f26 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java @@ -26,17 +26,20 @@ import java.util.Iterator; import java.util.List; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterators; import org.apache.jackrabbit.oak.commons.json.JsonObject; import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; import org.apache.jackrabbit.oak.segment.SegmentId; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.segment.file.JournalEntry; import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile; import org.apache.jackrabbit.oak.segment.file.tooling.BasicReadOnlyBlobStore; import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.jetbrains.annotations.NotNull; public final class Utils { @@ -74,8 +77,13 @@ public static List readRevisions(File store) { if (journal.exists()) { try (JournalReader journalReader = new JournalReader(journal)) { - Iterator revisionIterator = Iterators.transform(journalReader, - entry -> entry.getRevision()); + Iterator revisionIterator = Iterators.transform(journalReader, new Function() { + @NotNull + @Override + public String apply(JournalEntry entry) { + return entry.getRevision(); + } + }); return newArrayList(revisionIterator); } catch (Exception e) { e.printStackTrace(); diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java index da7f6ecdf1d..502b69e5a5b 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/ReaderCacheTest.java @@ -26,18 +26,24 @@ import java.util.ArrayList; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.Test; +import org.apache.jackrabbit.guava.common.base.Function; + public class ReaderCacheTest { @Test public void empty() { final AtomicInteger counter = new AtomicInteger(); - Function loader = input -> { + Function loader = new Function() { + @Override @NotNull + public String apply(@Nullable Integer input) { counter.incrementAndGet(); return valueOf(input); + } }; StringCache c = new StringCache(0); for (int repeat = 0; repeat < 10; repeat++) { @@ -55,9 +61,12 @@ public void empty() { public void largeEntries() { final AtomicInteger counter = new AtomicInteger(); final String large = new String(new char[1024]); - Function loader = input -> { + Function loader = new Function() { + @Override @Nullable + public String apply(@Nullable Integer input) { counter.incrementAndGet(); return large + input; + } }; StringCache c = new StringCache(1024); for (int repeat = 0; repeat < 10; repeat++) { @@ -75,7 +84,12 @@ public void largeEntries() { @Test public void clear() { final AtomicInteger counter = new AtomicInteger(); - Function uniqueLoader = input -> valueOf(counter.incrementAndGet()); + Function uniqueLoader = new Function() { + @Override @Nullable + public String apply(@Nullable Integer input) { + return valueOf(counter.incrementAndGet()); + } + }; StringCache c = new StringCache(0); // load a new entry assertEquals("1", c.get(0, 0, 0, uniqueLoader)); @@ -93,7 +107,12 @@ public void randomized() { int segmentCount = 10; for (int i = 0; i < segmentCount; i++) { final int x = i; - Function loader = input -> "loader #" + x + " offset " + input; + Function loader = new Function() { + @Override @Nullable + public String apply(@Nullable Integer input) { + return "loader #" + x + " offset " + input; + } + }; loaderList.add(loader); } StringCache c = new StringCache(10); diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java index 05cbe069265..0d4a5f89b07 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java @@ -34,8 +34,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Functions; import org.apache.jackrabbit.guava.common.util.concurrent.ListenableFuture; import org.apache.jackrabbit.guava.common.util.concurrent.ListeningExecutorService; diff --git a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java index f25fdb9938c..8469917df50 100644 --- a/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java +++ b/oak-store-composite/src/test/java/org/apache/jackrabbit/oak/composite/CompositeChildrenCountTest.java @@ -18,6 +18,7 @@ */ package org.apache.jackrabbit.oak.composite; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -34,6 +35,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.junit.Test; import java.util.List; @@ -222,7 +224,13 @@ public Iterable getChildNodeEntries() { Iterable childrenIterable = cycle(new MemoryChildNodeEntry("child", EMPTY_NODE)); return asCountingIterable(limit(childrenIterable, childrenCount == MAX_VALUE ? 1000 : (int) childrenCount)); } else { - return asCountingIterable(transform(asList(children), input -> new MemoryChildNodeEntry(input, EMPTY_NODE))); + return asCountingIterable(transform(asList(children), new Function() { + @Nullable + @Override + public ChildNodeEntry apply(@Nullable String input) { + return new MemoryChildNodeEntry(input, EMPTY_NODE); + } + })); } } @@ -238,9 +246,13 @@ public NodeBuilder builder() { } private Iterable asCountingIterable(Iterable input) { - return Iterables.transform(input, inp -> { + return Iterables.transform(input, new Function() { + @Nullable + @Override + public T apply(@Nullable T input) { fetchedChildren++; - return inp; + return input; + } }); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java index ea64bb6cc89..56040cf770a 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Branch.java @@ -31,6 +31,7 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentSkipListMap; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -278,7 +279,12 @@ public boolean apply(Map.Entry input) { return !input.getValue().isRebase() && input.getKey().compareRevisionTime(r) <= 0; } - }), input -> input.getValue().getModifiedPaths()); + }), new Function, Iterable>() { + @Override + public Iterable apply(Map.Entry input) { + return input.getValue().getModifiedPaths(); + } + }); return Iterables.concat(paths); } @@ -406,7 +412,12 @@ protected boolean isRebase() { @Override Iterable getModifiedPaths() { Iterable> paths = transform(previous.values(), - branchCommit -> branchCommit.getModifiedPaths()); + new Function>() { + @Override + public Iterable apply(BranchCommit branchCommit) { + return branchCommit.getModifiedPaths(); + } + }); return Iterables.concat(paths); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java index 1f2214542b1..b3016219cdd 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Commit.java @@ -29,6 +29,7 @@ import java.util.TreeSet; import java.util.concurrent.TimeUnit; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.oak.commons.json.JsopStream; @@ -890,8 +891,16 @@ private boolean isBundled(Path path) { return bundledNodes.containsKey(path); } + private static final Function KEY_TO_NAME = + new Function() { + @Override + public String apply(UpdateOp.Key input) { + return input.getName(); + } + }; + private static boolean hasContentChanges(UpdateOp op) { return filter(transform(op.getChanges().keySet(), - input -> input.getName()), Utils.PROPERTY_OR_DELETED).iterator().hasNext(); + KEY_TO_NAME), Utils.PROPERTY_OR_DELETED).iterator().hasNext(); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java index a9cc56fb4d0..215c5007014 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeState.java @@ -47,6 +47,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -572,19 +573,24 @@ private AbstractDocumentNodeState getSecondaryNodeState(){ private Iterable getChildNodeEntries(@NotNull String name, int limit) { Iterable children = store.getChildNodes(this, name, limit); - return Iterables.transform(children, input -> { + return Iterables.transform(children, new Function() { + @Override + public ChildNodeEntry apply(final AbstractDocumentNodeState input) { return new AbstractChildNodeEntry() { + @NotNull @Override public String getName() { return input.getPath().getName(); } + @NotNull @Override public NodeState getNodeState() { return input; } }; - }); + } + }); } private static Map asMap(Iterable props){ @@ -748,18 +754,24 @@ private AbstractDocumentNodeState createBundledState(String childNodeName, Match } private Iterator getBundledChildren(){ - return Iterators.transform(bundlingContext.getBundledChildNodeNames().iterator(), childNodeName -> { + return Iterators.transform(bundlingContext.getBundledChildNodeNames().iterator(), + new Function() { + @Override + public ChildNodeEntry apply(final String childNodeName) { return new AbstractChildNodeEntry() { + @NotNull @Override public String getName() { return childNodeName; } + @NotNull @Override public NodeState getNodeState() { return createBundledState(childNodeName, bundlingContext.matcher.next(childNodeName)); } }; + } }); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java index f5ed5650755..88e7d142085 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java @@ -67,7 +67,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Function; import javax.jcr.PropertyType; @@ -127,7 +126,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.base.Strings; @@ -493,6 +492,19 @@ public String serialize(Blob blob) { } }; + /** + * A predicate, which takes a String and returns {@code true} if the String + * is a serialized binary value of a {@link DocumentPropertyState}. The + * apply method will throw an IllegalArgumentException if the String is + * malformed. + */ + private final Function binarySize = new Function() { + @Override + public Long apply(@Nullable String input) { + return getBinarySize(input); + } + }; + private final Clock clock; private final Checkpoints checkpoints; @@ -1617,7 +1629,7 @@ private String docAsString(String id, boolean cached) { return e.toString(); } } - }::apply); + }); } @Nullable @@ -2217,7 +2229,12 @@ public Iterable checkpoints() { public boolean apply(Map.Entry cp) { return cp.getValue().getExpiryTime() > now; } - }), cp -> cp.getKey().toString()); + }), new Function, String>() { + @Override + public String apply(Map.Entry cp) { + return cp.getKey().toString(); + } + }); } @Nullable @@ -2656,7 +2673,7 @@ private void backgroundSplit() { continue; } cleanCollisions(doc, collisionGarbageBatchSize); - Iterator it = doc.split(this, head, input -> getBinarySize(input)).iterator(); + Iterator it = doc.split(this, head, binarySize).iterator(); while(it.hasNext()) { UpdateOp op = it.next(); Path path = doc.getPath(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java index 0378c78624a..1b04f62fa54 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBranch.java @@ -31,6 +31,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Sets; @@ -751,7 +752,12 @@ private void checkForConflicts() throws CommitFailedException { NodeDocument doc = Utils.getRootDocument(store.getDocumentStore()); Set collisions = Sets.newHashSet(doc.getLocalMap(COLLISIONS).keySet()); Set commits = Sets.newHashSet(Iterables.transform(b.getCommits(), - input -> input.asTrunkRevision())); + new Function() { + @Override + public Revision apply(Revision input) { + return input.asTrunkRevision(); + } + })); Set conflicts = Sets.intersection(collisions, commits); if (!conflicts.isEmpty()) { throw new CommitFailedException(STATE, 2, diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java index a06669a4788..7d649a34c3b 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java @@ -24,6 +24,7 @@ import javax.management.openmbean.CompositeData; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.api.stats.RepositoryStatistics; @@ -95,7 +96,12 @@ public String[] getInactiveClusterNodes() { public boolean apply(ClusterNodeInfoDocument input) { return !input.isActive(); } - }), input -> input.getClusterId() + "=" + input.getCreated()), String.class); + }), new Function() { + @Override + public String apply(ClusterNodeInfoDocument input) { + return input.getClusterId() + "=" + input.getCreated(); + } + }), String.class); } @Override @@ -106,7 +112,12 @@ public String[] getActiveClusterNodes() { public boolean apply(ClusterNodeInfoDocument input) { return input.isActive(); } - }), input -> input.getClusterId() + "=" + input.getLeaseEndTime()), String.class); + }), new Function() { + @Override + public String apply(ClusterNodeInfoDocument input) { + return input.getClusterId() + "=" + input.getLeaseEndTime(); + } + }), String.class); } @Override @@ -117,7 +128,12 @@ public String[] getLastKnownRevisions() { public boolean apply(Revision input) { return input.getClusterId() != getClusterId(); } - }), input -> input.getClusterId() + "=" + input.toString()), String.class); + }), new Function() { + @Override + public String apply(Revision input) { + return input.getClusterId() + "=" + input.toString(); + } + }), String.class); } @Override diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java index 2e66eee3fde..f0e9cd56d29 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingBcSweeper2.java @@ -30,7 +30,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; import org.apache.jackrabbit.oak.commons.TimeDurationFormatter; import org.apache.jackrabbit.oak.plugins.document.util.Utils; @@ -39,6 +38,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; /** @@ -161,7 +161,7 @@ public Map.Entry apply(NodeDocument doc) { } return immutableEntry(doc.getPath(), sweepOne(doc)); } - }::apply), new Predicate>() { + }), new Predicate>() { @Override public boolean apply(Map.Entry input) { return input.getValue() != null; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java index 1c76f63e122..d823247fbb0 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocument.java @@ -33,8 +33,8 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; import org.apache.jackrabbit.guava.common.collect.ImmutableList; @@ -1351,12 +1351,16 @@ Iterable getPreviousDocs(@NotNull final String property, } // didn't find entry -> scan through remaining head ranges - return filter(transform(getPreviousRanges().headMap(revision).entrySet(), input -> { + return filter(transform(getPreviousRanges().headMap(revision).entrySet(), + new Function, NodeDocument>() { + @Override + public NodeDocument apply(Map.Entry input) { if (input.getValue().includes(revision)) { return getPreviousDoc(input.getKey(), input.getValue()); } return null; - }), new Predicate() { + } + }), new Predicate() { @Override public boolean apply(@Nullable NodeDocument input) { return input != null && input.getValueMap(property).containsKey(revision); @@ -1655,14 +1659,17 @@ private Iterable> changesFor(final List range if (ranges.isEmpty()) { return Collections.emptyList(); } - - final Function>> rangeToChanges = input -> { + final Function>> rangeToChanges = + new Function>>() { + @Override + public Iterable> apply(Range input) { NodeDocument doc = getPreviousDoc(input.high, input); if (doc == null) { return Collections.emptyList(); } return doc.getVisibleChanges(property, readRev); - }; + } + }; Iterable> changes; if (ranges.size() == 1) { @@ -1675,7 +1682,7 @@ public Iterator> iterator() { } }; } else { - changes = Iterables.concat(transform(copyOf(ranges), rangeToChanges::apply)); + changes = Iterables.concat(transform(copyOf(ranges), rangeToChanges)); } return filter(changes, new Predicate>() { @Override @@ -1779,7 +1786,12 @@ String resolveCommitValue(Revision revision) { @NotNull RevisionVector getSweepRevisions() { return new RevisionVector(transform(getLocalMap(SWEEP_REV).values(), - s -> Revision.fromString(s))); + new Function() { + @Override + public Revision apply(String s) { + return Revision.fromString(s); + } + })); } //-------------------------< UpdateOp modifiers >--------------------------- diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java index 16a00272278..d36022c733d 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeper.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.oak.commons.TimeDurationFormatter; @@ -164,13 +165,18 @@ private Revision performSweep(Iterable documents, private Iterable> sweepOperations( final Iterable docs) { - return filter(transform(docs, doc -> immutableEntry(doc.getPath(), sweepOne(doc))), - new Predicate>() { - @Override - public boolean apply(Map.Entry input) { - return input.getValue() != null; - } - }); + return filter(transform(docs, + new Function>() { + @Override + public Map.Entry apply(NodeDocument doc) { + return immutableEntry(doc.getPath(), sweepOne(doc)); + } + }), new Predicate>() { + @Override + public boolean apply(Map.Entry input) { + return input.getValue() != null; + } + }); } private UpdateOp sweepOne(NodeDocument doc) throws DocumentStoreException { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java index 5af7684ecdb..37ccbb92cdd 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/PropertyHistory.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.TreeMap; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicates; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; import org.apache.jackrabbit.guava.common.collect.Iterators; @@ -57,7 +58,11 @@ public PropertyHistory(@NotNull NodeDocument doc, @Override public Iterator iterator() { - return ensureOrder(filter(transform(doc.getPreviousRanges().entrySet(), input -> { + return ensureOrder(filter(transform(doc.getPreviousRanges().entrySet(), + new Function, Map.Entry>() { + @Nullable + @Override + public Map.Entry apply(Map.Entry input) { Revision r = input.getKey(); int h = input.getValue().height; String prevId = Utils.getPreviousIdFor(mainPath, r, h); @@ -67,7 +72,8 @@ public Iterator iterator() { return null; } return new SimpleImmutableEntry(r, prev); - }), Predicates.notNull())); + } + }), Predicates.notNull())); } /** diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java index d9c0e45c626..2644ee71bb3 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/SplitOperations.java @@ -26,7 +26,6 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import java.util.function.Function; import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; import org.apache.jackrabbit.oak.plugins.document.util.Utils; @@ -35,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Supplier; import org.apache.jackrabbit.guava.common.base.Suppliers; @@ -233,7 +233,7 @@ private void populateSplitRevs() { } private boolean hasBinaryPropertyForSplit(Iterable values) { - return doc.hasBinary() && any(transform(values, binarySize::apply), BINARY_FOR_SPLIT_THRESHOLD); + return doc.hasBinary() && any(transform(values, binarySize), BINARY_FOR_SPLIT_THRESHOLD); } /** diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java index 8a489f697a6..be79d07f61a 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UnsavedModifications.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.base.Supplier; @@ -119,7 +120,12 @@ public Iterable getPaths(@NotNull final Revision start) { public boolean apply(Map.Entry input) { return start.compareRevisionTime(input.getValue()) < 1; } - }), input -> input.getKey()); + }), new Function, Path>() { + @Override + public Path apply(Map.Entry input) { + return input.getKey(); + } + }); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java index 9dbf9c1639d..730f46fa8b7 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java @@ -40,6 +40,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Stopwatch; @@ -50,6 +51,7 @@ import org.apache.jackrabbit.guava.common.collect.Sets; import org.apache.jackrabbit.oak.commons.sort.StringSort; +import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType; import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key; import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation; import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation.Type; @@ -2269,11 +2271,22 @@ private Iterator previousDocIdsFor(NodeDocument doc) { // documents only. final Path path = doc.getPath(); return Iterators.transform(prevRanges.entrySet().iterator(), - input -> Utils.getPreviousIdFor(path, input.getKey(), input.getValue().getHeight())); + new Function, String>() { + @Override + public String apply(Map.Entry input) { + int h = input.getValue().getHeight(); + return Utils.getPreviousIdFor(path, input.getKey(), h); + } + }); } else { // need to fetch the previous documents to get their ids return Iterators.transform(doc.getAllPreviousDocs(), - input -> input.getId()); + new Function() { + @Override + public String apply(NodeDocument input) { + return input.getId(); + } + }); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java index 531850c408c..f87d3e557df 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java @@ -93,6 +93,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Maps; import com.mongodb.BasicDBObject; import com.mongodb.MongoException; @@ -1364,10 +1365,20 @@ public List createOrUpdate(Collection collection, } } catch (MongoException e) { throw handleException(e, collection, Iterables.transform(updateOps, - input -> input.getId())); + new Function() { + @Override + public String apply(UpdateOp input) { + return input.getId(); + } + })); } finally { stats.doneCreateOrUpdate(watch.elapsed(TimeUnit.NANOSECONDS), - collection, Lists.transform(updateOps, input -> input.getId())); + collection, Lists.transform(updateOps, new Function() { + @Override + public String apply(UpdateOp input) { + return input.getId(); + } + })); } List resultList = new ArrayList(results.values()); log("createOrUpdate returns", resultList); @@ -1527,7 +1538,12 @@ private Map bulkUpdate(Collection collectio } private static Map createMap(List updateOps) { - return Maps.uniqueIndex(updateOps, input -> input.getId()); + return Maps.uniqueIndex(updateOps, new Function() { + @Override + public String apply(UpdateOp input) { + return input.getId(); + } + }); } private Map findDocuments(Collection collection, Set keys) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java index 62bc6c6508c..f20f51d5513 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java @@ -16,11 +16,13 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.jackrabbit.oak.plugins.document.mongo; import static com.mongodb.client.model.Filters.eq; import static com.mongodb.client.model.Filters.exists; import static com.mongodb.client.model.Filters.gt; +import static com.mongodb.client.model.Filters.not; import static com.mongodb.client.model.Filters.or; import static com.mongodb.client.model.Projections.include; import static java.util.Optional.empty; @@ -71,6 +73,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Joiner; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.StandardSystemProperty; @@ -334,13 +337,17 @@ protected Iterable identifyGarbage(final Set gcTypes // queries alone (15min is still long). Iterable iterable = filter(transform(getNodeCollection().find(query) .maxTime(15, TimeUnit.MINUTES).hint(hint), - input -> store.convertFromDBObject(NODES, input)), - new Predicate() { - @Override - public boolean apply(NodeDocument input) { - return !isDefaultNoBranchSplitNewerThan(input, sweepRevs); - } - }); + new Function() { + @Override + public NodeDocument apply(BasicDBObject input) { + return store.convertFromDBObject(NODES, input); + } + }), new Predicate() { + @Override + public boolean apply(NodeDocument input) { + return !isDefaultNoBranchSplitNewerThan(input, sweepRevs); + } + }); allResults = concat(allResults, iterable); } return allResults; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java index af0e49dd7c4..8639e54e0da 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/NodeCache.java @@ -46,11 +46,14 @@ import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.apache.jackrabbit.oak.stats.TimerStats; import org.h2.mvstore.MVMap; +import org.h2.mvstore.WriteBuffer; import org.h2.mvstore.type.DataType; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; + class NodeCache implements Cache, GenerationCache, EvictionListener { @@ -159,7 +162,10 @@ private V asyncReadIfPresent(K key) { } private void broadcast(final K key, final V value) { - cache.broadcast(type, buffer -> { + cache.broadcast(type, new Function() { + @Override + @Nullable + public Void apply(@Nullable WriteBuffer buffer) { keyType.write(buffer, key); if (value == null) { buffer.put((byte) 0); @@ -168,7 +174,8 @@ private void broadcast(final K key, final V value) { valueType.write(buffer, value); } return null; - }); + } + }); } private void write(final K key, final V value) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java index 6f4a0252bdf..141f81d223e 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/persistentCache/PersistentCache.java @@ -25,7 +25,6 @@ import java.util.TreeSet; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; import org.apache.jackrabbit.guava.common.cache.Cache; import org.apache.jackrabbit.oak.cache.CacheValue; @@ -47,17 +46,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; + /** * A persistent cache for the document store. */ public class PersistentCache implements Broadcaster.Listener { - + static final Logger LOG = LoggerFactory.getLogger(PersistentCache.class); private static final String FILE_PREFIX = "cache-"; private static final String FILE_SUFFIX = ".data"; private static final AtomicInteger COUNTER = new AtomicInteger(); - + private boolean cacheNodes = true; private boolean cacheChildren = true; private boolean cacheDiff = true; @@ -69,7 +70,7 @@ public class PersistentCache implements Broadcaster.Listener { private boolean asyncDiffCache = false; private HashMap caches = new HashMap(); - + private final String directory; private MapFactory writeStore; private MapFactory readStore; @@ -86,7 +87,7 @@ public class PersistentCache implements Broadcaster.Listener { private DynamicBroadcastConfig broadcastConfig; private CacheActionDispatcher writeDispatcher; private Thread writeDispatcherThread; - + { ByteBuffer bb = ByteBuffer.wrap(new byte[16]); UUID uuid = UUID.randomUUID(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java index c7f20564bff..82c09e213dd 100755 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java @@ -88,6 +88,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.ImmutableMap; @@ -445,7 +446,12 @@ private List internalCreateOrUpdate(Collection collec } } stats.doneCreateOrUpdate(watch.elapsed(TimeUnit.NANOSECONDS), - collection, Lists.transform(updateOps, input -> input.getId())); + collection, Lists.transform(updateOps, new Function() { + @Override + public String apply(UpdateOp input) { + return input.getId(); + } + })); return new ArrayList(results.values()); } @@ -1860,7 +1866,12 @@ public Iterator iterator() { Iterator res = db.queryAsIterator(ch, tmd, from, to, excludeKeyPatterns, conditions, limit, sortBy); returned.add(res); - Iterator tmp = Iterators.transform(res, input -> convertFromDBObject(collection, input)); + Iterator tmp = Iterators.transform(res, new Function() { + @Override + public T apply(RDBRow input) { + return convertFromDBObject(collection, input); + } + }); return CloseableIterator.wrap(tmp, (Closeable) res); } catch (SQLException ex) { throw new RuntimeException(ex); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java index 4ee8a1e41a7..5caa65d8758 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java @@ -61,6 +61,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Strings; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; @@ -431,7 +432,7 @@ public Set update(Connection connection, RDBTableMe } private static void assertNoDuplicatedIds(List documents) { - if (newHashSet(transform(documents, input -> input.getId())).size() < documents.size()) { + if (newHashSet(transform(documents, idExtractor)).size() < documents.size()) { throw new IllegalArgumentException("There are duplicated ids in the document list"); } } @@ -1126,4 +1127,11 @@ public int compare(T o1, T o2) { }); return result; } + + private static final Function idExtractor = new Function() { + @Override + public String apply(Document input) { + return input.getId(); + } + }; } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java index 84581d2f319..bc8dc15d63e 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/secondary/DelegatingDocumentNodeState.java @@ -19,7 +19,7 @@ package org.apache.jackrabbit.oak.plugins.document.secondary; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.api.PropertyState; @@ -178,8 +178,13 @@ public NodeState getChildNode(@NotNull String name) throws IllegalArgumentExcept @NotNull @Override public Iterable getChildNodeEntries() { - return Iterables.transform(delegate.getChildNodeEntries(), - input -> new MemoryChildNodeEntry(input.getName(), decorate(input.getName(), input.getNodeState()))); + return Iterables.transform(delegate.getChildNodeEntries(), new Function() { + @Nullable + @Override + public ChildNodeEntry apply(ChildNodeEntry input) { + return new MemoryChildNodeEntry(input.getName(), decorate(input.getName(), input.getNodeState())); + } + }); } @NotNull diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java index d0844c6238b..53ab10cc255 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java @@ -38,6 +38,7 @@ import java.util.TreeMap; import java.util.stream.Collectors; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; import org.apache.jackrabbit.oak.commons.OakVersion; @@ -982,7 +983,12 @@ public static boolean isHiddenPath(@NotNull String path) { */ public static Iterable asStringValueIterable( @NotNull Iterable values) { - return transform(values, input -> new StringValue(input)); + return transform(values, new Function() { + @Override + public StringValue apply(String input) { + return new StringValue(input); + } + }); } /** diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java index 48f617e654f..52cb467a4f1 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoVersionGCSupportDefaultNoBranchTest.java @@ -35,7 +35,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.PathUtils; @@ -59,6 +58,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Iterators; diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java index 184a4f3b520..d498ab1d18d 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/TestUtils.java @@ -18,8 +18,9 @@ import java.util.Iterator; import java.util.Map; -import java.util.function.Function; +import java.util.NavigableMap; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Functions; import org.apache.jackrabbit.guava.common.base.Predicate; diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java index 4b89c52d7c9..58c1f0ef160 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java @@ -97,6 +97,7 @@ import static org.junit.Assume.assumeThat; import static org.junit.Assume.assumeTrue; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.cache.Cache; import org.apache.jackrabbit.guava.common.collect.AbstractIterator; @@ -3274,7 +3275,12 @@ public void invalidateCacheOnMissingPreviousDocument() throws Exception { Long modCount = foo.getModCount(); assertNotNull(modCount); List prevIds = Lists.newArrayList(Iterators.transform( - foo.getPreviousDocLeaves(), input -> input.getId())); + foo.getPreviousDocLeaves(), new Function() { + @Override + public String apply(NodeDocument input) { + return input.getId(); + } + })); // run gc on another document node store createSecondaryStore(LeaseCheckMode.LENIENT); diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java index 425bd0a1da5..dcb3a92aaf3 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MemoryChildNodeEntry.java @@ -22,6 +22,7 @@ import java.util.Map.Entry; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.spi.state.AbstractChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; @@ -32,8 +33,16 @@ */ public class MemoryChildNodeEntry extends AbstractChildNodeEntry { - public static > Iterable iterable(Iterable set) { - return Iterables.transform(set, entry -> new MemoryChildNodeEntry(entry.getKey(), entry.getValue())); + public static > Iterable iterable( + Iterable set) { + return Iterables.transform( + set, + new Function, ChildNodeEntry>() { + @Override + public ChildNodeEntry apply(Entry entry) { + return new MemoryChildNodeEntry(entry.getKey(), entry.getValue()); + } + }); } private final String name; diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java index 05900f27449..3ae7072f62c 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/ModifiedNodeState.java @@ -30,8 +30,8 @@ import java.util.Map; import java.util.Map.Entry; -import java.util.function.Function; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; @@ -39,6 +39,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.apache.jackrabbit.guava.common.base.Predicate; import org.apache.jackrabbit.guava.common.base.Predicates; @@ -51,7 +52,17 @@ public class ModifiedNodeState extends AbstractNodeState { /** * Mapping from a PropertyState instance to its name. */ - private static final Function GET_NAME = input -> (input != null) ? input.getName() : null; + private static final Function GET_NAME = + new Function() { + @Override @Nullable + public String apply(@Nullable PropertyState input) { + if (input != null) { + return input.getName(); + } else { + return null; + } + } + }; /** * Unwraps the given {@code NodeState} instance into the given internals @@ -169,7 +180,7 @@ static Iterable getProperties( properties = newHashMap(properties); } Predicate predicate = Predicates.compose( - not(in(properties.keySet())), GET_NAME::apply); + not(in(properties.keySet())), GET_NAME); return concat( filter(base.getProperties(), predicate), filter(properties.values(), notNull())); @@ -350,7 +361,7 @@ public Iterable getChildNodeEntries() { return base.getChildNodeEntries(); // shortcut } else { Predicate predicate = Predicates.compose( - not(in(nodes.keySet())), ChildNodeEntry.GET_NAME::apply); + not(in(nodes.keySet())), ChildNodeEntry.GET_NAME); return concat( filter(base.getChildNodeEntries(), predicate), iterable(filterValues(nodes, NodeState.EXISTS).entrySet())); diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java index 933da8170b4..1d286c4e772 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/plugins/memory/MultiPropertyState.java @@ -21,12 +21,15 @@ import static org.apache.jackrabbit.guava.common.base.Preconditions.checkArgument; import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; +import java.math.BigDecimal; import java.util.List; import javax.jcr.PropertyType; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; +import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.value.Conversions.Converter; import org.jetbrains.annotations.NotNull; @@ -59,29 +62,89 @@ protected MultiPropertyState(String name, Iterable values) { private S convertTo(Type type) { switch (type.tag()) { case PropertyType.STRING: - return (S) Iterables.transform(values, value -> getConverter(value).toString()); + return (S) Iterables.transform(values, new Function() { + @Override + public String apply(T value) { + return getConverter(value).toString(); + } + }); case PropertyType.BINARY: - return (S) Iterables.transform(values, value -> getConverter(value).toBinary()); + return (S) Iterables.transform(values, new Function() { + @Override + public Blob apply(T value) { + return getConverter(value).toBinary(); + } + }); case PropertyType.LONG: - return (S) Iterables.transform(values, value -> getConverter(value).toLong()); + return (S) Iterables.transform(values, new Function() { + @Override + public Long apply(T value) { + return getConverter(value).toLong(); + } + }); case PropertyType.DOUBLE: - return (S) Iterables.transform(values, value -> getConverter(value).toDouble()); + return (S) Iterables.transform(values, new Function() { + @Override + public Double apply(T value) { + return getConverter(value).toDouble(); + } + }); case PropertyType.DATE: - return (S) Iterables.transform(values, value -> getConverter(value).toDate()); + return (S) Iterables.transform(values, new Function() { + @Override + public String apply(T value) { + return getConverter(value).toDate(); + } + }); case PropertyType.BOOLEAN: - return (S) Iterables.transform(values, value -> getConverter(value).toBoolean()); + return (S) Iterables.transform(values, new Function() { + @Override + public Boolean apply(T value) { + return getConverter(value).toBoolean(); + } + }); case PropertyType.NAME: - return (S) Iterables.transform(values, value -> getConverter(value).toString()); + return (S) Iterables.transform(values, new Function() { + @Override + public String apply(T value) { + return getConverter(value).toString(); + } + }); case PropertyType.PATH: - return (S) Iterables.transform(values, value -> getConverter(value).toString()); + return (S) Iterables.transform(values, new Function() { + @Override + public String apply(T value) { + return getConverter(value).toString(); + } + }); case PropertyType.REFERENCE: - return (S) Iterables.transform(values, value -> getConverter(value).toString()); + return (S) Iterables.transform(values, new Function() { + @Override + public String apply(T value) { + return getConverter(value).toString(); + } + }); case PropertyType.WEAKREFERENCE: - return (S) Iterables.transform(values, value -> getConverter(value).toString()); + return (S) Iterables.transform(values, new Function() { + @Override + public String apply(T value) { + return getConverter(value).toString(); + } + }); case PropertyType.URI: - return (S) Iterables.transform(values, value -> getConverter(value).toString()); + return (S) Iterables.transform(values, new Function() { + @Override + public String apply(T value) { + return getConverter(value).toString(); + } + }); case PropertyType.DECIMAL: - return (S) Iterables.transform(values, value -> getConverter(value).toDecimal()); + return (S) Iterables.transform(values, new Function() { + @Override + public BigDecimal apply(T value) { + return getConverter(value).toDecimal(); + } + }); default: throw new IllegalArgumentException("Unknown type:" + type); } } diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java index 1c28f616963..393618e5da7 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/commit/ProgressNotificationEditor.java @@ -21,8 +21,7 @@ import static org.apache.jackrabbit.oak.commons.PathUtils.concat; -import java.util.function.Function; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.spi.state.NodeState; diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java index d9ee3af66b0..d015779ff43 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/AbstractNodeState.java @@ -33,6 +33,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; /** @@ -288,7 +289,14 @@ public long getChildNodeCount(long max) { @Override public Iterable getChildNodeNames() { - return Iterables.transform(getChildNodeEntries(), input -> input.getName()); + return Iterables.transform( + getChildNodeEntries(), + new Function() { + @Override + public String apply(ChildNodeEntry input) { + return input.getName(); + } + }); } /** diff --git a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java index 0a59387fa01..c032e831044 100644 --- a/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java +++ b/oak-store-spi/src/main/java/org/apache/jackrabbit/oak/spi/state/ChildNodeEntry.java @@ -16,9 +16,11 @@ */ package org.apache.jackrabbit.oak.spi.state; -import java.util.function.Function; + +import org.apache.jackrabbit.guava.common.base.Function; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; /** * A {@code ChildNodeEntry} instance represents the child node states of a @@ -52,11 +54,15 @@ public interface ChildNodeEntry { * Mapping from a ChildNodeEntry instance to its name. */ Function GET_NAME = - input -> { + new Function() { + @Override @Nullable + public String apply(@Nullable ChildNodeEntry input) { if (input != null) { return input.getName(); } else { return null; } - }; + } + }; + } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java index a55904eebc2..cf6b27da998 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositoryUpgrade.java @@ -59,7 +59,7 @@ import javax.jcr.nodetype.PropertyDefinitionTemplate; import javax.jcr.security.Privilege; - +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.guava.common.collect.HashBiMap; import org.apache.jackrabbit.guava.common.collect.ImmutableList; @@ -144,6 +144,7 @@ import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermEnum; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -776,8 +777,15 @@ private void copyCustomPrivileges(PrivilegeManager pMgr) throws RepositoryExcept while (it.hasNext()) { Privilege aggrPriv = it.next(); - List aggrNames = Lists.transform(ImmutableList.copyOf(aggrPriv.getDeclaredAggregatePrivileges()), - input -> (input == null) ? null : input.getName()); + List aggrNames = Lists.transform( + ImmutableList.copyOf(aggrPriv.getDeclaredAggregatePrivileges()), + new Function() { + @Nullable + @Override + public String apply(@Nullable Privilege input) { + return (input == null) ? null : input.getName(); + } + }); if (allAggregatesRegistered(pMgr, aggrNames)) { pMgr.registerPrivilege(aggrPriv.getName(), aggrPriv.isAbstract(), aggrNames.toArray(new String[aggrNames.size()])); it.remove(); diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java index 631e261cb4a..ea5e28835f7 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/SameNameSiblingsEditor.java @@ -44,6 +44,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.base.Predicate; /** @@ -171,7 +172,12 @@ private static Iterable filterChildren(NodeState parent, final Predicate public boolean apply(ChildNodeEntry input) { return predicate.apply(input.getNodeState()); } - }), input -> input.getName()); + }), new Function() { + @Override + public String apply(ChildNodeEntry input) { + return input.getName(); + } + }); } /** diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java index 1337c03fe6e..1e058825c9b 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/checkpoint/CheckpointRetriever.java @@ -18,19 +18,21 @@ */ package org.apache.jackrabbit.oak.upgrade.checkpoint; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.jackrabbit.oak.plugins.document.DocumentCheckpointRetriever; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.segment.CheckpointAccessor; import org.apache.jackrabbit.oak.segment.SegmentNodeStore; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.upgrade.cli.node.FileStoreUtils; +import org.jetbrains.annotations.Nullable; import java.util.Collections; import java.util.List; - public final class CheckpointRetriever { public static class Checkpoint implements Comparable { @@ -86,7 +88,12 @@ public static List getCheckpoints(NodeStore nodeStore) { } private static List getCheckpoints(NodeState checkpointRoot) { - return Lists.newArrayList(Iterables.transform(checkpointRoot.getChildNodeEntries(), - input -> Checkpoint.createFromSegmentNode(input.getName(), input.getNodeState()))); + return Lists.newArrayList(Iterables.transform(checkpointRoot.getChildNodeEntries(), new Function() { + @Nullable + @Override + public Checkpoint apply(@Nullable ChildNodeEntry input) { + return Checkpoint.createFromSegmentNode(input.getName(), input.getNodeState()); + } + })); } } diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java index 97145a2aefe..eef40994b99 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java @@ -37,7 +37,9 @@ import javax.jcr.RepositoryException; import javax.jcr.Session; import javax.jcr.SimpleCredentials; +import javax.jcr.Value; +import org.apache.jackrabbit.guava.common.base.Function; import org.apache.jackrabbit.guava.common.collect.Lists; import org.apache.commons.codec.digest.DigestUtils; import org.apache.jackrabbit.oak.api.Blob; @@ -59,6 +61,7 @@ import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer; import org.apache.jackrabbit.oak.upgrade.cli.container.SegmentNodeStoreContainer; import org.apache.jackrabbit.oak.upgrade.cli.parser.CliArgumentException; +import org.jetbrains.annotations.Nullable; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -186,13 +189,17 @@ public static void verifyContent(Session session) throws RepositoryException { Node nodeType = session.getNode("/jcr:system/jcr:nodeTypes/sling:OrderedFolder"); assertEquals("rep:NodeType", nodeType.getProperty("jcr:primaryType").getString()); - List values = Lists.transform(Arrays.asList(nodeType.getProperty("rep:protectedProperties").getValues()), input -> { + List values = Lists.transform(Arrays.asList(nodeType.getProperty("rep:protectedProperties").getValues()), new Function() { + @Nullable + @Override + public String apply(@Nullable Value input) { try { return input.getString(); } catch (RepositoryException e) { return null; } - }); + } + }); assertTrue(values.contains("jcr:mixinTypes")); assertTrue(values.contains("jcr:primaryType")); assertEquals("false", nodeType.getProperty("jcr:isAbstract").getString()); From 28acbac8c1c2ab3733a598c0391211386a6ca4ea Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:30:19 +0200 Subject: [PATCH 54/86] Revert "OAK-10960: blob-cloud, segment: update netty version to 4.1.111 (#1590)" This reverts commit cd35db705fe768ede6991c6a5d25de034081114b. --- oak-blob-cloud-azure/pom.xml | 2 +- oak-blob-cloud/pom.xml | 2 +- oak-segment-azure/pom.xml | 2 +- oak-segment-tar/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/oak-blob-cloud-azure/pom.xml b/oak-blob-cloud-azure/pom.xml index 3a1a12fcced..4d102b1a723 100644 --- a/oak-blob-cloud-azure/pom.xml +++ b/oak-blob-cloud-azure/pom.xml @@ -31,7 +31,7 @@ bundle - 4.1.111.Final + 4.1.109.Final diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml index 4515d7138ab..b6fc94844d2 100644 --- a/oak-blob-cloud/pom.xml +++ b/oak-blob-cloud/pom.xml @@ -31,7 +31,7 @@ bundle - 4.1.111.Final + 4.1.107.Final 1.12.761 diff --git a/oak-segment-azure/pom.xml b/oak-segment-azure/pom.xml index ea5b073be4b..a975c5988de 100644 --- a/oak-segment-azure/pom.xml +++ b/oak-segment-azure/pom.xml @@ -33,7 +33,7 @@ Oak Segment Azure - 4.1.111.Final + 4.1.109.Final diff --git a/oak-segment-tar/pom.xml b/oak-segment-tar/pom.xml index b9e732bde2d..848cf0af680 100644 --- a/oak-segment-tar/pom.xml +++ b/oak-segment-tar/pom.xml @@ -33,7 +33,7 @@ Oak Segment Tar - 4.1.111.Final + 4.1.104.Final 1.4.2 From 52fa8997f24c1f55f3588ed96836a93819d5f54a Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:03 +0200 Subject: [PATCH 55/86] Revert "OAK-10959: webapp: update Tomcat dependency to 9.0.90 (#1589)" This reverts commit 5fb55e05e0ba137024e07d66dc3856864b87eacf. --- oak-examples/webapp/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-examples/webapp/pom.xml b/oak-examples/webapp/pom.xml index 8af0f35cf8d..c07b8d1f6a2 100644 --- a/oak-examples/webapp/pom.xml +++ b/oak-examples/webapp/pom.xml @@ -35,7 +35,7 @@ Web application that hosts and serves a Jackrabbit Oak content repository - 9.0.90 + 9.0.89 true From 063c26b20283ebedd9576ea77802521427f86239 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:04 +0200 Subject: [PATCH 56/86] Revert "OAK-10954: Update spotbugs plugin to 4.8.6.2 (#1588)" This reverts commit b098fd8930ec91cd4ea15769c21db8e4ee47864d. --- oak-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-parent/pom.xml b/oak-parent/pom.xml index 483409b135c..29598811d04 100644 --- a/oak-parent/pom.xml +++ b/oak-parent/pom.xml @@ -331,7 +331,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.8.6.2 + 4.8.5.0 org.apache.maven.plugins From 50e951fd263ae34387343346aa432b90588f5a2c Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:05 +0200 Subject: [PATCH 57/86] Revert "OAK-10949: blob-cloud, segment-aws: update aws SDK to 1.12.761 (dependencies reference vulnerable amazon ion-java version) (#1581)" This reverts commit cafbd29d520c247a74e88314b1f518c95fb74443. --- oak-blob-cloud/pom.xml | 2 +- oak-segment-aws/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml index b6fc94844d2..dd7d4479db3 100644 --- a/oak-blob-cloud/pom.xml +++ b/oak-blob-cloud/pom.xml @@ -32,7 +32,7 @@ 4.1.107.Final - 1.12.761 + 1.12.353 diff --git a/oak-segment-aws/pom.xml b/oak-segment-aws/pom.xml index 06105369ab6..e6ed369c425 100644 --- a/oak-segment-aws/pom.xml +++ b/oak-segment-aws/pom.xml @@ -33,7 +33,7 @@ Oak Segment AWS - 1.12.761 + 1.12.353 1.0.392 From 212836774f4c0de6fcf5f4073d4a722af2fc8791 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:06 +0200 Subject: [PATCH 58/86] Revert "OAK-10685: remove unused import of java.io.UnsupportedEncodingException" This reverts commit 36cfcfa08e66ef2aecc5cec3c08e6dd08db9038e. --- .../oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java | 1 + 1 file changed, 1 insertion(+) diff --git a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java index 33add8dafa4..0eba902e7fc 100644 --- a/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java +++ b/oak-blob-cloud-azure/src/main/java/org/apache/jackrabbit/oak/blob/cloud/azure/blobstorage/AzureBlobStoreBackend.java @@ -29,6 +29,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; From 6ea10b7955081e67bfbac4c5d3cdfde2525d6565 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:07 +0200 Subject: [PATCH 59/86] Revert "OAK-10848: commons: remove use of slf4j.event.Level in SystemPropertySupplier implementation (#1580)" This reverts commit ba530fe46563757474a4ed1740aa3396e5f051bd. --- .../properties/SystemPropertySupplier.java | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java index 30ad2c59616..c1b0da0e592 100644 --- a/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java +++ b/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/properties/SystemPropertySupplier.java @@ -25,6 +25,7 @@ import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.slf4j.event.Level; /** * Utility class for consistent handling of system properties. @@ -49,7 +50,7 @@ public class SystemPropertySupplier implements Supplier { private final Function parser; private Logger log = LOG; - private String successLogLevel = "INFO"; + private Level successLogLevel = Level.INFO; private Predicate validator = (a) -> true; private Function sysPropReader = System::getProperty; private BiFunction setMessageFormatter = (a, b) -> { @@ -99,14 +100,22 @@ public SystemPropertySupplier formatSetMessage(@NotNull BiFunction logSuccessAs(String successLogLevel) { - String newLevel; + Level newLevel; switch (Objects.requireNonNull(successLogLevel)) { case "DEBUG": + newLevel = Level.DEBUG; + break; case "ERROR": + newLevel = Level.ERROR; + break; case "INFO": + newLevel = Level.INFO; + break; case "TRACE": + newLevel = Level.TRACE; + break; case "WARN": - newLevel = successLogLevel; + newLevel = Level.WARN; break; default: throw new IllegalArgumentException("unsupported log level: " + successLogLevel); @@ -153,19 +162,19 @@ public T get() { if (!ret.equals(defaultValue)) { String msg = setMessageFormatter.apply(propName, ret); switch (successLogLevel) { - case "INFO": + case INFO: log.info(msg); break; - case "DEBUG": + case DEBUG: log.debug(msg); break; - case "ERROR": + case ERROR: log.error(msg); break; - case "TRACE": + case TRACE: log.trace(msg); break; - case "WARN": + case WARN: log.warn(msg); break; default: From 73ce197f5cf6dff5a638adcc79824e4ccefd2430 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:08 +0200 Subject: [PATCH 60/86] Revert "OAK-10905 | Add license header to AsyncCheckpointService (#1579)" This reverts commit 7f112faca966859e8bf117bcbd4665892f00a330. --- .../plugins/index/AsyncCheckpointService.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java index 460e5fe93f2..4bc3f90bdac 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java @@ -1,19 +1,3 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.jackrabbit.oak.plugins.index; import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; From 556d346909c057b4214d5b2077663424d37d7a46 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:09 +0200 Subject: [PATCH 61/86] Revert "OAK-10905 | Add a configurable async checkpoint creator service (#1560)" This reverts commit f12ccb1716e6677b54ad3f06955cc007479584bf. --- .../plugins/index/AsyncCheckpointCreator.java | 143 ------------------ .../plugins/index/AsyncCheckpointService.java | 104 ------------- .../oak/plugins/index/IndexUtils.java | 45 ------ .../index/AsyncCheckpointCreatorTest.java | 60 -------- .../index/AsyncCheckpointServiceTest.java | 131 ---------------- .../oak/plugins/index/IndexUtilsTest.java | 99 +----------- 6 files changed, 3 insertions(+), 579 deletions(-) delete mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java delete mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java delete mode 100644 oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java delete mode 100644 oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java deleted file mode 100644 index d4a1ab6de8a..00000000000 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreator.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index; - - -import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.apache.jackrabbit.util.ISO8601; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.Calendar; -import java.util.Map; -import java.util.Set; -import java.util.TimeZone; - -/** - * This class is responsible for creating and deleting checkpoints asynchronously. - * The number of minimum concurrent checkpoints to keep in the system, along with the default lifetime of a checkpoint - * can be configured. - * When executed, this class should create one checkpoint in a single run with a configurable name. - * Following the creation of the checkpoint, it should try to delete checkpoints with the given name, - * in case the total number of such checkpoints is greater than the configured minimum concurrent checkpoints. - * By default, this task is registered using AsyncCheckpointService - */ -public class AsyncCheckpointCreator implements Runnable { - - /** - * Name of service property which determines the name of this Async task - */ - public static final String PROP_ASYNC_NAME = "oak.checkpoint.async"; - - private final String name; - private final long checkpointLifetimeInSeconds; - private final long minConcurrentCheckpoints; - private final long maxConcurrentCheckpoints; - private final NodeStore store; - public static final String CHECKPOINT_CREATOR_KEY = "creator"; - public static final String CHECKPOINT_CREATED_KEY = "created"; - public static final String CHECKPOINT_CREATED_TIMESTAMP_KEY= "created-epoch"; - public static final String CHECKPOINT_THREAD_KEY = "thread"; - public static final String CHECKPOINT_NAME_KEY = "name"; - private static final Logger log = LoggerFactory - .getLogger(AsyncCheckpointCreator.class); - - public AsyncCheckpointCreator(@NotNull NodeStore store, @NotNull String name, long checkpointLifetimeInSeconds, long minConcurrentCheckpoints, long maxConcurrentCheckpoints) { - this.store = store; - this.name = name; - this.checkpointLifetimeInSeconds = checkpointLifetimeInSeconds; - this.minConcurrentCheckpoints = minConcurrentCheckpoints; - // maxConcurrentCheckpoints should at least be 1 more than minConcurrentCheckpoints - if (maxConcurrentCheckpoints <= minConcurrentCheckpoints) { - log.warn("[{}] Max concurrent checkpoints is less than or equal to min concurrent checkpoints. " + - "Setting max concurrent checkpoints to min concurrent checkpoints + 1.", this.name); - this.maxConcurrentCheckpoints = minConcurrentCheckpoints + 1; - } else { - this.maxConcurrentCheckpoints = maxConcurrentCheckpoints; - } - } - - public String getName() { - return name; - } - - protected long getCheckpointLifetimeInSeconds() { - return checkpointLifetimeInSeconds; - } - - protected long getMinConcurrentCheckpoints() { - return minConcurrentCheckpoints; - } - - protected long getMaxConcurrentCheckpoints() { - return maxConcurrentCheckpoints; - } - - @Override - public void run() { - Map> filteredCheckpointMap = IndexUtils.getFilteredCheckpoints(store, - entry -> name.equals(entry.getValue().get(CHECKPOINT_NAME_KEY))); - // If the number of checkpoints created by this task are equal to or greater than the maxConcurrentCheckpoints, skip - // creation of a new checkpoint. - // This could happen in case the deletion of older checkpoints failed in multiple previous runs. - if (filteredCheckpointMap.size() >= maxConcurrentCheckpoints) { - log.warn("[{}] Skipping checkpoint creation as the number of concurrent checkpoints is already at max limit {}", name, maxConcurrentCheckpoints); - } else { - Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - String creationTimeStamp = String.valueOf(cal.getTimeInMillis()); - String creationTimeISOFormat = ISO8601.format(cal); - String checkpoint = store.checkpoint(checkpointLifetimeInSeconds * 1000, Map.of( - CHECKPOINT_CREATOR_KEY, AsyncCheckpointCreator.class.getSimpleName(), - CHECKPOINT_CREATED_KEY, creationTimeISOFormat, - CHECKPOINT_CREATED_TIMESTAMP_KEY, creationTimeStamp, - CHECKPOINT_THREAD_KEY, Thread.currentThread().getName(), - CHECKPOINT_NAME_KEY, name)); - log.info("[{}] Created checkpoint {} with creation time {}", name, checkpoint, creationTimeISOFormat); - } - - // Get a list of checkpoints filtered on the basis of CHECKPOINT_NAME_KEY (name). This is done using the - // getFilteredCheckpoints in the IndexUtils, which gets all the checkpoints in the node store and then filters the list based on - // the provided predicate using the checkpoint info map associated with the checkpoints. - // The filtered checkpoints list is then sorted based on the CHECKPOINT_CREATED_TIMESTAMP_KEY (created-epoch). - // Both the initial filtering and sorting is done based on the information from the associated checkpoint info map of a given checkpoint. - Set sortedCheckpointSet = IndexUtils.getSortedCheckpointMap(IndexUtils.getFilteredCheckpoints(store, - entry -> name.equals(entry.getValue().get(CHECKPOINT_NAME_KEY))), CHECKPOINT_CREATED_TIMESTAMP_KEY).keySet(); - int counter = sortedCheckpointSet.size(); - // Iterate over the sortedCheckpointSet which is sorted based on the creation timestamp -> the oldest first - // We try and delete the checkpoints as long as the counter is greater than minConcurrentCheckpoints - // This ensures that the system always has concurrent checkpoints equal to or greater than the configured minConcurrentCheckpoints - for (String cp : sortedCheckpointSet) { - // Delete the checkpoint as long as the checkpoint count is greater than concurrentCheckpoints - if (counter > minConcurrentCheckpoints) { - if(store.release(cp) ) { - log.info("[{}] Deleted checkpoint {}", name, cp); - } else { - log.warn("[{}] Unable to delete checkpoint {}. Removal will be attempted again in next run.", name, cp); - } - } else { - break; - } - // Decrement the counter irrespective of the outcome of the checkpoint deletion. - // If we don't decrement the counter in case of a failure while trying to delete a checkpoint, - // the next iteration will try to delete a comparatively newer checkpoint and keep the older one in the system (which we don't want). - // The checkpoint that failed to get deleted in this case should get deleted in the next run. - counter --; - } - - } - -} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java deleted file mode 100644 index 4bc3f90bdac..00000000000 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointService.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.apache.jackrabbit.oak.plugins.index; - -import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; -import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.apache.jackrabbit.oak.spi.whiteboard.CompositeRegistration; -import org.apache.jackrabbit.oak.spi.whiteboard.Registration; -import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; -import org.osgi.framework.BundleContext; -import org.osgi.service.component.annotations.Component; -import org.osgi.service.component.annotations.ConfigurationPolicy; -import org.osgi.service.component.annotations.Reference; -import org.osgi.service.component.annotations.Activate; -import org.osgi.service.component.annotations.Deactivate; -import org.osgi.service.metatype.annotations.AttributeDefinition; -import org.osgi.service.metatype.annotations.Designate; -import org.osgi.service.metatype.annotations.ObjectClassDefinition; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.ScheduleExecutionInstanceTypes.RUN_ON_LEADER; -import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.scheduleWithFixedDelay; - -@Component( - configurationPolicy = ConfigurationPolicy.REQUIRE) -@Designate(ocd = AsyncCheckpointService.Configuration.class, factory = true) -public class AsyncCheckpointService { - - @ObjectClassDefinition( - name = "Apache Jackrabbit Oak Async Checkpoint Service", - description = "Configures the async checkpoint services which performs periodic creation and deletion of checkpoints" - ) - @interface Configuration { - - @AttributeDefinition( - name = "Checkpoint Creator Identifier", - description = "Unique identifier to be used for creating checkpoints" - ) - String name() default "checkpoint-async"; - - @AttributeDefinition( - name = "Enable", - description = "Flag to enable/disable the checkpoints creation task" - ) - boolean enable() default false; - - @AttributeDefinition( - name = "Minimum Concurrent Checkpoints", - description = "Minimum number of concurrent checkpoints to keep in system" - ) - long minConcurrentCheckpoints() default 2; - - @AttributeDefinition( - name = "Maximum Concurrent Checkpoints", - description = "Maximum number of concurrent checkpoints to keep in system. " + - "This limit is to prevent multiple checkpoint creation in case the deletion of older " + - "checkpoints fails multiple times. This should always be greater than Minimum Concurrent Checkpoints" - ) - long maxConcurrentCheckpoints() default 10; - - @AttributeDefinition( - name = "Checkpoint Lifetime", - description = "Lifetime of a checkpoint in seconds" - ) - long checkpointLifetime() default 60 * 60 * 24; - - @AttributeDefinition( - name = "Time Interval", - description = "Time interval between consecutive job runs in seconds. This would be the time interval between two consecutive checkpoints creation." - ) - long timeIntervalBetweenCheckpoints() default 60 * 15; - - } - - private final List regs = new ArrayList<>(); - @Reference - private NodeStore nodeStore; - - @Activate - public void activate(BundleContext bundleContext, AsyncCheckpointService.Configuration config) { - Whiteboard whiteboard = new OsgiWhiteboard(bundleContext); - if (config.enable()) { - AsyncCheckpointCreator task = new AsyncCheckpointCreator(nodeStore, config.name(), config.checkpointLifetime(), config.minConcurrentCheckpoints(), config.maxConcurrentCheckpoints()); - registerAsyncCheckpointCreator(whiteboard, task, config.timeIntervalBetweenCheckpoints()); - } - } - - private void registerAsyncCheckpointCreator(Whiteboard whiteboard, AsyncCheckpointCreator task, long delayInSeconds) { - Map config = Map.of( - AsyncCheckpointCreator.PROP_ASYNC_NAME, task.getName(), - "scheduler.name", AsyncCheckpointCreator.class.getName() + "-" + task.getName() - ); - regs.add(scheduleWithFixedDelay(whiteboard, task, config, delayInSeconds, RUN_ON_LEADER, true)); - } - - @Deactivate - public void deactivate() throws IOException { - new CompositeRegistration(regs).unregister(); - } - - -} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java index 076ef54029b..a6f64f0f42f 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUtils.java @@ -31,14 +31,9 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.UNIQUE_PROPERTY_NAME; import java.util.Collection; -import java.util.Comparator; import java.util.Map; import java.util.Set; -import java.util.LinkedHashMap; -import java.util.function.Predicate; -import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import javax.jcr.RepositoryException; @@ -54,7 +49,6 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.plugins.tree.TreeUtil; -import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -287,43 +281,4 @@ public static String getCaller(@Nullable String[] ignoredJavaPackages) { // if every element is ignored, we assume it's an internal request return "(internal)"; } - - /** - * Returns a Map consisting of checkpoints and checkpointInfoMap filtered based on a given predicate - * which can utilise checkpoint info map for filtering - * @param store - * @param predicate predicate used for filtering of checkpoints - * @return Map> filteredCheckpoint Map - */ - public static Map> getFilteredCheckpoints(NodeStore store, Predicate>> predicate) { - return StreamSupport.stream(store.checkpoints().spliterator(), false) - .collect(Collectors.toMap(str -> str, - store::checkpointInfo)) - .entrySet() - .stream() - .filter(predicate) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - /** - * Returns a map with checkpoint name as key and checkpoint metadata map as value, sorted on the basis of particular key in the metadata map. - * For example - given the following checkpoints in the system along with their associated metadata maps - - * checkpoint3 - {created-epoch=123458, creator=creator1} - * checkpoint1 - {created-epoch=123456, creator=creator2} - * checkpoint2 - {created-epoch=123457, creator=creator3} - * This method should return - - * {checkpoint1={created-epoch=123456, creator=creator2}, - * checkpoint2={created-epoch=123457, creator=creator3}, - * checkpoint3={created-epoch=123458, creator=creator1}} - * @param checkpointMap - the map consisting of checkpoints as keys and checkpoint metadata map as values - * @param keyForComparator - key in the metadata map of the checkpoint that can be used as comparator to sort on checkpoint creation time. - * @return Map> sorted checkpoint map - */ - public static Map> getSortedCheckpointMap(Map> checkpointMap, - String keyForComparator) { - return checkpointMap.entrySet().stream() - .filter(entry -> entry.getValue().containsKey(keyForComparator)) - .sorted(Comparator.comparingLong(entry -> Long.parseLong(entry.getValue().get(keyForComparator)))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (o, n) -> n, LinkedHashMap::new)); - } } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java deleted file mode 100644 index c569c78b598..00000000000 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointCreatorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.plugins.index; - -import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; -import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.junit.Assert; -import org.junit.Test; - -import java.util.*; - -public class AsyncCheckpointCreatorTest { - - @Test - public void testAsync() { - NodeStore store = new MemoryNodeStore(); - int minConcurrentCheckpoints = 3; - int maxConcurrentCheckpoints = 5; - AsyncCheckpointCreator task = new AsyncCheckpointCreator(store, "test", 1000, minConcurrentCheckpoints, maxConcurrentCheckpoints); - Map checkpointMap = new LinkedHashMap<>(); - for (int i = 0; i < minConcurrentCheckpoints; i++) { - List checkpointList = new ArrayList<>(); - task.run(); - for(String checkpoint : store.checkpoints()) { - if (!checkpointMap.containsValue(checkpoint)) { - checkpointMap.put(i + 1, checkpoint); - } - checkpointList.add(checkpoint); - } - Assert.assertEquals(i + 1, checkpointList.size()); - } - // Task run post the minConcurrentCheckpoints should not result in additional - // checkpoints since the oldest checkpoint should get deleted - for (int j = 0; j < 2 ; j++) { - List checkpointList = new ArrayList<>(); - task.run(); - for(String checkpoint : store.checkpoints()) { - checkpointList.add(checkpoint); - } - Assert.assertFalse(checkpointList.contains(checkpointMap.get(j + 1))); - Assert.assertEquals(minConcurrentCheckpoints, checkpointList.size()); - } - } -} diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java deleted file mode 100644 index 0bb148d010c..00000000000 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncCheckpointServiceTest.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.plugins.index; - -import org.apache.jackrabbit.guava.common.collect.ImmutableMap; -import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; -import org.apache.jackrabbit.oak.spi.state.Clusterable; -import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.apache.sling.testing.mock.osgi.MockOsgi; -import org.apache.sling.testing.mock.osgi.junit.OsgiContext; -import org.jetbrains.annotations.NotNull; -import org.junit.Rule; -import org.junit.Test; - -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class AsyncCheckpointServiceTest { - - @Rule - public final OsgiContext context = new OsgiContext(); - - private final MemoryNodeStore nodeStore = new AsyncCheckpointServiceTest.FakeClusterableMemoryNodeStore(); - private final AsyncCheckpointService service = new AsyncCheckpointService(); - - /** - * This test is used to verify the registration of the AsyncCheckpointService with the OSGI context. - * The test verifies this with 3 different configurations of the AsyncCheckpointService, 2 of them enabled and 1 disabled. - */ - @Test - public void asyncReg() { - injectDefaultServices(); - // Create 3 configurations of the AsyncCheckpointService, 2 of them enabled and 1 disabled. - String name1 = "checkpoint-async-test-1"; - String name2 = "checkpoint-async-test-2"; - String name3 = "checkpoint-async-test-3"; - Map config1 = ImmutableMap.of( - "name", "checkpoint-async-test-1", - "enable", true, - "minConcurrentCheckpoints", 3L, - "maxConcurrentCheckpoints", 10L, - "checkpointLifetime", 60 * 60 * 24L, - "timeIntervalBetweenCheckpoints", 60 * 15L - ); - - Map config2 = ImmutableMap.of( - "name", "checkpoint-async-test-2", - "enable", false, - "minConcurrentCheckpoints", 3L, - "maxConcurrentCheckpoints", 10L, - "checkpointLifetime", 60 * 60 * 24L, - "timeIntervalBetweenCheckpoints", 60 * 15L - ); - - Map config3 = ImmutableMap.of( - "name", "checkpoint-async-test-3", - "enable", true, - "minConcurrentCheckpoints", 4L, - "maxConcurrentCheckpoints", 2L, - "checkpointLifetime", 60 * 24L, - "timeIntervalBetweenCheckpoints", 60 * 15L - ); - // Activate the service with the above 3 configurations. - MockOsgi.activate(service, context.bundleContext(), config1); - MockOsgi.activate(service, context.bundleContext(), config2); - MockOsgi.activate(service, context.bundleContext(), config3); - // Verify that the configs that are enabled are registered with the OSGI context and the one that is disabled is not. - assertEquals(1, context.getServices(Runnable.class, "(oak.checkpoint.async="+name1+")").length); - assertEquals(0, context.getServices(Runnable.class, "(oak.checkpoint.async="+name2+")").length); - assertEquals(1, context.getServices(Runnable.class, "(oak.checkpoint.async="+name3+")").length); - - // Get the instances fo the async tasks that are registered as per the enabled configs and verify the values of the - // configured minConcurrentCheckpoints and checkpointLifetimeInSeconds. - AsyncCheckpointCreator checkpointCreator1 = getCheckpointCreator("checkpoint-async-test-1"); - assertEquals(3, checkpointCreator1.getMinConcurrentCheckpoints()); - assertEquals(10, checkpointCreator1.getMaxConcurrentCheckpoints()); - assertEquals(60 * 60 * 24, checkpointCreator1.getCheckpointLifetimeInSeconds()); - AsyncCheckpointCreator checkpointCreator3 = getCheckpointCreator("checkpoint-async-test-3"); - assertEquals(4, checkpointCreator3.getMinConcurrentCheckpoints()); - assertEquals(5, checkpointCreator3.getMaxConcurrentCheckpoints()); - assertEquals(60 * 24, checkpointCreator3.getCheckpointLifetimeInSeconds()); - MockOsgi.deactivate(service, context.bundleContext()); - assertNull(context.getService(Runnable.class)); - } - - - private AsyncCheckpointCreator getCheckpointCreator(String name) { - return (AsyncCheckpointCreator) context.getServices(Runnable.class, "(oak.checkpoint.async="+name+")")[0]; - } - - private void injectDefaultServices() { - context.registerService(NodeStore.class, nodeStore); - MockOsgi.injectServices(service, context.bundleContext()); - } - - private static class FakeClusterableMemoryNodeStore extends MemoryNodeStore implements Clusterable { - @NotNull - @Override - public String getInstanceId() { - return "foo"; - } - - @Override - public String getVisibilityToken() { - return ""; - } - - @Override - public boolean isVisible(String visibilityToken, long maxWaitMillis) throws InterruptedException { - return true; - } - } -} diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java index a5b9036de95..5894946773d 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/IndexUtilsTest.java @@ -18,27 +18,14 @@ */ package org.apache.jackrabbit.oak.plugins.index; + import org.apache.jackrabbit.oak.api.Type; -import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.junit.Assert; import org.junit.Test; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.Set; -import java.util.Map; -import java.util.LinkedHashMap; -import java.util.Calendar; - import static org.apache.jackrabbit.guava.common.collect.Lists.newArrayList; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - +import static org.junit.Assert.*; public class IndexUtilsTest { @@ -48,10 +35,6 @@ public class IndexUtilsTest { // all packages used with Oak private static final String[] OAK_CLASSES_IGNORED = new String[] {"org.apache.jackrabbit", "java.lang", "sun.reflect", "jdk"}; - private static final String CHECKPOINT_CREATOR_KEY = "creator"; - - private static final String CHECKPOINT_CREATED_TIMESTAMP_KEY = "created"; - @Test public void asyncName() throws Exception { assertNull(IndexUtils.getAsyncLaneName(EMPTY_NODE, "/fooIndex")); @@ -73,81 +56,5 @@ public void getCaller() { String caller = IndexUtils.getCaller(OAK_CLASSES_IGNORED); assertTrue(caller.startsWith("org.junit.runners")); - } - - @Test - public void checkpointFilterAndSorting() throws Exception { - NodeStore store = null; - Set checkpointSet = new LinkedHashSet<>(); - try { - store = new MemoryNodeStore(); - - // Check all works fines if no checkpoints present - Map> noCheckpointMap = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process-non-existing".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY))); - assertEquals(0, noCheckpointMap.size()); - - // Create checkpoints - String checkpoint1 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process")); - checkpointSet.add(checkpoint1); - Thread.sleep(1000); - String checkpoint2 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process-2")); - checkpointSet.add(checkpoint2); - Thread.sleep(1000); - String checkpoint3 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process")); - checkpointSet.add(checkpoint3); - Thread.sleep(1000); - String checkpoint4 = store.checkpoint(1800000L, getMetaDataMap("checkpoint-helper-test-process")); - checkpointSet.add(checkpoint4); - - // Check for happy case - Map> filteredCheckpointMap = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process".equals(entry.getValue().get("creator"))); - - assertEquals(3, filteredCheckpointMap.size()); - for (String checkpoint : filteredCheckpointMap.keySet()) { - assertEquals("checkpoint-helper-test-process",filteredCheckpointMap.get(checkpoint).get(CHECKPOINT_CREATOR_KEY)); - } - // Check sorting now - Map> sortedCheckpointMap = IndexUtils.getSortedCheckpointMap(filteredCheckpointMap, CHECKPOINT_CREATED_TIMESTAMP_KEY); - assertEquals(3, sortedCheckpointMap.size()); - Iterator sortedCheckpointIt = sortedCheckpointMap.keySet().iterator(); - assertEquals(checkpoint1, sortedCheckpointIt.next()); - assertEquals(checkpoint3, sortedCheckpointIt.next()); - assertEquals(checkpoint4, sortedCheckpointIt.next()); - - // Check for negative edge cases - Map> filteredCheckpointMap2 = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process-non-existing".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY))); - assertEquals(0, filteredCheckpointMap2.size()); - - // Create a checkpoint with incorrect metadata - Map checkpointMetadata = getMetaDataMap("checkpoint-helper-test-process"); - checkpointMetadata.remove(CHECKPOINT_CREATED_TIMESTAMP_KEY); - String latestFilteredCheckpointWithNoTimestamp = store.checkpoint(1800000L, checkpointMetadata); - checkpointSet.add(latestFilteredCheckpointWithNoTimestamp); - - // Modify the predicate in the filter method here to also include the check that created exists in the checkpoint metadata - Map> filteredCheckpointMap3 = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY)) && entry.getValue().containsKey(CHECKPOINT_CREATED_TIMESTAMP_KEY)); - assertEquals(3, filteredCheckpointMap3.size()); - Assert.assertFalse(filteredCheckpointMap3.containsKey(latestFilteredCheckpointWithNoTimestamp)); - - Map checkpointMetadata2 = getMetaDataMap("checkpoint-helper-test-process-3"); - checkpointMetadata2.remove(CHECKPOINT_CREATED_TIMESTAMP_KEY); - String latestFilteredCheckpointWithNoTimestamp2 = store.checkpoint(1800000L, checkpointMetadata2); - checkpointSet.add(latestFilteredCheckpointWithNoTimestamp2); - Map> filteredCheckpointMap4 = IndexUtils.getFilteredCheckpoints(store, entry -> "checkpoint-helper-test-process-3".equals(entry.getValue().get(CHECKPOINT_CREATOR_KEY))); - assertEquals(1, filteredCheckpointMap4.size()); - Assert.assertTrue(filteredCheckpointMap4.containsKey(latestFilteredCheckpointWithNoTimestamp2)); - } finally { - for (String checkpoint : checkpointSet) { - store.release(checkpoint); - } - } - - } - - public static Map getMetaDataMap(String creator) { - Map checkpointMetaData = new LinkedHashMap<>(); - checkpointMetaData.put(CHECKPOINT_CREATOR_KEY, creator); - checkpointMetaData.put(CHECKPOINT_CREATED_TIMESTAMP_KEY, String.valueOf(Calendar.getInstance().getTimeInMillis())); - return checkpointMetaData; - } + } } \ No newline at end of file From de30c042501a17ed6b73e404026288044ec2d4a1 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:10 +0200 Subject: [PATCH 62/86] Revert "OAK-10705: oak-standalone: update dependencies (#1411)" This reverts commit dc10a1206566d27004aea9aa7f9612547d2078cd. --- oak-examples/standalone/pom.xml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/oak-examples/standalone/pom.xml b/oak-examples/standalone/pom.xml index 941cef1e26c..7ca0f1e9211 100644 --- a/oak-examples/standalone/pom.xml +++ b/oak-examples/standalone/pom.xml @@ -90,7 +90,7 @@ org.fusesource.jansi jansi - 2.4.1 + 1.11 true @@ -187,17 +187,18 @@ org.apache.felix org.apache.felix.http.proxy - 3.0.6 + 2.3.2 org.apache.felix org.apache.felix.http.bridge - 5.1.6 + 2.3.2 org.apache.felix org.apache.felix.webconsole - 5.0.0 + 4.2.10 + all org.apache.felix From 1ec3ad8dba6c42c0f83676658bdeda8ddff18e2f Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 11:31:11 +0200 Subject: [PATCH 63/86] Revert "OAK-10944: oak-auth-ldap: update commons-pool2 dependency to 2.12.0 (#1576)" This reverts commit 3d6d009d654fc8061baac1b66502e47f71fa0921. --- oak-auth-ldap/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-auth-ldap/pom.xml b/oak-auth-ldap/pom.xml index 2f52d9a4c74..024817e0781 100644 --- a/oak-auth-ldap/pom.xml +++ b/oak-auth-ldap/pom.xml @@ -85,7 +85,7 @@ org.apache.commons commons-pool2 - 2.12.0 + 2.8.0 org.apache.commons From 9c829f34cea2cd5c441d6a2e5a1880a8ad4a22a0 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 30 Jul 2024 15:28:28 +0200 Subject: [PATCH 64/86] OAK-10341 Tree store (bugfix for loggers) --- .../indexer/document/flatfile/pipelined/BoundedHistogram.java | 2 +- .../index/indexer/document/flatfile/pipelined/ConfigHelper.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java index ece82decacc..f10787942f4 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java @@ -32,7 +32,7 @@ * histogram are correct but if the histogram overflowed, it may be missing some entries. */ public class BoundedHistogram { - private static final Logger LOG = LoggerFactory.getLogger(PipelinedStrategy.class); + private static final Logger LOG = LoggerFactory.getLogger(BoundedHistogram.class); private final ConcurrentHashMap histogram = new ConcurrentHashMap<>(); private volatile boolean overflowed = false; private final String histogramName; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java index 9374af70b21..cad58706834 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java @@ -22,7 +22,7 @@ import org.slf4j.LoggerFactory; public class ConfigHelper { - private static final Logger LOG = LoggerFactory.getLogger(PipelinedStrategy.class); + private static final Logger LOG = LoggerFactory.getLogger(BoundedHistogram.class); public static int getSystemPropertyAsInt(String name, int defaultValue) { int result = Integer.getInteger(name, defaultValue); From cca4e59e14ed6c1c79603d1fdc8ab91813542c47 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 2 Aug 2024 12:05:02 +0200 Subject: [PATCH 65/86] OAK-10341 Tree store (PipelinedTreeStoreStrategy.java) --- .../pipelined/PipelinedTreeStoreStrategy.java | 507 +++++++++++++ .../pipelined/PipelinedTreeStoreTask.java | 206 ++++++ .../indexer/document/tree/store/PageFile.java | 4 +- .../pipelined/PipelinedTreeStoreIT.java | 680 ++++++++++++++++++ 4 files changed, 1395 insertions(+), 2 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java new file mode 100644 index 00000000000..7a4c21fce70 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -0,0 +1,507 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +import com.mongodb.MongoClientURI; +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.guava.common.base.Preconditions; +import org.apache.jackrabbit.guava.common.base.Stopwatch; +import org.apache.jackrabbit.guava.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.commons.IOUtils; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreSortStrategyBase; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; +import org.apache.jackrabbit.oak.plugins.document.NodeDocument; +import org.apache.jackrabbit.oak.plugins.document.RevisionVector; +import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; +import org.apache.jackrabbit.oak.plugins.index.FormattingUtils; +import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter; +import org.apache.jackrabbit.oak.plugins.index.IndexingReporter; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import static org.apache.jackrabbit.oak.plugins.index.IndexUtils.INDEXING_PHASE_LOGGER; + +/** + * Downloads the contents of the MongoDB repository dividing the tasks in a pipeline with the following stages: + *
    + *
  • Download - Downloads from Mongo all the documents in the node store. + *
  • Transform - Converts Mongo documents to node state entries. + *
  • Sort and save - Sorts the batch of node state entries and saves them to disk + *
  • Merge sorted files - Merge the intermediate sorted files into a single file (the final FlatFileStore). + *
+ *

+ *

Memory management

+ *

+ * For efficiency, the intermediate sorted files should be as large as possible given the memory constraints. + * This strategy accumulates the entries that will be stored in each of these files in memory until reaching a maximum + * configurable size, at which point it sorts the data and writes it to a file. The data is accumulated in instances of + * {@link NodeStateEntryBatch}. This class contains two data structures: + *

    + *
  • A {@link java.nio.ByteBuffer} for the binary representation of the entry, that is, the byte array that will be written to the file. + * This buffer contains length-prefixed byte arrays, that is, each entry is {@code }, where size is a 4 byte int. + *
  • An array of {@link SortKey} instances, which contain the paths of each entry and are used to sort the entries. Each element + * in this array also contains the position in the ByteBuffer of the serialized representation of the entry. + *
+ * This representation has several advantages: + *
    + *
  • It is compact, as a String object in the heap requires more memory than a length-prefixed byte array in the ByteBuffer. + *
  • Predictable memory usage - the memory used by the {@link java.nio.ByteBuffer} is fixed and allocated at startup + * (more on this later). The memory used by the array of {@link SortKey} is not bounded, but these objects are small, + * as they contain little more than the path of the entry, and we can easily put limits on the maximum number of entries + * kept in a buffer. + *
+ *

+ * The instances of {@link NodeStateEntryBatch} are created at launch time. We create {@code #transformThreads+1} buffers. + * This way, except for some rare situations, each transform thread will have its own buffer where to write the entries + * and there will be an extra buffer to be used by the Save-and-Sort thread, so that all the transform and sort threads + * can operate concurrently. + *

+ * These buffers are reused. Once the Save-and-Sort thread finishes processing a buffer, it clears it and sends it back + * to the transform threads. For this, we use two queues, one with empty buffers, from where the transform threads take + * their buffers when they need one, and another with full buffers, which are read by the Save-and-Sort thread. + *

+ * Reusing the buffers reduces significantly the pressure on the garbage collector and ensures that we do not run out + * of memory, as the largest blocks of memory are pre-allocated and reused. + *

+ * The total amount of memory used by the buffers is a configurable parameter (env variable {@link #OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB}). + * This memory is divided in {@code numberOfBuffers + 1 } regions, each of + * {@code regionSize = PIPELINED_WORKING_MEMORY_MB/(#numberOfBuffers + 1)} size. + * Each ByteBuffer is of {@code regionSize} big. The extra region is to account for the memory taken by the {@link SortKey} + * entries. There is also a maximum limit on the number of entries, which is calculated based on regionSize + * (we assume each {@link SortKey} entry requires 256 bytes). + *

+ * The transform threads will stop filling a buffer and enqueue it for sorting and saving once either the byte buffer is + * full or the number of entries in the buffer reaches the limit. + *

+ * + *

Retrials on broken MongoDB connections

+ */ +public class PipelinedTreeStoreStrategy extends IndexStoreSortStrategyBase { + public static final String OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB = "oak.indexer.pipelined.mongoDocBatchMaxSizeMB"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB = 4; + public static final String OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS = "oak.indexer.pipelined.mongoDocBatchMaxNumberOfDocuments"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS = 10000; + public static final String OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB = "oak.indexer.pipelined.mongoDocQueueReservedMemoryMB"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB = 128; + public static final String OAK_INDEXER_PIPELINED_TRANSFORM_THREADS = "oak.indexer.pipelined.transformThreads"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_TRANSFORM_THREADS = 2; + public static final String OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB = "oak.indexer.pipelined.workingMemoryMB"; + // 0 means autodetect + public static final int DEFAULT_OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB = 0; + // Between 1 and 100 + public static final String OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE = "oak.indexer.pipelined.sortBufferMemoryPercentage"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE = 25; + + private static final Logger LOG = LoggerFactory.getLogger(PipelinedTreeStoreStrategy.class); + // A MongoDB document is at most 16MB, so the buffer that holds node state entries must be at least that big + private static final int MIN_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB = 16; + private static final int MIN_AUTODETECT_WORKING_MEMORY_MB = 128; + private static final int MIN_ENTRY_BATCH_BUFFER_SIZE_MB = 32; + private static final int MAX_AUTODETECT_WORKING_MEMORY_MB = 4000; + + private static void printStatistics(ArrayBlockingQueue mongoDocQueue, + ArrayBlockingQueue emptyBuffersQueue, + ArrayBlockingQueue nonEmptyBuffersQueue, + TransformStageStatistics transformStageStatistics, + boolean printHistogramsAtInfo) { + + String queueSizeStats = MetricsFormatter.newBuilder() + .add("mongoDocQueue", mongoDocQueue.size()) + .add("emptyBuffersQueue", emptyBuffersQueue.size()) + .add("nonEmptyBuffersQueue", nonEmptyBuffersQueue.size()) + .build(); + + LOG.info("Queue sizes: {}", queueSizeStats); + LOG.info("Transform stats: {}", transformStageStatistics.formatStats()); + prettyPrintTransformStatisticsHistograms(transformStageStatistics, printHistogramsAtInfo); + } + + private static void prettyPrintTransformStatisticsHistograms(TransformStageStatistics transformStageStatistics, boolean printHistogramAtInfo) { + if (printHistogramAtInfo) { + LOG.info("Top hidden paths rejected: {}", transformStageStatistics.getHiddenPathsRejectedHistogram().prettyPrint()); + LOG.info("Top paths filtered: {}", transformStageStatistics.getFilteredPathsRejectedHistogram().prettyPrint()); + LOG.info("Top empty node state documents: {}", transformStageStatistics.getEmptyNodeStateHistogram().prettyPrint()); + } else { + LOG.debug("Top hidden paths rejected: {}", transformStageStatistics.getHiddenPathsRejectedHistogram().prettyPrint()); + LOG.debug("Top paths filtered: {}", transformStageStatistics.getFilteredPathsRejectedHistogram().prettyPrint()); + LOG.debug("Top empty node state documents: {}", transformStageStatistics.getEmptyNodeStateHistogram().prettyPrint()); + } + } + + private final MongoDocumentStore docStore; + private final MongoClientURI mongoClientURI; + private final DocumentNodeStore documentNodeStore; + private final RevisionVector rootRevision; + private final BlobStore blobStore; + private final List pathFilters; + private final StatisticsProvider statisticsProvider; + private final IndexingReporter indexingReporter; + private final int numberOfTransformThreads; + private final int mongoDocQueueSize; + private final int mongoDocBatchMaxSizeMB; + private final int mongoDocBatchMaxNumberOfDocuments; + private final int nseBuffersCount; + private final int nseBuffersSizeBytes; + + private long nodeStateEntriesExtracted; + + /** + * @param mongoClientURI URI of the Mongo cluster. + * @param pathPredicate Used by the transform stage to test if a node should be kept or discarded. + * @param pathFilters If non-empty, the download stage will use these filters to create a query that downloads + * only the matching MongoDB documents. + * @param statisticsProvider Used to collect statistics about the indexing process. + * @param indexingReporter Used to collect diagnostics, metrics and statistics and report them at the end of the indexing process. + */ + public PipelinedTreeStoreStrategy(MongoClientURI mongoClientURI, + MongoDocumentStore documentStore, + DocumentNodeStore documentNodeStore, + RevisionVector rootRevision, + Set preferredPathElements, + BlobStore blobStore, + File storeDir, + Compression algorithm, + Predicate pathPredicate, + List pathFilters, + String checkpoint, + StatisticsProvider statisticsProvider, + IndexingReporter indexingReporter) { + super(storeDir, algorithm, pathPredicate, preferredPathElements, checkpoint); + this.mongoClientURI = mongoClientURI; + this.docStore = documentStore; + this.documentNodeStore = documentNodeStore; + this.rootRevision = rootRevision; + this.blobStore = blobStore; + this.pathFilters = pathFilters; + this.statisticsProvider = statisticsProvider; + this.indexingReporter = indexingReporter; + Preconditions.checkState(documentStore.isReadOnly(), "Traverser can only be used with readOnly store"); + + int mongoDocQueueReservedMemoryMB = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB); + Preconditions.checkArgument(mongoDocQueueReservedMemoryMB >= MIN_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, + "Invalid value for property " + OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB + ": " + mongoDocQueueReservedMemoryMB + ". Must be >= " + MIN_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, String.valueOf(mongoDocQueueReservedMemoryMB)); + + this.mongoDocBatchMaxSizeMB = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB, DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB); + Preconditions.checkArgument(mongoDocBatchMaxSizeMB > 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB + ": " + mongoDocBatchMaxSizeMB + ". Must be > 0"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB, String.valueOf(mongoDocBatchMaxSizeMB)); + + this.mongoDocBatchMaxNumberOfDocuments = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS, DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS); + Preconditions.checkArgument(mongoDocBatchMaxNumberOfDocuments > 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS + ": " + mongoDocBatchMaxNumberOfDocuments + ". Must be > 0"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS, String.valueOf(mongoDocBatchMaxNumberOfDocuments)); + + this.numberOfTransformThreads = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, DEFAULT_OAK_INDEXER_PIPELINED_TRANSFORM_THREADS); + Preconditions.checkArgument(numberOfTransformThreads > 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_TRANSFORM_THREADS + ": " + numberOfTransformThreads + ". Must be > 0"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, String.valueOf(numberOfTransformThreads)); + + int sortBufferMemoryPercentage = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE, DEFAULT_OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE); + Preconditions.checkArgument(sortBufferMemoryPercentage > 0 && sortBufferMemoryPercentage <= 100, + "Invalid value for property " + OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE + ": " + numberOfTransformThreads + ". Must be between 1 and 100"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE, String.valueOf(sortBufferMemoryPercentage)); + + // mongo-dump <-> transform threads + Preconditions.checkArgument(mongoDocQueueReservedMemoryMB >= 8 * mongoDocBatchMaxSizeMB, + "Invalid values for properties " + OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB + " and " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB + + ": " + OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB + " must be at least 8x " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB + + ", but are " + mongoDocQueueReservedMemoryMB + " and " + mongoDocBatchMaxSizeMB + ", respectively" + ); + this.mongoDocQueueSize = mongoDocQueueReservedMemoryMB / mongoDocBatchMaxSizeMB; + + // Derived values for transform <-> sort-save + int nseWorkingMemoryMB = readNSEBuffersReservedMemory(); + this.nseBuffersCount = 1 + numberOfTransformThreads; + long nseWorkingMemoryBytes = (long) nseWorkingMemoryMB * FileUtils.ONE_MB; + // The working memory is divided in the following regions: + // - #transforThreads NSE Binary buffers + // - x1 Memory reserved for the array created by the sort-batch thread with the keys of the entries + // in the batch that is being sorted + long memoryReservedForSortKeysArray = estimateMaxSizeOfSortKeyArray(nseWorkingMemoryBytes, nseBuffersCount, sortBufferMemoryPercentage); + long memoryReservedForBuffers = nseWorkingMemoryBytes - memoryReservedForSortKeysArray; + + // A ByteBuffer can be at most Integer.MAX_VALUE bytes long + this.nseBuffersSizeBytes = limitToIntegerRange(memoryReservedForBuffers / nseBuffersCount); + + if (nseBuffersSizeBytes < MIN_ENTRY_BATCH_BUFFER_SIZE_MB * FileUtils.ONE_MB) { + throw new IllegalArgumentException("Entry batch buffer size too small: " + nseBuffersSizeBytes + + " bytes. Must be at least " + MIN_ENTRY_BATCH_BUFFER_SIZE_MB + " MB. " + + "To increase the size of the buffers, either increase the size of the working memory region " + + "(system property " + OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB + ") or decrease the number of transform " + + "threads (" + OAK_INDEXER_PIPELINED_TRANSFORM_THREADS + ")"); + } + + LOG.info("MongoDocumentQueue: [ reservedMemory: {} MB, batchMaxSize: {} MB, queueSize: {} (reservedMemory/batchMaxSize) ]", + mongoDocQueueReservedMemoryMB, + mongoDocBatchMaxSizeMB, + mongoDocQueueSize); + LOG.info("NodeStateEntryBuffers: [ workingMemory: {} MB, numberOfBuffers: {}, bufferSize: {}, sortBufferReservedMemory: {} ]", + nseWorkingMemoryMB, + nseBuffersCount, + IOUtils.humanReadableByteCountBin(nseBuffersSizeBytes), + IOUtils.humanReadableByteCountBin(memoryReservedForSortKeysArray) + ); + } + + static long estimateMaxSizeOfSortKeyArray(long nseWorkingMemoryBytes, long nseBuffersCount, int sortBufferMemoryPercentage) { + // We reserve a percentage of the size of a buffer for the sort keys array. That is, we are assuming that for every line + // in the sort buffer, the memory needed to store the SortKey of the path section of the line will not be more + // than sortBufferMemoryPercentage of the total size of the line in average + // Estimate memory needed by the sort keys array. We assume each entry requires 256 bytes. + long approxNseBufferSize = limitToIntegerRange(nseWorkingMemoryBytes / nseBuffersCount); + return approxNseBufferSize * sortBufferMemoryPercentage / 100; + } + + private int readNSEBuffersReservedMemory() { + int workingMemoryMB = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, DEFAULT_OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB); + Preconditions.checkArgument(workingMemoryMB >= 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB + ": " + workingMemoryMB + ". Must be >= 0"); + indexingReporter.addConfig(OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, workingMemoryMB); + if (workingMemoryMB == 0) { + return autodetectWorkingMemoryMB(); + } else { + return workingMemoryMB; + } + } + + private int autodetectWorkingMemoryMB() { + int maxHeapSizeMB = (int) (Runtime.getRuntime().maxMemory() / FileUtils.ONE_MB); + int workingMemoryMB = maxHeapSizeMB - 2048; + LOG.info("Auto detecting working memory. Maximum heap size: {} MB, selected working memory: {} MB", maxHeapSizeMB, workingMemoryMB); + if (workingMemoryMB > MAX_AUTODETECT_WORKING_MEMORY_MB) { + LOG.warn("Auto-detected value for working memory too high, setting to the maximum allowed for auto-detection: {} MB", MAX_AUTODETECT_WORKING_MEMORY_MB); + return MAX_AUTODETECT_WORKING_MEMORY_MB; + } + if (workingMemoryMB < MIN_AUTODETECT_WORKING_MEMORY_MB) { + LOG.warn("Auto-detected value for working memory too low, setting to the minimum allowed for auto-detection: {} MB", MIN_AUTODETECT_WORKING_MEMORY_MB); + return MIN_AUTODETECT_WORKING_MEMORY_MB; + } + return workingMemoryMB; + } + + private static int limitToIntegerRange(long bufferSizeBytes) { + if (bufferSizeBytes > Integer.MAX_VALUE) { + // Probably not necessary to subtract 16, just a safeguard to avoid boundary conditions. + int truncatedBufferSize = Integer.MAX_VALUE - 16; + LOG.warn("Computed buffer size too big: {}, exceeds Integer.MAX_VALUE. Truncating to: {}", bufferSizeBytes, truncatedBufferSize); + return truncatedBufferSize; + } else { + return (int) bufferSizeBytes; + } + } + + @Override + public File createSortedStoreFile() throws IOException { + int numberOfThreads = 1 + numberOfTransformThreads + 1; // dump, transform, sort threads + ExecutorService threadPool = Executors.newFixedThreadPool(numberOfThreads, + new ThreadFactoryBuilder().setDaemon(true).build() + ); + // This executor can wait for several tasks at the same time. We use this below to wait at the same time for + // all the tasks, so that if one of them fails, we can abort the whole pipeline. Otherwise, if we wait on + // Future instances, we can only wait on one of them, so that if any of the others fail, we have no easy way + // to detect this failure. + ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); + TreeStore treeStore = new TreeStore(this.getStoreDir(), null); + treeStore.getSession().init(); + try { + // download -> transform thread. + ArrayBlockingQueue mongoDocQueue = new ArrayBlockingQueue<>(mongoDocQueueSize); + + // transform <-> sort and save threads + // Queue with empty buffers, used by the transform task + ArrayBlockingQueue emptyBatchesQueue = new ArrayBlockingQueue<>(nseBuffersCount); + // Queue with buffers filled by the transform task, used by the sort and save task. +1 for the SENTINEL + ArrayBlockingQueue nonEmptyBatchesQueue = new ArrayBlockingQueue<>(nseBuffersCount + 1); + + TransformStageStatistics transformStageStatistics = new TransformStageStatistics(); + + // Create empty buffers + for (int i = 0; i < nseBuffersCount; i++) { + // No limits on the number of entries, only on their total size. This might be revised later. + emptyBatchesQueue.add(NodeStateEntryBatch.createNodeStateEntryBatch(nseBuffersSizeBytes, Integer.MAX_VALUE)); + } + + INDEXING_PHASE_LOGGER.info("[TASK:PIPELINED-DUMP:START] Starting to build TreeStore"); + Stopwatch start = Stopwatch.createStarted(); + + Future downloadFuture = ecs.submit(new PipelinedMongoDownloadTask( + mongoClientURI, + docStore, + (int) (mongoDocBatchMaxSizeMB * FileUtils.ONE_MB), + mongoDocBatchMaxNumberOfDocuments, + mongoDocQueue, + pathFilters, + statisticsProvider, + indexingReporter + )); + + ArrayList> transformFutures = new ArrayList<>(numberOfTransformThreads); + for (int i = 0; i < numberOfTransformThreads; i++) { + NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(blobStore); + transformFutures.add(ecs.submit(new PipelinedTransformTask( + docStore, + documentNodeStore, + rootRevision, + this.getPathPredicate(), + entryWriter, + mongoDocQueue, + emptyBatchesQueue, + nonEmptyBatchesQueue, + transformStageStatistics + ))); + } + + Future sortBatchFuture = ecs.submit(new PipelinedTreeStoreTask( + treeStore, + emptyBatchesQueue, + nonEmptyBatchesQueue, + statisticsProvider, + indexingReporter + )); + + try { + LOG.info("Waiting for tasks to complete"); + int tasksFinished = 0; + int transformTasksFinished = 0; + boolean monitorQueues = true; + while (tasksFinished < numberOfThreads) { + // Wait with a timeout to print statistics periodically + Future completedTask = ecs.poll(30, TimeUnit.SECONDS); + if (completedTask == null) { + // Timeout waiting for a task to complete + if (monitorQueues) { + try { + printStatistics(mongoDocQueue, emptyBatchesQueue, nonEmptyBatchesQueue, transformStageStatistics, false); + } catch (Exception e) { + LOG.warn("Error while logging queue sizes", e); + } + } + } else { + try { + Object result = completedTask.get(); + if (result instanceof PipelinedMongoDownloadTask.Result) { + PipelinedMongoDownloadTask.Result downloadResult = (PipelinedMongoDownloadTask.Result) result; + LOG.info("Download finished. Documents downloaded: {}", downloadResult.getDocumentsDownloaded()); + downloadFuture = null; + + } else if (result instanceof PipelinedTransformTask.Result) { + PipelinedTransformTask.Result transformResult = (PipelinedTransformTask.Result) result; + transformTasksFinished++; + nodeStateEntriesExtracted += transformResult.getEntryCount(); + LOG.info("Transform task {} finished. Entries processed: {}", + transformResult.getThreadId(), transformResult.getEntryCount()); + if (transformTasksFinished == numberOfTransformThreads) { + LOG.info("All transform tasks finished. Total entries processed: {}", nodeStateEntriesExtracted); + // No need to keep monitoring the queues, the download and transform threads are done. + monitorQueues = false; + // Terminate the sort thread. + nonEmptyBatchesQueue.put(PipelinedStrategy.SENTINEL_NSE_BUFFER); + transformStageStatistics.publishStatistics(statisticsProvider, indexingReporter); + transformFutures.clear(); + } + + } else if (result instanceof PipelinedSortBatchTask.Result) { + PipelinedSortBatchTask.Result sortTaskResult = (PipelinedSortBatchTask.Result) result; + LOG.info("Sort batch task finished. Entries processed: {}", sortTaskResult.getTotalEntries()); + // The buffers between transform and merge sort tasks are no longer needed, so remove them + // from the queues so they can be garbage collected. + // These buffers can be very large, so this is important to avoid running out of memory in + // the merge-sort phase + if (!nonEmptyBatchesQueue.isEmpty()) { + LOG.warn("emptyBatchesQueue is not empty. Size: {}", emptyBatchesQueue.size()); + } + emptyBatchesQueue.clear(); + printStatistics(mongoDocQueue, emptyBatchesQueue, nonEmptyBatchesQueue, transformStageStatistics, true); + sortBatchFuture = null; + + } else { + throw new RuntimeException("Unknown result type: " + result); + } + tasksFinished++; + } catch (ExecutionException ex) { + throw new RuntimeException(ex.getCause()); + } catch (Throwable ex) { + throw new RuntimeException(ex); + } + } + } + long elapsedSeconds = start.elapsed(TimeUnit.SECONDS); + INDEXING_PHASE_LOGGER.info("[TASK:PIPELINED-DUMP:END] Metrics: {}", MetricsFormatter.newBuilder() + .add("duration", FormattingUtils.formatToSeconds(elapsedSeconds)) + .add("durationSeconds", elapsedSeconds) + .add("nodeStateEntriesExtracted", nodeStateEntriesExtracted) + .build()); + indexingReporter.addTiming("Build TreeStore (Dump+Merge)", FormattingUtils.formatToSeconds(elapsedSeconds)); + + LOG.info("[INDEXING_REPORT:BUILD_TREE_STORE]\n{}", indexingReporter.generateReport()); + } catch (Throwable e) { + INDEXING_PHASE_LOGGER.info("[TASK:PIPELINED-DUMP:FAIL] Metrics: {}, Error: {}", + MetricsFormatter.createMetricsWithDurationOnly(start), e.toString() + ); + LOG.warn("Error dumping from MongoDB. Cancelling all tasks. Error: {}", e.toString()); + // Cancel in order + cancelFuture(downloadFuture); + for (Future transformTask : transformFutures) { + cancelFuture(transformTask); + } + cancelFuture(sortBatchFuture); + throw new RuntimeException(e); + } + treeStore.close(); + return getStoreDir(); + } finally { + LOG.info("Shutting down build FFS thread pool"); + new ExecutorCloser(threadPool).close(); + } + } + + private void cancelFuture(Future future) { + if (future != null) { + LOG.info("Cancelling future: {}", future); + future.cancel(true); + } + } + + @Override + public long getEntryCount() { + return nodeStateEntriesExtracted; + } +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java new file mode 100644 index 00000000000..bfb75203c6e --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCountBin; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.SENTINEL_NSE_BUFFER; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedUtils.formatAsPercentage; +import static org.apache.jackrabbit.oak.plugins.index.IndexUtils.INDEXING_PHASE_LOGGER; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Locale; +import java.util.Map.Entry; +import java.util.TreeMap; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import org.apache.jackrabbit.guava.common.base.Stopwatch; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedSortBatchTask.Result; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.plugins.index.IndexingReporter; +import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter; +import org.apache.jackrabbit.oak.plugins.index.MetricsUtils; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Receives batches of node state entries, sorts then in memory, and finally writes them to a tree store. + */ +public class PipelinedTreeStoreTask implements Callable { + + private static final Logger LOG = LoggerFactory.getLogger(PipelinedTreeStoreTask.class); + private static final String THREAD_NAME = "stree-store-task"; + + private final TreeStore treeStore; + private final BlockingQueue emptyBuffersQueue; + private final BlockingQueue nonEmptyBuffersQueue; + private final StatisticsProvider statisticsProvider; + private final IndexingReporter reporter; + + private long entriesProcessed = 0; + private long batchesProcessed = 0; + private long timeCreatingSortArrayMillis = 0; + private long timeSortingMillis = 0; + private long timeWritingMillis = 0; + + public PipelinedTreeStoreTask(TreeStore treeStore, + ArrayBlockingQueue emptyBuffersQueue, + ArrayBlockingQueue nonEmptyBuffersQueue, + StatisticsProvider statisticsProvider, + IndexingReporter reporter) { + this.treeStore = treeStore; + this.emptyBuffersQueue = emptyBuffersQueue; + this.nonEmptyBuffersQueue = nonEmptyBuffersQueue; + this.statisticsProvider = statisticsProvider; + this.reporter = reporter; + + } + + @Override + public Result call() throws Exception { + Stopwatch taskStartTime = Stopwatch.createStarted(); + String originalName = Thread.currentThread().getName(); + Thread.currentThread().setName(THREAD_NAME); + INDEXING_PHASE_LOGGER.info("[TASK:{}:START] Starting tree store task", THREAD_NAME.toUpperCase(Locale.ROOT)); + try { + while (true) { + LOG.info("Waiting for next batch"); + NodeStateEntryBatch nseBuffer = nonEmptyBuffersQueue.take(); + if (nseBuffer == SENTINEL_NSE_BUFFER) { + synchronized (treeStore) { + LOG.info("Final merge"); + Stopwatch start = Stopwatch.createStarted(); + treeStore.getSession().mergeRoots(Integer.MAX_VALUE); + long durationSeconds = start.elapsed(TimeUnit.SECONDS); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, durationSeconds); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, 0); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, 0); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, 0); + MetricsUtils.addMetricByteSize(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, 0); + LOG.info("Final merge done"); + } + long totalTimeMillis = taskStartTime.elapsed().toMillis(); + String timeCreatingSortArrayPercentage = formatAsPercentage(timeCreatingSortArrayMillis, totalTimeMillis); + String timeSortingPercentage = formatAsPercentage(timeSortingMillis, totalTimeMillis); + String timeWritingPercentage = formatAsPercentage(timeWritingMillis, totalTimeMillis); + String metrics = MetricsFormatter.newBuilder() + .add("batchesProcessed", batchesProcessed) + .add("entriesProcessed", entriesProcessed) + .add("timeCreatingSortArrayMillis", timeCreatingSortArrayMillis) + .add("timeCreatingSortArrayPercentage", timeCreatingSortArrayPercentage) + .add("timeSortingMillis", timeSortingMillis) + .add("timeSortingPercentage", timeSortingPercentage) + .add("timeWritingMillis", timeWritingMillis) + .add("timeWritingPercentage", timeWritingPercentage) + .add("totalTimeSeconds", totalTimeMillis / 1000) + .build(); + INDEXING_PHASE_LOGGER.info("[TASK:{}:END] Metrics: {}", THREAD_NAME.toUpperCase(Locale.ROOT), metrics); + MetricsUtils.addMetric(statisticsProvider, reporter, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE, + PipelinedUtils.toPercentage(timeCreatingSortArrayMillis, totalTimeMillis)); + MetricsUtils.addMetric(statisticsProvider, reporter, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE, + PipelinedUtils.toPercentage(timeSortingMillis, totalTimeMillis)); + MetricsUtils.addMetric(statisticsProvider, reporter, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE, + PipelinedUtils.toPercentage(timeWritingMillis, totalTimeMillis)); + return new Result(entriesProcessed); + } + sortAndSaveBatch(nseBuffer); + nseBuffer.reset(); + emptyBuffersQueue.put(nseBuffer); + } + } catch (Throwable t) { + INDEXING_PHASE_LOGGER.info("[TASK:{}:FAIL] Metrics: {}, Error: {}", + THREAD_NAME.toUpperCase(Locale.ROOT), + MetricsFormatter.createMetricsWithDurationOnly(taskStartTime), + t.toString()); + LOG.warn("Thread terminating with exception", t); + throw t; + } finally { + Thread.currentThread().setName(originalName); + } + } + + private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { + LOG.info("Going to sort batch in memory. Entries: {}, Size: {}", + nseb.numberOfEntries(), humanReadableByteCountBin(nseb.sizeOfEntriesBytes())); + TreeMap sortBuffer = buildTreeMap(nseb); + if (sortBuffer.isEmpty()) { + return; + } + Stopwatch sortClock = Stopwatch.createStarted(); + timeSortingMillis += sortClock.elapsed().toMillis(); + LOG.info("Sorted batch in {}. Saving to disk", sortClock); + Stopwatch saveClock = Stopwatch.createStarted(); + long textSize = 0; + batchesProcessed++; + synchronized (treeStore) { + Session session = treeStore.getSession(); + for (Entry e : sortBuffer.entrySet()) { + session.put(e.getKey(), e.getValue()); + textSize += e.getKey().length() + e.getValue().length() + 2; + entriesProcessed++; + } + session.checkpoint(); + } + timeWritingMillis += saveClock.elapsed().toMillis(); + LOG.info("Wrote batch of size {} (uncompressed) with {} entries in {} at {}", + humanReadableByteCountBin(textSize), + sortBuffer.size(), saveClock, + PipelinedUtils.formatAsTransferSpeedMBs(textSize, saveClock.elapsed().toMillis()) + ); + } + + private TreeMap buildTreeMap(NodeStateEntryBatch nseb) { + Stopwatch startTime = Stopwatch.createStarted(); + ByteBuffer buffer = nseb.getBuffer(); + int totalPathSize = 0; + TreeMap map = new TreeMap<>(); + while (buffer.hasRemaining()) { + int pathLength = buffer.getInt(); + totalPathSize += pathLength; + String path = new String(buffer.array(), buffer.position() + buffer.arrayOffset(), pathLength, StandardCharsets.UTF_8); + buffer.position(buffer.position() + pathLength); + int entryLength = buffer.getInt(); + String data = new String(buffer.array(), buffer.position() + buffer.arrayOffset(), entryLength, StandardCharsets.UTF_8); + addEntry(path, data, map); + buffer.position(buffer.position() + entryLength); + } + timeCreatingSortArrayMillis += startTime.elapsed().toMillis(); + LOG.info("Built tree map in {}. Entries: {}, Total size of path strings: {}", startTime, map.size(), + humanReadableByteCountBin(totalPathSize)); + return map; + } + + private static void addEntry(String path, String data, TreeMap target) { + target.put(path, data); + if (!path.equals("/")) { + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + target.put(parentPath + "\t" + nodeName, ""); + } + } +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java index 260519dfe76..707949212d9 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java @@ -166,11 +166,11 @@ private static String readString(ByteBuffer buff) { len = buff.getInt(); int pos = buff.position(); buff.position(buff.position() + len); - return new String(buff.array(), pos, len, StandardCharsets.UTF_8); + return new String(buff.array(), pos + buff.arrayOffset(), len, StandardCharsets.UTF_8); } else { int pos = buff.position(); buff.position(buff.position() + len); - return new String(buff.array(), pos, len, StandardCharsets.UTF_8); + return new String(buff.array(), pos + buff.arrayOffset(), len, StandardCharsets.UTF_8); } } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java new file mode 100644 index 00000000000..0074ebd5492 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java @@ -0,0 +1,680 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +import static java.lang.management.ManagementFactory.getPlatformMBeanServer; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelineITUtil.assertMetrics; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelineITUtil.contentDamPathFilter; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_PARALLEL_DUMP; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider; +import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; +import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory; +import org.apache.jackrabbit.oak.plugins.document.MongoUtils; +import org.apache.jackrabbit.oak.plugins.document.RevisionVector; +import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection; +import org.apache.jackrabbit.oak.plugins.document.util.Utils; +import org.apache.jackrabbit.oak.plugins.index.ConsoleIndexingReporter; +import org.apache.jackrabbit.oak.plugins.metric.MetricStatisticsProvider; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.RestoreSystemProperties; +import org.junit.rules.TemporaryFolder; + +public class PipelinedTreeStoreIT { + private static ScheduledExecutorService executorService; + @Rule + public final MongoConnectionFactory connectionFactory = new MongoConnectionFactory(); + @Rule + public final DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); + @Rule + public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties(); + @Rule + public final TemporaryFolder sortFolder = new TemporaryFolder(); + + + private MetricStatisticsProvider statsProvider; + private ConsoleIndexingReporter indexingReporter; + + @BeforeClass + public static void setup() throws IOException { + Assume.assumeTrue(MongoUtils.isAvailable()); + executorService = Executors.newSingleThreadScheduledExecutor(); + } + + @AfterClass + public static void teardown() { + if (executorService != null) { + executorService.shutdown(); + } + } + + @Before + public void before() { + MongoConnection c = connectionFactory.getConnection(); + if (c != null) { + c.getDatabase().drop(); + } + statsProvider = new MetricStatisticsProvider(getPlatformMBeanServer(), executorService); + indexingReporter = new ConsoleIndexingReporter(); + } + + @After + public void tear() { + MongoConnection c = connectionFactory.getConnection(); + if (c != null) { + c.getDatabase().drop(); + } + statsProvider.close(); + statsProvider = null; + indexingReporter = null; + } + + @Test + public void createFFS_mongoFiltering_include_excludes() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/content/dam/2023"), List.of("/content/dam/2023/02"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/2023|{\"p2\":\"v2023\"}", + "/content/dam/2023/01|{\"p1\":\"v202301\"}", + "/content/dam/2023/02|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_include_excludes2() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + // NOTE: If a path /a/b is in the excluded paths, the descendants of /a/b will not be downloaded but /a/b will + // be downloaded. This is an intentional limitation of the logic to compute the Mongo filter which was done + // to avoid the extra complexity of also filtering the root of the excluded tree. The transform stage would anyway + // filter out these additional documents. + List pathFilters = List.of(new PathFilter(List.of("/content/dam/1000", "/content/dam/2022"), List.of("/content/dam/2022/02", "/content/dam/2022/04"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/1000|{}", + "/content/dam/1000/12|{\"p1\":\"v100012\"}", + "/content/dam/2022|{}", + "/content/dam/2022/01|{\"p1\":\"v202201\"}", + "/content/dam/2022/01/01|{\"p1\":\"v20220101\"}", + "/content/dam/2022/02|{\"p1\":\"v202202\"}", + "/content/dam/2022/03|{\"p1\":\"v202203\"}", + "/content/dam/2022/04|{\"p1\":\"v202204\"}" + ), true); + } + + + @Test + public void createFFS_mongoFiltering_include_excludes3() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/etc", "/home", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_include_excludes_retryOnConnectionErrors() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/etc", "/home", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_include_excludes4() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + List pathFilters = List.of( + new PathFilter(List.of("/content/dam/1000"), List.of()), + new PathFilter(List.of("/content/dam/2022"), List.of("/content/dam/2022/01")) + ); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/1000|{}", + "/content/dam/1000/12|{\"p1\":\"v100012\"}", + "/content/dam/2022|{}", + "/content/dam/2022/01|{\"p1\":\"v202201\"}", + "/content/dam/2022/02|{\"p1\":\"v202202\"}", + "/content/dam/2022/02/01|{\"p1\":\"v20220201\"}", + "/content/dam/2022/02/02|{\"p1\":\"v20220202\"}", + "/content/dam/2022/02/03|{\"p1\":\"v20220203\"}", + "/content/dam/2022/02/04|{\"p1\":\"v20220204\"}", + "/content/dam/2022/03|{\"p1\":\"v202203\"}", + "/content/dam/2022/04|{\"p1\":\"v202204\"}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_multipleIndexes() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + PathFilter pathFilter = new PathFilter(List.of("/content/dam/1000", "/content/dam/2023", "/content/dam/2023/01"), List.of()); + List pathFilters = List.of(pathFilter); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/1000|{}", + "/content/dam/1000/12|{\"p1\":\"v100012\"}", + "/content/dam/2023|{\"p2\":\"v2023\"}", + "/content/dam/2023/01|{\"p1\":\"v202301\"}", + "/content/dam/2023/02|{}", + "/content/dam/2023/02/28|{\"p1\":\"v20230228\"}" + ), true); + } + + @Test + public void createFFS_filter_long_paths() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + // Create a filter on the node with the longest path + String longestLine = PipelineITUtil.EXPECTED_FFS.stream().max(Comparator.comparingInt(String::length)).get(); + String longestPath = longestLine.substring(0, longestLine.lastIndexOf("|")); + String parent = PathUtils.getParentPath(longestPath); + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of(parent), List.of())); + + // The results should contain all the parents of the node with the longest path + ArrayList expected = new ArrayList<>(); + expected.add(longestPath + "|{}"); + while (true) { + expected.add(parent + "|{}"); + if (parent.equals("/")) { + break; + } + parent = PathUtils.getParentPath(parent); + } + // The list above has the longest paths first, reverse it to match the order in the FFS + Collections.reverse(expected); + + testSuccessfulDownload(pathPredicate, pathFilters, expected, false); + } + + + @Test + public void createFFSCustomExcludePathsRegexRetryOnConnectionErrors() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "true", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "false" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + null, + excludedPathsRegexTestExpected); + } + + @Test + public void createFFSCustomExcludePathsRegexNoRetryOnConnectionError() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "false" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + null, + excludedPathsRegexTestExpected); + } + + @Test + public void createFFSCustomExcludePathsRegexRetryOnConnectionErrorsRegexFiltering() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "true", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + List.of(contentDamPathFilter), + excludedPathsRegexTestExpected); + } + + @Test + public void createFFSCustomExcludePathsRegexNoRetryOnConnectionErrorRegexFiltering() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + List.of(contentDamPathFilter), + excludedPathsRegexTestExpected); + } + + private void buildNodeStoreForExcludedRegexTest(DocumentNodeStore rwNodeStore) { + @NotNull NodeBuilder rootBuilder = rwNodeStore.getRoot().builder(); + @NotNull NodeBuilder contentDamBuilder = rootBuilder.child("content").child("dam"); + contentDamBuilder.child("a.jpg").child("jcr:content").child("metadata.xml"); + contentDamBuilder.child("a.jpg").child("jcr:content").child("metadata.text"); + contentDamBuilder.child("image_a.png").child("jcr:content").child("metadata.text"); + contentDamBuilder.child("image_a.png").child("jcr:content").child("metadata.xml"); + try { + rwNodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } catch (CommitFailedException e) { + throw new RuntimeException(e); + } + } + + private final List excludedPathsRegexTestExpected = List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/a.jpg|{}", + "/content/dam/image_a.png|{}", + "/content/dam/image_a.png/jcr:content|{}", + "/content/dam/image_a.png/jcr:content/metadata.text|{}" + ); + + private void testPipelinedStrategy(Map settings, + Consumer contentBuilder, + Predicate pathPredicate, + List pathFilters, + List expected) throws IOException { + settings.forEach(System::setProperty); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + DocumentNodeStore rwNodeStore = rwStore.documentNodeStore; + contentBuilder.accept(rwNodeStore); + MongoTestBackend roStore = createNodeStore(true); + + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, pathFilters); + File file = pipelinedStrategy.createSortedStoreFile(); + + assertTrue(file.exists()); + assertEquals(expected, readAllEntries(file)); + assertMetrics(statsProvider); + } + } + + private void testSuccessfulDownload(Predicate pathPredicate, List pathFilters) + throws CommitFailedException, IOException { + testSuccessfulDownload(pathPredicate, pathFilters, PipelineITUtil.EXPECTED_FFS, false); + } + + private void testSuccessfulDownload(Predicate pathPredicate, List mongoRegexPathFilter, List expected, boolean ignoreLongPaths) + throws CommitFailedException, IOException { + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, mongoRegexPathFilter); + File file = pipelinedStrategy.createSortedStoreFile(); + assertTrue(file.exists()); + List result = readAllEntries(file); + if (ignoreLongPaths) { + // Remove the long paths from the result. The filter on Mongo is best-effort, it will download long path + // documents, even if they do not match the includedPaths. + result = result.stream() + .filter(s -> { + String name = s.split("\\|")[0]; + return name.length() < Utils.PATH_LONG; + }) + .collect(Collectors.toList()); + + } + assertEquals(expected, result); + assertMetrics(statsProvider); + } + } + + @Test + public void createFFS_pathPredicateDoesNotMatch() throws Exception { + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + Predicate pathPredicate = s -> s.startsWith("/content/dam/does-not-exist"); + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, null); + + File file = pipelinedStrategy.createSortedStoreFile(); + + assertTrue(file.exists()); + assertEquals("[]", readAllEntries(file).toString()); + } + } + + @Test + public void createFFS_badNumberOfTransformThreads() throws CommitFailedException, IOException { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, "0"); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + assertThrows("Invalid value for property " + PipelinedStrategy.OAK_INDEXER_PIPELINED_TRANSFORM_THREADS + ": 0. Must be > 0", + IllegalArgumentException.class, + () -> createStrategy(roStore) + ); + } + } + + @Test + public void createFFS_badWorkingMemorySetting() throws CommitFailedException, IOException { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, "-1"); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + assertThrows("Invalid value for property " + PipelinedStrategy.OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB + ": -1. Must be >= 0", + IllegalArgumentException.class, + () -> createStrategy(roStore) + ); + } + } + + @Test + public void createFFS_smallNumberOfDocsPerBatch() throws Exception { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS, "2"); + + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + List pathFilters = null; + + testSuccessfulDownload(pathPredicate, pathFilters); + } + + @Test + public void createFFS_largeMongoDocuments() throws Exception { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB, "1"); + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, "32"); + + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + List pathFilters = null; + + MongoTestBackend rwStore = createNodeStore(false); + @NotNull NodeBuilder rootBuilder = rwStore.documentNodeStore.getRoot().builder(); + // This property does not fit in the reserved memory, but must still be processed without errors + String longString = RandomStringUtils.random((int) (10 * FileUtils.ONE_MB), true, true); + @NotNull NodeBuilder contentDamBuilder = rootBuilder.child("content").child("dam"); + contentDamBuilder.child("2021").child("01").setProperty("p1", "v202101"); + contentDamBuilder.child("2022").child("01").setProperty("p1", longString); + contentDamBuilder.child("2023").child("01").setProperty("p1", "v202301"); + rwStore.documentNodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + + List expected = List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/2021|{}", + "/content/dam/2021/01|{\"p1\":\"v202101\"}", + "/content/dam/2022|{}", + "/content/dam/2022/01|{\"p1\":\"" + longString + "\"}", + "/content/dam/2023|{}", + "/content/dam/2023/01|{\"p1\":\"v202301\"}" + ); + + MongoTestBackend roStore = createNodeStore(true); + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, pathFilters); + + File file = pipelinedStrategy.createSortedStoreFile(); + assertTrue(file.exists()); + assertArrayEquals(expected.toArray(new String[0]), readAllEntriesArray(file)); + assertMetrics(statsProvider); + } + + static String[] readAllEntriesArray(File dir) throws IOException { + return readAllEntries(dir).toArray(new String[0]); + } + + static List readAllEntries(File dir) throws IOException { + TreeStore treeStore = new TreeStore(dir, null); + ArrayList list = new ArrayList<>(); + Session session = treeStore.getSession(); + for (String k : session.keys()) { + String v = session.get(k); + if (!v.isEmpty()) { + list.add(k + "|" + v); + } + } + treeStore.close(); + return list; + } + + @Test + public void createFFS_mongoFiltering_custom_excluded_paths_1() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/etc", "/home", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_custom_excluded_paths_2() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_custom_excluded_paths_3() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home,/content/dam,/jcr:system"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of())); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFSNoMatches() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_PARALLEL_DUMP, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home,/content/dam,/jcr:system"); + + Predicate pathPredicate = t -> true; + List mongoRegexPathFilters = List.of(new PathFilter(List.of("/doesnotexist"), List.of())); + + // For an included path of /foo, the / should not be included. But the mongo regex filter is only best effort, + // and it will download the parents of all the included paths, even if they are empty. This is not a problem, + // because the filter at the transform stage will remove these paths. This test has no filter at the transform + // stage (pathPredicate is always true), so the / will be included in the result. + testSuccessfulDownload(pathPredicate, mongoRegexPathFilters, List.of("/|{}"), true); + } + + @Test(expected = IllegalArgumentException.class) + public void createFFS_mongoFiltering_custom_excluded_paths_cannot_exclude_root() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/"); + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of())); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + + @Ignore("This test is for manual execution only. It allocates two byte buffers of 2GB each, which might exceed the memory available in the CI") + public void createFFSWithPipelinedStrategy_veryLargeWorkingMemorySetting() throws Exception { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, "1"); + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, "8000"); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + Predicate pathPredicate = s -> s.startsWith("/content/dam"); + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, null); + pipelinedStrategy.createSortedStoreFile(); + } + } + + private MongoTestBackend createNodeStore(boolean b) { + return PipelineITUtil.createNodeStore(b, connectionFactory, builderProvider); + } + + private PipelinedTreeStoreStrategy createStrategy(MongoTestBackend roStore) { + return createStrategy(roStore, s -> true, null); + } + + private PipelinedTreeStoreStrategy createStrategy(MongoTestBackend backend, Predicate pathPredicate, List mongoRegexPathFilter) { + Set preferredPathElements = Set.of(); + RevisionVector rootRevision = backend.documentNodeStore.getRoot().getRootRevision(); + indexingReporter.setIndexNames(List.of("testIndex")); + return new PipelinedTreeStoreStrategy( + backend.mongoClientURI, + backend.mongoDocumentStore, + backend.documentNodeStore, + rootRevision, + preferredPathElements, + new MemoryBlobStore(), + sortFolder.getRoot(), + Compression.NONE, + pathPredicate, + mongoRegexPathFilter, + null, + statsProvider, + indexingReporter); + } +} From 5bf6a11378f4c4a3ec959fcf72003ed572d5961f Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Aug 2024 11:47:54 +0200 Subject: [PATCH 66/86] OAK-10341 Tree store (tests) --- .../pipelined/PipelinedTreeStoreStrategy.java | 2 +- .../pipelined/PipelinedTreeStoreTask.java | 3 + .../indexer/document/tree/store/Session.java | 21 ++- .../tree/store/utils/SortedStream.java | 10 +- .../pipelined/PipelinedTreeStoreIT.java | 1 - .../tree/store/CrudMultiRootTest.java | 163 ++++++++++++++++++ .../document/tree/store/MergeRootsTest.java | 134 ++++++++++++++ 7 files changed, 325 insertions(+), 9 deletions(-) create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 7a4c21fce70..7b002ae3254 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -339,7 +339,7 @@ public File createSortedStoreFile() throws IOException { // Future instances, we can only wait on one of them, so that if any of the others fail, we have no easy way // to detect this failure. ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); - TreeStore treeStore = new TreeStore(this.getStoreDir(), null); + TreeStore treeStore = new TreeStore(getStoreDir(), null); treeStore.getSession().init(); try { // download -> transform thread. diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index bfb75203c6e..2ea8db149bf 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -53,6 +53,8 @@ public class PipelinedTreeStoreTask implements Callable emptyBuffersQueue; private final BlockingQueue nonEmptyBuffersQueue; @@ -165,6 +167,7 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { entriesProcessed++; } session.checkpoint(); + session.mergeRoots(MERGE_BATCH); } timeWritingMillis += saveClock.elapsed().toMillis(); LOG.info("Wrote batch of size {} (uncompressed) with {} entries in {} at {}", diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index 510c2fb0c08..ca1577f56c1 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -420,6 +420,9 @@ private PageFile getFile(String key) { */ public void mergeRoots(int max) { List list = getRootFileNames(); + if (list.size() <= 1 || (max != Integer.MAX_VALUE && list.size() < max)) { + return; + } PageFile root = getFile(ROOT_NAME); String rootFileCopy = ROOT_NAME + "_" + updateId; root = copyPageFile(root); @@ -429,7 +432,7 @@ public void mergeRoots(int max) { PageFile newRoot = newPageFile(false); newRoot.setNextRoot(rootFileCopy); putFile(ROOT_NAME, newRoot); - while(it.hasNext()) { + while (it.hasNext()) { Entry e = it.next(); put(e.getKey(), e.getValue()); // we can remove files that are processed @@ -487,7 +490,6 @@ public void checkpoint() { flush(); root = copyPageFile(root); putFile(ROOT_NAME, root); - } else { flush(); root = copyPageFile(root); @@ -555,11 +557,22 @@ public Iterator> iterator(String largerThan) { return iterator(largerThan, Integer.MAX_VALUE); } + public Iterable> entrySet() { + return new Iterable>() { + + @Override + public Iterator> iterator() { + return iterator(); + } + + }; + } + private Iterator> iterator(String largerThan, int maxRootCount) { ArrayList streams = new ArrayList<>(); String next = ROOT_NAME; for (int i = 0; i < maxRootCount; i++) { - streams.add(new SortedStream(next, immutableRootIterator(next, largerThan))); + streams.add(new SortedStream(i, next, immutableRootIterator(next, largerThan))); next = getFile(next).getNextRoot(); if (next == null) { break; @@ -584,6 +597,8 @@ private void fetchNext() { break; } if (key.equals(lastKey)) { + s.next(); + pq.add(s); continue; } String value = s.currentValue(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java index 95df3ac239b..163e5dc6fd5 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java @@ -22,19 +22,21 @@ public class SortedStream implements Comparable { + private final int priority; private final String rootFileName; private Iterator it; private String currentKey; private String currentValue; - public SortedStream(String rootFileName, Iterator it) { + public SortedStream(int priority, String rootFileName, Iterator it) { + this.priority = priority; this.rootFileName = rootFileName; this.it = it; next(); } public String toString() { - return "file " + rootFileName + " key " + currentKey + " value " + currentValue; + return "priority " + priority + " file " + rootFileName + " key " + currentKey + " value " + currentValue; } public String currentKeyOrNull() { @@ -60,7 +62,7 @@ public void next() { public int compareTo(SortedStream o) { if (currentKey == null) { if (o.currentKey == null) { - return rootFileName.compareTo(o.rootFileName); + return Integer.compare(priority, o.priority); } return 1; } else if (o.currentKey == null) { @@ -68,7 +70,7 @@ public int compareTo(SortedStream o) { } int comp = currentKey.compareTo(o.currentKey); if (comp == 0) { - return rootFileName.compareTo(o.rootFileName); + return Integer.compare(priority, o.priority); } return comp; } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java index 0074ebd5492..5f4f73baebc 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java @@ -33,7 +33,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java new file mode 100644 index 00000000000..c2b3eab3347 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.junit.Test; + +/** + * Create, read, update, delete test. + */ +public class CrudMultiRootTest { + + @Test + public void memory() { + Store store = StoreBuilder.build(""); + test(store); + testAdd(store); + } + + @Test + public void file() { + Store store = StoreBuilder.build( + "type=file\n" + + "dir=target/files\n" + + "cacheSizeMB=1\n" + + "maxFileSizeBytes=100"); + store.setWriteCompression(Compression.NO); + test(store); + testAdd(store); + } + + @Test + public void fileLZ4() { + Store store = StoreBuilder.build( + "type=file\n" + + "dir=target/files"); + store.setWriteCompression(Compression.LZ4); + test(store); + testAdd(store); + } + + public void testAdd(Store store) { + store.removeAll(); + Session session = new Session(store); + session.init(); + int count = 100; + TreeMap map = new TreeMap<>(); + for (int i = 0; i < count; i++) { + map.put("key" + i, "v" + i); + session.put("key" + i, "v" + i); + session.flush(); + if (count % 10 == 0) { + session.checkpoint(); + session.mergeRoots(10); + } + } + session.mergeRoots(Integer.MAX_VALUE); + session.flush(); + + session = new Session(store); + TreeMap map2 = new TreeMap<>(); + for(String k : session.keys()) { + map2.put(k, session.get(k)); + } + assertEquals(map, map2); + } + + public void test(Store store) { + store.removeAll(); + Session session = new Session(store); + session.init(); + int count = 100; + TreeMap verify = new TreeMap<>(); + for (int i = 0; i < count; i++) { + session.put("hello" + i, "world" + i); + verify.put("hello" + i, "world" + i); + } + session.checkpoint(); + for (int i = 0; i < count; i++) { + if (i % 3 == 0) { + session.put("hello" + i, null); + verify.put("hello" + i, null); + } else if (i % 3 == 1) { + session.put("hello" + i, "World" + i); + verify.put("hello" + i, "World" + i); + } + } + + Iterator> it = verify.entrySet().iterator(); + Iterator it2 = session.keys().iterator(); + String previous = null; + for (int i = 0; i < count; i++) { + Entry e = it.next(); + String k = it2.next(); + assertEquals(e.getKey(), k); + assertEquals(e.getValue(), session.get(k)); + + Iterator it3 = session.keys(previous).iterator(); + assertTrue(it3.hasNext()); + assertEquals("previous: " + previous, k, it3.next()); + Iterator> it4 = session.iterator(previous); + assertTrue("previous: " + previous, it4.hasNext()); + Entry e4 = it4.next(); + assertEquals("previous " + previous, k, e4.getKey()); + assertEquals(e.getValue(), e4.getValue()); + if (it4.hasNext()) { + Entry e4b = it4.next(); + assertFalse("key returned twice " + e4b.getKey(), e4b.getKey().equals(k)); + } + + previous = k; + } + + session = new Session(store); + for (int i = 0; i < count; i++) { + String key = "hello" + i; + assertEquals("world" + i, session.get(key)); + session.put(key, "World " + i); + } + session.flush(); + + session = new Session(store); + for (int i = 0; i < count; i++) { + String key = "hello" + i; + assertEquals("World " + i, session.get(key)); + session.put(key, null); + } + session.flush(); + + session = new Session(store); + for (int i = 0; i < count; i++) { + String key = "hello" + i; + assertEquals(null, session.get(key)); + } + + for (int i = 0; i < 20; i++) { + session.checkpoint(); + } + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java new file mode 100644 index 00000000000..2aaf00185b3 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; + +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeMap; + +import org.junit.Test; + +public class MergeRootsTest { + + @Test + public void simpleTest() { + Store store = StoreBuilder.build(""); + Session session = new Session(store); + session.init(); + for (int i = 0; i < 10; i++) { + session.put("x" + i, "y" + i); + session.checkpoint(); + assertEquals(i + 2, session.getRootCount()); + } + assertEquals(11, session.getRootCount()); + session.mergeRoots(2); + assertEquals(10, session.getRootCount()); + for (int i = 0; i < 10; i++) { + assertEquals("y" + i, session.get("x" + i)); + } + } + + @Test + public void multipleRootAppendTest() { + Store store = StoreBuilder.build(""); + Session session = new Session(store); + session.init(); + for(int j = 1; j <= 3; j++) { + for (int i = 0; i < 10 * j; i++) { + session.put("x" + i, "y" + i + "j" + j); + } + session.checkpoint(); + } + for (int i = 0; i < 10 * 3; i++) { + if (!("y" + i + "j3").equals(session.get("x" + i))) { + assertEquals("y" + i + "j3", session.get("x" + i)); + } + } + session.mergeRoots(Integer.MAX_VALUE); + for (int i = 0; i < 10 * 3; i++) { + if (!("y" + i + "j3").equals(session.get("x" + i))) { + assertEquals("y" + i + "j3", session.get("x" + i)); + } + } + } + + @Test + public void multipleRootRandomOverwriteTest() { + Store store = StoreBuilder.build(""); + Session session = new Session(store); + session.init(); + TreeMap map = new TreeMap<>(); + Random r = new Random(42); + for(int j = 1; j <= 3; j++) { + for (int i = 0; i < 10; i++) { + String k = "x" + r.nextInt(30); + String v = "y" + i + "j" + j; + session.put(k, v); + map.put(k, v); + } + session.checkpoint(); + } + session.mergeRoots(Integer.MAX_VALUE); + for(Entry e : map.entrySet()) { + assertEquals(e.getValue(), session.get(e.getKey())); + } + TreeMap map2 = new TreeMap<>(); + + for(Entry e : map.entrySet()) { + map2.put(e.getKey(), e.getValue()); + } + assertEquals(map2, map); + } + + @Test + public void logStructuredMerge() { + Store store = StoreBuilder.build( + "type=memory\n" + + "maxFileSizeBytes=10000"); + Session session = new Session(store); + session.init(); + for (int batch = 1; batch <= 200; batch++) { + // System.out.println("batch " + batch); + if (batch % 10 == 0) { + session.mergeRoots(10); + // System.out.println("merged 10 roots; new root count: " + session.getRootCount()); + } + if (batch % 100 == 0) { + session.mergeRoots(20); + // System.out.println("merged 20 roots; new root count: " + session.getRootCount()); + // System.out.println(session.getInfo()); + } + session.flush(); + // System.out.println("info:"); + // System.out.println(session.getInfo()); + session.runGC(); + for (int j = 0; j < 100; j++) { + session.put("x" + j + " " + Math.random(), new String(new char[1000])); + } + session.checkpoint(); + } + assertEquals(3, session.getRootCount()); + session.mergeRoots(Integer.MAX_VALUE); + assertEquals(1, session.getRootCount()); + } + + +} From cf8c348c98cf3eee23c75cad92ed21bc31ba21c2 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 5 Aug 2024 18:20:32 +0200 Subject: [PATCH 67/86] OAK-10341 Tree store (use less memory) --- .../document/DocumentStoreIndexerBase.java | 31 ++++---- .../flatfile/FlatFileNodeStoreBuilder.java | 79 ++++++++++++------- .../pipelined/PipelinedTreeStoreStrategy.java | 6 +- .../indexer/document/tree/TreeStore.java | 6 +- .../document/tree/store/FileStore.java | 5 ++ .../indexer/document/tree/store/Session.java | 5 ++ .../FlatFileNodeStoreBuilderTest.java | 11 +-- .../document/flatfile/FlatFileStoreTest.java | 3 +- .../jackrabbit/oak/index/IndexCommand.java | 7 +- 9 files changed, 92 insertions(+), 61 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index a64d05227a1..50f24c8d985 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -30,7 +30,6 @@ import org.apache.jackrabbit.oak.index.IndexHelper; import org.apache.jackrabbit.oak.index.IndexerSupport; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileNodeStoreBuilder; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStore; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.IncrementalStoreBuilder; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; @@ -164,14 +163,14 @@ public NodeStateEntryTraverser create(TraversingRange traversingRange) { } } - private List buildFlatFileStoreList(NodeState checkpointedState, + private List buildFlatFileStoreList(NodeState checkpointedState, CompositeIndexer indexer, Predicate pathPredicate, Set preferredPathElements, boolean splitFlatFile, Set indexDefinitions, IndexingReporter reporter) throws IOException { - List storeList = new ArrayList<>(); + List storeList = new ArrayList<>(); Stopwatch flatFileStoreWatch = Stopwatch.createStarted(); int executionCount = 1; @@ -210,7 +209,7 @@ nodeStore, getMongoDocumentStore(), traversalLog)) } else { storeList.add(builder.build()); } - for (FlatFileStore item : storeList) { + for (IndexStore item : storeList) { closer.register(item); } } catch (CompositeException e) { @@ -303,16 +302,16 @@ public IndexStore buildStore(String initialCheckpoint, String finalCheckpoint) t * @deprecated replaced by {@link #buildStore()} */ @Deprecated - public FlatFileStore buildFlatFileStore() throws IOException, CommitFailedException { + public IndexStore buildFlatFileStore() throws IOException, CommitFailedException { NodeState checkpointedState = indexerSupport.retrieveNodeStateForCheckpoint(); Set indexDefinitions = indexerSupport.getIndexDefinitions(); Set preferredPathElements = indexerSupport.getPreferredPathElements(indexDefinitions); Predicate predicate = indexerSupport.getFilterPredicate(indexDefinitions, Function.identity()); - FlatFileStore flatFileStore = buildFlatFileStoreList(checkpointedState, null, predicate, + IndexStore indexStore = buildFlatFileStoreList(checkpointedState, null, predicate, preferredPathElements, IndexerConfiguration.parallelIndexEnabled(), indexDefinitions, indexingReporter).get(0); log.info("FlatFileStore built at {}. To use this flatFileStore in a reindex step, set System Property-{} with value {}", - flatFileStore.getStorePath(), OAK_INDEXER_SORTED_FILE_PATH, flatFileStore.getStorePath()); - return flatFileStore; + indexStore.getStorePath(), OAK_INDEXER_SORTED_FILE_PATH, indexStore.getStorePath()); + return indexStore; } public void reindexUsingTreeStore() throws CommitFailedException, IOException { @@ -378,7 +377,7 @@ public void reindex() throws CommitFailedException, IOException { closer.register(indexer); - List flatFileStores = buildFlatFileStoreList( + List indexStores = buildFlatFileStoreList( checkpointedState, indexer, indexer::shouldInclude, @@ -397,11 +396,11 @@ public void reindex() throws CommitFailedException, IOException { Stopwatch indexerWatch = Stopwatch.createStarted(); try { - if (flatFileStores.size() > 1) { - indexParallel(flatFileStores, indexer, progressReporter); - } else if (flatFileStores.size() == 1) { - FlatFileStore flatFileStore = flatFileStores.get(0); - for (NodeStateEntry entry : flatFileStore) { + if (indexStores.size() > 1) { + indexParallel(indexStores, indexer, progressReporter); + } else if (indexStores.size() == 1) { + IndexStore indexStore = indexStores.get(0); + for (NodeStateEntry entry : indexStore) { reportDocumentRead(entry.getPath(), progressReporter); indexer.index(entry); } @@ -447,12 +446,12 @@ public void reindex() throws CommitFailedException, IOException { } } - private void indexParallel(List storeList, CompositeIndexer indexer, IndexingProgressReporter progressReporter) + private void indexParallel(List storeList, CompositeIndexer indexer, IndexingProgressReporter progressReporter) throws IOException { ExecutorService service = Executors.newFixedThreadPool(IndexerConfiguration.indexThreadPoolSize()); List futureList = new ArrayList<>(); - for (FlatFileStore item : storeList) { + for (IndexStore item : storeList) { Future future = service.submit(() -> { for (NodeStateEntry entry : item) { reportDocumentRead(entry.getPath(), progressReporter); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index b06f76ce55d..212baaf61ca 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -19,18 +19,31 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile; -import com.mongodb.MongoClientURI; -import com.mongodb.client.MongoDatabase; -import org.apache.commons.lang3.StringUtils; -import org.apache.jackrabbit.guava.common.collect.Iterables; +import static java.util.Collections.unmodifiableSet; +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; +import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4; +import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + import org.apache.jackrabbit.guava.common.collect.Iterables; -import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.index.IndexHelper; import org.apache.jackrabbit.oak.index.IndexerSupport; import org.apache.jackrabbit.oak.index.indexer.document.CompositeException; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedTreeStoreStrategy; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreSortStrategy; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; @@ -47,21 +60,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static java.util.Collections.unmodifiableSet; -import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; -import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4; -import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP; +import com.mongodb.MongoClientURI; +import com.mongodb.client.MongoDatabase; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; /** * This class is where the strategy being selected for building FlatFileStore. @@ -121,7 +122,11 @@ public enum SortStrategyType { /** * System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} if set to this value would result in {@link PipelinedStrategy} being used. */ - PIPELINED + PIPELINED, + /** + * System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} if set to this value would result in {@link PipelinedTreeStoreStrategy} being used. + */ + PIPELINED_TREE, } public FlatFileNodeStoreBuilder(File workDir) { @@ -201,21 +206,28 @@ public FlatFileNodeStoreBuilder withIndexingReporter(IndexingReporter reporter) return this; } - public FlatFileStore build() throws IOException, CompositeException { + public IndexStore build() throws IOException, CompositeException { logFlags(); entryWriter = new NodeStateEntryWriter(blobStore); IndexStoreFiles indexStoreFiles = createdSortedStoreFiles(); File metadataFile = indexStoreFiles.metadataFile; - FlatFileStore store = new FlatFileStore(blobStore, indexStoreFiles.storeFiles.get(0), metadataFile, - new NodeStateEntryReader(blobStore), - unmodifiableSet(preferredPathElements), algorithm); + File file = indexStoreFiles.storeFiles.get(0); + IndexStore store; + if (file.isDirectory()) { + store = new TreeStore(file, + new NodeStateEntryReader(blobStore)); + } else { + store = new FlatFileStore(blobStore, file, metadataFile, + new NodeStateEntryReader(blobStore), + unmodifiableSet(preferredPathElements), algorithm); + } if (entryCount > 0) { store.setEntryCount(entryCount); } return store; } - public List buildList(IndexHelper indexHelper, IndexerSupport indexerSupport, + public List buildList(IndexHelper indexHelper, IndexerSupport indexerSupport, Set indexDefinitions) throws IOException, CompositeException { logFlags(); entryWriter = new NodeStateEntryWriter(blobStore); @@ -235,7 +247,7 @@ public List buildList(IndexHelper indexHelper, IndexerSupport ind log.info("Split flat file to result files '{}' is done, took {} ms", fileList, System.currentTimeMillis() - start); } - List storeList = new ArrayList<>(); + List storeList = new ArrayList<>(); for (File flatFileItem : fileList) { FlatFileStore store = new FlatFileStore(blobStore, flatFileItem, metadataFile, new NodeStateEntryReader(blobStore), unmodifiableSet(preferredPathElements), algorithm); @@ -326,7 +338,7 @@ IndexStoreSortStrategy createSortStrategy(File dir) { log.warn("TraverseWithSortStrategy is deprecated and will be removed in the near future. Use PipelinedStrategy instead."); return new TraverseWithSortStrategy(nodeStateEntryTraverserFactory, preferredPathElements, entryWriter, dir, algorithm, pathPredicate, checkpoint); - case PIPELINED: + case PIPELINED: { log.info("Using PipelinedStrategy"); List pathFilters = indexDefinitions.stream().map(IndexDefinition::getPathFilter).collect(Collectors.toList()); List indexNames = indexDefinitions.stream().map(IndexDefinition::getIndexName).collect(Collectors.toList()); @@ -334,7 +346,16 @@ IndexStoreSortStrategy createSortStrategy(File dir) { return new PipelinedStrategy(mongoClientURI, mongoDocumentStore, nodeStore, rootRevision, preferredPathElements, blobStore, dir, algorithm, pathPredicate, pathFilters, checkpoint, statisticsProvider, indexingReporter); - + } + case PIPELINED_TREE: { + log.info("Using PipelinedTreeStoreStrategy"); + List pathFilters = indexDefinitions.stream().map(IndexDefinition::getPathFilter).collect(Collectors.toList()); + List indexNames = indexDefinitions.stream().map(IndexDefinition::getIndexName).collect(Collectors.toList()); + indexingReporter.setIndexNames(indexNames); + return new PipelinedTreeStoreStrategy(mongoClientURI, mongoDocumentStore, nodeStore, rootRevision, + preferredPathElements, blobStore, dir, algorithm, pathPredicate, pathFilters, checkpoint, + statisticsProvider, indexingReporter); + } } throw new IllegalStateException("Not a valid sort strategy value " + sortStrategyType); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 7b002ae3254..6b070280135 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -259,7 +259,7 @@ public PipelinedTreeStoreStrategy(MongoClientURI mongoClientURI, long memoryReservedForBuffers = nseWorkingMemoryBytes - memoryReservedForSortKeysArray; // A ByteBuffer can be at most Integer.MAX_VALUE bytes long - this.nseBuffersSizeBytes = limitToIntegerRange(memoryReservedForBuffers / nseBuffersCount); + this.nseBuffersSizeBytes = limitToIntegerRange(memoryReservedForBuffers / nseBuffersCount / 2); if (nseBuffersSizeBytes < MIN_ENTRY_BATCH_BUFFER_SIZE_MB * FileUtils.ONE_MB) { throw new IllegalArgumentException("Entry batch buffer size too small: " + nseBuffersSizeBytes + @@ -365,8 +365,8 @@ public File createSortedStoreFile() throws IOException { Future downloadFuture = ecs.submit(new PipelinedMongoDownloadTask( mongoClientURI, docStore, - (int) (mongoDocBatchMaxSizeMB * FileUtils.ONE_MB), - mongoDocBatchMaxNumberOfDocuments, + (int) (mongoDocBatchMaxSizeMB * FileUtils.ONE_MB / 2), + mongoDocBatchMaxNumberOfDocuments / 2, mongoDocQueue, pathFilters, statisticsProvider, diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index cafc6bd0d5a..291fca335df 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -39,9 +39,9 @@ public class TreeStore implements IndexStore { private static final String STORE_TYPE = "TreeStore"; - private static final long CACHE_SIZE_NODE_MB = 64; - private static final long CACHE_SIZE_TREE_STORE_MB = 1024; - private static final long MAX_FILE_SIZE_MB = 64; + private static final long CACHE_SIZE_NODE_MB = 32; + private static final long CACHE_SIZE_TREE_STORE_MB = 32; + private static final long MAX_FILE_SIZE_MB = 8; private static final long MB = 1024 * 1024; private final Store store; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java index 2483684ed61..35369420198 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java @@ -30,9 +30,13 @@ import java.util.Set; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.TimeUuid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileStore implements Store { + private static final Logger LOG = LoggerFactory.getLogger(FileStore.class); + private final Properties config; private final String directory; private Compression compression = Compression.NO; @@ -48,6 +52,7 @@ public FileStore(Properties config) { this.directory = config.getProperty("dir"); this.maxFileSizeBytes = Long.parseLong(config.getProperty( Store.MAX_FILE_SIZE_BYTES, "" + Store.DEFAULT_MAX_FILE_SIZE_BYTES)); + LOG.info("Max file size {} bytes", maxFileSizeBytes); new File(directory).mkdirs(); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index ca1577f56c1..eb2696664e7 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -30,12 +30,16 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Position; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SortedStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Read and write keys and values. */ public class Session { + private static final Logger LOG = LoggerFactory.getLogger(Session.class); + public static final String CACHE_SIZE_MB = "cacheSizeMB"; private static final int DEFAULT_CACHE_SIZE_MB = 256; private static final int DEFAULT_MAX_ROOTS = 10; @@ -62,6 +66,7 @@ public Session(Store store) { long cacheSizeMB = Long.parseLong(store.getConfig().getProperty( CACHE_SIZE_MB, "" + DEFAULT_CACHE_SIZE_MB)); long cacheSizeBytes = cacheSizeMB * 1024 * 1024; + LOG.info("Cache size {} bytes", cacheSizeBytes); this.cache = new MemoryBoundCache<>(cacheSizeBytes) { private static final long serialVersionUID = 1L; diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java index c15c29003e3..679fcc62db3 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java @@ -36,6 +36,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.IndexerConfiguration; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; @@ -177,8 +178,8 @@ public void testBuildListSplitLZ4() throws CompositeException, IOException { public void assertBuild(String dir) throws CompositeException, IOException { FlatFileNodeStoreBuilder builder = new FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory( nodeStateEntryTraverserFactory); - try (FlatFileStore store = builder.build()) { - assertEquals(dir, store.getFlatFileStorePath()); + try (IndexStore store = builder.build()) { + assertEquals(dir, store.getStorePath()); } } @@ -201,13 +202,13 @@ public void assertBuildList(String dir, boolean split) throws CompositeException NodeState rootState = mock(NodeState.class); when(indexerSupport.retrieveNodeStateForCheckpoint()).thenReturn(rootState); - List storeList = builder.buildList(indexHelper, indexerSupport, mockIndexDefns()); + List storeList = builder.buildList(indexHelper, indexerSupport, mockIndexDefns()); if (split) { - assertEquals(new File(dir, "split").getAbsolutePath(), storeList.get(0).getFlatFileStorePath()); + assertEquals(new File(dir, "split").getAbsolutePath(), storeList.get(0).getStorePath()); assertTrue(storeList.size() > 1); } else { - assertEquals(dir, storeList.get(0).getFlatFileStorePath()); + assertEquals(dir, storeList.get(0).getStorePath()); assertEquals(1, storeList.size()); } } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java index fabe1273a88..1688dc849c1 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java @@ -21,6 +21,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverser; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; import org.jetbrains.annotations.NotNull; import org.junit.Rule; @@ -55,7 +56,7 @@ public class FlatFileStoreTest { private void runBasicTest() throws Exception { List paths = createTestPaths(); FlatFileNodeStoreBuilder spyBuilder = Mockito.spy(new FlatFileNodeStoreBuilder(folder.getRoot())); - FlatFileStore flatStore = spyBuilder.withBlobStore(new MemoryBlobStore()) + IndexStore flatStore = spyBuilder.withBlobStore(new MemoryBlobStore()) .withPreferredPathElements(preferred) .withPathPredicate(pathPredicate) .withNodeStateEntryTraverserFactory(range -> new NodeStateEntryTraverser("NS-1", null, null,null, range) { diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java index 82876eebc22..893914421b9 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java @@ -30,8 +30,7 @@ import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.index.async.AsyncIndexerLucene; import org.apache.jackrabbit.oak.index.indexer.document.DocumentStoreIndexer; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStore; -import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater; import org.apache.jackrabbit.oak.run.cli.CommonOptions; import org.apache.jackrabbit.oak.run.cli.DocumentBuilderCustomizer; @@ -256,8 +255,8 @@ private File reindex(IndexOptions idxOpts, ExtendedIndexHelper extendedIndexHelp indexer.reindexUsingTreeStore(); } else { if (idxOpts.buildFlatFileStoreSeparately()) { - FlatFileStore ffs = indexer.buildFlatFileStore(); - String pathToFFS = ffs.getFlatFileStorePath(); + IndexStore store = indexer.buildStore(); + String pathToFFS = store.getStorePath(); System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToFFS); } indexer.reindex(); From e325f514091b7820578e5e5adc21eca3452b0425 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 6 Aug 2024 14:44:27 +0200 Subject: [PATCH 68/86] OAK-10341 Tree store (fix memory cache calculation) --- .../tree/store/utils/MemoryBoundCache.java | 2 +- .../store/utils/MemoryBoundCacheTest.java | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCacheTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java index d5619e569e5..3381cd14f9a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java @@ -73,7 +73,7 @@ public void clear() { @Override public boolean removeEldestEntry(Map.Entry eldest) { - boolean removeEldest = size() > maxMemoryBytes; + boolean removeEldest = memoryUsed > maxMemoryBytes; if (removeEldest) { memoryUsed -= eldest.getValue().estimatedMemory(); } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCacheTest.java new file mode 100644 index 00000000000..6aacac9432c --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCacheTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class MemoryBoundCacheTest { + + @Test + public void test() { + MemoryBoundCache cache = new MemoryBoundCache<>(1000); + for (int i = 0; i < 200; i++) { + cache.put("k" + i, new MemoryValue("v" + i)); + } + assertEquals(101, cache.size()); + } + + static class MemoryValue implements MemoryBoundCache.MemoryObject { + + final String value; + + MemoryValue(String value) { + this.value = value; + } + + @Override + public long estimatedMemory() { + return 10; + } + + public String toString() { + return value; + } + + } +} From 7b3d10d113b39e3de0c9ee24f845493974d2d970 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 7 Aug 2024 11:33:30 +0200 Subject: [PATCH 69/86] OAK-10341 Tree store (blob prefetch) --- .../document/DocumentStoreIndexerBase.java | 2 +- .../document/flatfile/BlobPrefetcher.java | 93 +++++++++++++++++++ .../flatfile/FlatFileNodeStoreBuilder.java | 7 +- .../pipelined/PipelinedTreeStoreStrategy.java | 2 +- .../pipelined/PipelinedTreeStoreTask.java | 2 + .../indexer/document/tree/TreeStore.java | 67 ++++++++++--- .../document/tree/TreeStoreNodeState.java | 1 + .../indexer/document/tree/TreeStoreUtils.java | 6 +- .../indexer/document/tree/store/Session.java | 4 +- .../tree/store/utils/MemoryBoundCache.java | 29 +++--- .../pipelined/PipelinedTreeStoreIT.java | 2 +- .../document/tree/store/SessionCacheTest.java | 37 ++++++++ 12 files changed, 221 insertions(+), 31 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 50f24c8d985..830af3a36f1 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -316,7 +316,7 @@ public IndexStore buildFlatFileStore() throws IOException, CommitFailedException public void reindexUsingTreeStore() throws CommitFailedException, IOException { NodeStateEntryReader reader = new NodeStateEntryReader(indexHelper.getGCBlobStore()); - TreeStore treeStore = new TreeStore(new File("target/treeStore"), reader); + TreeStore treeStore = new TreeStore(new File("target/treeStore"), reader, false); // TODO this is mostly a copy of reindex() diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java new file mode 100644 index 00000000000..7c4608ba905 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BlobPrefetcher { + + private static final Logger LOG = LoggerFactory.getLogger(BlobPrefetcher.class); + + private static final String BLOB_PREFETCH_PROPERTY_NAME = "oak.index.BlobPrefetchSuffix"; + private static final String BLOB_PREFETCH_SUFFIX_DEFAULT = ""; + private static final int PRETCH_THREADS = 3; + + public static void prefetch(TreeStore treeStore) { + String prefix = System.getProperty( + BLOB_PREFETCH_PROPERTY_NAME, + BLOB_PREFETCH_SUFFIX_DEFAULT); + LOG.info("Prefetch prefix '{}'", prefix); + if (prefix.isEmpty()) { + return; + } + Iterator it = treeStore.pathIterator(); + ExecutorService executorService = Executors.newFixedThreadPool(1 + PRETCH_THREADS, new ThreadFactory() { + private final AtomicInteger threadNumber = new AtomicInteger(1); + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setName("BlobPrefetcher-" + threadNumber.getAndIncrement()); + return t; + } + }); + executorService.submit(() -> { + int count = 0; + while (it.hasNext()) { + String e = it.next(); + if (++count % 200_000 == 0) { + LOG.info("Count {} path {}", count, e); + } + if (e.endsWith(BLOB_PREFETCH_SUFFIX_DEFAULT)) { + executorService.submit(() -> { + NodeStateEntry nse = treeStore.getNodeStateEntry(e); + PropertyState p = nse.getNodeState().getProperty("jcr:data"); + if (p == null || p.isArray() || p.getType() != Type.BINARY) { + return; + } + Blob blob = p.getValue(Type.BINARY); + try { + InputStream in = blob.getNewStream(); + // read one byte only, in order to prefetch + in.read(); + in.close(); + } catch (IOException e1) { + LOG.warn("Prefetching failed", e); + } + }); + } + } + }); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index 212baaf61ca..d5b703235c4 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -214,8 +214,13 @@ public IndexStore build() throws IOException, CompositeException { File file = indexStoreFiles.storeFiles.get(0); IndexStore store; if (file.isDirectory()) { + // use a separate tree store (with a smaller cache) + // to avoid concurrency issues + TreeStore blobTreeStore = new TreeStore(file, + new NodeStateEntryReader(blobStore), true); + BlobPrefetcher.prefetch(blobTreeStore); store = new TreeStore(file, - new NodeStateEntryReader(blobStore)); + new NodeStateEntryReader(blobStore), false); } else { store = new FlatFileStore(blobStore, file, metadataFile, new NodeStateEntryReader(blobStore), diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 6b070280135..587bcd99fb3 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -339,7 +339,7 @@ public File createSortedStoreFile() throws IOException { // Future instances, we can only wait on one of them, so that if any of the others fail, we have no easy way // to detect this failure. ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); - TreeStore treeStore = new TreeStore(getStoreDir(), null); + TreeStore treeStore = new TreeStore(getStoreDir(), null, false); treeStore.getSession().init(); try { // download -> transform thread. diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index 2ea8db149bf..3a990f63734 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -95,6 +95,7 @@ public Result call() throws Exception { LOG.info("Final merge"); Stopwatch start = Stopwatch.createStarted(); treeStore.getSession().mergeRoots(Integer.MAX_VALUE); + treeStore.getSession().runGC(); long durationSeconds = start.elapsed(TimeUnit.SECONDS); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, durationSeconds); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, 0); @@ -168,6 +169,7 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { } session.checkpoint(); session.mergeRoots(MERGE_BATCH); + session.runGC(); } timeWritingMillis += saveClock.elapsed().toMillis(); LOG.info("Wrote batch of size {} (uncompressed) with {} entries in {} at {}", diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 291fca335df..787de676993 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -38,32 +38,77 @@ public class TreeStore implements IndexStore { private static final String STORE_TYPE = "TreeStore"; + private static final String TREE_STORE_CONFIG = "oak.treeStoreConfig"; - private static final long CACHE_SIZE_NODE_MB = 32; - private static final long CACHE_SIZE_TREE_STORE_MB = 32; - private static final long MAX_FILE_SIZE_MB = 8; + public static final long CACHE_SIZE_NODE_MB = 64; + private static final long CACHE_SIZE_TREE_STORE_MB = 64; + + private static final long MAX_FILE_SIZE_MB = 12; private static final long MB = 1024 * 1024; private final Store store; private final File directory; private final Session session; private final NodeStateEntryReader entryReader; - private final MemoryBoundCache nodeStateCache = - new MemoryBoundCache<>(CACHE_SIZE_NODE_MB * MB); + private final MemoryBoundCache nodeStateCache; private long entryCount; - public TreeStore(File directory, NodeStateEntryReader entryReader) { + public TreeStore(File directory, NodeStateEntryReader entryReader, boolean smallCache) { this.directory = directory; this.entryReader = entryReader; - String storeConfig = System.getProperty("oak.treeStoreConfig", + long cacheSizeNodeMB = CACHE_SIZE_NODE_MB; + long cacheSizeTreeStoreMB = CACHE_SIZE_TREE_STORE_MB; + if (smallCache) { + cacheSizeNodeMB = 16; + cacheSizeTreeStoreMB = 48; + } + nodeStateCache = new MemoryBoundCache<>(cacheSizeNodeMB * MB); + String storeConfig = System.getProperty(TREE_STORE_CONFIG, "type=file\n" + - Session.CACHE_SIZE_MB + "=" + CACHE_SIZE_TREE_STORE_MB + "\n" + + Session.CACHE_SIZE_MB + "=" + cacheSizeTreeStoreMB + "\n" + Store.MAX_FILE_SIZE_BYTES + "=" + MAX_FILE_SIZE_MB * MB + "\n" + "dir=" + directory.getAbsolutePath()); this.store = StoreBuilder.build(storeConfig); this.session = new Session(store); } + public Iterator pathIterator() { + Iterator> it = session.iterator(); + return new Iterator() { + + String current; + + { + fetch(); + } + + private void fetch() { + while (it.hasNext()) { + Entry e = it.next(); + if (e.getValue().isEmpty()) { + continue; + } + current = e.getKey(); + return; + } + current = null; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public String next() { + String result = current; + fetch(); + return result; + } + + }; + } + @Override public void close() throws IOException { session.flush(); @@ -108,7 +153,7 @@ public NodeStateEntry next() { }; } - NodeStateEntry getNodeStateEntry(String path) { + public NodeStateEntry getNodeStateEntry(String path) { return new NodeStateEntryBuilder(getNodeState(path), path).build(); } @@ -123,7 +168,7 @@ NodeState getNodeState(String path) { } String value = session.get(path); if (value == null || value.isEmpty()) { - result = new TreeStoreNodeState(EmptyNodeState.MISSING_NODE, path, this, path.length()); + result = new TreeStoreNodeState(EmptyNodeState.MISSING_NODE, path, this, path.length() * 2); } else { result = getNodeState(path, value); } @@ -138,7 +183,7 @@ TreeStoreNodeState getNodeState(String path, String value) { } String line = path + "|" + value; NodeStateEntry entry = entryReader.read(line); - result = new TreeStoreNodeState(entry.getNodeState(), path, this, line.length()); + result = new TreeStoreNodeState(entry.getNodeState(), path, this, path.length() * 2 + line.length() * 10); nodeStateCache.put(path, result); return result; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java index 3e875b44902..e6c04a5e3b3 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java @@ -50,6 +50,7 @@ public TreeStoreNodeState(NodeState delegate, String path, TreeStore treeStore, this.estimatedMemory = estimatedMemory; } + @Override public long estimatedMemory() { return estimatedMemory; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java index c345200c672..a6a52176457 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -38,7 +38,7 @@ public static void main(String... args) throws IOException { String dir = args[0]; MemoryBlobStore blobStore = new MemoryBlobStore(); NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); - try (TreeStore treeStore = new TreeStore(new File(dir), entryReader)) { + try (TreeStore treeStore = new TreeStore(new File(dir), entryReader, false)) { Session session = treeStore.getSession(); Store store = treeStore.getStore(); if (store.keySet().isEmpty()) { @@ -65,13 +65,13 @@ public static void main(String... args) throws IOException { String path = line.substring(0, index); String value = line.substring(index + 1); session.put(path, value); - + if (!path.equals("/")) { String nodeName = PathUtils.getName(path); String parentPath = PathUtils.getParentPath(path); session.put(parentPath + "\t" + nodeName, ""); } - + } } session.flush(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index eb2696664e7..3377afeef80 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -70,12 +70,13 @@ public Session(Store store) { this.cache = new MemoryBoundCache<>(cacheSizeBytes) { private static final long serialVersionUID = 1L; + @Override public boolean removeEldestEntry(Map.Entry eldest) { boolean result = super.removeEldestEntry(eldest); if (result) { String key = eldest.getKey(); PageFile value = eldest.getValue(); - if(value.isModified()) { + if (value.isModified()) { store.put(key, value); // not strictly needed as it's no longer referenced value.setModified(false); @@ -853,6 +854,7 @@ public String getInfo() { rootId, gc.mark(Collections.singletonList(r)).size(), r)); rootId++; } + buff.append(cache.toString()); return buff.toString(); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java index 3381cd14f9a..41f8c69ed07 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java @@ -20,31 +20,36 @@ import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class MemoryBoundCache extends LinkedHashMap { private static final long serialVersionUID = 1L; - private long maxMemoryBytes; - private long memoryUsed; + private volatile long maxMemoryBytes; + private AtomicLong memoryUsed = new AtomicLong(); public MemoryBoundCache(long maxMemoryBytes) { super(16, 0.75f, true); this.maxMemoryBytes = maxMemoryBytes; } + public String toString() { + return "cache entries:" + size() + " max:" + maxMemoryBytes + " used:" + memoryUsed; + } + public void setSize(int maxMemoryBytes) { this.maxMemoryBytes = maxMemoryBytes; } @Override - public V put(K key, V value) { + public synchronized V put(K key, V value) { V old = super.put(key, value); if (old != null) { - memoryUsed -= old.estimatedMemory(); + memoryUsed.addAndGet(-old.estimatedMemory()); } if (value != null) { - memoryUsed += value.estimatedMemory(); + memoryUsed.addAndGet(value.estimatedMemory()); } return old; } @@ -57,25 +62,25 @@ public void putAll(Map map) { } @Override - public V remove(Object key ) { + public synchronized V remove(Object key ) { V old = super.remove(key); if (old != null) { - memoryUsed -= old.estimatedMemory(); + memoryUsed.addAndGet(-old.estimatedMemory()); } return old; } @Override - public void clear() { + public synchronized void clear() { super.clear(); - memoryUsed = 0; + memoryUsed.set(0); } @Override - public boolean removeEldestEntry(Map.Entry eldest) { - boolean removeEldest = memoryUsed > maxMemoryBytes; + public synchronized boolean removeEldestEntry(Map.Entry eldest) { + boolean removeEldest = memoryUsed.get() > maxMemoryBytes; if (removeEldest) { - memoryUsed -= eldest.getValue().estimatedMemory(); + memoryUsed.addAndGet(-eldest.getValue().estimatedMemory()); } return removeEldest; } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java index 5f4f73baebc..24b1971e31c 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java @@ -532,7 +532,7 @@ static String[] readAllEntriesArray(File dir) throws IOException { } static List readAllEntries(File dir) throws IOException { - TreeStore treeStore = new TreeStore(dir, null); + TreeStore treeStore = new TreeStore(dir, null, false); ArrayList list = new ArrayList<>(); Session session = treeStore.getSession(); for (String k : session.keys()) { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java new file mode 100644 index 00000000000..8620c33c75d --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import org.junit.Test; + +public class SessionCacheTest { + + @Test + public void test() { + Store store = StoreBuilder.build("type:memory\n" + + Store.MAX_FILE_SIZE_BYTES + "=1000\n" + + Session.CACHE_SIZE_MB + "=1"); + Session s = new Session(store); + s.init(); + for (int i = 0; i < 1000000; i++) { + s.put("k" + i, "v" + i); + } + System.out.println(s.getInfo()); + } +} From bc04a80c3dc9d074eced98afb37120d28cc58c85 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 7 Aug 2024 21:49:11 +0200 Subject: [PATCH 70/86] OAK-10341 Tree store (blob prefetch) --- .../document/flatfile/BlobPrefetcher.java | 49 ++++++++++++------- .../pipelined/PipelinedTreeStoreTask.java | 39 +++++++++++---- .../indexer/document/tree/TreeStore.java | 6 ++- 3 files changed, 65 insertions(+), 29 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java index 7c4608ba905..7d6ad5175f7 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java @@ -23,8 +23,10 @@ import java.util.Iterator; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.PropertyState; @@ -40,7 +42,7 @@ public class BlobPrefetcher { private static final String BLOB_PREFETCH_PROPERTY_NAME = "oak.index.BlobPrefetchSuffix"; private static final String BLOB_PREFETCH_SUFFIX_DEFAULT = ""; - private static final int PRETCH_THREADS = 3; + private static final int PRETCH_THREADS = 16; public static void prefetch(TreeStore treeStore) { String prefix = System.getProperty( @@ -51,6 +53,7 @@ public static void prefetch(TreeStore treeStore) { return; } Iterator it = treeStore.pathIterator(); + Semaphore semaphore = new Semaphore(PRETCH_THREADS); ExecutorService executorService = Executors.newFixedThreadPool(1 + PRETCH_THREADS, new ThreadFactory() { private final AtomicInteger threadNumber = new AtomicInteger(1); @@ -61,31 +64,41 @@ public Thread newThread(Runnable r) { return t; } }); + AtomicLong prefetched = new AtomicLong(); executorService.submit(() -> { int count = 0; - while (it.hasNext()) { - String e = it.next(); - if (++count % 200_000 == 0) { - LOG.info("Count {} path {}", count, e); - } - if (e.endsWith(BLOB_PREFETCH_SUFFIX_DEFAULT)) { - executorService.submit(() -> { + try { + while (it.hasNext()) { + String e = it.next(); + if (++count % 200_000 == 0) { + LOG.info("Count {} path {}", count, e); + } + if (e.endsWith(BLOB_PREFETCH_SUFFIX_DEFAULT)) { NodeStateEntry nse = treeStore.getNodeStateEntry(e); PropertyState p = nse.getNodeState().getProperty("jcr:data"); if (p == null || p.isArray() || p.getType() != Type.BINARY) { - return; + continue; } Blob blob = p.getValue(Type.BINARY); - try { - InputStream in = blob.getNewStream(); - // read one byte only, in order to prefetch - in.read(); - in.close(); - } catch (IOException e1) { - LOG.warn("Prefetching failed", e); - } - }); + semaphore.acquire(); + executorService.submit(() -> { + try { + InputStream in = blob.getNewStream(); + // read one byte only, in order to prefetch + in.read(); + in.close(); + } catch (IOException e1) { + LOG.warn("Prefetching failed", e); + } + semaphore.release(); + prefetched.incrementAndGet(); + }); + } } + } catch (Exception e) { + LOG.warn("Prefetch error", e); + } finally { + LOG.info("Completed after {} nodes, {} prefetched", count, prefetched.get()); } }); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index 3a990f63734..a05bde7a1ef 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -51,9 +51,9 @@ public class PipelinedTreeStoreTask implements Callable { private static final Logger LOG = LoggerFactory.getLogger(PipelinedTreeStoreTask.class); - private static final String THREAD_NAME = "stree-store-task"; + private static final String THREAD_NAME = "tree-store-task"; - private static final int MERGE_BATCH = 10; + private static final int MERGE_BATCH = 5; private final TreeStore treeStore; private final BlockingQueue emptyBuffersQueue; @@ -66,6 +66,7 @@ public class PipelinedTreeStoreTask implements Callable emptyBuffersQueue, @@ -92,17 +93,25 @@ public Result call() throws Exception { NodeStateEntryBatch nseBuffer = nonEmptyBuffersQueue.take(); if (nseBuffer == SENTINEL_NSE_BUFFER) { synchronized (treeStore) { - LOG.info("Final merge"); + Session session = treeStore.getSession(); + LOG.info("Final merge(s)"); Stopwatch start = Stopwatch.createStarted(); - treeStore.getSession().mergeRoots(Integer.MAX_VALUE); - treeStore.getSession().runGC(); + while (session.getRootCount() > MERGE_BATCH) { + LOG.info("Merging {} roots; we have {} roots", + MERGE_BATCH, session.getRootCount()); + session.mergeRoots(MERGE_BATCH); + session.runGC(); + } + LOG.info("Final merge; we have {} roots", session.getRootCount()); + session.mergeRoots(Integer.MAX_VALUE); + session.runGC(); long durationSeconds = start.elapsed(TimeUnit.SECONDS); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, durationSeconds); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, 0); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, 0); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, 0); MetricsUtils.addMetricByteSize(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, 0); - LOG.info("Final merge done"); + LOG.info("Final merge done, new root count {}", session.getRootCount()); } long totalTimeMillis = taskStartTime.elapsed().toMillis(); String timeCreatingSortArrayPercentage = formatAsPercentage(timeCreatingSortArrayMillis, totalTimeMillis); @@ -160,6 +169,7 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { Stopwatch saveClock = Stopwatch.createStarted(); long textSize = 0; batchesProcessed++; + int sortBufferSize = sortBuffer.size(); synchronized (treeStore) { Session session = treeStore.getSession(); for (Entry e : sortBuffer.entrySet()) { @@ -167,14 +177,25 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { textSize += e.getKey().length() + e.getValue().length() + 2; entriesProcessed++; } + // free memory, before we merge, because merging needs memory as well + sortBuffer = null; session.checkpoint(); - session.mergeRoots(MERGE_BATCH); - session.runGC(); + unmergedRoots++; + LOG.info("Root count is {}, we have {} small unmerged roots", + session.getRootCount(), unmergedRoots); + if (unmergedRoots == MERGE_BATCH) { + LOG.info("Merging {} roots, as we have {} new ones; root count is {}", + MERGE_BATCH, unmergedRoots, session.getRootCount()); + session.mergeRoots(MERGE_BATCH); + session.runGC(); + LOG.info("New root count is {}", session.getRootCount()); + unmergedRoots = 0; + } } timeWritingMillis += saveClock.elapsed().toMillis(); LOG.info("Wrote batch of size {} (uncompressed) with {} entries in {} at {}", humanReadableByteCountBin(textSize), - sortBuffer.size(), saveClock, + sortBufferSize, saveClock, PipelinedUtils.formatAsTransferSpeedMBs(textSize, saveClock.elapsed().toMillis()) ); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 787de676993..b251f9aeb68 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -43,7 +43,7 @@ public class TreeStore implements IndexStore { public static final long CACHE_SIZE_NODE_MB = 64; private static final long CACHE_SIZE_TREE_STORE_MB = 64; - private static final long MAX_FILE_SIZE_MB = 12; + private static final long MAX_FILE_SIZE_MB = 8; private static final long MB = 1024 * 1024; private final Store store; @@ -60,7 +60,7 @@ public TreeStore(File directory, NodeStateEntryReader entryReader, boolean small long cacheSizeTreeStoreMB = CACHE_SIZE_TREE_STORE_MB; if (smallCache) { cacheSizeNodeMB = 16; - cacheSizeTreeStoreMB = 48; + cacheSizeTreeStoreMB = 32; } nodeStateCache = new MemoryBoundCache<>(cacheSizeNodeMB * MB); String storeConfig = System.getProperty(TREE_STORE_CONFIG, @@ -70,6 +70,8 @@ public TreeStore(File directory, NodeStateEntryReader entryReader, boolean small "dir=" + directory.getAbsolutePath()); this.store = StoreBuilder.build(storeConfig); this.session = new Session(store); + // we don not want to merge too early during the download + session.setMaxRoots(1000); } public Iterator pathIterator() { From b242328a76d31e0b09131b5a6b84b146e06736b3 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 9 Aug 2024 16:33:27 +0200 Subject: [PATCH 71/86] OAK-10341 Tree store (blob prefetch) --- .../document/DocumentStoreIndexerBase.java | 2 +- .../document/flatfile/BlobPrefetcher.java | 106 ---------- .../flatfile/FlatFileNodeStoreBuilder.java | 13 +- .../pipelined/PipelinedMongoDownloadTask.java | 2 +- .../pipelined/PipelinedTreeStoreStrategy.java | 16 +- .../pipelined/PipelinedTreeStoreTask.java | 103 +++++----- .../flatfile/pipelined/SortKeyPath.java | 43 ++++ .../indexer/document/tree/BlobPrefetcher.java | 183 ++++++++++++++++++ .../indexer/document/tree/TreeStore.java | 36 +++- .../indexer/document/tree/TreeStoreUtils.java | 2 +- .../indexer/document/tree/store/Session.java | 3 +- .../tree/store/utils/MemoryBoundCache.java | 5 + .../pipelined/PipelinedTreeStoreIT.java | 14 +- .../document/tree/BlobPrefetcherTest.java | 29 +++ .../indexer/document/tree/TreeStoreTest.java | 30 ++- .../document/tree/store/MergeRootsTest.java | 8 + 16 files changed, 419 insertions(+), 176 deletions(-) delete mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/SortKeyPath.java create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcher.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 830af3a36f1..9336c47cd16 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -316,7 +316,7 @@ public IndexStore buildFlatFileStore() throws IOException, CommitFailedException public void reindexUsingTreeStore() throws CommitFailedException, IOException { NodeStateEntryReader reader = new NodeStateEntryReader(indexHelper.getGCBlobStore()); - TreeStore treeStore = new TreeStore(new File("target/treeStore"), reader, false); + TreeStore treeStore = new TreeStore("reindex", new File("target/treeStore"), reader, false); // TODO this is mostly a copy of reindex() diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java deleted file mode 100644 index 7d6ad5175f7..00000000000 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/BlobPrefetcher.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.index.indexer.document.flatfile; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Iterator; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.api.PropertyState; -import org.apache.jackrabbit.oak.api.Type; -import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; -import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BlobPrefetcher { - - private static final Logger LOG = LoggerFactory.getLogger(BlobPrefetcher.class); - - private static final String BLOB_PREFETCH_PROPERTY_NAME = "oak.index.BlobPrefetchSuffix"; - private static final String BLOB_PREFETCH_SUFFIX_DEFAULT = ""; - private static final int PRETCH_THREADS = 16; - - public static void prefetch(TreeStore treeStore) { - String prefix = System.getProperty( - BLOB_PREFETCH_PROPERTY_NAME, - BLOB_PREFETCH_SUFFIX_DEFAULT); - LOG.info("Prefetch prefix '{}'", prefix); - if (prefix.isEmpty()) { - return; - } - Iterator it = treeStore.pathIterator(); - Semaphore semaphore = new Semaphore(PRETCH_THREADS); - ExecutorService executorService = Executors.newFixedThreadPool(1 + PRETCH_THREADS, new ThreadFactory() { - private final AtomicInteger threadNumber = new AtomicInteger(1); - - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r); - t.setName("BlobPrefetcher-" + threadNumber.getAndIncrement()); - return t; - } - }); - AtomicLong prefetched = new AtomicLong(); - executorService.submit(() -> { - int count = 0; - try { - while (it.hasNext()) { - String e = it.next(); - if (++count % 200_000 == 0) { - LOG.info("Count {} path {}", count, e); - } - if (e.endsWith(BLOB_PREFETCH_SUFFIX_DEFAULT)) { - NodeStateEntry nse = treeStore.getNodeStateEntry(e); - PropertyState p = nse.getNodeState().getProperty("jcr:data"); - if (p == null || p.isArray() || p.getType() != Type.BINARY) { - continue; - } - Blob blob = p.getValue(Type.BINARY); - semaphore.acquire(); - executorService.submit(() -> { - try { - InputStream in = blob.getNewStream(); - // read one byte only, in order to prefetch - in.read(); - in.close(); - } catch (IOException e1) { - LOG.warn("Prefetching failed", e); - } - semaphore.release(); - prefetched.incrementAndGet(); - }); - } - } - } catch (Exception e) { - LOG.warn("Prefetch error", e); - } finally { - LOG.info("Completed after {} nodes, {} prefetched", count, prefetched.get()); - } - }); - } - -} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index d5b703235c4..460d49fb019 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -62,6 +62,8 @@ import com.mongodb.MongoClientURI; import com.mongodb.client.MongoDatabase; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.BlobPrefetcher; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; /** @@ -216,11 +218,14 @@ public IndexStore build() throws IOException, CompositeException { if (file.isDirectory()) { // use a separate tree store (with a smaller cache) // to avoid concurrency issues - TreeStore blobTreeStore = new TreeStore(file, - new NodeStateEntryReader(blobStore), true); - BlobPrefetcher.prefetch(blobTreeStore); - store = new TreeStore(file, + TreeStore indexingTreeStore = new TreeStore( + "indexing", file, new NodeStateEntryReader(blobStore), false); + TreeStore prefetchTreeStore = new TreeStore( + "prefetch", file, + new NodeStateEntryReader(blobStore), true); + BlobPrefetcher.prefetch(prefetchTreeStore, indexingTreeStore); + store = indexingTreeStore; } else { store = new FlatFileStore(blobStore, file, metadataFile, new NodeStateEntryReader(blobStore), diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java index e8c5770abe3..e8de33a4685 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java @@ -743,7 +743,7 @@ void download(FindIterable mongoIterable) throws InterruptedExcept this.lastIdDownloaded = id; this.documentsDownloadedTotal++; downloadStatics.incrementDocumentsDownloadedTotal(); - if (this.documentsDownloadedTotal % 20000 == 0) { + if (this.documentsDownloadedTotal % 100_000 == 0) { reportProgress(id); } TRAVERSAL_LOG.trace(id); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 587bcd99fb3..2a4a9ef0d3d 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -255,11 +255,11 @@ public PipelinedTreeStoreStrategy(MongoClientURI mongoClientURI, // - #transforThreads NSE Binary buffers // - x1 Memory reserved for the array created by the sort-batch thread with the keys of the entries // in the batch that is being sorted - long memoryReservedForSortKeysArray = estimateMaxSizeOfSortKeyArray(nseWorkingMemoryBytes, nseBuffersCount, sortBufferMemoryPercentage); + long memoryReservedForSortKeysArray = estimateMaxSizeOfSortArray(nseWorkingMemoryBytes, nseBuffersCount, sortBufferMemoryPercentage); long memoryReservedForBuffers = nseWorkingMemoryBytes - memoryReservedForSortKeysArray; // A ByteBuffer can be at most Integer.MAX_VALUE bytes long - this.nseBuffersSizeBytes = limitToIntegerRange(memoryReservedForBuffers / nseBuffersCount / 2); + this.nseBuffersSizeBytes = limitToIntegerRange(memoryReservedForBuffers / nseBuffersCount); if (nseBuffersSizeBytes < MIN_ENTRY_BATCH_BUFFER_SIZE_MB * FileUtils.ONE_MB) { throw new IllegalArgumentException("Entry batch buffer size too small: " + nseBuffersSizeBytes + @@ -281,9 +281,9 @@ public PipelinedTreeStoreStrategy(MongoClientURI mongoClientURI, ); } - static long estimateMaxSizeOfSortKeyArray(long nseWorkingMemoryBytes, long nseBuffersCount, int sortBufferMemoryPercentage) { - // We reserve a percentage of the size of a buffer for the sort keys array. That is, we are assuming that for every line - // in the sort buffer, the memory needed to store the SortKey of the path section of the line will not be more + static long estimateMaxSizeOfSortArray(long nseWorkingMemoryBytes, long nseBuffersCount, int sortBufferMemoryPercentage) { + // We reserve a percentage of the size of a buffer for sorting. That is, we are assuming that for every line + // in the sort buffer, the memory needed to store the path section of the line will not be more // than sortBufferMemoryPercentage of the total size of the line in average // Estimate memory needed by the sort keys array. We assume each entry requires 256 bytes. long approxNseBufferSize = limitToIntegerRange(nseWorkingMemoryBytes / nseBuffersCount); @@ -339,7 +339,7 @@ public File createSortedStoreFile() throws IOException { // Future instances, we can only wait on one of them, so that if any of the others fail, we have no easy way // to detect this failure. ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); - TreeStore treeStore = new TreeStore(getStoreDir(), null, false); + TreeStore treeStore = new TreeStore("dump", getStoreDir(), null, false); treeStore.getSession().init(); try { // download -> transform thread. @@ -365,8 +365,8 @@ public File createSortedStoreFile() throws IOException { Future downloadFuture = ecs.submit(new PipelinedMongoDownloadTask( mongoClientURI, docStore, - (int) (mongoDocBatchMaxSizeMB * FileUtils.ONE_MB / 2), - mongoDocBatchMaxNumberOfDocuments / 2, + (int) (mongoDocBatchMaxSizeMB * FileUtils.ONE_MB), + mongoDocBatchMaxNumberOfDocuments, mongoDocQueue, pathFilters, statisticsProvider, diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index a05bde7a1ef..7638bb8def5 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -25,9 +25,9 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; import java.util.Locale; -import java.util.Map.Entry; -import java.util.TreeMap; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; @@ -53,7 +53,7 @@ public class PipelinedTreeStoreTask implements Callable emptyBuffersQueue; @@ -78,7 +78,6 @@ public PipelinedTreeStoreTask(TreeStore treeStore, this.nonEmptyBuffersQueue = nonEmptyBuffersQueue; this.statisticsProvider = statisticsProvider; this.reporter = reporter; - } @Override @@ -86,23 +85,21 @@ public Result call() throws Exception { Stopwatch taskStartTime = Stopwatch.createStarted(); String originalName = Thread.currentThread().getName(); Thread.currentThread().setName(THREAD_NAME); - INDEXING_PHASE_LOGGER.info("[TASK:{}:START] Starting tree store task", THREAD_NAME.toUpperCase(Locale.ROOT)); + INDEXING_PHASE_LOGGER.info("[TASK:{}:START] Starting sort-and-save task", THREAD_NAME.toUpperCase(Locale.ROOT)); try { while (true) { - LOG.info("Waiting for next batch"); NodeStateEntryBatch nseBuffer = nonEmptyBuffersQueue.take(); if (nseBuffer == SENTINEL_NSE_BUFFER) { synchronized (treeStore) { Session session = treeStore.getSession(); - LOG.info("Final merge(s)"); Stopwatch start = Stopwatch.createStarted(); while (session.getRootCount() > MERGE_BATCH) { - LOG.info("Merging {} roots; we have {} roots", + LOG.info("Merging {} roots; there are {} roots", MERGE_BATCH, session.getRootCount()); session.mergeRoots(MERGE_BATCH); session.runGC(); } - LOG.info("Final merge; we have {} roots", session.getRootCount()); + LOG.info("Final merge; {} roots", session.getRootCount()); session.mergeRoots(Integer.MAX_VALUE); session.runGC(); long durationSeconds = start.elapsed(TimeUnit.SECONDS); @@ -111,7 +108,7 @@ public Result call() throws Exception { MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, 0); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, 0); MetricsUtils.addMetricByteSize(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, 0); - LOG.info("Final merge done, new root count {}", session.getRootCount()); + LOG.info("Final merge done, {} roots", session.getRootCount()); } long totalTimeMillis = taskStartTime.elapsed().toMillis(); String timeCreatingSortArrayPercentage = formatAsPercentage(timeCreatingSortArrayMillis, totalTimeMillis); @@ -156,72 +153,80 @@ public Result call() throws Exception { } } + private ArrayList buildSortArray(NodeStateEntryBatch nseb) { + Stopwatch startTime = Stopwatch.createStarted(); + ByteBuffer buffer = nseb.getBuffer(); + int totalPathSize = 0; + ArrayList sortBuffer = new ArrayList<>(nseb.numberOfEntries()); + while (buffer.hasRemaining()) { + // Read the next key from the buffer + int pathLength = buffer.getInt(); + totalPathSize += pathLength; + // Create the String directly from the buffer without creating an intermediate byte[] + String path = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), pathLength, StandardCharsets.UTF_8); + buffer.position(buffer.position() + pathLength); + int valuePosition = buffer.position(); + // Skip the json + int entryLength = buffer.getInt(); + buffer.position(buffer.position() + entryLength); + SortKeyPath entry = new SortKeyPath(path, valuePosition); + sortBuffer.add(entry); + } + timeCreatingSortArrayMillis += startTime.elapsed().toMillis(); + LOG.info("Built sort array in {}. Entries: {}, Total size of path strings: {}", + startTime, sortBuffer.size(), humanReadableByteCountBin(totalPathSize)); + return sortBuffer; + } + private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { + ByteBuffer buffer = nseb.getBuffer(); LOG.info("Going to sort batch in memory. Entries: {}, Size: {}", nseb.numberOfEntries(), humanReadableByteCountBin(nseb.sizeOfEntriesBytes())); - TreeMap sortBuffer = buildTreeMap(nseb); + ArrayList sortBuffer = buildSortArray(nseb); if (sortBuffer.isEmpty()) { return; } Stopwatch sortClock = Stopwatch.createStarted(); + Collections.sort(sortBuffer); timeSortingMillis += sortClock.elapsed().toMillis(); - LOG.info("Sorted batch in {}. Saving to disk", sortClock); + LOG.info("Sorted batch in {}. Saving.", sortClock); Stopwatch saveClock = Stopwatch.createStarted(); long textSize = 0; batchesProcessed++; - int sortBufferSize = sortBuffer.size(); synchronized (treeStore) { Session session = treeStore.getSession(); - for (Entry e : sortBuffer.entrySet()) { - session.put(e.getKey(), e.getValue()); - textSize += e.getKey().length() + e.getValue().length() + 2; + for (SortKeyPath entry : sortBuffer) { entriesProcessed++; + // Retrieve the entry from the buffer + int posInBuffer = entry.getBufferPos(); + buffer.position(posInBuffer); + int valueLength = buffer.getInt(); + String value = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), valueLength, StandardCharsets.UTF_8); + textSize += entry.getPath().length() + value.length() + 2; + addEntry(entry.getPath(), value, session); } - // free memory, before we merge, because merging needs memory as well - sortBuffer = null; session.checkpoint(); unmergedRoots++; LOG.info("Root count is {}, we have {} small unmerged roots", session.getRootCount(), unmergedRoots); if (unmergedRoots == MERGE_BATCH) { - LOG.info("Merging {} roots, as we have {} new ones; root count is {}", - MERGE_BATCH, unmergedRoots, session.getRootCount()); session.mergeRoots(MERGE_BATCH); session.runGC(); - LOG.info("New root count is {}", session.getRootCount()); + LOG.info("Merged {} roots, root count is now {}", + unmergedRoots, session.getRootCount()); unmergedRoots = 0; } + timeWritingMillis += saveClock.elapsed().toMillis(); + batchesProcessed++; + LOG.info("Wrote batch of size {} (uncompressed) in {} at {}", + humanReadableByteCountBin(textSize), + saveClock, + PipelinedUtils.formatAsTransferSpeedMBs(textSize, saveClock.elapsed().toMillis()) + ); } - timeWritingMillis += saveClock.elapsed().toMillis(); - LOG.info("Wrote batch of size {} (uncompressed) with {} entries in {} at {}", - humanReadableByteCountBin(textSize), - sortBufferSize, saveClock, - PipelinedUtils.formatAsTransferSpeedMBs(textSize, saveClock.elapsed().toMillis()) - ); - } - - private TreeMap buildTreeMap(NodeStateEntryBatch nseb) { - Stopwatch startTime = Stopwatch.createStarted(); - ByteBuffer buffer = nseb.getBuffer(); - int totalPathSize = 0; - TreeMap map = new TreeMap<>(); - while (buffer.hasRemaining()) { - int pathLength = buffer.getInt(); - totalPathSize += pathLength; - String path = new String(buffer.array(), buffer.position() + buffer.arrayOffset(), pathLength, StandardCharsets.UTF_8); - buffer.position(buffer.position() + pathLength); - int entryLength = buffer.getInt(); - String data = new String(buffer.array(), buffer.position() + buffer.arrayOffset(), entryLength, StandardCharsets.UTF_8); - addEntry(path, data, map); - buffer.position(buffer.position() + entryLength); - } - timeCreatingSortArrayMillis += startTime.elapsed().toMillis(); - LOG.info("Built tree map in {}. Entries: {}, Total size of path strings: {}", startTime, map.size(), - humanReadableByteCountBin(totalPathSize)); - return map; } - private static void addEntry(String path, String data, TreeMap target) { + public static void addEntry(String path, String data, Session target) { target.put(path, data); if (!path.equals("/")) { String nodeName = PathUtils.getName(path); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/SortKeyPath.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/SortKeyPath.java new file mode 100644 index 00000000000..0edf84d9d24 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/SortKeyPath.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +public class SortKeyPath implements Comparable { + + private final String path; + private final int valuePosition; + + public SortKeyPath(String path, int valuePosition) { + this.path = path; + this.valuePosition = valuePosition; + } + + public int getBufferPos() { + return valuePosition; + } + + public String getPath() { + return path; + } + + @Override + public int compareTo(SortKeyPath o) { + return path.compareTo(o.path); + } +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcher.java new file mode 100644 index 00000000000..83c926f771a --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcher.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.utils.Hash; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.utils.HyperLogLog; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BlobPrefetcher { + + private static final Logger LOG = LoggerFactory.getLogger(BlobPrefetcher.class); + + private static final String BLOB_PREFETCH_PROPERTY_NAME = "oak.index.BlobPrefetchSuffix"; + private static final String BLOB_PREFETCH_SUFFIX_DEFAULT = ""; + private static final int PRETCH_THREADS = 16; + private static final long DATASTORE_CACHE_SIZE = 16 * 1024 * 1024 * 1024L; + + public static void prefetch(TreeStore prefetchStore, TreeStore indexStore) { + String prefix = System.getProperty( + BLOB_PREFETCH_PROPERTY_NAME, + BLOB_PREFETCH_SUFFIX_DEFAULT); + LOG.info("Prefetch prefix '{}', prefetch {}, index {}", + prefix, prefetchStore, indexStore); + if (prefix.isEmpty()) { + return; + } + Semaphore semaphore = new Semaphore(PRETCH_THREADS); + ExecutorService executorService = Executors.newFixedThreadPool(2 + PRETCH_THREADS, new ThreadFactory() { + private final AtomicInteger threadNumber = new AtomicInteger(1); + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setName("BlobPrefetcher-" + threadNumber.getAndIncrement()); + return t; + } + }); + AtomicLong downloadMax = new AtomicLong(); + executorService.submit( + iterator(prefetchStore, indexStore, + null, null, false, downloadMax, prefix)); + executorService.submit( + iterator(prefetchStore, indexStore, + executorService, semaphore, true, downloadMax, prefix)); + } + + static Runnable iterator(TreeStore prefetchStore, + TreeStore indexStore, + ExecutorService executorService, + Semaphore semaphore, boolean download, + AtomicLong downloadMax, + String prefix) { + return () -> { + Iterator it = prefetchStore.pathIterator(); + HyperLogLog estimateUniqueCount = new HyperLogLog(1024, 0); + AtomicLong prefetched = new AtomicLong(); + int count = 0; + try { + long totalCount = 0; + long inlinedCount = 0; + long totalSize = 0; + long maxSize = 0; + while (it.hasNext()) { + String path = it.next(); + if (++count % 1_000_000 == 0) { + int available = semaphore == null ? -1 : semaphore.availablePermits(); + LOG.info("Iterated {} path {} inlinedCount {} totalCount {} " + + "totalSize {} maxSize {} indexing {} download {} max {} available {}", + count, path, inlinedCount, totalCount, + totalSize, maxSize, indexStore.toString(), + download, downloadMax.get(), + available + ); + } + if (!download) { + while (true) { + String indexingPath = indexStore.getHighestReadKey(); + if (indexingPath.compareTo(path) >= 0) { + break; + } + Thread.sleep(1000); + } + } + if (!path.endsWith(prefix)) { + continue; + } + NodeStateEntry nse = prefetchStore.getNodeStateEntry(path); + PropertyState p = nse.getNodeState().getProperty("jcr:data"); + if (p == null || p.isArray() || p.getType() != Type.BINARY) { + continue; + } + Blob blob = p.getValue(Type.BINARY); + if (blob.isInlined()) { + inlinedCount++; + continue; + } + estimateUniqueCount.add(longHash(blob)); + totalCount++; + totalSize += blob.length(); + if (download) { + while (totalSize - DATASTORE_CACHE_SIZE > downloadMax.get()) { + Thread.sleep(1000); + } + } else { + downloadMax.set(totalSize); + } + maxSize = Math.max(maxSize, blob.length()); + if (!download) { + continue; + } + String indexingPos = indexStore.getHighestReadKey(); + if (indexingPos.compareTo(path) >= 0) { + // we are behind indexing! + // do not download, in order to catch up + LOG.debug("Indexing is ahead; ignoring {}", path); + continue; + } + semaphore.acquire(); + executorService.submit(() -> { + try { + LOG.debug("Prefetching {} took {} ms", path); + InputStream in = blob.getNewStream(); + // read one byte only, in order to prefetch + in.read(); + in.close(); + } catch (IOException e) { + LOG.warn("Prefetching failed", path, e); + } + semaphore.release(); + prefetched.incrementAndGet(); + }); + } + } catch (Exception e) { + LOG.warn("Prefetch error", e); + } finally { + LOG.info("Completed after {} nodes, {} prefetched, {} unique", + count, prefetched.get(), + estimateUniqueCount.estimate()); + } + }; + } + + private static long longHash(Blob blob) { + // the String.hashCode is only 32 bit + // because of that, we mix it with the length + // and then we use a secondary hash + // otherwise the estimation is way off + int h = blob.getContentIdentity().hashCode(); + return Hash.hash64(h | (blob.length() << 32)); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index b251f9aeb68..10f2e5aea5c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -34,34 +34,43 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class TreeStore implements IndexStore { + private static final Logger LOG = LoggerFactory.getLogger(TreeStore.class); + private static final String STORE_TYPE = "TreeStore"; private static final String TREE_STORE_CONFIG = "oak.treeStoreConfig"; - public static final long CACHE_SIZE_NODE_MB = 64; + public static final long CACHE_SIZE_NODE_MB = 128; private static final long CACHE_SIZE_TREE_STORE_MB = 64; private static final long MAX_FILE_SIZE_MB = 8; private static final long MB = 1024 * 1024; + private final String name; private final Store store; + private final long cacheSizeTreeStoreMB; private final File directory; private final Session session; private final NodeStateEntryReader entryReader; private final MemoryBoundCache nodeStateCache; private long entryCount; + private volatile String highestReadKey = ""; - public TreeStore(File directory, NodeStateEntryReader entryReader, boolean smallCache) { + public TreeStore(String name, File directory, NodeStateEntryReader entryReader, boolean smallCache) { + this.name = name; this.directory = directory; this.entryReader = entryReader; long cacheSizeNodeMB = CACHE_SIZE_NODE_MB; long cacheSizeTreeStoreMB = CACHE_SIZE_TREE_STORE_MB; if (smallCache) { - cacheSizeNodeMB = 16; + cacheSizeNodeMB = 32; cacheSizeTreeStoreMB = 32; } + this.cacheSizeTreeStoreMB = cacheSizeTreeStoreMB; nodeStateCache = new MemoryBoundCache<>(cacheSizeNodeMB * MB); String storeConfig = System.getProperty(TREE_STORE_CONFIG, "type=file\n" + @@ -72,6 +81,14 @@ public TreeStore(File directory, NodeStateEntryReader entryReader, boolean small this.session = new Session(store); // we don not want to merge too early during the download session.setMaxRoots(1000); + LOG.info("Open " + toString()); + } + + @Override + public String toString() { + return name + + " cache " + cacheSizeTreeStoreMB + + " at " + highestReadKey; } public Iterator pathIterator() { @@ -91,6 +108,9 @@ private void fetch() { continue; } current = e.getKey(); + if (current.compareTo(highestReadKey) > 0) { + highestReadKey = current; + } return; } current = null; @@ -155,6 +175,10 @@ public NodeStateEntry next() { }; } + public String getHighestReadKey() { + return highestReadKey; + } + public NodeStateEntry getNodeStateEntry(String path) { return new NodeStateEntryBuilder(getNodeState(path), path).build(); } @@ -174,6 +198,9 @@ NodeState getNodeState(String path) { } else { result = getNodeState(path, value); } + if (path.compareTo(highestReadKey) > 0) { + highestReadKey = path; + } nodeStateCache.put(path, result); return result; } @@ -186,6 +213,9 @@ TreeStoreNodeState getNodeState(String path, String value) { String line = path + "|" + value; NodeStateEntry entry = entryReader.read(line); result = new TreeStoreNodeState(entry.getNodeState(), path, this, path.length() * 2 + line.length() * 10); + if (path.compareTo(highestReadKey) > 0) { + highestReadKey = path; + } nodeStateCache.put(path, result); return result; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java index a6a52176457..78c06c15e37 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -38,7 +38,7 @@ public static void main(String... args) throws IOException { String dir = args[0]; MemoryBlobStore blobStore = new MemoryBlobStore(); NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); - try (TreeStore treeStore = new TreeStore(new File(dir), entryReader, false)) { + try (TreeStore treeStore = new TreeStore("utils", new File(dir), entryReader, false)) { Session session = treeStore.getSession(); Store store = treeStore.getStore(); if (store.keySet().isEmpty()) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index 3377afeef80..d527c8ff095 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -71,7 +71,7 @@ public Session(Store store) { private static final long serialVersionUID = 1L; @Override - public boolean removeEldestEntry(Map.Entry eldest) { + public synchronized boolean removeEldestEntry(Map.Entry eldest) { boolean result = super.removeEldestEntry(eldest); if (result) { String key = eldest.getKey(); @@ -839,6 +839,7 @@ private String getNextKey(String largerThan, String fileName) { // garbage collection public void runGC() { + flush(); new GarbageCollection(store).run(getRootFileNames()); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java index 41f8c69ed07..421818a8006 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java @@ -42,6 +42,11 @@ public void setSize(int maxMemoryBytes) { this.maxMemoryBytes = maxMemoryBytes; } + @Override + public synchronized V get(Object key) { + return super.get(key); + } + @Override public synchronized V put(K key, V value) { V old = super.put(key, value); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java index 24b1971e31c..d260430fca1 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -50,6 +51,8 @@ import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider; @@ -532,7 +535,16 @@ static String[] readAllEntriesArray(File dir) throws IOException { } static List readAllEntries(File dir) throws IOException { - TreeStore treeStore = new TreeStore(dir, null, false); + TreeStore treeStore = new TreeStore("test", dir, + new NodeStateEntryReader(new MemoryBlobStore()), false); + Iterator it = treeStore.iterator(); + while (it.hasNext()) { + System.out.println("highest " + treeStore.getHighestReadKey()); + System.out.println("path " + it.next().getPath()); + treeStore.getNodeStateEntry("/zzz"); + } + + ArrayList list = new ArrayList<>(); Session session = treeStore.getSession(); for (String k : session.keys()) { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java new file mode 100644 index 00000000000..565c810292e --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import org.junit.Test; + +public class BlobPrefetcherTest { + + @Test + public void test() { + + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java index b9d2c591cae..b7a7baf3ac0 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -20,12 +20,22 @@ import static org.junit.Assert.assertEquals; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedTreeStoreTask; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; public class TreeStoreTest { + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + @Test - public void test() { + public void convertPathTest() { assertEquals("\t", TreeStore.toChildNodeEntry("/")); assertEquals("/\tabc", TreeStore.toChildNodeEntry("/abc")); assertEquals("/hello\tworld", TreeStore.toChildNodeEntry("/hello/world")); @@ -33,4 +43,22 @@ public void test() { assertEquals("/\tabc", TreeStore.toChildNodeEntry("/", "abc")); assertEquals("/hello\tworld", TreeStore.toChildNodeEntry("/hello", "world")); } + + @Test + public void buildAndIterateTest() throws IOException { + File testFolder = temporaryFolder.newFolder(); + TreeStore store = new TreeStore("test", testFolder, null, true); + try { + store.getSession().init(); + PipelinedTreeStoreTask.addEntry("/", "{}", store.getSession()); + PipelinedTreeStoreTask.addEntry("/content", "{}", store.getSession()); + Iterator it = store.pathIterator(); + assertEquals("/", it.next()); + assertEquals("/content", it.next()); + } finally { + store.close(); + } + } + + } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java index 2aaf00185b3..921461dd8ca 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java @@ -28,6 +28,14 @@ public class MergeRootsTest { + @Test + public void gcTest() { + Store store = StoreBuilder.build(""); + Session session = new Session(store); + session.init(); + session.runGC(); + } + @Test public void simpleTest() { Store store = StoreBuilder.build(""); From 962dab6b8286e5ecad4eeb6b89b2e1e5feaaffc1 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 14 Aug 2024 14:23:53 +0200 Subject: [PATCH 72/86] OAK-10341 Tree store (node prefetch) --- .../document/DocumentStoreIndexerBase.java | 2 +- .../flatfile/FlatFileNodeStoreBuilder.java | 9 +- .../pipelined/PipelinedTreeStoreStrategy.java | 2 +- .../pipelined/PipelinedTreeStoreTask.java | 13 +- .../{BlobPrefetcher.java => Prefetcher.java} | 169 ++++++++++++------ .../indexer/document/tree/TreeStore.java | 57 ++++-- .../document/tree/TreeStoreNodeState.java | 4 +- .../indexer/document/tree/TreeStoreUtils.java | 2 +- .../indexer/document/tree/store/PageFile.java | 5 +- .../indexer/document/tree/store/Session.java | 37 ++-- .../tree/store/utils/MemoryBoundCache.java | 27 +-- .../tree/store/utils/MemoryObject.java} | 20 +-- .../document/tree/store/utils/SieveCache.java | 140 +++++++++++++++ .../pipelined/PipelinedTreeStoreIT.java | 12 +- .../document/tree/StorePrefetcherTest.java | 74 ++++++++ .../indexer/document/tree/TreeStoreTest.java | 2 +- ...moryBoundCacheTest.java => CacheTest.java} | 39 +++- .../tree/store/utils/ConcurrentCacheTest.java | 109 +++++++++++ 18 files changed, 572 insertions(+), 151 deletions(-) rename oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/{BlobPrefetcher.java => Prefetcher.java} (50%) rename oak-run-commons/src/{test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java => main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java} (71%) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SieveCache.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/StorePrefetcherTest.java rename oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/{MemoryBoundCacheTest.java => CacheTest.java} (54%) create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentCacheTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 9336c47cd16..291bfd37cb0 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -316,7 +316,7 @@ public IndexStore buildFlatFileStore() throws IOException, CommitFailedException public void reindexUsingTreeStore() throws CommitFailedException, IOException { NodeStateEntryReader reader = new NodeStateEntryReader(indexHelper.getGCBlobStore()); - TreeStore treeStore = new TreeStore("reindex", new File("target/treeStore"), reader, false); + TreeStore treeStore = new TreeStore("reindex", new File("target/treeStore"), reader, 16); // TODO this is mostly a copy of reindex() diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index 460d49fb019..919fa9623bc 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -63,7 +63,7 @@ import com.mongodb.MongoClientURI; import com.mongodb.client.MongoDatabase; -import org.apache.jackrabbit.oak.index.indexer.document.tree.BlobPrefetcher; +import org.apache.jackrabbit.oak.index.indexer.document.tree.Prefetcher; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; /** @@ -220,11 +220,12 @@ public IndexStore build() throws IOException, CompositeException { // to avoid concurrency issues TreeStore indexingTreeStore = new TreeStore( "indexing", file, - new NodeStateEntryReader(blobStore), false); + new NodeStateEntryReader(blobStore), 10); TreeStore prefetchTreeStore = new TreeStore( "prefetch", file, - new NodeStateEntryReader(blobStore), true); - BlobPrefetcher.prefetch(prefetchTreeStore, indexingTreeStore); + new NodeStateEntryReader(blobStore), 3); + Prefetcher prefetcher = new Prefetcher(prefetchTreeStore, indexingTreeStore); + prefetcher.startPrefetch(); store = indexingTreeStore; } else { store = new FlatFileStore(blobStore, file, metadataFile, diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 2a4a9ef0d3d..8332c542333 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -339,7 +339,7 @@ public File createSortedStoreFile() throws IOException { // Future instances, we can only wait on one of them, so that if any of the others fail, we have no easy way // to detect this failure. ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); - TreeStore treeStore = new TreeStore("dump", getStoreDir(), null, false); + TreeStore treeStore = new TreeStore("dump", getStoreDir(), null, 1); treeStore.getSession().init(); try { // download -> transform thread. diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index 7638bb8def5..60c4a573946 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -53,7 +53,8 @@ public class PipelinedTreeStoreTask implements Callable emptyBuffersQueue; @@ -99,9 +100,13 @@ public Result call() throws Exception { session.mergeRoots(MERGE_BATCH); session.runGC(); } - LOG.info("Final merge; {} roots", session.getRootCount()); - session.mergeRoots(Integer.MAX_VALUE); - session.runGC(); + if (SKIP_FINAL_MERGE) { + LOG.info("Final merge is skipped"); + } else { + LOG.info("Final merge; {} roots", session.getRootCount()); + session.mergeRoots(Integer.MAX_VALUE); + session.runGC(); + } long durationSeconds = start.elapsed(TimeUnit.SECONDS); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, durationSeconds); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, 0); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java similarity index 50% rename from oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcher.java rename to oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java index 83c926f771a..4652635e73c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcher.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java @@ -25,6 +25,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -37,26 +38,43 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class BlobPrefetcher { +/** + * The prefetcher, in a separate threads, reads ahead of indexing, such that the + * nodestore cache and datastore cache is filled. + * + * There are 3 threads (in addition to the threads that read from the datastore): + * - TRACK_INDEXING - tries to be at the same position as the indexing + * - DATASTORE_PREFETCH - reads ahead about 1 GB, such that the datastore cache is filled + * - NODESTORE_CACHE_FILLER - reads ahead about 32'000 entries, such that the node store cache is filled + */ +public class Prefetcher { - private static final Logger LOG = LoggerFactory.getLogger(BlobPrefetcher.class); + private static final Logger LOG = LoggerFactory.getLogger(Prefetcher.class); private static final String BLOB_PREFETCH_PROPERTY_NAME = "oak.index.BlobPrefetchSuffix"; private static final String BLOB_PREFETCH_SUFFIX_DEFAULT = ""; private static final int PRETCH_THREADS = 16; - private static final long DATASTORE_CACHE_SIZE = 16 * 1024 * 1024 * 1024L; - public static void prefetch(TreeStore prefetchStore, TreeStore indexStore) { - String prefix = System.getProperty( + private final TreeStore prefetchStore; + private final TreeStore indexStore; + private final ExecutorService executorService; + private final AtomicLong downloadMax = new AtomicLong(); + private final AtomicLong iterateCount = new AtomicLong(); + private final Semaphore semaphore = new Semaphore(PRETCH_THREADS); + + private String blobSuffix; + + private volatile long blobReadAheadSize = 4 * 1024 * 1024 * 1024L; + private volatile long nodeReadAheadCount = 64 * 1024; + private volatile long maxBlobSize; + + public Prefetcher(TreeStore prefetchStore, TreeStore indexStore) { + blobSuffix = System.getProperty( BLOB_PREFETCH_PROPERTY_NAME, BLOB_PREFETCH_SUFFIX_DEFAULT); - LOG.info("Prefetch prefix '{}', prefetch {}, index {}", - prefix, prefetchStore, indexStore); - if (prefix.isEmpty()) { - return; - } - Semaphore semaphore = new Semaphore(PRETCH_THREADS); - ExecutorService executorService = Executors.newFixedThreadPool(2 + PRETCH_THREADS, new ThreadFactory() { + this.prefetchStore = prefetchStore; + this.indexStore = indexStore; + this.executorService = Executors.newFixedThreadPool(3 + PRETCH_THREADS, new ThreadFactory() { private final AtomicInteger threadNumber = new AtomicInteger(1); @Override @@ -66,53 +84,86 @@ public Thread newThread(Runnable r) { return t; } }); - AtomicLong downloadMax = new AtomicLong(); + } + + public void setBlobSuffix(String blobSuffix) { + this.blobSuffix = blobSuffix; + } + + public void setBlobReadAheadSize(long blobReadAheadSize) { + this.blobReadAheadSize = blobReadAheadSize; + } + + public void setNodeReadAheadCount(long nodeReadAheadCount) { + this.nodeReadAheadCount = nodeReadAheadCount; + } + + public boolean shutdown() throws InterruptedException { + executorService.shutdown(); + return executorService.awaitTermination(1, TimeUnit.SECONDS); + } + + public void startPrefetch() { + LOG.info("Prefetch suffix '{}', prefetch {}, index {}", + blobSuffix, prefetchStore, indexStore); + if (blobSuffix.isEmpty()) { + return; + } + executorService.submit( + iterator(PrefetchType.TRACK_INDEXING)); executorService.submit( - iterator(prefetchStore, indexStore, - null, null, false, downloadMax, prefix)); + iterator(PrefetchType.NODESTORE_CACHE_FILLER)); executorService.submit( - iterator(prefetchStore, indexStore, - executorService, semaphore, true, downloadMax, prefix)); + iterator(PrefetchType.BLOB_PREFETCH)); } - static Runnable iterator(TreeStore prefetchStore, - TreeStore indexStore, - ExecutorService executorService, - Semaphore semaphore, boolean download, - AtomicLong downloadMax, - String prefix) { + public void sleep(String status) throws InterruptedException { + Thread.sleep(10); + } + + Runnable iterator(PrefetchType prefetchType) { return () -> { Iterator it = prefetchStore.pathIterator(); - HyperLogLog estimateUniqueCount = new HyperLogLog(1024, 0); + HyperLogLog estimatedUniqueBlobCount = new HyperLogLog(1024, 0); AtomicLong prefetched = new AtomicLong(); - int count = 0; + long count = 0; try { - long totalCount = 0; - long inlinedCount = 0; - long totalSize = 0; - long maxSize = 0; + long totalBlobCount = 0; + long inlinedBlobCount = 0; + long totalBlobSize = 0; while (it.hasNext()) { String path = it.next(); - if (++count % 1_000_000 == 0) { - int available = semaphore == null ? -1 : semaphore.availablePermits(); - LOG.info("Iterated {} path {} inlinedCount {} totalCount {} " + - "totalSize {} maxSize {} indexing {} download {} max {} available {}", - count, path, inlinedCount, totalCount, - totalSize, maxSize, indexStore.toString(), - download, downloadMax.get(), - available - ); + if (++count % 100_000 == 0) { + int available = semaphore.availablePermits(); + LOG.info("Iterated {} type {} inlinedCount {} totalCount {} " + + "totalSize {} maxSize {} max {} availableThreads {} " + + "indexing {} prefetch {} path {}", + count, prefetchType, inlinedBlobCount, totalBlobCount, + totalBlobSize, maxBlobSize, downloadMax.get(), available, + indexStore.toString(), prefetchStore.toString(), path); } - if (!download) { + if (prefetchType == PrefetchType.TRACK_INDEXING) { + iterateCount.set(count); while (true) { String indexingPath = indexStore.getHighestReadKey(); if (indexingPath.compareTo(path) >= 0) { break; } - Thread.sleep(1000); + sleep("wait for indexing to progress"); } } - if (!path.endsWith(prefix)) { + if (prefetchType == PrefetchType.NODESTORE_CACHE_FILLER) { + while (count - nodeReadAheadCount > iterateCount.get()) { + sleep("wait in node cache fillter"); + } + // this will fill the page cache of the index store + String value = indexStore.getSession().get(path); + // this will not cause a cache miss + TreeStoreNodeState entry = indexStore.buildNodeState(path, value); + indexStore.prefillCache(path, entry); + continue; + } + if (!path.endsWith(blobSuffix)) { continue; } NodeStateEntry nse = prefetchStore.getNodeStateEntry(path); @@ -122,30 +173,30 @@ static Runnable iterator(TreeStore prefetchStore, } Blob blob = p.getValue(Type.BINARY); if (blob.isInlined()) { - inlinedCount++; + inlinedBlobCount++; continue; } - estimateUniqueCount.add(longHash(blob)); - totalCount++; - totalSize += blob.length(); - if (download) { - while (totalSize - DATASTORE_CACHE_SIZE > downloadMax.get()) { - Thread.sleep(1000); - } - } else { - downloadMax.set(totalSize); - } - maxSize = Math.max(maxSize, blob.length()); - if (!download) { + estimatedUniqueBlobCount.add(longHash(blob)); + totalBlobCount++; + totalBlobSize += blob.length(); + maxBlobSize = Math.max(maxBlobSize, blob.length()); + if (prefetchType == PrefetchType.TRACK_INDEXING) { + downloadMax.set(totalBlobSize); continue; } + if (prefetchType != PrefetchType.BLOB_PREFETCH) { + throw new IllegalStateException("Incorrect type: " + prefetchType); + } String indexingPos = indexStore.getHighestReadKey(); if (indexingPos.compareTo(path) >= 0) { // we are behind indexing! // do not download, in order to catch up - LOG.debug("Indexing is ahead; ignoring {}", path); + LOG.info("Indexing is ahead; ignoring {}", path); continue; } + while (totalBlobSize - blobReadAheadSize > downloadMax.get()) { + sleep("wait in downloader"); + } semaphore.acquire(); executorService.submit(() -> { try { @@ -166,7 +217,7 @@ static Runnable iterator(TreeStore prefetchStore, } finally { LOG.info("Completed after {} nodes, {} prefetched, {} unique", count, prefetched.get(), - estimateUniqueCount.estimate()); + estimatedUniqueBlobCount.estimate()); } }; } @@ -180,4 +231,10 @@ private static long longHash(Blob blob) { return Hash.hash64(h | (blob.length() << 32)); } + static enum PrefetchType { + TRACK_INDEXING, + BLOB_PREFETCH, + NODESTORE_CACHE_FILLER + } + } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 10f2e5aea5c..8b42447cb3a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.Map.Entry; +import java.util.concurrent.atomic.AtomicLong; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; @@ -31,7 +32,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SieveCache; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; @@ -44,10 +45,10 @@ public class TreeStore implements IndexStore { private static final String STORE_TYPE = "TreeStore"; private static final String TREE_STORE_CONFIG = "oak.treeStoreConfig"; - public static final long CACHE_SIZE_NODE_MB = 128; + public static final long CACHE_SIZE_NODE_MB = 64; private static final long CACHE_SIZE_TREE_STORE_MB = 64; - private static final long MAX_FILE_SIZE_MB = 8; + private static final long MAX_FILE_SIZE_MB = 2; private static final long MB = 1024 * 1024; private final String name; @@ -56,22 +57,22 @@ public class TreeStore implements IndexStore { private final File directory; private final Session session; private final NodeStateEntryReader entryReader; - private final MemoryBoundCache nodeStateCache; + private final SieveCache nodeStateCache; private long entryCount; private volatile String highestReadKey = ""; + private final AtomicLong nodeCacheHits = new AtomicLong(); + private final AtomicLong nodeCacheMisses = new AtomicLong(); + private final AtomicLong nodeCacheFills = new AtomicLong(); + private int iterationCount; - public TreeStore(String name, File directory, NodeStateEntryReader entryReader, boolean smallCache) { + public TreeStore(String name, File directory, NodeStateEntryReader entryReader, long cacheSizeFactor) { this.name = name; this.directory = directory; this.entryReader = entryReader; - long cacheSizeNodeMB = CACHE_SIZE_NODE_MB; - long cacheSizeTreeStoreMB = CACHE_SIZE_TREE_STORE_MB; - if (smallCache) { - cacheSizeNodeMB = 32; - cacheSizeTreeStoreMB = 32; - } + long cacheSizeNodeMB = cacheSizeFactor * CACHE_SIZE_NODE_MB; + long cacheSizeTreeStoreMB = cacheSizeFactor * CACHE_SIZE_TREE_STORE_MB; this.cacheSizeTreeStoreMB = cacheSizeTreeStoreMB; - nodeStateCache = new MemoryBoundCache<>(cacheSizeNodeMB * MB); + nodeStateCache = new SieveCache<>(cacheSizeFactor * cacheSizeNodeMB * MB); String storeConfig = System.getProperty(TREE_STORE_CONFIG, "type=file\n" + Session.CACHE_SIZE_MB + "=" + cacheSizeTreeStoreMB + "\n" + @@ -88,7 +89,10 @@ public TreeStore(String name, File directory, NodeStateEntryReader entryReader, public String toString() { return name + " cache " + cacheSizeTreeStoreMB + - " at " + highestReadKey; + " at " + highestReadKey + + " cache-hits " + nodeCacheHits.get() + + " cache-misses " + nodeCacheMisses.get() + + " cache-fills " + nodeCacheFills.get(); } public Iterator pathIterator() { @@ -151,10 +155,16 @@ public Iterator iterator() { private void fetch() { while (it.hasNext()) { Entry e = it.next(); + if (++iterationCount % 1_000_000 == 0) { + LOG.info("Fetching {} in {}", iterationCount, TreeStore.this.toString()); + } if (e.getValue().isEmpty()) { continue; } current = getNodeStateEntry(e.getKey(), e.getValue()); + if (current.getPath().compareTo(highestReadKey) > 0) { + highestReadKey = current.getPath(); + } return; } current = null; @@ -190,8 +200,10 @@ NodeStateEntry getNodeStateEntry(String path, String value) { NodeState getNodeState(String path) { TreeStoreNodeState result = nodeStateCache.get(path); if (result != null) { + nodeCacheHits.incrementAndGet(); return result; } + nodeCacheMisses.incrementAndGet(); String value = session.get(path); if (value == null || value.isEmpty()) { result = new TreeStoreNodeState(EmptyNodeState.MISSING_NODE, path, this, path.length() * 2); @@ -208,11 +220,11 @@ NodeState getNodeState(String path) { TreeStoreNodeState getNodeState(String path, String value) { TreeStoreNodeState result = nodeStateCache.get(path); if (result != null) { + nodeCacheHits.incrementAndGet(); return result; } - String line = path + "|" + value; - NodeStateEntry entry = entryReader.read(line); - result = new TreeStoreNodeState(entry.getNodeState(), path, this, path.length() * 2 + line.length() * 10); + nodeCacheMisses.incrementAndGet(); + result = buildNodeState(path, value); if (path.compareTo(highestReadKey) > 0) { highestReadKey = path; } @@ -220,6 +232,19 @@ TreeStoreNodeState getNodeState(String path, String value) { return result; } + TreeStoreNodeState buildNodeState(String path, String value) { + String line = path + "|" + value; + NodeStateEntry entry = entryReader.read(line); + return new TreeStoreNodeState(entry.getNodeState(), path, this, path.length() * 2 + line.length() * 10); + } + + public void prefillCache(String path, TreeStoreNodeState nse) { + TreeStoreNodeState old = nodeStateCache.put(path, nse); + if (old == null) { + nodeCacheFills.incrementAndGet(); + } + } + /** * The child node entry for the given path. * diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java index e6c04a5e3b3..3016feb108d 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java @@ -26,6 +26,7 @@ import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryObject; import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; @@ -34,9 +35,8 @@ import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; -public class TreeStoreNodeState implements NodeState, MemoryBoundCache.MemoryObject { +public class TreeStoreNodeState implements NodeState, MemoryObject { private final NodeState delegate; private final String path; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java index 78c06c15e37..17d0adf5f75 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -38,7 +38,7 @@ public static void main(String... args) throws IOException { String dir = args[0]; MemoryBlobStore blobStore = new MemoryBlobStore(); NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); - try (TreeStore treeStore = new TreeStore("utils", new File(dir), entryReader, false)) { + try (TreeStore treeStore = new TreeStore("utils", new File(dir), entryReader, 16)) { Session session = treeStore.getSession(); Store store = treeStore.getStore(); if (store.keySet().isEmpty()) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java index 707949212d9..8cc5f01b22d 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java @@ -23,14 +23,15 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryObject; /** * A B-tree page (leaf, or inner node). * An inner node contains one more value than keys. * A leaf page has the same number of keys and values. */ -public class PageFile implements MemoryBoundCache.MemoryObject { +public class PageFile implements MemoryObject { private static final boolean VERIFY_SIZE = false; private static final int INITIAL_SIZE_IN_BYTES = 24; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index d527c8ff095..ded3438d246 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -21,7 +21,6 @@ import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.PriorityQueue; @@ -68,21 +67,16 @@ public Session(Store store) { long cacheSizeBytes = cacheSizeMB * 1024 * 1024; LOG.info("Cache size {} bytes", cacheSizeBytes); this.cache = new MemoryBoundCache<>(cacheSizeBytes) { + private static final long serialVersionUID = 1L; @Override - public synchronized boolean removeEldestEntry(Map.Entry eldest) { - boolean result = super.removeEldestEntry(eldest); - if (result) { - String key = eldest.getKey(); - PageFile value = eldest.getValue(); - if (value.isModified()) { - store.put(key, value); - // not strictly needed as it's no longer referenced - value.setModified(false); - } + public void entryWasRemoved(String key, PageFile value) { + if (value.isModified()) { + store.put(key, value); + // not strictly needed as it's no longer referenced + value.setModified(false); } - return result; } }; } @@ -481,7 +475,7 @@ public void checkpoint() { updateId++; } PageFile root = getFile(ROOT_NAME); - cache.remove(ROOT_NAME); + // cache.remove(ROOT_NAME); String rootFileCopy = ROOT_NAME + "_" + updateId; root = copyPageFile(root); root.setFileName(rootFileCopy); @@ -512,16 +506,15 @@ public void flush() { // and points to a page that doesn't exist yet - // but we don't try to ensure this completely; // stored inner nodes might point to pages that are not stored yet - Entry changedRoot = null; - for(Entry e : cache.entrySet()) { - String k = e.getKey(); - PageFile v = e.getValue(); - if (!v.isModified()) { + PageFile changedRoot = null; + for(String k : cache.keys()) { + PageFile v = cache.get(k); + if (v == null || !v.isModified()) { continue; } if (k.equals(ROOT_NAME)) { // don't store the changed root yet - changedRoot = e; + changedRoot = v; } else { store.put(k, v); // here we have to reset the flag @@ -530,11 +523,9 @@ public void flush() { } // now store the changed root if (changedRoot != null) { - String k = changedRoot.getKey(); - PageFile v = changedRoot.getValue(); - store.put(k, v); + store.put(ROOT_NAME, changedRoot); // here we have to reset the flag - v.setModified(false); + changedRoot.setModified(false); } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java index 421818a8006..ba94c9c3864 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java @@ -18,12 +18,14 @@ */ package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; -public class MemoryBoundCache - extends LinkedHashMap { +public class MemoryBoundCache + extends LinkedHashMap { private static final long serialVersionUID = 1L; private volatile long maxMemoryBytes; @@ -47,6 +49,10 @@ public synchronized V get(Object key) { return super.get(key); } + public synchronized Set keys() { + return new HashSet<>(super.keySet()); + } + @Override public synchronized V put(K key, V value) { V old = super.put(key, value); @@ -60,7 +66,7 @@ public synchronized V put(K key, V value) { } @Override - public void putAll(Map map) { + public synchronized void putAll(Map map) { for(Map.Entry e : map.entrySet()) { put(e.getKey(), e.getValue()); } @@ -81,23 +87,18 @@ public synchronized void clear() { memoryUsed.set(0); } + public void entryWasRemoved(K key, V value) { + // nothing to do + } + @Override public synchronized boolean removeEldestEntry(Map.Entry eldest) { boolean removeEldest = memoryUsed.get() > maxMemoryBytes; if (removeEldest) { memoryUsed.addAndGet(-eldest.getValue().estimatedMemory()); + entryWasRemoved(eldest.getKey(), eldest.getValue()); } return removeEldest; } - public interface MemoryObject { - /** - * Get the estimate memory size. The value must not change afterwards, otherwise - * the memory calculation is wrong. - * - * @return the memory in bytes - */ - long estimatedMemory(); - } - } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java similarity index 71% rename from oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java rename to oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java index 565c810292e..4726c9f2b75 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/BlobPrefetcherTest.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java @@ -16,14 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.jackrabbit.oak.index.indexer.document.tree; +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; -import org.junit.Test; - -public class BlobPrefetcherTest { - - @Test - public void test() { - - } -} +public interface MemoryObject { + /** + * Get the estimate memory size. The value must not change afterwards, otherwise + * the memory calculation is wrong. + * + * @return the memory in bytes + */ + long estimatedMemory(); +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SieveCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SieveCache.java new file mode 100644 index 00000000000..5aba5003085 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SieveCache.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Sieve cache. It is faster and has a higher hit-rate than LRU. + * See https://cachemon.github.io/SIEVE-website/ + * + * @param the key type + * @param the value type + */ +public class SieveCache { + + private final ConcurrentHashMap> map = new ConcurrentHashMap<>(); + private final ConcurrentLinkedDeque> queue = new ConcurrentLinkedDeque<>(); + private final AtomicReference>> hand = new AtomicReference<>(); + + private volatile long maxMemoryBytes; + private AtomicLong memoryUsed = new AtomicLong(); + + public SieveCache(long maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + Iterator> it = queue.iterator(); + hand.set(it); + } + + public void setSize(int maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + } + + public V get(K key) { + Value v = map.get(key); + if (v == null) { + return null; + } + v.visited = true; + return v.value; + } + + public Set keys() { + return new HashSet<>(map.keySet()); + } + + public V put(K key, V value) { + Value v = new Value<>(key, value); + memoryUsed.addAndGet(value.estimatedMemory()); + Value old = map.put(key, v); + V result = null; + if (old == null) { + queue.add(v); + } else { + memoryUsed.addAndGet(-old.value.estimatedMemory()); + result = old.value; + } + while (memoryUsed.get() > maxMemoryBytes) { + Value evict; + synchronized (hand) { + if (memoryUsed.get() < maxMemoryBytes || map.isEmpty()) { + break; + } + Iterator> it = hand.get(); + while (true) { + if (it.hasNext()) { + evict = it.next(); + if (!evict.visited) { + break; + } + evict.visited = false; + } else { + Iterator> it2 = queue.iterator(); + it = hand.compareAndExchange(it, it2); + } + } + it.remove(); + evict = map.remove(evict.key); + } + if (evict != null) { + memoryUsed.addAndGet(-evict.value.estimatedMemory()); + entryWasRemoved(evict.key, evict.value); + } + } + return result; + } + + public void entryWasRemoved(K key, V value) { + // nothing to do + } + + public String toString() { + return "cache entries:" + map.size() + + " queue:" + queue.size() + + " max:" + maxMemoryBytes + + " used:" + memoryUsed; + } + + public int size() { + return map.size(); + } + + private static class Value { + private final K key; + private final V value; + volatile boolean visited; + + public Value(K key, V value) { + this.key = key; + this.value = value; + } + + @Override + public String toString() { + return "(" + key + ":" + value + ")"; + } + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java index d260430fca1..b7411de068c 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java @@ -36,7 +36,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -51,7 +50,6 @@ import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.commons.PathUtils; -import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; @@ -536,15 +534,7 @@ static String[] readAllEntriesArray(File dir) throws IOException { static List readAllEntries(File dir) throws IOException { TreeStore treeStore = new TreeStore("test", dir, - new NodeStateEntryReader(new MemoryBlobStore()), false); - Iterator it = treeStore.iterator(); - while (it.hasNext()) { - System.out.println("highest " + treeStore.getHighestReadKey()); - System.out.println("path " + it.next().getPath()); - treeStore.getNodeStateEntry("/zzz"); - } - - + new NodeStateEntryReader(new MemoryBlobStore()), 1); ArrayList list = new ArrayList<>(); Session session = treeStore.getSession(); for (String k : session.keys()) { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/StorePrefetcherTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/StorePrefetcherTest.java new file mode 100644 index 00000000000..69c3b4c21ab --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/StorePrefetcherTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class StorePrefetcherTest { + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + @Test + public void test() throws IOException, InterruptedException { + BlobStore blobStore = null; + NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); + File folder = temporaryFolder.newFolder(); + TreeStore indexStore = new TreeStore("index", + folder, entryReader, 1); + indexStore.getSession().init(); + indexStore.getSession().put("/", "{}"); + indexStore.getSession().put("/content", "{}"); + indexStore.getSession().put("/test.txt", "{}"); + indexStore.getSession().put("/test.txt/jcr:content", "{}"); + indexStore.getSession().checkpoint(); + TreeStore prefetchStore = new TreeStore( + "prefetch", folder, entryReader, 1); + Prefetcher prefetcher = new Prefetcher(prefetchStore, indexStore) { + @Override + public void sleep(String status) throws InterruptedException { + Thread.sleep(0, 1); + } + }; + prefetcher.setBlobReadAheadSize(1); + prefetcher.setNodeReadAheadCount(1); + prefetcher.setBlobSuffix("/test.txt"); + prefetcher.startPrefetch(); + Iterator it = indexStore.iterator(); + while (it.hasNext()) { + Thread.sleep(100); + NodeStateEntry e = it.next(); + assertTrue(e.getPath().compareTo(indexStore.getHighestReadKey()) <= 0); + } + prefetcher.shutdown(); + System.out.println(indexStore.toString()); + assertTrue("Expected shutdown after 1 second", prefetcher.shutdown()); + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java index b7a7baf3ac0..eb59b2b3d9f 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -47,7 +47,7 @@ public void convertPathTest() { @Test public void buildAndIterateTest() throws IOException { File testFolder = temporaryFolder.newFolder(); - TreeStore store = new TreeStore("test", testFolder, null, true); + TreeStore store = new TreeStore("test", testFolder, null, 1); try { store.getSession().init(); PipelinedTreeStoreTask.addEntry("/", "{}", store.getSession()); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java similarity index 54% rename from oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCacheTest.java rename to oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java index 6aacac9432c..76b7875bba5 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCacheTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java @@ -20,30 +20,57 @@ import static org.junit.Assert.assertEquals; +import java.util.concurrent.atomic.AtomicLong; + import org.junit.Test; -public class MemoryBoundCacheTest { +public class CacheTest { @Test - public void test() { + public void memoryBoundCacheTest() { MemoryBoundCache cache = new MemoryBoundCache<>(1000); for (int i = 0; i < 200; i++) { - cache.put("k" + i, new MemoryValue("v" + i)); + cache.put("k" + i, new MemoryValue("v" + i, 10)); } assertEquals(101, cache.size()); } - static class MemoryValue implements MemoryBoundCache.MemoryObject { + @Test + public void sieveCacheTest() { + AtomicLong removed = new AtomicLong(); + SieveCache cache = new SieveCache<>(1000) { + @Override + public void entryWasRemoved(String key, MemoryValue value) { + removed.incrementAndGet(); + } + }; + for (int i = 0; i < 200; i++) { + cache.put("k" + i, new MemoryValue("v" + i, 10)); + } + assertEquals(100, removed.get()); + assertEquals(100, cache.size()); + for (int j = 0; j < 10; j++) { + for (int i = 0; i < 50; i++) { + cache.put("k" + i, new MemoryValue("v" + i, 10)); + } + } + assertEquals(150, removed.get()); + assertEquals(100, cache.size()); + } + + static class MemoryValue implements MemoryObject { final String value; + final int memory; - MemoryValue(String value) { + MemoryValue(String value, int memory) { this.value = value; + this.memory = memory; } @Override public long estimatedMemory() { - return 10; + return memory; } public String toString() { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentCacheTest.java new file mode 100644 index 00000000000..35e91dcaefd --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentCacheTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertTrue; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Test; + +public class ConcurrentCacheTest { + + @Test + public void testRandomOperations() throws Exception { + Random r = new Random(1); + int maxMemory = 1000; +// final MemoryBoundCache cache = +// new MemoryBoundCache<>(maxMemory); + final SieveCache cache = + new SieveCache<>(maxMemory); + final Exception[] ex = new Exception[1]; + int size = 3; + Thread[] threads = new Thread[size]; + final AtomicBoolean stop = new AtomicBoolean(); + for (int i = 0; i < size; i++) { + Thread t = new Thread() { + @Override + public void run() { + while (!stop.get()) { + try { + cache.get(r.nextInt(maxMemory)); + cache.keys(); + cache.put(r.nextInt(maxMemory), new MemoryValue("1", 1 + r.nextInt(10))); + cache.size(); + } catch (Exception e) { + ex[0] = e; + } + } + } + }; + t.start(); + threads[i] = t; + } + try { + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() < start + 1000) { + for (int i = 0; i < 100000 && ex[0] == null; i++) { + cache.put(r.nextInt(maxMemory), new MemoryValue("1", 1 + r.nextInt(10))); + } + } + } finally { + stop.set(true); + for (Thread t : threads) { + t.join(); + } + } + if (ex[0] != null) { + throw ex[0]; + } + int memorySum = 0; + for(Integer k : cache.keys()) { + memorySum += cache.get(k).memory; + } + System.out.println(cache.toString()); + assertTrue(memorySum >= 0); + assertTrue("totalMemory: " + memorySum, memorySum <= maxMemory); + assertTrue(cache.size() >= memorySum / 10); + assertTrue(cache.size() < memorySum); + } + + static class MemoryValue implements MemoryObject { + + final String value; + final int memory; + + MemoryValue(String value, int memory) { + this.value = value; + this.memory = memory; + } + + @Override + public long estimatedMemory() { + return memory; + } + + public String toString() { + return value; + } + + } +} + From 927aafc70594883e4596151c33f0fe6141c9c88b Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 15 Aug 2024 12:33:10 +0200 Subject: [PATCH 73/86] OAK-10341 Tree store (incremental) --- .../jackrabbit/oak/index/IndexOptions.java | 9 +- .../document/DocumentStoreIndexerBase.java | 52 +--- .../pipelined/PipelinedTreeStoreTask.java | 11 +- .../IncrementalStoreBuilder.java | 4 +- .../MergeIncrementalTreeStore.java | 244 ++++++++++++++++++ .../document/indexstore/IndexStoreUtils.java | 19 +- .../indexer/document/tree/TreeStore.java | 9 + .../indexer/document/tree/store/Session.java | 2 +- .../MergeIncrementalTreeStoreTest.java | 134 ++++++++++ .../indexer/document/tree/TreeStoreTest.java | 5 +- .../jackrabbit/oak/index/IndexCommand.java | 14 +- 11 files changed, 420 insertions(+), 83 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java index e4b21efd138..976d544ae15 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java @@ -56,7 +56,6 @@ public class IndexOptions implements OptionsBean { private final OptionSpec docTraversal; private final OptionSpec enableCowCor; private final OptionSpec buildFlatFileStoreSeparately; - private final OptionSpec useTreeStore; private final OptionSpec consistencyCheck; private final OptionSpec asyncDelay; protected OptionSet options; @@ -114,8 +113,6 @@ public IndexOptions(OptionParser parser){ enableCowCor = parser.accepts("enable-cow-cor", "Enables COW/COR during async indexing using oak-run"); buildFlatFileStoreSeparately = parser.accepts("build-flatfilestore-separately", "Builds FlatFileStore as a separate step and then uses it as part of the doc-traversal-mode for reindexing"); - useTreeStore = parser.accepts("use-tree-store", "Use a pre-built tree store"); - indexImportDir = parser.accepts("index-import-dir", "Directory containing index files. This " + "is required when --index-import operation is selected") .requiredIf(importIndex) @@ -141,7 +138,7 @@ public String title() { @Override public String description() { - return "The index command supports the following operations. Most operations are read only.\n" + + return "The index command supports the following operations. Most operations are read only.\n" + "BloStore related options must be provided, as operations access the binaries stored there.\n" + "If no explicit operation is selected, --index-info and --index-definitions operation are performed.\n" + "Use --index-paths to restrict the set of indexes on which the operation needs to be run."; @@ -236,10 +233,6 @@ public boolean buildFlatFileStoreSeparately() { return options.has(buildFlatFileStoreSeparately); } - public boolean useTreeStore() { - return options.has(useTreeStore); - } - public String getCheckpoint(){ return checkpoint.value(options); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 36c1bf60674..9ff9a2eae85 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -33,8 +33,6 @@ import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.IncrementalStoreBuilder; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; -import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.RevisionVector; @@ -240,6 +238,12 @@ nodeStore, getMongoDocumentStore(), traversalLog)) return storeList; } + public IndexStore buildTreeStore() throws IOException, CommitFailedException { + System.setProperty(FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE, + FlatFileNodeStoreBuilder.SortStrategyType.PIPELINED_TREE.name()); + return buildFlatFileStore(); + } + public IndexStore buildStore() throws IOException, CommitFailedException { return buildFlatFileStore(); } @@ -317,50 +321,6 @@ public IndexStore buildFlatFileStore() throws IOException, CommitFailedException return indexStore; } - public void reindexUsingTreeStore() throws CommitFailedException, IOException { - NodeStateEntryReader reader = new NodeStateEntryReader(indexHelper.getGCBlobStore()); - TreeStore treeStore = new TreeStore("reindex", new File("target/treeStore"), reader, 16); - - // TODO this is mostly a copy of reindex() - - IndexingProgressReporter progressReporter = - new IndexingProgressReporter(IndexUpdateCallback.NOOP, NodeTraversalCallback.NOOP); - configureEstimators(progressReporter); - - NodeState checkpointedState = indexerSupport.retrieveNodeStateForCheckpoint(); - NodeStore copyOnWriteStore = new MemoryNodeStore(checkpointedState); - indexerSupport.switchIndexLanesAndReindexFlag(copyOnWriteStore); - - NodeBuilder builder = copyOnWriteStore.getRoot().builder(); - CompositeIndexer indexer = prepareIndexers(copyOnWriteStore, builder, progressReporter); - if (indexer.isEmpty()) { - return; - } - - closer.register(indexer); - - progressReporter.reset(); - - progressReporter.reindexingTraversalStart("/"); - - preIndexOperations(indexer.getIndexers()); - - Stopwatch indexerWatch = Stopwatch.createStarted(); - - for (NodeStateEntry entry : treeStore) { - reportDocumentRead(entry.getPath(), progressReporter); - indexer.index(entry); - } - - progressReporter.reindexingTraversalEnd(); - progressReporter.logReport(); - log.info("Completed the indexing in {}", indexerWatch); - - copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); - - indexerSupport.postIndexWork(copyOnWriteStore); - } - public void reindex() throws CommitFailedException, IOException { INDEXING_PHASE_LOGGER.info("[TASK:FULL_INDEX_CREATION:START] Starting indexing job"); Stopwatch indexJobWatch = Stopwatch.createStarted(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index 60c4a573946..203bcbe099a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -34,7 +34,6 @@ import java.util.concurrent.TimeUnit; import org.apache.jackrabbit.guava.common.base.Stopwatch; -import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedSortBatchTask.Result; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; @@ -208,7 +207,7 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { int valueLength = buffer.getInt(); String value = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), valueLength, StandardCharsets.UTF_8); textSize += entry.getPath().length() + value.length() + 2; - addEntry(entry.getPath(), value, session); + treeStore.putNode(entry.getPath(), value); } session.checkpoint(); unmergedRoots++; @@ -231,12 +230,4 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { } } - public static void addEntry(String path, String data, Session target) { - target.put(path, data); - if (!path.equals("/")) { - String nodeName = PathUtils.getName(path); - String parentPath = PathUtils.getParentPath(path); - target.put(parentPath + "\t" + nodeName, ""); - } - } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java index 13907ab6ea5..e6b09886685 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java @@ -74,7 +74,9 @@ public enum IncrementalSortStrategyType { * Incremental store having nodes updated between initial and final checkpoint */ - INCREMENTAL_FFS_STORE + INCREMENTAL_FFS_STORE, + + INCREMENTAL_TREE_STORE } public IncrementalStoreBuilder(File workDir, IndexHelper indexHelper, diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java new file mode 100644 index 00000000000..985f01ac3ef --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.incrementalstore; + +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreMetadata; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreMetadataOperatorImpl; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + + +public class MergeIncrementalTreeStore implements MergeIncrementalStore { + + private static final String MERGE_BASE_AND_INCREMENTAL_TREE_STORE = "MergeBaseAndIncrementalTreeStore"; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final Logger LOG = LoggerFactory.getLogger(MergeIncrementalTreeStore.class); + + private final File baseDir; + private final File incrementalFile; + private final File mergedDir; + private final Compression algorithm; + + private final static Map OPERATION_MAP = Arrays.stream(IncrementalStoreOperand.values()) + .collect(Collectors.toUnmodifiableMap(IncrementalStoreOperand::toString, k -> IncrementalStoreOperand.valueOf(k.name()))); + + + public MergeIncrementalTreeStore(File baseDir, File incrementalFile, File mergedDir, Compression algorithm) throws IOException { + this.baseDir = baseDir; + this.incrementalFile = incrementalFile; + this.mergedDir = mergedDir; + this.algorithm = algorithm; + if (mergedDir.exists()) { + LOG.warn("merged dir {} exists; it will be replaced", mergedDir.getAbsolutePath()); + } else { + Files.createDirectories(mergedDir.toPath()); + } + } + + @Override + public void doMerge() throws IOException { + LOG.info("base dir " + baseDir.getAbsolutePath()); + LOG.info("incremental file " + incrementalFile.getAbsolutePath()); + LOG.info("merged dir " + mergedDir.getAbsolutePath()); + mergeMetadataFiles(); + mergeIndexStore(); + } + + @Override + public String getStrategyName() { + return MERGE_BASE_AND_INCREMENTAL_TREE_STORE; + } + + /** + * Merges multiple index store files. + * + * This method is a little verbose, but I think this is fine + * as we are not getting consistent data from checkpoint diff + * and we need to handle cases differently. + */ + private void mergeIndexStore() throws IOException { + TreeStore baseStore = new TreeStore("base", baseDir, new NodeStateEntryReader(null), 10); + TreeStore mergedStore = new TreeStore("merged", mergedDir, new NodeStateEntryReader(null), 10); + mergedStore.getSession().init(); + Iterator> baseIt = baseStore.getSession().iterator(); + try (BufferedReader incrementalReader = IndexStoreUtils.createReader(incrementalFile, algorithm)) { + StoreEntry base = StoreEntry.readFromTreeStore(baseIt); + StoreEntry increment = StoreEntry.readFromReader(incrementalReader); + while (base != null || increment != null) { + // which one to advance at the end of the loop + boolean advanceBase, advanceIncrement; + // the entry to write (or null, in case of a delete) + StoreEntry write; + if (base == null) { + // base EOF: we expect ADD + if (increment.operation != IncrementalStoreOperand.ADD) { + LOG.warn( + "Expected ADD but got {} for incremental path {} value {}. " + + "Merging will proceed, but this is unexpected.", + increment.operation, increment.path, increment.value); + } + write = increment; + advanceBase = false; + advanceIncrement = true; + } else if (increment == null) { + // increment EOF: copy from base + write = base; + advanceBase = true; + advanceIncrement = false; + } else { + // both base and increment (normal case) + int compare = base.path.compareTo(increment.path); + if (compare < 0) { + // base path is smaller + write = base; + advanceBase = true; + advanceIncrement = false; + } else if (compare > 0) { + // increment path is smaller: we expect ADD + if (increment.operation != IncrementalStoreOperand.ADD) { + LOG.warn("Expected ADD but got {} for incremental path {} value {}. " + + "Merging will proceed, but this is unexpected.", + increment.operation, increment.path, increment.value); + } + write = increment; + advanceBase = false; + advanceIncrement = true; + } else { + // both paths are the same: we expect modify or delete + write = increment; + advanceBase = true; + advanceIncrement = true; + switch (increment.operation) { + case ADD: + LOG.warn("Expected MODIFY/DELETE but got {} for incremental path {} value {}. " + + "Merging will proceed, but this is unexpected.", + increment.operation, increment.path, increment.value); + break; + case MODIFY: + break; + case DELETE: + write = null; + } + } + } + if (write != null) { + mergedStore.getSession().put(write.path, write.value); + } + if (advanceBase) { + base = StoreEntry.readFromTreeStore(baseIt); + } + if (advanceIncrement) { + increment = StoreEntry.readFromReader(incrementalReader); + } + } + } + baseStore.close(); + mergedStore.getSession().flush(); + mergedStore.close(); + } + + static class StoreEntry { + final String path; + final String value; + final IncrementalStoreOperand operation; + + StoreEntry(String path, String value, IncrementalStoreOperand operation) { + this.path = path; + this.value = value; + this.operation = operation; + } + + static StoreEntry readFromTreeStore(Iterator> it) { + while (it.hasNext()) { + Map.Entry e = it.next(); + if (!e.getValue().isEmpty()) { + return new StoreEntry(e.getKey(), e.getValue(), null); + } + } + return null; + } + + static StoreEntry readFromReader(BufferedReader reader) throws IOException { + String line = reader.readLine(); + if (line == null) { + return null; + } + String[] parts = IncrementalFlatFileStoreNodeStateEntryWriter.getParts(line); + return new StoreEntry(parts[0], parts[1], OPERATION_MAP.get(parts[3])); + } + } + + + private IndexStoreMetadata getIndexStoreMetadataForMergedFile() throws IOException { + File baseFFSMetadataFile = IndexStoreUtils.getMetadataFile(baseDir, algorithm); + File incrementalMetadataFile = IndexStoreUtils.getMetadataFile(incrementalFile, algorithm); + + if (baseFFSMetadataFile.exists() && incrementalMetadataFile.exists()) { + IndexStoreMetadata indexStoreMetadata = new IndexStoreMetadataOperatorImpl() + .getIndexStoreMetadata(baseFFSMetadataFile, algorithm, new TypeReference<>() { + }); + IncrementalIndexStoreMetadata incrementalIndexStoreMetadata = new IndexStoreMetadataOperatorImpl() + .getIndexStoreMetadata(incrementalMetadataFile, algorithm, new TypeReference<>() { + }); + return mergeIndexStores(indexStoreMetadata, incrementalIndexStoreMetadata); + } else { + throw new RuntimeException("either one or both metadataFiles don't exist at path: " + + baseFFSMetadataFile.getAbsolutePath() + ", " + incrementalMetadataFile.getAbsolutePath()); + } + } + + private void mergeMetadataFiles() throws IOException { + try (BufferedWriter writer = IndexStoreUtils.createWriter(IndexStoreUtils.getMetadataFile(mergedDir, algorithm), algorithm)) { + JSON_MAPPER.writeValue(writer, getIndexStoreMetadataForMergedFile()); + } + } + + /** + * We only merge indexStore and incrementalStore if: + * 1. IndexStore's checkpoint equals incrementalStore's before checkpoint. + * 2. IndexStore's preferredPaths equals incrementalStore's preferredPaths. + */ + private IndexStoreMetadata mergeIndexStores(IndexStoreMetadata indexStoreMetadata, + IncrementalIndexStoreMetadata incrementalIndexStoreMetadata) { + checkState(indexStoreMetadata.getCheckpoint().equals(incrementalIndexStoreMetadata.getBeforeCheckpoint())); + return new IndexStoreMetadata(incrementalIndexStoreMetadata.getAfterCheckpoint(), indexStoreMetadata.getStoreType(), + getStrategyName(), Collections.emptySet()); + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java index cdb3c1f78d3..c6170dabaae 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java @@ -117,10 +117,15 @@ public static File getMetadataFile(File indexStoreFile, Compression algorithm) { if (algorithm.equals(Compression.NONE)) { metadataFile = new File(indexStoreFile.getAbsolutePath() + METADATA_SUFFIX); } else { - String fileName = indexStoreFile.getName(); - String compressionSuffix = getCompressionSuffix(indexStoreFile); - checkState(algorithm.addSuffix("").equals(compressionSuffix)); - String fileNameWithoutCompressionSuffix = fileName.substring(0, fileName.lastIndexOf(".")); + String fileNameWithoutCompressionSuffix; + if (indexStoreFile.isFile()) { + String fileName = indexStoreFile.getName(); + String compressionSuffix = getCompressionSuffix(indexStoreFile); + checkState(algorithm.addSuffix("").equals(compressionSuffix)); + fileNameWithoutCompressionSuffix = fileName.substring(0, fileName.lastIndexOf(".")); + } else { + fileNameWithoutCompressionSuffix = indexStoreFile.getName(); + } metadataFile = new File(algorithm.addSuffix(indexStoreFile.getParent() + "/" + fileNameWithoutCompressionSuffix + METADATA_SUFFIX)); } @@ -128,7 +133,11 @@ public static File getMetadataFile(File indexStoreFile, Compression algorithm) { } private static String getCompressionSuffix(File file) { - return file.getName().substring(file.getName().lastIndexOf(".")); + int lastDot = file.getName().lastIndexOf("."); + if (lastDot < 0) { + return ""; + } + return file.getName().substring(lastDot); } /** diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 8b42447cb3a..f0ae214ca35 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -271,6 +271,15 @@ public static String toChildNodeEntry(String parentPath, String childName) { return parentPath + "\t" + childName; } + public void putNode(String path, String json) { + session.put(path, json); + if (!path.equals("/")) { + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + session.put(parentPath + "\t" + nodeName, ""); + } + } + public Session getSession() { return session; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index ded3438d246..6bd4b661787 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -559,7 +559,7 @@ public Iterable> entrySet() { @Override public Iterator> iterator() { - return iterator(); + return Session.this.iterator(); } }; diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java new file mode 100644 index 00000000000..de719333b0c --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile; + +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.MergeIncrementalTreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; + +public class MergeIncrementalTreeStoreTest { + private static final String BUILD_TARGET_FOLDER = "target"; + private static final Compression algorithm = Compression.GZIP; + @Rule + public TemporaryFolder folder = new TemporaryFolder(new File(BUILD_TARGET_FOLDER)); + + @Test + public void test1() throws IOException { + File baseDir = folder.newFolder("base"); + File baseMetadata = folder.newFile("base.metadata.gz"); + + TreeStore base = new TreeStore("base", baseDir, null, 1); + base.getSession().init(); + base.putNode("/tmp", "{prop1=\"foo\"}"); + base.putNode("/tmp/a", "{prop2=\"foo\"}"); + base.putNode("/tmp/a/b", "{prop3=\"foo\"}"); + base.putNode("/tmp/b", "{prop1=\"foo\"}"); + base.putNode("/tmp/b/c", "{prop2=\"foo\"}"); + base.putNode("/tmp/c", "{prop3=\"foo\"}"); + base.close(); + + File increment = folder.newFile("inc.gz"); + File incrementMetadata = folder.newFile("inc.metadata.gz"); + File mergedDir = folder.newFolder("merged"); + File mergedMetadata = folder.newFile("merged.metadata.gz"); + + try (BufferedWriter baseBW = IndexStoreUtils.createWriter(baseMetadata, algorithm)) { + baseBW.write("{\"checkpoint\":\"" + "r0" + "\",\"storeType\":\"TreeStore\"," + + "\"strategy\":\"" + "BaseFFSCreationStrategy" + "\"}"); + baseBW.newLine(); + } + + try (BufferedWriter baseInc = IndexStoreUtils.createWriter(increment, algorithm)) { + baseInc.write("/tmp/a|{prop2=\"fooModified\"}|r1|M"); + baseInc.newLine(); + baseInc.write("/tmp/b|{prop1=\"foo\"}|r1|D"); + baseInc.newLine(); + baseInc.write("/tmp/b/c/d|{prop2=\"fooNew\"}|r1|A"); + baseInc.newLine(); + baseInc.write("/tmp/c|{prop3=\"fooModified\"}|r1|M"); + baseInc.newLine(); + baseInc.write("/tmp/d|{prop3=\"bar\"}|r1|A"); + baseInc.newLine(); + baseInc.write("/tmp/e|{prop3=\"bar\"}|r1|A"); + } + try (BufferedWriter baseInc = IndexStoreUtils.createWriter(incrementMetadata, algorithm)) { + baseInc.write("{\"beforeCheckpoint\":\"" + "r0" + "\",\"afterCheckpoint\":\"" + "r1" + "\"," + + "\"storeType\":\"" + "IncrementalFFSType" + "\"," + + "\"strategy\":\"" + "pipelineStrategy" + "\"," + + "\"preferredPaths\":[]}"); + baseInc.newLine(); + } + + List expectedMergedList = new LinkedList<>(); + + expectedMergedList.add("/tmp|{prop1=\"foo\"}"); + expectedMergedList.add("/tmp/a|{prop2=\"fooModified\"}"); + expectedMergedList.add("/tmp/a/b|{prop3=\"foo\"}"); + expectedMergedList.add("/tmp/b/c|{prop2=\"foo\"}"); + expectedMergedList.add("/tmp/b/c/d|{prop2=\"fooNew\"}"); + expectedMergedList.add("/tmp/c|{prop3=\"fooModified\"}"); + expectedMergedList.add("/tmp/d|{prop3=\"bar\"}"); + expectedMergedList.add("/tmp/e|{prop3=\"bar\"}"); + + MergeIncrementalTreeStore merge = new MergeIncrementalTreeStore( + baseDir, increment, mergedDir, algorithm); + merge.doMerge(); + List expectedMergedMetadataList = new LinkedList<>(); + expectedMergedMetadataList.add("{\"checkpoint\":\"" + "r1" + "\",\"storeType\":\"TreeStore\"," + + "\"strategy\":\"" + merge.getStrategyName() + "\",\"preferredPaths\":[]}"); + + TreeStore merged = new TreeStore("merged", mergedDir, null, 1); + int i = 0; + for (Entry e : merged.getSession().entrySet()) { + if (e.getValue().isEmpty()) { + continue; + } + String expected = expectedMergedList.get(i++); + String actual = e.getKey() + "|" + e.getValue(); + Assert.assertEquals(expected, actual); + } + assertEquals(expectedMergedList.size(), i); + merged.close(); + + try (BufferedReader br = IndexStoreUtils.createReader(mergedMetadata, algorithm)) { + for (String line : expectedMergedMetadataList) { + String actual = br.readLine(); + System.out.println(actual); + Assert.assertEquals(line, actual); + } + Assert.assertNull(br.readLine()); + } + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java index eb59b2b3d9f..3c9debaaea7 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.util.Iterator; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedTreeStoreTask; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -50,8 +49,8 @@ public void buildAndIterateTest() throws IOException { TreeStore store = new TreeStore("test", testFolder, null, 1); try { store.getSession().init(); - PipelinedTreeStoreTask.addEntry("/", "{}", store.getSession()); - PipelinedTreeStoreTask.addEntry("/content", "{}", store.getSession()); + store.putNode("/", "{}"); + store.putNode("/content", "{}"); Iterator it = store.pathIterator(); assertEquals("/", it.next()); assertEquals("/content", it.next()); diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java index 893914421b9..333e994e342 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java @@ -251,16 +251,12 @@ private File reindex(IndexOptions idxOpts, ExtendedIndexHelper extendedIndexHelp if (opts.getCommonOpts().isMongo() && idxOpts.isDocTraversalMode()) { log.info("Using Document order traversal to perform reindexing"); try (DocumentStoreIndexer indexer = new DocumentStoreIndexer(extendedIndexHelper, indexerSupport)) { - if (idxOpts.useTreeStore()) { - indexer.reindexUsingTreeStore(); - } else { - if (idxOpts.buildFlatFileStoreSeparately()) { - IndexStore store = indexer.buildStore(); - String pathToFFS = store.getStorePath(); - System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToFFS); - } - indexer.reindex(); + if (idxOpts.buildFlatFileStoreSeparately()) { + IndexStore store = indexer.buildStore(); + String pathToFFS = store.getStorePath(); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToFFS); } + indexer.reindex(); } } else { try (OutOfBandIndexer indexer = new OutOfBandIndexer(extendedIndexHelper, indexerSupport)) { From 144fde89ee0fca203baef89c7e54ba8ce19c9571 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 16 Aug 2024 12:05:36 +0200 Subject: [PATCH 74/86] OAK-10341 Tree store (pack files) --- .../MergeIncrementalTreeStore.java | 1 - .../indexer/document/tree/Prefetcher.java | 2 +- .../indexer/document/tree/TreeStore.java | 2 +- .../document/tree/store/utils/FilePacker.java | 138 ++++++++++++++ .../indexer/document/tree/TreeStoreTest.java | 2 +- .../tree/store/utils/FilePackerTest.java | 173 ++++++++++++++++++ 6 files changed, 314 insertions(+), 4 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java index 985f01ac3ef..a44e1f123f8 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -43,7 +43,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; - public class MergeIncrementalTreeStore implements MergeIncrementalStore { private static final String MERGE_BASE_AND_INCREMENTAL_TREE_STORE = "MergeBaseAndIncrementalTreeStore"; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java index 4652635e73c..9ecbf8af834 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java @@ -123,7 +123,7 @@ public void sleep(String status) throws InterruptedException { Runnable iterator(PrefetchType prefetchType) { return () -> { - Iterator it = prefetchStore.pathIterator(); + Iterator it = prefetchStore.iteratorOverPaths(); HyperLogLog estimatedUniqueBlobCount = new HyperLogLog(1024, 0); AtomicLong prefetched = new AtomicLong(); long count = 0; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index f0ae214ca35..a1642738596 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -95,7 +95,7 @@ public String toString() { " cache-fills " + nodeCacheFills.get(); } - public Iterator pathIterator() { + public Iterator iteratorOverPaths() { Iterator> it = session.iterator(); return new Iterator() { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java new file mode 100644 index 00000000000..3582a8592de --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class FilePacker { + + /** + * Packs all the files in the source directory into a target file. + * The file header is "PACK". + * + * @param deleteSource whether the source files are deleted while copying + */ + public static void pack(File sourceDirectory, File targetFile, boolean deleteSource) throws IOException { + if (!sourceDirectory.exists() || !sourceDirectory.isDirectory()) { + throw new IOException("Source directory doesn't exist or is a file: " + sourceDirectory.getAbsolutePath()); + } + List list = Files.list(sourceDirectory.toPath()). + map(p -> p.toFile()). + filter(f -> f.isFile()). + map(f -> new FileEntry(f)). + collect(Collectors.toList()); + RandomAccessFile target = new RandomAccessFile(targetFile, "rw"); + target.writeUTF("PACK"); + for (FileEntry f : list) { + target.write(1); + byte[] name = f.fileName.getBytes(StandardCharsets.UTF_8); + target.writeInt(name.length); + target.write(name); + target.writeLong(f.length); + } + target.write(0); + for (FileEntry f : list) { + File file = new File(sourceDirectory, f.fileName); + FileInputStream in = new FileInputStream(file); + in.getChannel().transferTo(0, f.length, target.getChannel()); + in.close(); + if (deleteSource) { + // delete after copying to avoid using twice the space + file.delete(); + } + } + target.close(); + } + + /** + * Unpack a target file target file. The target directory is created if needed. + * Existing files are overwritten. + * + * @param sourceFile the pack file + * @param targetDirectory the target directory + * @param deleteSource whether the source file is truncated while copying, + * and finally deleted. + */ + public static void unpack(File sourceFile, File targetDirectory, boolean deleteSource) throws IOException { + if (!sourceFile.exists() || !sourceFile.isFile()) { + throw new IOException("Source file doesn't exist or is not a file: " + sourceFile.getAbsolutePath()); + } + if (targetDirectory.exists()) { + if (targetDirectory.isFile()) { + throw new IOException("Target file exists: " + targetDirectory.getAbsolutePath()); + } + } else { + targetDirectory.mkdirs(); + } + RandomAccessFile source = new RandomAccessFile(sourceFile, "rw"); + if (!"PACK".equals(source.readUTF())) { + source.close(); + throw new IOException("File header is not 'PACK': " + sourceFile.getAbsolutePath()); + } + ArrayList list = new ArrayList<>(); + long offset = 0; + while (source.read() > 0) { + byte[] name = new byte[source.readInt()]; + source.readFully(name); + long length = source.readLong(); + list.add(new FileEntry(new String(name, StandardCharsets.UTF_8), length, offset)); + offset += length; + } + long start = source.getFilePointer(); + for (int i = list.size() - 1; i >= 0; i--) { + FileEntry f = list.get(i); + source.seek(start + f.offset); + FileOutputStream out = new FileOutputStream(new File(targetDirectory, f.fileName)); + source.getChannel().transferTo(source.getFilePointer(), f.length, out.getChannel()); + out.close(); + if (deleteSource) { + // truncate the source to avoid using twice the space + source.setLength(start + f.offset); + } + } + source.close(); + if (deleteSource) { + sourceFile.delete(); + } + } + + static class FileEntry { + final String fileName; + final long length; + long offset; + FileEntry(String fileName, long length, long offset) { + this.fileName = fileName; + this.length = length; + this.offset = offset; + } + FileEntry(File f) { + this.fileName = f.getName(); + this.length = f.length(); + } + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java index 3c9debaaea7..425cb713915 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -51,7 +51,7 @@ public void buildAndIterateTest() throws IOException { store.getSession().init(); store.putNode("/", "{}"); store.putNode("/content", "{}"); - Iterator it = store.pathIterator(); + Iterator it = store.iteratorOverPaths(); assertEquals("/", it.next()); assertEquals("/content", it.next()); } finally { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java new file mode 100644 index 00000000000..6e2c991bb45 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Random; + +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; +import java.util.zip.CheckedOutputStream; + +public class FilePackerTest { + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + @Test + public void headerMismatch() throws IOException { + File pack = temporaryFolder.newFile("pack"); + File dir = temporaryFolder.newFolder("sourceDir"); + RandomAccessFile f = new RandomAccessFile(pack, "rw"); + f.writeUTF("test"); + try { + FilePacker.unpack(pack, dir, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("File header is not 'PACK'")); + } + f.close(); + pack.delete(); + } + + @Test + public void sourceIsDirectory() throws IOException { + File dir = temporaryFolder.newFolder("source"); + try { + FilePacker.unpack(dir, dir, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("Source file doesn't exist or is not a file")); + } + dir.delete(); + } + + @Test + public void sourceMissing() throws IOException { + File dir = temporaryFolder.newFolder("source"); + try { + FilePacker.unpack(new File(dir, "missing"), dir, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("Source file doesn't exist or is not a file")); + } + dir.delete(); + } + + @Test + public void targetDirectoryIsFile() throws IOException { + File target = temporaryFolder.newFile(); + try { + FilePacker.unpack(target, target, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("Target file exists")); + } + target.delete(); + } + + @Test + public void packUnpack() throws IOException { + packUnpack(false); + packUnpack(true); + } + + public void packUnpack(boolean delete) throws IOException { + File dir = temporaryFolder.newFolder("sourceDir"); + File pack = temporaryFolder.newFile("pack"); + ArrayList list = new ArrayList<>(); + Random r = new Random(1); + for (int i = 0; i < 5; i++) { + FileEntry f = new FileEntry(); + f.file = File.createTempFile("test", ".txt", dir); + list.add(f); + CRC32 crc = new CRC32(); + CheckedOutputStream out = new CheckedOutputStream( + new BufferedOutputStream(new FileOutputStream(f.file)), + crc); + // at most ~3 MB + int s = i * i * 50; + f.length = i * (s * s + r.nextInt(100000)); + for (int j = 0; j < f.length; j++) { + out.write(r.nextInt()); + } + f.contentHash = crc.getValue(); + out.close(); + } + // for debugging + // System.out.println(pack.getAbsolutePath()); + // System.out.println(dir.getAbsolutePath()); + FilePacker.pack(dir, pack, delete); + + for (FileEntry f : list) { + if (delete) { + assertFalse(f.file.exists()); + } else { + f.file.delete(); + } + } + dir.delete(); + + FilePacker.unpack(pack, dir, delete); + if (delete) { + assertFalse(pack.exists()); + } else { + assertTrue(pack.exists()); + } + for (FileEntry f : list) { + if (!f.file.exists()) { + fail(); + } + CRC32 crc = new CRC32(); + CheckedInputStream in = new CheckedInputStream( + new BufferedInputStream(new FileInputStream(f.file)), crc); + while (in.read() >= 0) + ; + in.close(); + assertEquals(f.contentHash, crc.getValue()); + } + // cleanup + for (FileEntry f : list) { + f.file.delete(); + } + dir.delete(); + pack.delete(); + } + + static class FileEntry { + File file; + long length; + long contentHash; + } +} From 64639a584e541fbedd91662a85b3c93df359437c Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 16 Aug 2024 17:20:55 +0200 Subject: [PATCH 75/86] OAK-10341 Tree store (pack files) --- .../MergeIncrementalTreeStore.java | 40 ++++++++++--------- .../document/indexstore/IndexStoreUtils.java | 13 ++---- .../indexer/document/tree/TreeStore.java | 4 ++ .../MergeIncrementalTreeStoreTest.java | 10 ++++- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java index a44e1f123f8..0a02f8eb028 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -24,7 +24,6 @@ import java.io.BufferedWriter; import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; @@ -37,6 +36,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreMetadataOperatorImpl; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,34 +49,36 @@ public class MergeIncrementalTreeStore implements MergeIncrementalStore { private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); private static final Logger LOG = LoggerFactory.getLogger(MergeIncrementalTreeStore.class); - private final File baseDir; + private final File baseFile; private final File incrementalFile; - private final File mergedDir; + private final File mergedFile; private final Compression algorithm; private final static Map OPERATION_MAP = Arrays.stream(IncrementalStoreOperand.values()) .collect(Collectors.toUnmodifiableMap(IncrementalStoreOperand::toString, k -> IncrementalStoreOperand.valueOf(k.name()))); - - public MergeIncrementalTreeStore(File baseDir, File incrementalFile, File mergedDir, Compression algorithm) throws IOException { - this.baseDir = baseDir; + public MergeIncrementalTreeStore(File baseFile, File incrementalFile, File mergedFile, Compression algorithm) throws IOException { + this.baseFile = baseFile; this.incrementalFile = incrementalFile; - this.mergedDir = mergedDir; + this.mergedFile = mergedFile; this.algorithm = algorithm; - if (mergedDir.exists()) { - LOG.warn("merged dir {} exists; it will be replaced", mergedDir.getAbsolutePath()); - } else { - Files.createDirectories(mergedDir.toPath()); - } } @Override public void doMerge() throws IOException { - LOG.info("base dir " + baseDir.getAbsolutePath()); - LOG.info("incremental file " + incrementalFile.getAbsolutePath()); - LOG.info("merged dir " + mergedDir.getAbsolutePath()); + LOG.info("Merging " + baseFile.getAbsolutePath()); + LOG.info("and " + incrementalFile.getAbsolutePath()); + File baseDir = new File(baseFile.getAbsolutePath() + ".files"); + LOG.info("Unpacking into " + baseDir.getAbsolutePath()); + FilePacker.unpack(baseFile, baseDir, true); + LOG.info("Merging " + baseDir.getAbsolutePath()); + File mergedDir = new File(mergedFile.getAbsolutePath() + ".files"); + LOG.info("into " + mergedDir.getAbsolutePath()); mergeMetadataFiles(); - mergeIndexStore(); + mergeIndexStore(baseDir, mergedDir); + LOG.info("Packing into " + mergedFile.getAbsolutePath()); + FilePacker.pack(mergedDir, mergedFile, true); + LOG.info("Done"); } @Override @@ -91,7 +93,7 @@ public String getStrategyName() { * as we are not getting consistent data from checkpoint diff * and we need to handle cases differently. */ - private void mergeIndexStore() throws IOException { + private void mergeIndexStore(File baseDir, File mergedDir) throws IOException { TreeStore baseStore = new TreeStore("base", baseDir, new NodeStateEntryReader(null), 10); TreeStore mergedStore = new TreeStore("merged", mergedDir, new NodeStateEntryReader(null), 10); mergedStore.getSession().init(); @@ -205,7 +207,7 @@ static StoreEntry readFromReader(BufferedReader reader) throws IOException { private IndexStoreMetadata getIndexStoreMetadataForMergedFile() throws IOException { - File baseFFSMetadataFile = IndexStoreUtils.getMetadataFile(baseDir, algorithm); + File baseFFSMetadataFile = IndexStoreUtils.getMetadataFile(baseFile, algorithm); File incrementalMetadataFile = IndexStoreUtils.getMetadataFile(incrementalFile, algorithm); if (baseFFSMetadataFile.exists() && incrementalMetadataFile.exists()) { @@ -223,7 +225,7 @@ private IndexStoreMetadata getIndexStoreMetadataForMergedFile() throws IOExcepti } private void mergeMetadataFiles() throws IOException { - try (BufferedWriter writer = IndexStoreUtils.createWriter(IndexStoreUtils.getMetadataFile(mergedDir, algorithm), algorithm)) { + try (BufferedWriter writer = IndexStoreUtils.createWriter(IndexStoreUtils.getMetadataFile(mergedFile, algorithm), algorithm)) { JSON_MAPPER.writeValue(writer, getIndexStoreMetadataForMergedFile()); } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java index c6170dabaae..5886cef32db 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java @@ -117,15 +117,10 @@ public static File getMetadataFile(File indexStoreFile, Compression algorithm) { if (algorithm.equals(Compression.NONE)) { metadataFile = new File(indexStoreFile.getAbsolutePath() + METADATA_SUFFIX); } else { - String fileNameWithoutCompressionSuffix; - if (indexStoreFile.isFile()) { - String fileName = indexStoreFile.getName(); - String compressionSuffix = getCompressionSuffix(indexStoreFile); - checkState(algorithm.addSuffix("").equals(compressionSuffix)); - fileNameWithoutCompressionSuffix = fileName.substring(0, fileName.lastIndexOf(".")); - } else { - fileNameWithoutCompressionSuffix = indexStoreFile.getName(); - } + String fileName = indexStoreFile.getName(); + String compressionSuffix = getCompressionSuffix(indexStoreFile); + checkState(algorithm.addSuffix("").equals(compressionSuffix)); + String fileNameWithoutCompressionSuffix = fileName.substring(0, fileName.lastIndexOf(".")); metadataFile = new File(algorithm.addSuffix(indexStoreFile.getParent() + "/" + fileNameWithoutCompressionSuffix + METADATA_SUFFIX)); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index a1642738596..bbc47ee6cd5 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -85,6 +85,10 @@ public TreeStore(String name, File directory, NodeStateEntryReader entryReader, LOG.info("Open " + toString()); } + public void init() { + session.init(); + } + @Override public String toString() { return name + diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java index de719333b0c..647bcfb2ef9 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java @@ -22,6 +22,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.MergeIncrementalTreeStore; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -46,10 +47,11 @@ public class MergeIncrementalTreeStoreTest { @Test public void test1() throws IOException { File baseDir = folder.newFolder("base"); + File baseFile = folder.newFile("base.gz"); File baseMetadata = folder.newFile("base.metadata.gz"); TreeStore base = new TreeStore("base", baseDir, null, 1); - base.getSession().init(); + base.init(); base.putNode("/tmp", "{prop1=\"foo\"}"); base.putNode("/tmp/a", "{prop2=\"foo\"}"); base.putNode("/tmp/a/b", "{prop3=\"foo\"}"); @@ -58,8 +60,11 @@ public void test1() throws IOException { base.putNode("/tmp/c", "{prop3=\"foo\"}"); base.close(); + FilePacker.pack(baseDir, baseFile, true); + File increment = folder.newFile("inc.gz"); File incrementMetadata = folder.newFile("inc.metadata.gz"); + File mergedFile = folder.newFile("merged.gz"); File mergedDir = folder.newFolder("merged"); File mergedMetadata = folder.newFile("merged.metadata.gz"); @@ -102,12 +107,13 @@ public void test1() throws IOException { expectedMergedList.add("/tmp/e|{prop3=\"bar\"}"); MergeIncrementalTreeStore merge = new MergeIncrementalTreeStore( - baseDir, increment, mergedDir, algorithm); + baseFile, increment, mergedFile, algorithm); merge.doMerge(); List expectedMergedMetadataList = new LinkedList<>(); expectedMergedMetadataList.add("{\"checkpoint\":\"" + "r1" + "\",\"storeType\":\"TreeStore\"," + "\"strategy\":\"" + merge.getStrategyName() + "\",\"preferredPaths\":[]}"); + FilePacker.unpack(mergedFile, mergedDir, true); TreeStore merged = new TreeStore("merged", mergedDir, null, 1); int i = 0; for (Entry e : merged.getSession().entrySet()) { From a32bf4444e8df6a617324446222223f928397e66 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 20 Aug 2024 09:22:37 +0200 Subject: [PATCH 76/86] OAK-10341 Tree store (tests) --- .../flatfile/pipelined/ConfigHelper.java | 2 +- .../pipelined/PipelinedTreeStoreStrategy.java | 9 +- .../indexer/document/tree/store/Session.java | 10 +- ...oundCache.java => ConcurrentLRUCache.java} | 4 +- .../document/tree/store/utils/TimeUuid.java | 6 +- .../document/tree/store/CompressionTest.java | 47 +++++++ .../document/tree/store/PageFileTest.java | 64 +++++++++ .../document/tree/store/RandomizedTest.java | 95 ++++++++++++++ .../document/tree/store/SessionCacheTest.java | 8 +- .../document/tree/store/utils/CacheTest.java | 2 +- .../tree/store/utils/TimeUuidTest.java | 121 ++++++++++++++++++ .../jackrabbit/oak/index/IndexCommand.java | 4 +- 12 files changed, 356 insertions(+), 16 deletions(-) rename oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/{MemoryBoundCache.java => ConcurrentLRUCache.java} (96%) create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CompressionTest.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFileTest.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuidTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java index cad58706834..bc9121709bc 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java @@ -22,7 +22,7 @@ import org.slf4j.LoggerFactory; public class ConfigHelper { - private static final Logger LOG = LoggerFactory.getLogger(BoundedHistogram.class); + private static final Logger LOG = LoggerFactory.getLogger(ConfigHelper.class); public static int getSystemPropertyAsInt(String name, int defaultValue) { int result = Integer.getInteger(name, defaultValue); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 8332c542333..1906bb137b2 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -338,6 +338,7 @@ public File createSortedStoreFile() throws IOException { // all the tasks, so that if one of them fails, we can abort the whole pipeline. Otherwise, if we wait on // Future instances, we can only wait on one of them, so that if any of the others fail, we have no easy way // to detect this failure. + @SuppressWarnings("rawtypes") ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); TreeStore treeStore = new TreeStore("dump", getStoreDir(), null, 1); treeStore.getSession().init(); @@ -362,6 +363,7 @@ public File createSortedStoreFile() throws IOException { INDEXING_PHASE_LOGGER.info("[TASK:PIPELINED-DUMP:START] Starting to build TreeStore"); Stopwatch start = Stopwatch.createStarted(); + @SuppressWarnings("unchecked") Future downloadFuture = ecs.submit(new PipelinedMongoDownloadTask( mongoClientURI, docStore, @@ -376,7 +378,8 @@ public File createSortedStoreFile() throws IOException { ArrayList> transformFutures = new ArrayList<>(numberOfTransformThreads); for (int i = 0; i < numberOfTransformThreads; i++) { NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(blobStore); - transformFutures.add(ecs.submit(new PipelinedTransformTask( + @SuppressWarnings("unchecked") + Future future = ecs.submit(new PipelinedTransformTask( docStore, documentNodeStore, rootRevision, @@ -386,9 +389,11 @@ public File createSortedStoreFile() throws IOException { emptyBatchesQueue, nonEmptyBatchesQueue, transformStageStatistics - ))); + )); + transformFutures.add(future); } + @SuppressWarnings("unchecked") Future sortBatchFuture = ecs.submit(new PipelinedTreeStoreTask( treeStore, emptyBatchesQueue, diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java index 6bd4b661787..b9bcbe6ad20 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java @@ -26,7 +26,7 @@ import java.util.PriorityQueue; import java.util.Properties; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryBoundCache; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.ConcurrentLRUCache; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Position; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SortedStream; import org.slf4j.Logger; @@ -51,7 +51,7 @@ public class Session { static final boolean MULTI_ROOT = true; private final Store store; - private final MemoryBoundCache cache; + private final ConcurrentLRUCache cache; private long updateId; private int maxRoots = DEFAULT_MAX_ROOTS; private long fileReadCount; @@ -66,7 +66,7 @@ public Session(Store store) { CACHE_SIZE_MB, "" + DEFAULT_CACHE_SIZE_MB)); long cacheSizeBytes = cacheSizeMB * 1024 * 1024; LOG.info("Cache size {} bytes", cacheSizeBytes); - this.cache = new MemoryBoundCache<>(cacheSizeBytes) { + this.cache = new ConcurrentLRUCache<>(cacheSizeBytes) { private static final long serialVersionUID = 1L; @@ -842,6 +842,10 @@ public String getInfo() { GarbageCollection gc = new GarbageCollection(store); int rootId = 0; for (String r : getRootFileNames()) { + if (store.getIfExists(r) == null) { + // not yet stored - ignore + continue; + } buff.append(String.format("root #%d contains %d files (file name %s)\n", rootId, gc.mark(Collections.singletonList(r)).size(), r)); rootId++; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java similarity index 96% rename from oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java rename to oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java index ba94c9c3864..578355824cc 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryBoundCache.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java @@ -24,14 +24,14 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; -public class MemoryBoundCache +public class ConcurrentLRUCache extends LinkedHashMap { private static final long serialVersionUID = 1L; private volatile long maxMemoryBytes; private AtomicLong memoryUsed = new AtomicLong(); - public MemoryBoundCache(long maxMemoryBytes) { + public ConcurrentLRUCache(long maxMemoryBytes) { super(16, 0.75f, true); this.maxMemoryBytes = maxMemoryBytes; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java index 5cbae014936..4614c32a127 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java @@ -43,7 +43,7 @@ public class TimeUuid implements Comparable { // least significant bits private final long lsb; - private TimeUuid(long msb, long lsb) { + public TimeUuid(long msb, long lsb) { this.msb = msb; this.lsb = lsb; } @@ -136,7 +136,7 @@ public boolean equals(Object obj) { * @param lastMillisAndCount the last returned value * @return the new value */ - private static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisAndCount) { + public static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisAndCount) { long result = now << 12; while (true) { long last = lastMillisAndCount.get(); @@ -151,7 +151,7 @@ private static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisA } } - private static TimeUuid newUuid(long millisAndCount, + static TimeUuid newUuid(long millisAndCount, long random) { long millis = millisAndCount >>> 12; long counter = millisAndCount & ((1L << 12) - 1); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CompressionTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CompressionTest.java new file mode 100644 index 00000000000..42406eac875 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CompressionTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Random; + +import org.junit.Test; + +public class CompressionTest { + + @Test + public void randomized() { + Random r = new Random(); + for (int i = 0; i < 2000; i++) { + byte[] data = new byte[r.nextInt(1000)]; + for (int j = 0; j < data.length; j++) { + // less random first, and then more random + data[j] = (byte) r.nextInt(1 + (i / 10)); + } + byte[] comp = Compression.LZ4.compress(data); + byte[] test = Compression.LZ4.expand(comp); + assertEquals(data.length, test.length); + assertTrue(Arrays.equals(data, test)); + } + + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFileTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFileTest.java new file mode 100644 index 00000000000..57122da06f8 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFileTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class PageFileTest { + + @Test + public void serializeLeafNode() { + PageFile f = new PageFile(false, 1_000_000); + f.appendRecord("test", null); + f.appendRecord("", ""); + f.appendRecord(new String(new char[8000]) + "x", new String(new char[80000]) + "y"); + f.appendRecord("a", "b"); + f.setUpdate(-3); + byte[] data = f.toBytes(); + PageFile f2 = PageFile.fromBytes(data, 1_000_000); + assertTrue(f.isInnerNode() == f2.isInnerNode()); + for (int i = 0; i < f.getKeys().size(); i++) { + assertEquals(f.getKey(i), f2.getKey(i)); + assertEquals(f.getValue(i), f2.getValue(i)); + } + assertEquals(f.getUpdate(), f2.getUpdate()); + } + + @Test + public void serializeInnerNode() { + PageFile f = new PageFile(true, 1_000_000); + f.appendRecord("test", null); + f.appendRecord("", ""); + f.appendRecord(new String(new char[8000]) + "x", new String(new char[80000]) + "y"); + f.appendRecord("a", "b"); + f.setUpdate(-3); + byte[] data = f.toBytes(); + PageFile f2 = PageFile.fromBytes(data, 1_000_000); + assertTrue(f.isInnerNode() == f2.isInnerNode()); + for (int i = 0; i < f.getKeys().size(); i++) { + assertEquals(f.getKey(i), f2.getKey(i)); + assertEquals(f.getValue(i), f2.getValue(i)); + } + assertEquals(f.getUpdate(), f2.getUpdate()); + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java new file mode 100644 index 00000000000..287a131b176 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Random; + +import org.junit.Test; + +public class RandomizedTest { + + @Test + public void test() { + Store store = StoreBuilder.build( + "type=memory\n" + + "cacheSizeMB=1\n" + + "maxFileSizeBytes=100"); + Session session = new Session(store); + session.init(); + HashMap verify = new HashMap<>(); + Random r = new Random(1); + boolean verifyAllAlways = false; + + int count = 100000; + int size = 100; + for (int i = 0; i < count; i++) { + String key = "" + r.nextInt(size); + String value; + boolean remove = r.nextBoolean(); + if (remove) { + value = null; + } else { + value = "x" + r.nextInt(size); + } + verify(verify, session, key); + log("#" + i + " put " + key + "=" + (value == null ? "null" : value.toString().replace('\n', ' '))); + verify.put(key, value); + session.put(key, value); + verify(verify, session, key); + if (r.nextInt(size) == 0) { + log("flush"); + session.flush(); + } + if (verifyAllAlways) { + for(String k : verify.keySet()) { + verify(verify, session, k); + } + } + } + + String min = session.getMinKey(); + assertEquals("0", min); + String max = session.getMaxKey(); + assertEquals("99", max); + String median = session.getApproximateMedianKey(min, max); + assertEquals("53", median); + } + + private void verify(HashMap verify, Session session, String key) { + String a = verify.get(key); + String b = session.get(key); + if (a == null || b == null) { + if (a != null) { + fail(key + " a: " + a + " b: " + b); + } + } else { + assertEquals( + key + " a: " + a + " b: " + b, + a.toString(), b.toString()); + } + } + + private void log(String msg) { + // System.out.println(msg); + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java index 8620c33c75d..7bfacd631f6 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java @@ -18,6 +18,8 @@ */ package org.apache.jackrabbit.oak.index.indexer.document.tree.store; +import static org.junit.Assert.assertEquals; + import org.junit.Test; public class SessionCacheTest { @@ -29,9 +31,11 @@ public void test() { Session.CACHE_SIZE_MB + "=1"); Session s = new Session(store); s.init(); - for (int i = 0; i < 1000000; i++) { + for (int i = 0; i < 50_000; i++) { s.put("k" + i, "v" + i); } - System.out.println(s.getInfo()); + s.flush(); + assertEquals("root #0 contains 1756 files (file name root)\n" + + "cache entries:1049 max:1048576 used:1049000", s.getInfo()); } } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java index 76b7875bba5..ba13518be8c 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java @@ -28,7 +28,7 @@ public class CacheTest { @Test public void memoryBoundCacheTest() { - MemoryBoundCache cache = new MemoryBoundCache<>(1000); + ConcurrentLRUCache cache = new ConcurrentLRUCache<>(1000); for (int i = 0; i < 200; i++) { cache.put("k" + i, new MemoryValue("v" + i, 10)); } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuidTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuidTest.java new file mode 100644 index 00000000000..623cc93f883 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuidTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.Test; + +public class TimeUuidTest { + + @Test + public void stringRepresentation() { + TimeUuid u = new TimeUuid(112989318798340096L, -5844840652695650548L); + assertEquals("2024-08-19T15:09:41.859Z 000 aee2f464c7503f0c", u.toHumanReadableString()); + + assertEquals("01916b2fd263000aee2f464c7503f0c", u.toShortString()); + assertEquals(0x1916b2fd263L, u.getTimestampPart()); + assertEquals(0L, u.getCounterPart()); + assertEquals(0xaee2f464c7503f0cL, u.getRandomPart()); + } + + @Test + public void convert() { + Random r = new Random(1); + for (int i = 0; i < 1000; i++) { + long msb = r.nextLong(); + long lsb = r.nextLong(); + TimeUuid a = TimeUuid.newUuid(msb, lsb); + TimeUuid b = TimeUuid.newUuid(msb, lsb); + assertEquals(a.toString(), b.toString()); + } + } + + @Test + public void compare() { + Random r = new Random(1); + TimeUuid lastY = TimeUuid.newUuid(0, 0); + for (int i = 0; i < 1000; i++) { + long a = r.nextBoolean() ? r.nextInt(10) : r.nextLong(); + long b = r.nextBoolean() ? r.nextInt(10) : r.nextLong(); + TimeUuid y = TimeUuid.newUuid(a, b); + assertEquals((int) Math.signum(y.compareTo(lastY)), + (int) Math.signum(y.toString().compareTo(lastY.toString()))); + if (y.compareTo(lastY) == 0) { + assertEquals(y.hashCode(), lastY.hashCode()); + assertTrue(y.equals(lastY)); + } else { + assertFalse(y.equals(lastY)); + } + lastY = y; + } + } + + @Test + public void versionAndVariant() { + TimeUuid x = TimeUuid.newUuid(); + UUID y = new UUID(x.getMostSignificantBits(), x.getLeastSignificantBits()); + assertEquals(7, y.version()); + assertEquals(2, y.variant()); + } + + @Test + public void incremental() { + TimeUuid last = TimeUuid.newUuid(); + for (int i = 0; i < 1000; i++) { + TimeUuid x = TimeUuid.newUuid(); + assertTrue(x.compareTo(last) >= 0); + last = x; + } + } + + @Test + public void getMillisIncreasing() { + AtomicLong lastMillis = new AtomicLong(); + assertEquals((10 << 12) + 0, TimeUuid.getMillisAndCountIncreasing(10, lastMillis)); + assertEquals((10 << 12) + 0, lastMillis.get()); + assertEquals((10 << 12) + 1, TimeUuid.getMillisAndCountIncreasing(9, lastMillis)); + assertEquals((10 << 12) + 1, lastMillis.get()); + assertEquals((11 << 12) + 0, TimeUuid.getMillisAndCountIncreasing(11, lastMillis)); + assertEquals((11 << 12) + 0, lastMillis.get()); + } + + @Test + public void fastGeneration() { + int size = 1 << 14; + ArrayList list = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + list.add(TimeUuid.newUuid()); + } + for (int i = 1; i < size; i++) { + TimeUuid a = list.get(i - 1); + TimeUuid b = list.get(i); + assertFalse(a.equals(b)); + assertTrue(a.compareTo(b) < 0); + } + } + +} diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java index cf5c37df8b8..e845f79abfe 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java @@ -253,8 +253,8 @@ private File reindex(IndexOptions idxOpts, ExtendedIndexHelper extendedIndexHelp try (DocumentStoreIndexer indexer = new DocumentStoreIndexer(extendedIndexHelper, indexerSupport)) { if (idxOpts.buildFlatFileStoreSeparately()) { IndexStore store = indexer.buildStore(); - String pathToFFS = store.getStorePath(); - System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToFFS); + String pathToStore = store.getStorePath(); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToStore); } indexer.reindex(); } From fb515ef90106c6912f93565de7194bfe4f0aeb30 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 22 Aug 2024 12:21:34 +0200 Subject: [PATCH 77/86] OAK-10341 Tree store (incremental) --- .../document/DocumentStoreIndexerBase.java | 2 +- .../pipelined/PipelinedTreeStoreStrategy.java | 5 +- .../pipelined/PipelinedTreeStoreTask.java | 6 +- .../IncrementalStoreBuilder.java | 5 +- .../MergeIncrementalTreeStore.java | 3 +- .../indexer/document/tree/TreeStore.java | 12 +- .../indexer/document/tree/TreeStoreUtils.java | 4 +- .../tree/store/GarbageCollection.java | 2 +- .../store/{Session.java => TreeSession.java} | 16 ++- .../document/tree/store/utils/FilePacker.java | 47 +++++++- .../MergeIncrementalTreeStoreTest.java | 3 +- .../pipelined/PipelinedTreeStoreIT.java | 4 +- .../tree/store/CrudMultiRootTest.java | 12 +- .../document/tree/store/MergeRootsTest.java | 10 +- .../document/tree/store/RandomizedTest.java | 4 +- .../document/tree/store/SessionCacheTest.java | 4 +- .../tree/store/utils/FilePackerTest.java | 5 +- .../oak/index/merge/IndexStoreCommand.java | 112 ++++++++++++++++++ .../jackrabbit/oak/run/AvailableModes.java | 4 +- .../plugins/document/DocumentNodeStore.java | 15 ++- 20 files changed, 224 insertions(+), 51 deletions(-) rename oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/{Session.java => TreeSession.java} (98%) create mode 100644 oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexStoreCommand.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index cdd1ea932a0..0d4edbaf3af 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -316,7 +316,7 @@ public IndexStore buildFlatFileStore() throws IOException, CommitFailedException Predicate predicate = indexerSupport.getFilterPredicate(indexDefinitions, Function.identity()); IndexStore indexStore = buildFlatFileStoreList(checkpointedState, null, predicate, preferredPathElements, IndexerConfiguration.parallelIndexEnabled(), indexDefinitions, indexingReporter).get(0); - log.info("FlatFileStore built at {}. To use this flatFileStore in a reindex step, set System Property-{} with value {}", + log.info("Store built at {}. To use this store in a reindex step, set the system property {} to {}", indexStore.getStorePath(), OAK_INDEXER_SORTED_FILE_PATH, indexStore.getStorePath()); return indexStore; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 1906bb137b2..dbbd0c63948 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -340,7 +340,8 @@ public File createSortedStoreFile() throws IOException { // to detect this failure. @SuppressWarnings("rawtypes") ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); - TreeStore treeStore = new TreeStore("dump", getStoreDir(), null, 1); + File resultDir = getStoreDir(); + TreeStore treeStore = new TreeStore("dump", resultDir, null, 1); treeStore.getSession().init(); try { // download -> transform thread. @@ -491,7 +492,7 @@ public File createSortedStoreFile() throws IOException { throw new RuntimeException(e); } treeStore.close(); - return getStoreDir(); + return resultDir; } finally { LOG.info("Shutting down build FFS thread pool"); new ExecutorCloser(threadPool).close(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index 203bcbe099a..257852aaf0d 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -36,7 +36,7 @@ import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedSortBatchTask.Result; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; import org.apache.jackrabbit.oak.plugins.index.IndexingReporter; import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter; import org.apache.jackrabbit.oak.plugins.index.MetricsUtils; @@ -91,7 +91,7 @@ public Result call() throws Exception { NodeStateEntryBatch nseBuffer = nonEmptyBuffersQueue.take(); if (nseBuffer == SENTINEL_NSE_BUFFER) { synchronized (treeStore) { - Session session = treeStore.getSession(); + TreeSession session = treeStore.getSession(); Stopwatch start = Stopwatch.createStarted(); while (session.getRootCount() > MERGE_BATCH) { LOG.info("Merging {} roots; there are {} roots", @@ -198,7 +198,7 @@ private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { long textSize = 0; batchesProcessed++; synchronized (treeStore) { - Session session = treeStore.getSession(); + TreeSession session = treeStore.getSession(); for (SortKeyPath entry : sortBuffer) { entriesProcessed++; // Retrieve the entry from the buffer diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java index e6b09886685..03565746e60 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java @@ -111,8 +111,9 @@ public IncrementalStoreBuilder withBlobStore(BlobStore blobStore) { public IndexStore build() throws IOException, CompositeException { logFlags(); File dir = createStoreDir(); - - if (Objects.requireNonNull(sortStrategyType) == IncrementalSortStrategyType.INCREMENTAL_FFS_STORE) { + Objects.requireNonNull(sortStrategyType); + if (sortStrategyType == IncrementalSortStrategyType.INCREMENTAL_FFS_STORE || + sortStrategyType == IncrementalSortStrategyType.INCREMENTAL_TREE_STORE) { IncrementalFlatFileStoreNodeStateEntryWriter entryWriter = new IncrementalFlatFileStoreNodeStateEntryWriter(blobStore); IncrementalIndexStoreSortStrategy strategy = new IncrementalFlatFileStoreStrategy( indexHelper.getNodeStore(), diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java index 0a02f8eb028..6883da6014c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -36,6 +36,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreMetadataOperatorImpl; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +78,7 @@ public void doMerge() throws IOException { mergeMetadataFiles(); mergeIndexStore(baseDir, mergedDir); LOG.info("Packing into " + mergedFile.getAbsolutePath()); - FilePacker.pack(mergedDir, mergedFile, true); + FilePacker.pack(mergedDir, TreeSession.getFileNameRegex(), mergedFile, true); LOG.info("Done"); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index bbc47ee6cd5..2ac566e170f 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -29,7 +29,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry.NodeStateEntryBuilder; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SieveCache; @@ -42,6 +42,8 @@ public class TreeStore implements IndexStore { private static final Logger LOG = LoggerFactory.getLogger(TreeStore.class); + public static final String DIRECTORY_NAME = "tree"; + private static final String STORE_TYPE = "TreeStore"; private static final String TREE_STORE_CONFIG = "oak.treeStoreConfig"; @@ -55,7 +57,7 @@ public class TreeStore implements IndexStore { private final Store store; private final long cacheSizeTreeStoreMB; private final File directory; - private final Session session; + private final TreeSession session; private final NodeStateEntryReader entryReader; private final SieveCache nodeStateCache; private long entryCount; @@ -75,11 +77,11 @@ public TreeStore(String name, File directory, NodeStateEntryReader entryReader, nodeStateCache = new SieveCache<>(cacheSizeFactor * cacheSizeNodeMB * MB); String storeConfig = System.getProperty(TREE_STORE_CONFIG, "type=file\n" + - Session.CACHE_SIZE_MB + "=" + cacheSizeTreeStoreMB + "\n" + + TreeSession.CACHE_SIZE_MB + "=" + cacheSizeTreeStoreMB + "\n" + Store.MAX_FILE_SIZE_BYTES + "=" + MAX_FILE_SIZE_MB * MB + "\n" + "dir=" + directory.getAbsolutePath()); this.store = StoreBuilder.build(storeConfig); - this.session = new Session(store); + this.session = new TreeSession(store); // we don not want to merge too early during the download session.setMaxRoots(1000); LOG.info("Open " + toString()); @@ -284,7 +286,7 @@ public void putNode(String path, String json) { } } - public Session getSession() { + public TreeSession getSession() { return session; } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java index 17d0adf5f75..bc0cdb43c99 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -28,7 +28,7 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; @@ -39,7 +39,7 @@ public static void main(String... args) throws IOException { MemoryBlobStore blobStore = new MemoryBlobStore(); NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); try (TreeStore treeStore = new TreeStore("utils", new File(dir), entryReader, 16)) { - Session session = treeStore.getSession(); + TreeSession session = treeStore.getSession(); Store store = treeStore.getStore(); if (store.keySet().isEmpty()) { session.init(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java index ff6f3bbe48c..176cb443e83 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java @@ -53,7 +53,7 @@ HashSet mark(List rootFiles) { private void mark(String fileName, HashSet used) { used.add(fileName); - if (fileName.startsWith(Session.LEAF_PREFIX)) { + if (fileName.startsWith(TreeSession.LEAF_PREFIX)) { return; } // root or inner node diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java similarity index 98% rename from oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java rename to oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java index b9bcbe6ad20..5501a57a691 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Session.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java @@ -35,9 +35,9 @@ /** * Read and write keys and values. */ -public class Session { +public class TreeSession { - private static final Logger LOG = LoggerFactory.getLogger(Session.class); + private static final Logger LOG = LoggerFactory.getLogger(TreeSession.class); public static final String CACHE_SIZE_MB = "cacheSizeMB"; private static final int DEFAULT_CACHE_SIZE_MB = 256; @@ -56,11 +56,17 @@ public class Session { private int maxRoots = DEFAULT_MAX_ROOTS; private long fileReadCount; - public Session() { + public TreeSession() { this(new MemoryStore(new Properties())); } - public Session(Store store) { + public static String getFileNameRegex() { + return "(" + ROOT_NAME + ".*|" + + LEAF_PREFIX + ".*|" + + INNER_NODE_PREFIX + ".*)"; + } + + public TreeSession(Store store) { this.store = store; long cacheSizeMB = Long.parseLong(store.getConfig().getProperty( CACHE_SIZE_MB, "" + DEFAULT_CACHE_SIZE_MB)); @@ -559,7 +565,7 @@ public Iterable> entrySet() { @Override public Iterator> iterator() { - return Session.this.iterator(); + return TreeSession.this.iterator(); } }; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java index 3582a8592de..ee6fcf721ee 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java @@ -31,23 +31,58 @@ public class FilePacker { + /** + * The header of pack files ("PACK"). + */ + public static final String PACK_HEADER = "PACK"; + + public static void main(String... args) throws IOException { + if (args.length <= 2) { + System.out.println("Usage:\n" + + " java -jar target/oak-run-commons-*.jar " + + FilePacker.class.getCanonicalName() + " -d \n" + + " expands a file into the contained files"); + return; + } + if (args[0].equals("-d")) { + File packFile = new File(args[1]); + unpack(packFile, packFile.getParentFile(), false); + } + } + + /** + * Check whether the file starts with the magic header. + * @param file + * @return + * @throws IOException + */ + public static boolean isPackFile(File file) throws IOException { + RandomAccessFile source = new RandomAccessFile(file, "rw"); + byte[] magic = new byte[4]; + source.readFully(magic); + source.close(); + return PACK_HEADER.equals(new String(magic, StandardCharsets.UTF_8)); + } + /** * Packs all the files in the source directory into a target file. - * The file header is "PACK". * + * @param sourceDirectory the source directory + * @param fileNameRegex the file name regular expression * @param deleteSource whether the source files are deleted while copying */ - public static void pack(File sourceDirectory, File targetFile, boolean deleteSource) throws IOException { + public static void pack(File sourceDirectory, String fileNameRegex, File targetFile, boolean deleteSource) throws IOException { if (!sourceDirectory.exists() || !sourceDirectory.isDirectory()) { throw new IOException("Source directory doesn't exist or is a file: " + sourceDirectory.getAbsolutePath()); } List list = Files.list(sourceDirectory.toPath()). map(p -> p.toFile()). filter(f -> f.isFile()). + filter(f -> f.getName().matches(fileNameRegex)). map(f -> new FileEntry(f)). collect(Collectors.toList()); RandomAccessFile target = new RandomAccessFile(targetFile, "rw"); - target.writeUTF("PACK"); + target.write(PACK_HEADER.getBytes(StandardCharsets.UTF_8)); for (FileEntry f : list) { target.write(1); byte[] name = f.fileName.getBytes(StandardCharsets.UTF_8); @@ -90,9 +125,11 @@ public static void unpack(File sourceFile, File targetDirectory, boolean deleteS targetDirectory.mkdirs(); } RandomAccessFile source = new RandomAccessFile(sourceFile, "rw"); - if (!"PACK".equals(source.readUTF())) { + byte[] magic = new byte[4]; + source.readFully(magic); + if (!PACK_HEADER.equals(new String(magic, StandardCharsets.UTF_8))) { source.close(); - throw new IOException("File header is not 'PACK': " + sourceFile.getAbsolutePath()); + throw new IOException("File header is not '" + PACK_HEADER + "': " + sourceFile.getAbsolutePath()); } ArrayList list = new ArrayList<>(); long offset = 0; diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java index 647bcfb2ef9..6ca08001a1c 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java @@ -22,6 +22,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.MergeIncrementalTreeStore; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; import org.junit.Assert; import org.junit.Rule; @@ -60,7 +61,7 @@ public void test1() throws IOException { base.putNode("/tmp/c", "{prop3=\"foo\"}"); base.close(); - FilePacker.pack(baseDir, baseFile, true); + FilePacker.pack(baseDir, TreeSession.getFileNameRegex(), baseFile, true); File increment = folder.newFile("inc.gz"); File incrementMetadata = folder.newFile("inc.metadata.gz"); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java index b7411de068c..3318cc77c28 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java @@ -52,7 +52,7 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; -import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Session; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory; @@ -536,7 +536,7 @@ static List readAllEntries(File dir) throws IOException { TreeStore treeStore = new TreeStore("test", dir, new NodeStateEntryReader(new MemoryBlobStore()), 1); ArrayList list = new ArrayList<>(); - Session session = treeStore.getSession(); + TreeSession session = treeStore.getSession(); for (String k : session.keys()) { String v = session.get(k); if (!v.isEmpty()) { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java index c2b3eab3347..4c7b7a30a39 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java @@ -64,7 +64,7 @@ public void fileLZ4() { public void testAdd(Store store) { store.removeAll(); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); int count = 100; TreeMap map = new TreeMap<>(); @@ -80,7 +80,7 @@ public void testAdd(Store store) { session.mergeRoots(Integer.MAX_VALUE); session.flush(); - session = new Session(store); + session = new TreeSession(store); TreeMap map2 = new TreeMap<>(); for(String k : session.keys()) { map2.put(k, session.get(k)); @@ -90,7 +90,7 @@ public void testAdd(Store store) { public void test(Store store) { store.removeAll(); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); int count = 100; TreeMap verify = new TreeMap<>(); @@ -134,7 +134,7 @@ public void test(Store store) { previous = k; } - session = new Session(store); + session = new TreeSession(store); for (int i = 0; i < count; i++) { String key = "hello" + i; assertEquals("world" + i, session.get(key)); @@ -142,7 +142,7 @@ public void test(Store store) { } session.flush(); - session = new Session(store); + session = new TreeSession(store); for (int i = 0; i < count; i++) { String key = "hello" + i; assertEquals("World " + i, session.get(key)); @@ -150,7 +150,7 @@ public void test(Store store) { } session.flush(); - session = new Session(store); + session = new TreeSession(store); for (int i = 0; i < count; i++) { String key = "hello" + i; assertEquals(null, session.get(key)); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java index 921461dd8ca..a7295f19125 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java @@ -31,7 +31,7 @@ public class MergeRootsTest { @Test public void gcTest() { Store store = StoreBuilder.build(""); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); session.runGC(); } @@ -39,7 +39,7 @@ public void gcTest() { @Test public void simpleTest() { Store store = StoreBuilder.build(""); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); for (int i = 0; i < 10; i++) { session.put("x" + i, "y" + i); @@ -57,7 +57,7 @@ public void simpleTest() { @Test public void multipleRootAppendTest() { Store store = StoreBuilder.build(""); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); for(int j = 1; j <= 3; j++) { for (int i = 0; i < 10 * j; i++) { @@ -81,7 +81,7 @@ public void multipleRootAppendTest() { @Test public void multipleRootRandomOverwriteTest() { Store store = StoreBuilder.build(""); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); TreeMap map = new TreeMap<>(); Random r = new Random(42); @@ -111,7 +111,7 @@ public void logStructuredMerge() { Store store = StoreBuilder.build( "type=memory\n" + "maxFileSizeBytes=10000"); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); for (int batch = 1; batch <= 200; batch++) { // System.out.println("batch " + batch); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java index 287a131b176..67e92056aa4 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java @@ -34,7 +34,7 @@ public void test() { "type=memory\n" + "cacheSizeMB=1\n" + "maxFileSizeBytes=100"); - Session session = new Session(store); + TreeSession session = new TreeSession(store); session.init(); HashMap verify = new HashMap<>(); Random r = new Random(1); @@ -75,7 +75,7 @@ public void test() { assertEquals("53", median); } - private void verify(HashMap verify, Session session, String key) { + private void verify(HashMap verify, TreeSession session, String key) { String a = verify.get(key); String b = session.get(key); if (a == null || b == null) { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java index 7bfacd631f6..95e4303a4ec 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java @@ -28,8 +28,8 @@ public class SessionCacheTest { public void test() { Store store = StoreBuilder.build("type:memory\n" + Store.MAX_FILE_SIZE_BYTES + "=1000\n" + - Session.CACHE_SIZE_MB + "=1"); - Session s = new Session(store); + TreeSession.CACHE_SIZE_MB + "=1"); + TreeSession s = new TreeSession(store); s.init(); for (int i = 0; i < 50_000; i++) { s.put("k" + i, "v" + i); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java index 6e2c991bb45..25fdc19b67e 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java @@ -33,6 +33,7 @@ import java.util.ArrayList; import java.util.Random; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -110,7 +111,7 @@ public void packUnpack(boolean delete) throws IOException { Random r = new Random(1); for (int i = 0; i < 5; i++) { FileEntry f = new FileEntry(); - f.file = File.createTempFile("test", ".txt", dir); + f.file = File.createTempFile("root_", ".txt", dir); list.add(f); CRC32 crc = new CRC32(); CheckedOutputStream out = new CheckedOutputStream( @@ -128,7 +129,7 @@ public void packUnpack(boolean delete) throws IOException { // for debugging // System.out.println(pack.getAbsolutePath()); // System.out.println(dir.getAbsolutePath()); - FilePacker.pack(dir, pack, delete); + FilePacker.pack(dir, TreeSession.getFileNameRegex(), pack, delete); for (FileEntry f : list) { if (delete) { diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexStoreCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexStoreCommand.java new file mode 100644 index 00000000000..c28a0e8a6d3 --- /dev/null +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexStoreCommand.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import static java.util.Arrays.asList; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.LZ4Compression; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; +import org.apache.jackrabbit.oak.run.commons.Command; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +public class IndexStoreCommand implements Command { + + public final static String INDEX_STORE = "index-store"; + + @SuppressWarnings("unchecked") + @Override + public void execute(String... args) throws IOException { + OptionParser parser = new OptionParser(); + OptionSpec helpSpec = parser.acceptsAll( + asList("h", "?", "help"), "show help").forHelp(); + OptionSet options = parser.parse(args); + parser.nonOptions( + "An index store file").ofType(File.class); + if (options.has(helpSpec) + || options.nonOptionArguments().isEmpty()) { + System.out.println("Mode: " + INDEX_STORE); + System.out.println(); + parser.printHelpOn(System.out); + return; + } + for (String fileName : ((List) options.nonOptionArguments())) { + File file = new File(fileName); + if (!file.exists()) { + System.out.println("File not found: " + fileName); + return; + } + if (FilePacker.isPackFile(file)) { + File treeFile = new File(file.getAbsoluteFile().getParent(), "tree"); + treeFile.mkdirs(); + FilePacker.unpack(file, treeFile, false); + file = treeFile; + } + if (file.isDirectory()) { + System.out.println("Tree store " + fileName); + listTreeStore(file); + } else if (file.isFile()) { + System.out.println("Flat file " + fileName); + listFlatFile(file); + } + } + } + + public void listFlatFile(File file) throws IOException { + BufferedReader reader; + if (file.getName().endsWith(".lz4")) { + reader = IndexStoreUtils.createReader(file, new LZ4Compression()); + } else if (file.getName().endsWith(".gz")) { + reader = IndexStoreUtils.createReader(file, Compression.GZIP); + } else { + reader = IndexStoreUtils.createReader(file, Compression.NONE); + } + while (true) { + String line = reader.readLine(); + if (line == null) { + break; + } + System.out.println(line); + } + reader.close(); + } + + public static void listTreeStore(File directory) throws IOException { + TreeStore treeStore = new TreeStore("tree", directory, null, 1); + Iterator it = treeStore.iteratorOverPaths(); + while (it.hasNext()) { + String path = it.next(); + String node = treeStore.getSession().get(path); + System.out.println(path + "|" + node); + } + treeStore.close(); + } + +} diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java index 6c723cb93f1..cb6f6e34fde 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java @@ -23,6 +23,7 @@ import org.apache.jackrabbit.oak.exporter.NodeStateExportCommand; import org.apache.jackrabbit.oak.index.IndexCommand; import org.apache.jackrabbit.oak.index.merge.IndexDiffCommand; +import org.apache.jackrabbit.oak.index.merge.IndexStoreCommand; import org.apache.jackrabbit.oak.run.commons.Command; import org.apache.jackrabbit.oak.run.commons.Modes; @@ -51,8 +52,9 @@ public final class AvailableModes { .put("garbage", new GarbageCommand()) .put("help", new HelpCommand()) .put("history", new HistoryCommand()) - .put("index-merge", new IndexMergeCommand()) .put("index-diff", new IndexDiffCommand()) + .put("index-merge", new IndexMergeCommand()) + .put(IndexStoreCommand.INDEX_STORE, new IndexStoreCommand()) .put(IndexCommand.NAME, new IndexCommand()) .put(IOTraceCommand.NAME, new IOTraceCommand()) .put(JsonIndexCommand.INDEX, new JsonIndexCommand()) diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java index 62c60d55d71..c6579aa1f92 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java @@ -3405,9 +3405,18 @@ private String diffImpl(AbstractDocumentNodeState from, AbstractDocumentNodeStat fromRev = from.getRootRevision(); toRev = to.getRootRevision(); } catch (RuntimeException e) { - LOG.warn("diffJournalChildren failed with " + - e.getClass().getSimpleName() + - ", falling back to classic diff", e); + // avoid filling the log file with stack traces for a known issue + // see OAK-6016 and OAK-6011 + if (e instanceof IllegalStateException && + "Root document does not have a lastRev entry for local clusterId 0".equals(e.getMessage())) { + LOG.warn("diffJournalChildren failed with " + + e.getClass().getSimpleName() + + ", falling back to classic diff"); + } else { + LOG.warn("diffJournalChildren failed with " + + e.getClass().getSimpleName() + + ", falling back to classic diff", e); + } } } if (diff == null) { From 7a31307818050896c9eefb9aac226dd42e0c8436 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 23 Aug 2024 15:10:54 +0200 Subject: [PATCH 78/86] OAK-10341 Tree store (compress) --- .../jackrabbit/oak/index/indexer/document/tree/TreeStore.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 2ac566e170f..649263d4d41 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -30,6 +30,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Compression; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SieveCache; @@ -50,7 +51,7 @@ public class TreeStore implements IndexStore { public static final long CACHE_SIZE_NODE_MB = 64; private static final long CACHE_SIZE_TREE_STORE_MB = 64; - private static final long MAX_FILE_SIZE_MB = 2; + private static final long MAX_FILE_SIZE_MB = 4; private static final long MB = 1024 * 1024; private final String name; @@ -81,6 +82,7 @@ public TreeStore(String name, File directory, NodeStateEntryReader entryReader, Store.MAX_FILE_SIZE_BYTES + "=" + MAX_FILE_SIZE_MB * MB + "\n" + "dir=" + directory.getAbsolutePath()); this.store = StoreBuilder.build(storeConfig); + store.setWriteCompression(Compression.LZ4); this.session = new TreeSession(store); // we don not want to merge too early during the download session.setMaxRoots(1000); From 6776860cb643b2409034447d17e23e3554d2be2e Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 26 Aug 2024 15:00:39 +0200 Subject: [PATCH 79/86] OAK-10341 Tree store (traverse included paths) --- .../document/DocumentStoreIndexerBase.java | 1 + .../flatfile/FlatFileNodeStoreBuilder.java | 8 +- .../document/tree/PathIteratorFilter.java | 129 ++++++++++++++++++ .../indexer/document/tree/Prefetcher.java | 2 +- .../indexer/document/tree/TreeStore.java | 78 +++++++---- .../document/tree/PathIteratorFilterTest.java | 122 +++++++++++++++++ .../indexer/document/tree/TreeStoreTest.java | 67 +++++++++ 7 files changed, 378 insertions(+), 29 deletions(-) create mode 100644 oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java create mode 100644 oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilterTest.java diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 0d4edbaf3af..0b2ed63e83c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -66,6 +66,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index 919fa9623bc..ea7397c1c83 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -216,14 +216,18 @@ public IndexStore build() throws IOException, CompositeException { File file = indexStoreFiles.storeFiles.get(0); IndexStore store; if (file.isDirectory()) { - // use a separate tree store (with a smaller cache) - // to avoid concurrency issues TreeStore indexingTreeStore = new TreeStore( "indexing", file, new NodeStateEntryReader(blobStore), 10); + indexingTreeStore.setIndexDefinitions(indexDefinitions); + + // use a separate tree store (with a smaller cache) + // for prefetching, to avoid cache evictions TreeStore prefetchTreeStore = new TreeStore( "prefetch", file, new NodeStateEntryReader(blobStore), 3); + prefetchTreeStore.setIndexDefinitions(indexDefinitions); + Prefetcher prefetcher = new Prefetcher(prefetchTreeStore, indexingTreeStore); prefetcher.startPrefetch(); store = indexingTreeStore; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java new file mode 100644 index 00000000000..ec6f31708a6 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; + +public class PathIteratorFilter { + + private final boolean includeAll; + private final TreeSet includedPaths; + + private String cachedMatchingPrefix; + + PathIteratorFilter(SortedSet includedPaths) { + this.includedPaths = new TreeSet<>(includedPaths); + this.includeAll = includedPaths.contains(PathUtils.ROOT_PATH); + } + + public PathIteratorFilter() { + this.includedPaths = new TreeSet<>(); + this.includeAll = true; + } + + /** + * Extract all the path filters from a set of index definitions. + * + * @param indexDefs the index definitions + * @return the list of path filters + */ + public static List extractPathFilters(Set indexDefs) { + return indexDefs.stream().map(IndexDefinition::getPathFilter).collect(Collectors.toList()); + } + + /** + * Extract a list of included paths from a path filter. Only the top-most + * entries are retained. Excluded path are ignored. + * + * @param pathFilters the path filters + * @return the set of included path, sorted by path + */ + public static SortedSet getAllIncludedPaths(List pathFilters) { + TreeSet set = new TreeSet<>(); + // convert to a flat set + for (PathFilter f : pathFilters) { + for (String p : f.getIncludedPaths()) { + set.add(p); + } + } + // only keep entries where the parent isn't in the set + TreeSet result = new TreeSet<>(); + for (String path : set) { + boolean parentExists = false; + String p = path; + while (!PathUtils.denotesRoot(p)) { + p = PathUtils.getParentPath(p); + if (set.contains(p)) { + parentExists = true; + break; + } + } + if (!parentExists) { + result.add(path); + } + } + return result; + } + + public boolean includes(String path) { + if (includeAll) { + return true; + } + String cache = cachedMatchingPrefix; + if (cache != null && path.startsWith(cache)) { + return true; + } + String p = path; + while (!PathUtils.denotesRoot(p)) { + if (includedPaths.contains(p)) { + // add a final slash, so that we only accept children + String newCache = p; + if (!newCache.endsWith("/")) { + newCache += "/"; + } + cachedMatchingPrefix = newCache; + return true; + } + p = PathUtils.getParentPath(p); + } + return false; + } + + /** + * Get the next higher included path, or null if none. + * + * @param path the path + * @return the next included path, or null + */ + public String nextIncludedPath(String path) { + if (includeAll) { + return null; + } + return includedPaths.higher(path); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java index 9ecbf8af834..e5ecade5061 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java @@ -133,7 +133,7 @@ Runnable iterator(PrefetchType prefetchType) { long totalBlobSize = 0; while (it.hasNext()) { String path = it.next(); - if (++count % 100_000 == 0) { + if (++count % 1_000_000 == 0) { int available = semaphore.availablePermits(); LOG.info("Iterated {} type {} inlinedCount {} totalCount {} " + "totalSize {} maxSize {} max {} availableThreads {} " + diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 649263d4d41..0d6a8cc6d2e 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -21,7 +21,10 @@ import java.io.File; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedSet; import java.util.concurrent.atomic.AtomicLong; import org.apache.jackrabbit.oak.commons.PathUtils; @@ -34,7 +37,9 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SieveCache; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,6 +72,7 @@ public class TreeStore implements IndexStore { private final AtomicLong nodeCacheMisses = new AtomicLong(); private final AtomicLong nodeCacheFills = new AtomicLong(); private int iterationCount; + private PathIteratorFilter filter = new PathIteratorFilter(); public TreeStore(String name, File directory, NodeStateEntryReader entryReader, long cacheSizeFactor) { this.name = name; @@ -93,6 +99,16 @@ public void init() { session.init(); } + public void setIndexDefinitions(Set indexDefs) { + if (indexDefs == null) { + return; + } + List pathFilters = PathIteratorFilter.extractPathFilters(indexDefs); + SortedSet includedPaths = PathIteratorFilter.getAllIncludedPaths(pathFilters); + LOG.info("Included paths {}", includedPaths.toString()); + filter = new PathIteratorFilter(includedPaths); + } + @Override public String toString() { return name + @@ -104,9 +120,10 @@ public String toString() { } public Iterator iteratorOverPaths() { - Iterator> it = session.iterator(); + final Iterator> firstIterator = session.iterator(); return new Iterator() { + Iterator> it = firstIterator; String current; { @@ -116,10 +133,34 @@ public Iterator iteratorOverPaths() { private void fetch() { while (it.hasNext()) { Entry e = it.next(); - if (e.getValue().isEmpty()) { + String key = e.getKey(); + String value = e.getValue(); + // if the value is empty (not null!) this is a child node reference, + // without node data + if (value.isEmpty()) { continue; } - current = e.getKey(); + if (!filter.includes(key)) { + // if the path is not, see if there is a next included path + String next = filter.nextIncludedPath(key); + if (next == null) { + // it was the last one + break; + } + // this node itself could be a match + key = next; + value = session.get(key); + // for the next fetch operation, we use the new iterator + it = session.iterator(next); + if (value == null || value.isEmpty()) { + // no such node, or it's a child node reference + continue; + } + } + if (++iterationCount % 1_000_000 == 0) { + LOG.info("Fetching {} in {}", iterationCount, TreeStore.this.toString()); + } + current = key; if (current.compareTo(highestReadKey) > 0) { highestReadKey = current; } @@ -143,15 +184,9 @@ public String next() { }; } - @Override - public void close() throws IOException { - session.flush(); - store.close(); - } - @Override public Iterator iterator() { - Iterator> it = session.iterator(); + Iterator it = iteratorOverPaths(); return new Iterator() { NodeStateEntry current; @@ -161,21 +196,7 @@ public Iterator iterator() { } private void fetch() { - while (it.hasNext()) { - Entry e = it.next(); - if (++iterationCount % 1_000_000 == 0) { - LOG.info("Fetching {} in {}", iterationCount, TreeStore.this.toString()); - } - if (e.getValue().isEmpty()) { - continue; - } - current = getNodeStateEntry(e.getKey(), e.getValue()); - if (current.getPath().compareTo(highestReadKey) > 0) { - highestReadKey = current.getPath(); - } - return; - } - current = null; + current = it.hasNext() ? getNodeStateEntry(it.next()) : null; } @Override @@ -193,6 +214,12 @@ public NodeStateEntry next() { }; } + @Override + public void close() throws IOException { + session.flush(); + store.close(); + } + public String getHighestReadKey() { return highestReadKey; } @@ -317,7 +344,6 @@ public String getIndexStoreType() { @Override public boolean isIncremental() { - // TODO support diff and use Session.checkpoint() / flush(). return false; } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilterTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilterTest.java new file mode 100644 index 00000000000..940672a5759 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilterTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.junit.Test; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; + +public class PathIteratorFilterTest { + + @Test + public void convert() { + List list = new ArrayList<>(); + // no includes + assertEquals("[]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // root + list.add(new PathFilter(List.of("/"), Collections.emptyList())); + assertEquals("[/]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // root is higher than /content, so we only need to retain root + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + assertEquals("[/]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + list.clear(); + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + assertEquals("[/content]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // /content is higher than /content/abc, so we only keep /content + list.add(new PathFilter(List.of("/content/abc"), Collections.emptyList())); + assertEquals("[/content]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // /libs is new + list.add(new PathFilter(List.of("/lib"), Collections.emptyList())); + assertEquals("[/content, /lib]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // root overrides everything + list.add(new PathFilter(List.of("/"), Collections.emptyList())); + assertEquals("[/]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + } + + @Test + public void emptySet() { + List list = new ArrayList<>(); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertFalse(filter.includes("/")); + assertNull(filter.nextIncludedPath("/")); + } + + @Test + public void all() { + List list = new ArrayList<>(); + list.add(new PathFilter(List.of("/"), Collections.emptyList())); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertTrue(filter.includes("/")); + assertTrue(filter.includes("/content")); + assertTrue(filter.includes("/var")); + assertNull(filter.nextIncludedPath("/")); + } + + @Test + public void content() { + List list = new ArrayList<>(); + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertFalse(filter.includes("/")); + assertEquals("/content", filter.nextIncludedPath("/")); + assertTrue(filter.includes("/content")); + assertTrue(filter.includes("/content/abc")); + assertTrue(filter.includes("/content/def")); + assertFalse(filter.includes("/var")); + assertNull(filter.nextIncludedPath("/var")); + } + + @Test + public void contentAndEtc() { + List list = new ArrayList<>(); + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + list.add(new PathFilter(List.of("/etc"), Collections.emptyList())); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertFalse(filter.includes("/")); + assertEquals("/content", filter.nextIncludedPath("/")); + assertTrue(filter.includes("/content")); + assertTrue(filter.includes("/content/abc")); + assertTrue(filter.includes("/content/def")); + assertFalse(filter.includes("/content1")); + assertEquals("/etc", filter.nextIncludedPath("/content1")); + assertTrue(filter.includes("/etc")); + assertTrue(filter.includes("/etc/test")); + assertFalse(filter.includes("/tmp")); + assertNull(filter.nextIncludedPath("/tmp")); + + assertEquals("/content", filter.nextIncludedPath("/")); + assertEquals("/etc", filter.nextIncludedPath("/content1")); + assertNull(filter.nextIncludedPath("/etc")); + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java index 425cb713915..6c26d03277d 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -19,14 +19,30 @@ package org.apache.jackrabbit.oak.index.indexer.document.tree; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import java.io.File; import java.io.IOException; +import java.util.HashSet; import java.util.Iterator; +import java.util.Set; +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.OakInitializer; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.plugins.name.NamespaceEditorProvider; +import org.apache.jackrabbit.oak.plugins.nodetype.TypeEditorProvider; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfo; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfoProvider; +import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; public class TreeStoreTest { @@ -54,10 +70,61 @@ public void buildAndIterateTest() throws IOException { Iterator it = store.iteratorOverPaths(); assertEquals("/", it.next()); assertEquals("/content", it.next()); + assertFalse(it.hasNext()); } finally { store.close(); } } + @Test + public void includedPathTest() throws IOException { + File testFolder = temporaryFolder.newFolder(); + TreeStore store = new TreeStore("test", testFolder, null, 1); + try { + store.getSession().init(); + store.putNode("/", "{}"); + store.putNode("/content", "{}"); + store.putNode("/content/abc", "{}"); + store.putNode("/jcr:system", "{}"); + store.putNode("/jcr:system/jcr:version", "{}"); + store.putNode("/var/abc", "{}"); + store.putNode("/var/def", "{}"); + store.putNode("/zero", "{}"); + + Set defs = inMemoryIndexDefinitions("/content", "/var", "/tmp"); + store.setIndexDefinitions(defs); + + Iterator it = store.iteratorOverPaths(); + assertEquals("/content", it.next()); + assertEquals("/content/abc", it.next()); + assertEquals("/var/abc", it.next()); + assertEquals("/var/def", it.next()); + assertFalse(it.hasNext()); + } finally { + store.close(); + } + } + + private static Set inMemoryIndexDefinitions(String... includedPaths) { + NodeStore store = new MemoryNodeStore(); + EditorHook hook = new EditorHook( + new CompositeEditorProvider(new NamespaceEditorProvider(), new TypeEditorProvider())); + OakInitializer.initialize(store, new InitialContent(), hook); + + Set defns = new HashSet<>(); + IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder(); + defnBuilder.includedPaths(includedPaths); + defnBuilder.indexRule("dam:Asset"); + defnBuilder.aggregateRule("dam:Asset"); + IndexDefinition defn = IndexDefinition.newBuilder(store.getRoot(), defnBuilder.build(), "/foo").build(); + defns.add(defn); + + NodeTypeInfoProvider mockNodeTypeInfoProvider = Mockito.mock(NodeTypeInfoProvider.class); + NodeTypeInfo mockNodeTypeInfo = Mockito.mock(NodeTypeInfo.class, "dam:Asset"); + Mockito.when(mockNodeTypeInfo.getNodeTypeName()).thenReturn("dam:Asset"); + Mockito.when(mockNodeTypeInfoProvider.getNodeTypeInfo("dam:Asset")).thenReturn(mockNodeTypeInfo); + + return defns; + } } From 7f80b987e3a70b38460414e49efca06a2575f003 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 27 Aug 2024 14:52:44 +0200 Subject: [PATCH 80/86] OAK-10341 Tree store --- .../oak/index/indexer/document/DocumentStoreIndexerBase.java | 1 - .../document/flatfile/pipelined/PipelinedTransformTask.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 0b2ed63e83c..0d4edbaf3af 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -66,7 +66,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java index fd8701a22bc..9a4cda22b26 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java @@ -160,7 +160,7 @@ public Result call() throws Exception { for (NodeDocument nodeDoc : nodeDocumentBatch) { statistics.incrementMongoDocumentsTraversed(); mongoObjectsProcessed++; - if (mongoObjectsProcessed % 50000 == 0) { + if (mongoObjectsProcessed % 200_000 == 0) { LOG.info("Mongo objects: {}, total entries: {}, current batch: {}, Size: {}/{} MB", mongoObjectsProcessed, totalEntryCount, nseBatch.numberOfEntries(), nseBatch.sizeOfEntriesBytes() / FileUtils.ONE_MB, From 0d522d7e677d15bdc1ea9303241a112d88f00895 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 29 Aug 2024 15:27:17 +0200 Subject: [PATCH 81/86] OAK-10341 Tree store --- .../java/org/apache/jackrabbit/oak/index/IndexOptions.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java index 976d544ae15..90e4b8b7a40 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java @@ -138,7 +138,7 @@ public String title() { @Override public String description() { - return "The index command supports the following operations. Most operations are read only.\n" + + return "The index command supports the following operations. Most operations are read only.\n" + "BloStore related options must be provided, as operations access the binaries stored there.\n" + "If no explicit operation is selected, --index-info and --index-definitions operation are performed.\n" + "Use --index-paths to restrict the set of indexes on which the operation needs to be run."; @@ -272,5 +272,4 @@ private static Set collectionOperationNames(Set actionOpts){ } return result; } - } From d312ca8c1b4d35e8b02d21ab4f15cd9e82822fdd Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 30 Aug 2024 09:20:06 +0200 Subject: [PATCH 82/86] OAK-10341 Tree store --- .../MergeIncrementalTreeStore.java | 2 +- .../indexer/document/tree/TreeStore.java | 17 +++++++++++ .../MergeIncrementalTreeStoreTest.java | 12 ++++++-- .../indexer/document/tree/TreeStoreTest.java | 30 +++++++++++++++++++ 4 files changed, 58 insertions(+), 3 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java index 6883da6014c..181cfda60c5 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -160,7 +160,7 @@ private void mergeIndexStore(File baseDir, File mergedDir) throws IOException { } } if (write != null) { - mergedStore.getSession().put(write.path, write.value); + mergedStore.putNode(write.path, write.value); } if (advanceBase) { base = StoreEntry.readFromTreeStore(baseIt); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 0d6a8cc6d2e..39610209ef1 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -295,6 +295,23 @@ public static String toChildNodeEntry(String path) { return parentPath + "\t" + nodeName; } + /** + * Convert a child node entry to parent and node name. + * This method is used for tooling and testing only. + * It does the reverse of toChildNodeEntry(parentPath, childName) + * + * @param child node entry, e.g. /helloworld + * @return the parent path and the child node name, e.g. ["/hello" "world"] + * @throws IllegalArgumentException if this is not a child node entry + */ + public static String[] toParentAndChildNodeName(String key) { + int index = key.lastIndexOf('\t'); + if (index < 0) { + throw new IllegalArgumentException("Not a child node entry: " + key); + } + return new String[] { key.substring(0, index), key.substring(index + 1) }; + } + /** * The child node entry for the given parent and child. * diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java index 6ca08001a1c..9ccb49804ae 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java @@ -30,11 +30,13 @@ import org.junit.rules.TemporaryFolder; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; import java.io.IOException; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; @@ -46,7 +48,7 @@ public class MergeIncrementalTreeStoreTest { public TemporaryFolder folder = new TemporaryFolder(new File(BUILD_TARGET_FOLDER)); @Test - public void test1() throws IOException { + public void merge() throws IOException { File baseDir = folder.newFolder("base"); File baseFile = folder.newFile("base.gz"); File baseMetadata = folder.newFile("base.metadata.gz"); @@ -116,16 +118,22 @@ public void test1() throws IOException { FilePacker.unpack(mergedFile, mergedDir, true); TreeStore merged = new TreeStore("merged", mergedDir, null, 1); + HashSet paths = new HashSet<>(); int i = 0; for (Entry e : merged.getSession().entrySet()) { + String key = e.getKey(); if (e.getValue().isEmpty()) { + String[] parts = TreeStore.toParentAndChildNodeName(key); + String childEntry = TreeStore.toChildNodeEntry(parts[0], parts[1]); + assertTrue(paths.add(childEntry)); continue; } String expected = expectedMergedList.get(i++); - String actual = e.getKey() + "|" + e.getValue(); + String actual = key + "|" + e.getValue(); Assert.assertEquals(expected, actual); } assertEquals(expectedMergedList.size(), i); + assertEquals(expectedMergedList.size(), paths.size()); merged.close(); try (BufferedReader br = IndexStoreUtils.createReader(mergedMetadata, algorithm)) { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java index 6c26d03277d..8009cfda932 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; @@ -59,6 +60,35 @@ public void convertPathTest() { assertEquals("/hello\tworld", TreeStore.toChildNodeEntry("/hello", "world")); } + @Test + public void convertPathAndReverseTest() { + // normal case + String path = "/hello"; + String childNodeName = "world"; + String key = TreeStore.toChildNodeEntry(path, childNodeName); + assertEquals("/hello\tworld", key); + String[] parts = TreeStore.toParentAndChildNodeName(key); + assertEquals(path, parts[0]); + assertEquals(childNodeName, parts[1]); + + // root node + path = "/"; + childNodeName = "test"; + key = TreeStore.toChildNodeEntry(path, childNodeName); + parts = TreeStore.toParentAndChildNodeName(key); + assertEquals(path, parts[0]); + assertEquals(childNodeName, parts[1]); + + // failure case + key = "/"; + try { + parts = TreeStore.toParentAndChildNodeName(key); + fail(); + } catch (IllegalArgumentException e) { + // expected + } + } + @Test public void buildAndIterateTest() throws IOException { File testFolder = temporaryFolder.newFolder(); From a1bf3142dc78a87995dcb9a6e2b5a9f9fae436a9 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 3 Sep 2024 09:26:23 +0200 Subject: [PATCH 83/86] OAK-10341 Tree store --- .../incrementalstore/MergeIncrementalTreeStore.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java index 181cfda60c5..2e87d49e727 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -67,19 +67,17 @@ public MergeIncrementalTreeStore(File baseFile, File incrementalFile, File merge @Override public void doMerge() throws IOException { - LOG.info("Merging " + baseFile.getAbsolutePath()); - LOG.info("and " + incrementalFile.getAbsolutePath()); + LOG.info("Merging {} and {}", baseFile.getAbsolutePath(), incrementalFile.getAbsolutePath()); File baseDir = new File(baseFile.getAbsolutePath() + ".files"); - LOG.info("Unpacking into " + baseDir.getAbsolutePath()); + LOG.info("Unpacking to {}", baseDir.getAbsolutePath()); FilePacker.unpack(baseFile, baseDir, true); - LOG.info("Merging " + baseDir.getAbsolutePath()); File mergedDir = new File(mergedFile.getAbsolutePath() + ".files"); - LOG.info("into " + mergedDir.getAbsolutePath()); + LOG.info("Merging to {}", mergedDir.getAbsolutePath()); mergeMetadataFiles(); mergeIndexStore(baseDir, mergedDir); - LOG.info("Packing into " + mergedFile.getAbsolutePath()); + LOG.info("Packing to {}", mergedFile.getAbsolutePath()); FilePacker.pack(mergedDir, TreeSession.getFileNameRegex(), mergedFile, true); - LOG.info("Done"); + LOG.info("Completed"); } @Override From cee6866c67cbec2395b4ffda60ceed5859ebaeaf Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 4 Sep 2024 14:45:22 +0200 Subject: [PATCH 84/86] OAK-10341 Tree store (use same blob prefetching configuration as for FlatFileStore) --- ...eadOfTimeBlobDownloadingFlatFileStore.java | 10 ++++- .../flatfile/FlatFileNodeStoreBuilder.java | 43 ++++++++++++------- .../indexer/document/tree/Prefetcher.java | 14 ++---- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java index 767737dd898..849f1d0bc60 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java @@ -82,7 +82,15 @@ private AheadOfTimeBlobDownloadingFlatFileStore(FlatFileStore ffs, CompositeInde } } - static boolean isEnabledForIndexes(String indexesEnabledPrefix, List indexPaths) { + /** + * Whether blob downloading is needed for the given indexes. + * + * @param indexesEnabledPrefix the comma-separated list of prefixes of the index + * definitions that benefit from the download + * @param indexPaths the index paths + * @return true if any of the indexes start with any of the prefixes + */ + public static boolean isEnabledForIndexes(String indexesEnabledPrefix, List indexPaths) { List enableForIndexes = splitAndTrim(indexesEnabledPrefix); for (String indexPath : indexPaths) { if (enableForIndexes.stream().anyMatch(indexPath::startsWith)) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index a50c6c42213..14706e7d4cb 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -42,6 +42,7 @@ import org.apache.jackrabbit.oak.index.indexer.document.CompositeException; import org.apache.jackrabbit.oak.index.indexer.document.CompositeIndexer; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedTreeStoreStrategy; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; @@ -235,21 +236,7 @@ public IndexStore build(IndexHelper indexHelper, CompositeIndexer indexer) throw File file = indexStoreFiles.storeFiles.get(0); IndexStore store; if (file.isDirectory()) { - TreeStore indexingTreeStore = new TreeStore( - "indexing", file, - new NodeStateEntryReader(blobStore), 10); - indexingTreeStore.setIndexDefinitions(indexDefinitions); - - // use a separate tree store (with a smaller cache) - // for prefetching, to avoid cache evictions - TreeStore prefetchTreeStore = new TreeStore( - "prefetch", file, - new NodeStateEntryReader(blobStore), 3); - prefetchTreeStore.setIndexDefinitions(indexDefinitions); - - Prefetcher prefetcher = new Prefetcher(prefetchTreeStore, indexingTreeStore); - prefetcher.startPrefetch(); - store = indexingTreeStore; + store = buildTreeStoreForIndexing(indexHelper, file); } else { store = new FlatFileStore(blobStore, file, metadataFile, new NodeStateEntryReader(blobStore), @@ -268,6 +255,32 @@ public IndexStore build(IndexHelper indexHelper, CompositeIndexer indexer) throw return store; } + public IndexStore buildTreeStoreForIndexing(IndexHelper indexHelper, File file) { + TreeStore indexingTreeStore = new TreeStore( + "indexing", file, + new NodeStateEntryReader(blobStore), 10); + indexingTreeStore.setIndexDefinitions(indexDefinitions); + + // use a separate tree store (with a smaller cache) + // for prefetching, to avoid cache evictions + TreeStore prefetchTreeStore = new TreeStore( + "prefetch", file, + new NodeStateEntryReader(blobStore), 3); + prefetchTreeStore.setIndexDefinitions(indexDefinitions); + String blobPrefetchEnableForIndexes = ConfigHelper.getSystemPropertyAsString( + AheadOfTimeBlobDownloadingFlatFileStore.BLOB_PREFETCH_ENABLE_FOR_INDEXES_PREFIXES, ""); + Prefetcher prefetcher = new Prefetcher(prefetchTreeStore, indexingTreeStore); + String blobSuffix = ""; + if (AheadOfTimeBlobDownloadingFlatFileStore.isEnabledForIndexes( + blobPrefetchEnableForIndexes, indexHelper.getIndexPaths())) { + blobSuffix = ConfigHelper.getSystemPropertyAsString( + AheadOfTimeBlobDownloadingFlatFileStore.BLOB_PREFETCH_BINARY_NODES_SUFFIX, ""); + } + prefetcher.setBlobSuffix(blobSuffix); + prefetcher.startPrefetch(); + return indexingTreeStore; + } + public List buildList(IndexHelper indexHelper, IndexerSupport indexerSupport, Set indexDefinitions) throws IOException, CompositeException { logFlags(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java index e5ecade5061..779227244bc 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java @@ -51,8 +51,6 @@ public class Prefetcher { private static final Logger LOG = LoggerFactory.getLogger(Prefetcher.class); - private static final String BLOB_PREFETCH_PROPERTY_NAME = "oak.index.BlobPrefetchSuffix"; - private static final String BLOB_PREFETCH_SUFFIX_DEFAULT = ""; private static final int PRETCH_THREADS = 16; private final TreeStore prefetchStore; @@ -69,9 +67,6 @@ public class Prefetcher { private volatile long maxBlobSize; public Prefetcher(TreeStore prefetchStore, TreeStore indexStore) { - blobSuffix = System.getProperty( - BLOB_PREFETCH_PROPERTY_NAME, - BLOB_PREFETCH_SUFFIX_DEFAULT); this.prefetchStore = prefetchStore; this.indexStore = indexStore; this.executorService = Executors.newFixedThreadPool(3 + PRETCH_THREADS, new ThreadFactory() { @@ -106,15 +101,14 @@ public boolean shutdown() throws InterruptedException { public void startPrefetch() { LOG.info("Prefetch suffix '{}', prefetch {}, index {}", blobSuffix, prefetchStore, indexStore); - if (blobSuffix.isEmpty()) { - return; - } executorService.submit( iterator(PrefetchType.TRACK_INDEXING)); executorService.submit( iterator(PrefetchType.NODESTORE_CACHE_FILLER)); - executorService.submit( - iterator(PrefetchType.BLOB_PREFETCH)); + if (!blobSuffix.isEmpty()) { + executorService.submit( + iterator(PrefetchType.BLOB_PREFETCH)); + } } public void sleep(String status) throws InterruptedException { From 95cd63eda0472d4326651492162b82836635a3b6 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Thu, 5 Sep 2024 17:05:35 +0200 Subject: [PATCH 85/86] OAK-10341 Tree store (javadocs) --- .../document/tree/PathIteratorFilter.java | 7 +++++++ .../indexer/document/tree/TreeStore.java | 5 +++++ .../document/tree/TreeStoreNodeState.java | 6 ++++++ .../indexer/document/tree/TreeStoreUtils.java | 21 +++++++++++++++++++ .../document/tree/store/Compression.java | 3 +++ .../document/tree/store/FileStore.java | 6 ++++++ .../tree/store/GarbageCollection.java | 3 +++ .../indexer/document/tree/store/LogStore.java | 3 +++ .../document/tree/store/MemoryStore.java | 3 +++ .../document/tree/store/SlowStore.java | 4 ++++ .../document/tree/store/StatsStore.java | 4 ++++ .../indexer/document/tree/store/Store.java | 2 +- .../document/tree/store/StoreBuilder.java | 3 +++ .../document/tree/store/TreeSession.java | 2 +- .../tree/store/utils/ConcurrentLRUCache.java | 7 +++++++ .../document/tree/store/utils/FilePacker.java | 11 ++++++++++ .../tree/store/utils/MemoryObject.java | 4 ++++ .../document/tree/store/utils/Position.java | 5 +++++ .../tree/store/utils/SortedStream.java | 6 ++++++ 19 files changed, 103 insertions(+), 2 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java index ec6f31708a6..8cf3bafc0f3 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java @@ -28,6 +28,13 @@ import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.apache.jackrabbit.oak.spi.filter.PathFilter; +/** + * A utility class that allows skipping nodes that are not included in the index + * definition. + * + * The use case is to speed up indexing by only traversing over the nodes that + * are included in the set of indexes. + */ public class PathIteratorFilter { private final boolean includeAll; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 39610209ef1..c3b4f63e509 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -44,6 +44,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * The tree store is similar to the flat file store, but instead of storing all + * key-value pairs in a single file, it stores the entries in multiple files + * (except if there are very few nodes). + */ public class TreeStore implements IndexStore { private static final Logger LOG = LoggerFactory.getLogger(TreeStore.class); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java index 3016feb108d..72da39575ed 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java @@ -36,6 +36,12 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +/** + * A node state of an Oak node that is stored in a tree store. + * + * This is mostly a wrapper. It allows iterating over the children and reading + * children directly. + */ public class TreeStoreNodeState implements NodeState, MemoryObject { private final NodeState delegate; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java index bc0cdb43c99..f2eb586b622 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.jackrabbit.oak.index.indexer.document.tree; import java.io.BufferedReader; @@ -32,6 +50,9 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; +/** + * A command line utility for the tree store. + */ public class TreeStoreUtils { public static void main(String... args) throws IOException { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java index 83a14363a52..5d922e6d9c2 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java @@ -24,6 +24,9 @@ import net.jpountz.lz4.LZ4Factory; import net.jpountz.lz4.LZ4FastDecompressor; +/** + * The enum allows to disable or enable compression of the storage files. + */ public enum Compression { NO { @Override diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java index 35369420198..5894f589267 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java @@ -33,6 +33,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * A storage backend for the tree store that stores files on the local file + * system. + * + * This is the main (read-write) storage backend. + */ public class FileStore implements Store { private static final Logger LOG = LoggerFactory.getLogger(FileStore.class); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java index 176cb443e83..a1ec6c3a608 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java @@ -23,6 +23,9 @@ /** * Remove unreferenced files from the store. + * + * Only root files and inner nodes are read. This is possible because leaf pages + * do not have references to other files. */ public class GarbageCollection { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java index ae47cdfebf0..8224b23bea5 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java @@ -22,6 +22,9 @@ import java.util.Properties; import java.util.Set; +/** + * A wrapper for storage backends that allows to log store and read operations. + */ public class LogStore implements Store { private final Properties config; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java index 146c04fcdee..9e7c238cab3 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java @@ -22,6 +22,9 @@ import java.util.Properties; import java.util.Set; +/** + * An in-memory storage backend for the tree store. + */ public class MemoryStore implements Store { private final Properties config; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java index f931e4beca7..28a0b69d0c9 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java @@ -21,6 +21,10 @@ import java.util.Properties; import java.util.Set; +/** + * A wrapper store to simulate a slow backend store. It can be used for + * simulations. It is not intended to be used for real-world situations. + */ public class SlowStore implements Store { private final Properties config; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java index 601ba775555..8f375efe8b3 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java @@ -24,6 +24,10 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +/** + * A wrapper store that allows capturing performance counters for a storage + * backend. + */ public class StatsStore implements Store { private final Properties config; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java index 5696b826f7b..101a706c40e 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java @@ -22,7 +22,7 @@ import java.util.Set; /** - * Storage for files. + * Storage for files in a tree store. */ public interface Store { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java index c390c2a496b..0aaada8d266 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java @@ -22,6 +22,9 @@ import java.io.StringReader; import java.util.Properties; +/** + * A helper class to build storage backends for a tree store. + */ public class StoreBuilder { /** diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java index 5501a57a691..a1ecef2bcc6 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; /** - * Read and write keys and values. + * A session that allows reading and writing keys and values in a tree store. */ public class TreeSession { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java index 578355824cc..9883d9b621d 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java @@ -24,6 +24,13 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; +/** + * A synchronized LRU cache. The cache size is limited by the amount of memory + * (and not number of entries). + * + * @param the key type + * @param the value type + */ public class ConcurrentLRUCache extends LinkedHashMap { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java index ee6fcf721ee..36c2038b668 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java @@ -29,6 +29,17 @@ import java.util.List; import java.util.stream.Collectors; +/** + * A utility class that allows converting the files of a tree store into one + * file (pack the files), and back from a file to a list of files (unpack the + * files). This is a bit similar to a zip file, however + * + * - each entry is already compressed, so no additional compression is needed; + * - only files in the same directory can be processed; + * - the pack file starts with a header that contains the list of files; + * - while packing, the files are (optionally) deleted, so that this doesn't require twice the disk space; + * - while unpacking, the pack file is (optionally) truncated, also to conserve disk space. + */ public class FilePacker { /** diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java index 4726c9f2b75..99ed6cb8ea6 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java @@ -18,7 +18,11 @@ */ package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; +/** + * A interface for memory-bound cache objects. + */ public interface MemoryObject { + /** * Get the estimate memory size. The value must not change afterwards, otherwise * the memory calculation is wrong. diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java index 2fc6b6e51ad..f50cb060c90 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java @@ -20,6 +20,11 @@ import org.apache.jackrabbit.oak.index.indexer.document.tree.store.PageFile; +/** + * A position of an entry in a page file. + * + * This class is used to iterate over entries in a page file. + */ public class Position { public PageFile file; public int valuePos; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java index 163e5dc6fd5..84bfac2aefa 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java @@ -20,6 +20,12 @@ import java.util.Iterator; +/** + * A helper class to iterate over key-value pairs in a tree store, in ascending + * key order. The class helps merging multiple streams of key-value pairs. + * + * Internally, it is backed by an iterator over positions in the key-value pair. + */ public class SortedStream implements Comparable { private final int priority; From 785a118f546683e5ccd0a438ab9b10605f66bbfe Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Sep 2024 09:00:09 +0200 Subject: [PATCH 86/86] OAK-10341 Tree store (code review) --- .../document/DocumentStoreIndexerBase.java | 8 +++- .../flatfile/FlatFileNodeStoreBuilder.java | 41 +++++++++---------- .../pipelined/PipelinedTransformTask.java | 2 +- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 921fafc3866..4add3d822b0 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -240,9 +240,13 @@ nodeStore, getMongoDocumentStore(), traversalLog)) } public IndexStore buildTreeStore() throws IOException, CommitFailedException { - System.setProperty(FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE, + String old = System.setProperty(FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE, FlatFileNodeStoreBuilder.SortStrategyType.PIPELINED_TREE.name()); - return buildFlatFileStore(); + try { + return buildFlatFileStore(); + } finally { + System.setProperty(FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE, old); + } } public IndexStore buildStore() throws IOException, CommitFailedException { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index 14706e7d4cb..caf6cd70d89 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -19,22 +19,9 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile; -import static java.util.Collections.unmodifiableSet; -import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; -import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4; -import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - +import com.mongodb.MongoClientURI; +import com.mongodb.client.MongoDatabase; +import org.apache.commons.lang3.StringUtils; import org.apache.jackrabbit.guava.common.collect.Iterables; import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.index.IndexHelper; @@ -48,6 +35,8 @@ import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreSortStrategy; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.Prefetcher; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.RevisionVector; import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; @@ -62,11 +51,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.mongodb.MongoClientURI; -import com.mongodb.client.MongoDatabase; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; -import org.apache.jackrabbit.oak.index.indexer.document.tree.Prefetcher; -import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import static java.util.Collections.unmodifiableSet; +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; +import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4; +import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP; /** * This class is where the strategy being selected for building FlatFileStore. @@ -324,7 +323,7 @@ public List buildList(IndexHelper indexHelper, IndexerSupport indexe private IndexStoreFiles createdSortedStoreFiles() throws IOException, CompositeException { // Check system property defined path String sortedFilePath = System.getProperty(OAK_INDEXER_SORTED_FILE_PATH); - if (sortedFilePath != null && !sortedFilePath.isEmpty()) { + if (StringUtils.isNotBlank(sortedFilePath)) { File sortedDir = new File(sortedFilePath); log.info("Attempting to read from provided sorted files directory [{}] (via system property '{}')", sortedDir.getAbsolutePath(), OAK_INDEXER_SORTED_FILE_PATH); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java index 9a4cda22b26..d230201d5bf 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java @@ -160,7 +160,7 @@ public Result call() throws Exception { for (NodeDocument nodeDoc : nodeDocumentBatch) { statistics.incrementMongoDocumentsTraversed(); mongoObjectsProcessed++; - if (mongoObjectsProcessed % 200_000 == 0) { + if (mongoObjectsProcessed % 50_000 == 0) { LOG.info("Mongo objects: {}, total entries: {}, current batch: {}, Size: {}/{} MB", mongoObjectsProcessed, totalEntryCount, nseBatch.numberOfEntries(), nseBatch.sizeOfEntriesBytes() / FileUtils.ONE_MB,