diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index 95c87c56999..4add3d822b0 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -239,6 +239,16 @@ nodeStore, getMongoDocumentStore(), traversalLog)) return storeList; } + public IndexStore buildTreeStore() throws IOException, CommitFailedException { + String old = System.setProperty(FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE, + FlatFileNodeStoreBuilder.SortStrategyType.PIPELINED_TREE.name()); + try { + return buildFlatFileStore(); + } finally { + System.setProperty(FlatFileNodeStoreBuilder.OAK_INDEXER_SORT_STRATEGY_TYPE, old); + } + } + public IndexStore buildStore() throws IOException, CommitFailedException { return buildFlatFileStore(); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java index 767737dd898..849f1d0bc60 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/AheadOfTimeBlobDownloadingFlatFileStore.java @@ -82,7 +82,15 @@ private AheadOfTimeBlobDownloadingFlatFileStore(FlatFileStore ffs, CompositeInde } } - static boolean isEnabledForIndexes(String indexesEnabledPrefix, List indexPaths) { + /** + * Whether blob downloading is needed for the given indexes. + * + * @param indexesEnabledPrefix the comma-separated list of prefixes of the index + * definitions that benefit from the download + * @param indexPaths the index paths + * @return true if any of the indexes start with any of the prefixes + */ + public static boolean isEnabledForIndexes(String indexesEnabledPrefix, List indexPaths) { List enableForIndexes = splitAndTrim(indexesEnabledPrefix); for (String indexPath : indexPaths) { if (enableForIndexes.stream().anyMatch(indexPath::startsWith)) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java index 949253ba467..caf6cd70d89 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java @@ -29,10 +29,14 @@ import org.apache.jackrabbit.oak.index.indexer.document.CompositeException; import org.apache.jackrabbit.oak.index.indexer.document.CompositeIndexer; import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper; import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedTreeStoreStrategy; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreSortStrategy; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.Prefetcher; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.RevisionVector; import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; @@ -122,7 +126,11 @@ public enum SortStrategyType { /** * System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} if set to this value would result in {@link PipelinedStrategy} being used. */ - PIPELINED + PIPELINED, + /** + * System property {@link #OAK_INDEXER_SORT_STRATEGY_TYPE} if set to this value would result in {@link PipelinedTreeStoreStrategy} being used. + */ + PIPELINED_TREE, } public FlatFileNodeStoreBuilder(File workDir) { @@ -224,20 +232,52 @@ public IndexStore build(IndexHelper indexHelper, CompositeIndexer indexer) throw entryWriter = new NodeStateEntryWriter(blobStore); IndexStoreFiles indexStoreFiles = createdSortedStoreFiles(); File metadataFile = indexStoreFiles.metadataFile; - FlatFileStore store = new FlatFileStore(blobStore, indexStoreFiles.storeFiles.get(0), metadataFile, - new NodeStateEntryReader(blobStore), - unmodifiableSet(preferredPathElements), algorithm); + File file = indexStoreFiles.storeFiles.get(0); + IndexStore store; + if (file.isDirectory()) { + store = buildTreeStoreForIndexing(indexHelper, file); + } else { + store = new FlatFileStore(blobStore, file, metadataFile, + new NodeStateEntryReader(blobStore), + unmodifiableSet(preferredPathElements), algorithm); + } if (entryCount > 0) { store.setEntryCount(entryCount); } if (indexer == null || indexHelper == null) { return store; } - if (withAheadOfTimeBlobDownloading) { - return AheadOfTimeBlobDownloadingFlatFileStore.wrap(store, indexer, indexHelper); - } else { - return store; + if (withAheadOfTimeBlobDownloading && store instanceof FlatFileStore) { + FlatFileStore ffs = (FlatFileStore) store; + return AheadOfTimeBlobDownloadingFlatFileStore.wrap(ffs, indexer, indexHelper); + } + return store; + } + + public IndexStore buildTreeStoreForIndexing(IndexHelper indexHelper, File file) { + TreeStore indexingTreeStore = new TreeStore( + "indexing", file, + new NodeStateEntryReader(blobStore), 10); + indexingTreeStore.setIndexDefinitions(indexDefinitions); + + // use a separate tree store (with a smaller cache) + // for prefetching, to avoid cache evictions + TreeStore prefetchTreeStore = new TreeStore( + "prefetch", file, + new NodeStateEntryReader(blobStore), 3); + prefetchTreeStore.setIndexDefinitions(indexDefinitions); + String blobPrefetchEnableForIndexes = ConfigHelper.getSystemPropertyAsString( + AheadOfTimeBlobDownloadingFlatFileStore.BLOB_PREFETCH_ENABLE_FOR_INDEXES_PREFIXES, ""); + Prefetcher prefetcher = new Prefetcher(prefetchTreeStore, indexingTreeStore); + String blobSuffix = ""; + if (AheadOfTimeBlobDownloadingFlatFileStore.isEnabledForIndexes( + blobPrefetchEnableForIndexes, indexHelper.getIndexPaths())) { + blobSuffix = ConfigHelper.getSystemPropertyAsString( + AheadOfTimeBlobDownloadingFlatFileStore.BLOB_PREFETCH_BINARY_NODES_SUFFIX, ""); } + prefetcher.setBlobSuffix(blobSuffix); + prefetcher.startPrefetch(); + return indexingTreeStore; } public List buildList(IndexHelper indexHelper, IndexerSupport indexerSupport, @@ -351,7 +391,7 @@ IndexStoreSortStrategy createSortStrategy(File dir) { log.warn("TraverseWithSortStrategy is deprecated and will be removed in the near future. Use PipelinedStrategy instead."); return new TraverseWithSortStrategy(nodeStateEntryTraverserFactory, preferredPathElements, entryWriter, dir, algorithm, pathPredicate, checkpoint); - case PIPELINED: + case PIPELINED: { log.info("Using PipelinedStrategy"); List pathFilters = indexDefinitions.stream().map(IndexDefinition::getPathFilter).collect(Collectors.toList()); List indexNames = indexDefinitions.stream().map(IndexDefinition::getIndexName).collect(Collectors.toList()); @@ -359,7 +399,16 @@ IndexStoreSortStrategy createSortStrategy(File dir) { return new PipelinedStrategy(mongoClientURI, mongoDocumentStore, nodeStore, rootRevision, preferredPathElements, blobStore, dir, algorithm, pathPredicate, pathFilters, checkpoint, statisticsProvider, indexingReporter); - + } + case PIPELINED_TREE: { + log.info("Using PipelinedTreeStoreStrategy"); + List pathFilters = indexDefinitions.stream().map(IndexDefinition::getPathFilter).collect(Collectors.toList()); + List indexNames = indexDefinitions.stream().map(IndexDefinition::getIndexName).collect(Collectors.toList()); + indexingReporter.setIndexNames(indexNames); + return new PipelinedTreeStoreStrategy(mongoClientURI, mongoDocumentStore, nodeStore, rootRevision, + preferredPathElements, blobStore, dir, algorithm, pathPredicate, pathFilters, checkpoint, + statisticsProvider, indexingReporter); + } } throw new IllegalStateException("Not a valid sort strategy value " + sortStrategyType); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java index ece82decacc..f10787942f4 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/BoundedHistogram.java @@ -32,7 +32,7 @@ * histogram are correct but if the histogram overflowed, it may be missing some entries. */ public class BoundedHistogram { - private static final Logger LOG = LoggerFactory.getLogger(PipelinedStrategy.class); + private static final Logger LOG = LoggerFactory.getLogger(BoundedHistogram.class); private final ConcurrentHashMap histogram = new ConcurrentHashMap<>(); private volatile boolean overflowed = false; private final String histogramName; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java index 9374af70b21..bc9121709bc 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/ConfigHelper.java @@ -22,7 +22,7 @@ import org.slf4j.LoggerFactory; public class ConfigHelper { - private static final Logger LOG = LoggerFactory.getLogger(PipelinedStrategy.class); + private static final Logger LOG = LoggerFactory.getLogger(ConfigHelper.class); public static int getSystemPropertyAsInt(String name, int defaultValue) { int result = Integer.getInteger(name, defaultValue); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java index fd8701a22bc..d230201d5bf 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTransformTask.java @@ -160,7 +160,7 @@ public Result call() throws Exception { for (NodeDocument nodeDoc : nodeDocumentBatch) { statistics.incrementMongoDocumentsTraversed(); mongoObjectsProcessed++; - if (mongoObjectsProcessed % 50000 == 0) { + if (mongoObjectsProcessed % 50_000 == 0) { LOG.info("Mongo objects: {}, total entries: {}, current batch: {}, Size: {}/{} MB", mongoObjectsProcessed, totalEntryCount, nseBatch.numberOfEntries(), nseBatch.sizeOfEntriesBytes() / FileUtils.ONE_MB, diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java new file mode 100644 index 00000000000..dbbd0c63948 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -0,0 +1,513 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +import com.mongodb.MongoClientURI; +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.guava.common.base.Preconditions; +import org.apache.jackrabbit.guava.common.base.Stopwatch; +import org.apache.jackrabbit.guava.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.commons.IOUtils; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryWriter; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreSortStrategyBase; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; +import org.apache.jackrabbit.oak.plugins.document.NodeDocument; +import org.apache.jackrabbit.oak.plugins.document.RevisionVector; +import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; +import org.apache.jackrabbit.oak.plugins.index.FormattingUtils; +import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter; +import org.apache.jackrabbit.oak.plugins.index.IndexingReporter; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import static org.apache.jackrabbit.oak.plugins.index.IndexUtils.INDEXING_PHASE_LOGGER; + +/** + * Downloads the contents of the MongoDB repository dividing the tasks in a pipeline with the following stages: + *
    + *
  • Download - Downloads from Mongo all the documents in the node store. + *
  • Transform - Converts Mongo documents to node state entries. + *
  • Sort and save - Sorts the batch of node state entries and saves them to disk + *
  • Merge sorted files - Merge the intermediate sorted files into a single file (the final FlatFileStore). + *
+ *

+ *

Memory management

+ *

+ * For efficiency, the intermediate sorted files should be as large as possible given the memory constraints. + * This strategy accumulates the entries that will be stored in each of these files in memory until reaching a maximum + * configurable size, at which point it sorts the data and writes it to a file. The data is accumulated in instances of + * {@link NodeStateEntryBatch}. This class contains two data structures: + *

    + *
  • A {@link java.nio.ByteBuffer} for the binary representation of the entry, that is, the byte array that will be written to the file. + * This buffer contains length-prefixed byte arrays, that is, each entry is {@code }, where size is a 4 byte int. + *
  • An array of {@link SortKey} instances, which contain the paths of each entry and are used to sort the entries. Each element + * in this array also contains the position in the ByteBuffer of the serialized representation of the entry. + *
+ * This representation has several advantages: + *
    + *
  • It is compact, as a String object in the heap requires more memory than a length-prefixed byte array in the ByteBuffer. + *
  • Predictable memory usage - the memory used by the {@link java.nio.ByteBuffer} is fixed and allocated at startup + * (more on this later). The memory used by the array of {@link SortKey} is not bounded, but these objects are small, + * as they contain little more than the path of the entry, and we can easily put limits on the maximum number of entries + * kept in a buffer. + *
+ *

+ * The instances of {@link NodeStateEntryBatch} are created at launch time. We create {@code #transformThreads+1} buffers. + * This way, except for some rare situations, each transform thread will have its own buffer where to write the entries + * and there will be an extra buffer to be used by the Save-and-Sort thread, so that all the transform and sort threads + * can operate concurrently. + *

+ * These buffers are reused. Once the Save-and-Sort thread finishes processing a buffer, it clears it and sends it back + * to the transform threads. For this, we use two queues, one with empty buffers, from where the transform threads take + * their buffers when they need one, and another with full buffers, which are read by the Save-and-Sort thread. + *

+ * Reusing the buffers reduces significantly the pressure on the garbage collector and ensures that we do not run out + * of memory, as the largest blocks of memory are pre-allocated and reused. + *

+ * The total amount of memory used by the buffers is a configurable parameter (env variable {@link #OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB}). + * This memory is divided in {@code numberOfBuffers + 1 } regions, each of + * {@code regionSize = PIPELINED_WORKING_MEMORY_MB/(#numberOfBuffers + 1)} size. + * Each ByteBuffer is of {@code regionSize} big. The extra region is to account for the memory taken by the {@link SortKey} + * entries. There is also a maximum limit on the number of entries, which is calculated based on regionSize + * (we assume each {@link SortKey} entry requires 256 bytes). + *

+ * The transform threads will stop filling a buffer and enqueue it for sorting and saving once either the byte buffer is + * full or the number of entries in the buffer reaches the limit. + *

+ * + *

Retrials on broken MongoDB connections

+ */ +public class PipelinedTreeStoreStrategy extends IndexStoreSortStrategyBase { + public static final String OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB = "oak.indexer.pipelined.mongoDocBatchMaxSizeMB"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB = 4; + public static final String OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS = "oak.indexer.pipelined.mongoDocBatchMaxNumberOfDocuments"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS = 10000; + public static final String OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB = "oak.indexer.pipelined.mongoDocQueueReservedMemoryMB"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB = 128; + public static final String OAK_INDEXER_PIPELINED_TRANSFORM_THREADS = "oak.indexer.pipelined.transformThreads"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_TRANSFORM_THREADS = 2; + public static final String OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB = "oak.indexer.pipelined.workingMemoryMB"; + // 0 means autodetect + public static final int DEFAULT_OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB = 0; + // Between 1 and 100 + public static final String OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE = "oak.indexer.pipelined.sortBufferMemoryPercentage"; + public static final int DEFAULT_OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE = 25; + + private static final Logger LOG = LoggerFactory.getLogger(PipelinedTreeStoreStrategy.class); + // A MongoDB document is at most 16MB, so the buffer that holds node state entries must be at least that big + private static final int MIN_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB = 16; + private static final int MIN_AUTODETECT_WORKING_MEMORY_MB = 128; + private static final int MIN_ENTRY_BATCH_BUFFER_SIZE_MB = 32; + private static final int MAX_AUTODETECT_WORKING_MEMORY_MB = 4000; + + private static void printStatistics(ArrayBlockingQueue mongoDocQueue, + ArrayBlockingQueue emptyBuffersQueue, + ArrayBlockingQueue nonEmptyBuffersQueue, + TransformStageStatistics transformStageStatistics, + boolean printHistogramsAtInfo) { + + String queueSizeStats = MetricsFormatter.newBuilder() + .add("mongoDocQueue", mongoDocQueue.size()) + .add("emptyBuffersQueue", emptyBuffersQueue.size()) + .add("nonEmptyBuffersQueue", nonEmptyBuffersQueue.size()) + .build(); + + LOG.info("Queue sizes: {}", queueSizeStats); + LOG.info("Transform stats: {}", transformStageStatistics.formatStats()); + prettyPrintTransformStatisticsHistograms(transformStageStatistics, printHistogramsAtInfo); + } + + private static void prettyPrintTransformStatisticsHistograms(TransformStageStatistics transformStageStatistics, boolean printHistogramAtInfo) { + if (printHistogramAtInfo) { + LOG.info("Top hidden paths rejected: {}", transformStageStatistics.getHiddenPathsRejectedHistogram().prettyPrint()); + LOG.info("Top paths filtered: {}", transformStageStatistics.getFilteredPathsRejectedHistogram().prettyPrint()); + LOG.info("Top empty node state documents: {}", transformStageStatistics.getEmptyNodeStateHistogram().prettyPrint()); + } else { + LOG.debug("Top hidden paths rejected: {}", transformStageStatistics.getHiddenPathsRejectedHistogram().prettyPrint()); + LOG.debug("Top paths filtered: {}", transformStageStatistics.getFilteredPathsRejectedHistogram().prettyPrint()); + LOG.debug("Top empty node state documents: {}", transformStageStatistics.getEmptyNodeStateHistogram().prettyPrint()); + } + } + + private final MongoDocumentStore docStore; + private final MongoClientURI mongoClientURI; + private final DocumentNodeStore documentNodeStore; + private final RevisionVector rootRevision; + private final BlobStore blobStore; + private final List pathFilters; + private final StatisticsProvider statisticsProvider; + private final IndexingReporter indexingReporter; + private final int numberOfTransformThreads; + private final int mongoDocQueueSize; + private final int mongoDocBatchMaxSizeMB; + private final int mongoDocBatchMaxNumberOfDocuments; + private final int nseBuffersCount; + private final int nseBuffersSizeBytes; + + private long nodeStateEntriesExtracted; + + /** + * @param mongoClientURI URI of the Mongo cluster. + * @param pathPredicate Used by the transform stage to test if a node should be kept or discarded. + * @param pathFilters If non-empty, the download stage will use these filters to create a query that downloads + * only the matching MongoDB documents. + * @param statisticsProvider Used to collect statistics about the indexing process. + * @param indexingReporter Used to collect diagnostics, metrics and statistics and report them at the end of the indexing process. + */ + public PipelinedTreeStoreStrategy(MongoClientURI mongoClientURI, + MongoDocumentStore documentStore, + DocumentNodeStore documentNodeStore, + RevisionVector rootRevision, + Set preferredPathElements, + BlobStore blobStore, + File storeDir, + Compression algorithm, + Predicate pathPredicate, + List pathFilters, + String checkpoint, + StatisticsProvider statisticsProvider, + IndexingReporter indexingReporter) { + super(storeDir, algorithm, pathPredicate, preferredPathElements, checkpoint); + this.mongoClientURI = mongoClientURI; + this.docStore = documentStore; + this.documentNodeStore = documentNodeStore; + this.rootRevision = rootRevision; + this.blobStore = blobStore; + this.pathFilters = pathFilters; + this.statisticsProvider = statisticsProvider; + this.indexingReporter = indexingReporter; + Preconditions.checkState(documentStore.isReadOnly(), "Traverser can only be used with readOnly store"); + + int mongoDocQueueReservedMemoryMB = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB); + Preconditions.checkArgument(mongoDocQueueReservedMemoryMB >= MIN_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, + "Invalid value for property " + OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB + ": " + mongoDocQueueReservedMemoryMB + ". Must be >= " + MIN_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, String.valueOf(mongoDocQueueReservedMemoryMB)); + + this.mongoDocBatchMaxSizeMB = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB, DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB); + Preconditions.checkArgument(mongoDocBatchMaxSizeMB > 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB + ": " + mongoDocBatchMaxSizeMB + ". Must be > 0"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB, String.valueOf(mongoDocBatchMaxSizeMB)); + + this.mongoDocBatchMaxNumberOfDocuments = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS, DEFAULT_OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS); + Preconditions.checkArgument(mongoDocBatchMaxNumberOfDocuments > 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS + ": " + mongoDocBatchMaxNumberOfDocuments + ". Must be > 0"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS, String.valueOf(mongoDocBatchMaxNumberOfDocuments)); + + this.numberOfTransformThreads = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, DEFAULT_OAK_INDEXER_PIPELINED_TRANSFORM_THREADS); + Preconditions.checkArgument(numberOfTransformThreads > 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_TRANSFORM_THREADS + ": " + numberOfTransformThreads + ". Must be > 0"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, String.valueOf(numberOfTransformThreads)); + + int sortBufferMemoryPercentage = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE, DEFAULT_OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE); + Preconditions.checkArgument(sortBufferMemoryPercentage > 0 && sortBufferMemoryPercentage <= 100, + "Invalid value for property " + OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE + ": " + numberOfTransformThreads + ". Must be between 1 and 100"); + this.indexingReporter.addConfig(OAK_INDEXER_PIPELINED_SORT_BUFFER_MEMORY_PERCENTAGE, String.valueOf(sortBufferMemoryPercentage)); + + // mongo-dump <-> transform threads + Preconditions.checkArgument(mongoDocQueueReservedMemoryMB >= 8 * mongoDocBatchMaxSizeMB, + "Invalid values for properties " + OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB + " and " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB + + ": " + OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB + " must be at least 8x " + OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB + + ", but are " + mongoDocQueueReservedMemoryMB + " and " + mongoDocBatchMaxSizeMB + ", respectively" + ); + this.mongoDocQueueSize = mongoDocQueueReservedMemoryMB / mongoDocBatchMaxSizeMB; + + // Derived values for transform <-> sort-save + int nseWorkingMemoryMB = readNSEBuffersReservedMemory(); + this.nseBuffersCount = 1 + numberOfTransformThreads; + long nseWorkingMemoryBytes = (long) nseWorkingMemoryMB * FileUtils.ONE_MB; + // The working memory is divided in the following regions: + // - #transforThreads NSE Binary buffers + // - x1 Memory reserved for the array created by the sort-batch thread with the keys of the entries + // in the batch that is being sorted + long memoryReservedForSortKeysArray = estimateMaxSizeOfSortArray(nseWorkingMemoryBytes, nseBuffersCount, sortBufferMemoryPercentage); + long memoryReservedForBuffers = nseWorkingMemoryBytes - memoryReservedForSortKeysArray; + + // A ByteBuffer can be at most Integer.MAX_VALUE bytes long + this.nseBuffersSizeBytes = limitToIntegerRange(memoryReservedForBuffers / nseBuffersCount); + + if (nseBuffersSizeBytes < MIN_ENTRY_BATCH_BUFFER_SIZE_MB * FileUtils.ONE_MB) { + throw new IllegalArgumentException("Entry batch buffer size too small: " + nseBuffersSizeBytes + + " bytes. Must be at least " + MIN_ENTRY_BATCH_BUFFER_SIZE_MB + " MB. " + + "To increase the size of the buffers, either increase the size of the working memory region " + + "(system property " + OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB + ") or decrease the number of transform " + + "threads (" + OAK_INDEXER_PIPELINED_TRANSFORM_THREADS + ")"); + } + + LOG.info("MongoDocumentQueue: [ reservedMemory: {} MB, batchMaxSize: {} MB, queueSize: {} (reservedMemory/batchMaxSize) ]", + mongoDocQueueReservedMemoryMB, + mongoDocBatchMaxSizeMB, + mongoDocQueueSize); + LOG.info("NodeStateEntryBuffers: [ workingMemory: {} MB, numberOfBuffers: {}, bufferSize: {}, sortBufferReservedMemory: {} ]", + nseWorkingMemoryMB, + nseBuffersCount, + IOUtils.humanReadableByteCountBin(nseBuffersSizeBytes), + IOUtils.humanReadableByteCountBin(memoryReservedForSortKeysArray) + ); + } + + static long estimateMaxSizeOfSortArray(long nseWorkingMemoryBytes, long nseBuffersCount, int sortBufferMemoryPercentage) { + // We reserve a percentage of the size of a buffer for sorting. That is, we are assuming that for every line + // in the sort buffer, the memory needed to store the path section of the line will not be more + // than sortBufferMemoryPercentage of the total size of the line in average + // Estimate memory needed by the sort keys array. We assume each entry requires 256 bytes. + long approxNseBufferSize = limitToIntegerRange(nseWorkingMemoryBytes / nseBuffersCount); + return approxNseBufferSize * sortBufferMemoryPercentage / 100; + } + + private int readNSEBuffersReservedMemory() { + int workingMemoryMB = ConfigHelper.getSystemPropertyAsInt(OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, DEFAULT_OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB); + Preconditions.checkArgument(workingMemoryMB >= 0, + "Invalid value for property " + OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB + ": " + workingMemoryMB + ". Must be >= 0"); + indexingReporter.addConfig(OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, workingMemoryMB); + if (workingMemoryMB == 0) { + return autodetectWorkingMemoryMB(); + } else { + return workingMemoryMB; + } + } + + private int autodetectWorkingMemoryMB() { + int maxHeapSizeMB = (int) (Runtime.getRuntime().maxMemory() / FileUtils.ONE_MB); + int workingMemoryMB = maxHeapSizeMB - 2048; + LOG.info("Auto detecting working memory. Maximum heap size: {} MB, selected working memory: {} MB", maxHeapSizeMB, workingMemoryMB); + if (workingMemoryMB > MAX_AUTODETECT_WORKING_MEMORY_MB) { + LOG.warn("Auto-detected value for working memory too high, setting to the maximum allowed for auto-detection: {} MB", MAX_AUTODETECT_WORKING_MEMORY_MB); + return MAX_AUTODETECT_WORKING_MEMORY_MB; + } + if (workingMemoryMB < MIN_AUTODETECT_WORKING_MEMORY_MB) { + LOG.warn("Auto-detected value for working memory too low, setting to the minimum allowed for auto-detection: {} MB", MIN_AUTODETECT_WORKING_MEMORY_MB); + return MIN_AUTODETECT_WORKING_MEMORY_MB; + } + return workingMemoryMB; + } + + private static int limitToIntegerRange(long bufferSizeBytes) { + if (bufferSizeBytes > Integer.MAX_VALUE) { + // Probably not necessary to subtract 16, just a safeguard to avoid boundary conditions. + int truncatedBufferSize = Integer.MAX_VALUE - 16; + LOG.warn("Computed buffer size too big: {}, exceeds Integer.MAX_VALUE. Truncating to: {}", bufferSizeBytes, truncatedBufferSize); + return truncatedBufferSize; + } else { + return (int) bufferSizeBytes; + } + } + + @Override + public File createSortedStoreFile() throws IOException { + int numberOfThreads = 1 + numberOfTransformThreads + 1; // dump, transform, sort threads + ExecutorService threadPool = Executors.newFixedThreadPool(numberOfThreads, + new ThreadFactoryBuilder().setDaemon(true).build() + ); + // This executor can wait for several tasks at the same time. We use this below to wait at the same time for + // all the tasks, so that if one of them fails, we can abort the whole pipeline. Otherwise, if we wait on + // Future instances, we can only wait on one of them, so that if any of the others fail, we have no easy way + // to detect this failure. + @SuppressWarnings("rawtypes") + ExecutorCompletionService ecs = new ExecutorCompletionService<>(threadPool); + File resultDir = getStoreDir(); + TreeStore treeStore = new TreeStore("dump", resultDir, null, 1); + treeStore.getSession().init(); + try { + // download -> transform thread. + ArrayBlockingQueue mongoDocQueue = new ArrayBlockingQueue<>(mongoDocQueueSize); + + // transform <-> sort and save threads + // Queue with empty buffers, used by the transform task + ArrayBlockingQueue emptyBatchesQueue = new ArrayBlockingQueue<>(nseBuffersCount); + // Queue with buffers filled by the transform task, used by the sort and save task. +1 for the SENTINEL + ArrayBlockingQueue nonEmptyBatchesQueue = new ArrayBlockingQueue<>(nseBuffersCount + 1); + + TransformStageStatistics transformStageStatistics = new TransformStageStatistics(); + + // Create empty buffers + for (int i = 0; i < nseBuffersCount; i++) { + // No limits on the number of entries, only on their total size. This might be revised later. + emptyBatchesQueue.add(NodeStateEntryBatch.createNodeStateEntryBatch(nseBuffersSizeBytes, Integer.MAX_VALUE)); + } + + INDEXING_PHASE_LOGGER.info("[TASK:PIPELINED-DUMP:START] Starting to build TreeStore"); + Stopwatch start = Stopwatch.createStarted(); + + @SuppressWarnings("unchecked") + Future downloadFuture = ecs.submit(new PipelinedMongoDownloadTask( + mongoClientURI, + docStore, + (int) (mongoDocBatchMaxSizeMB * FileUtils.ONE_MB), + mongoDocBatchMaxNumberOfDocuments, + mongoDocQueue, + pathFilters, + statisticsProvider, + indexingReporter + )); + + ArrayList> transformFutures = new ArrayList<>(numberOfTransformThreads); + for (int i = 0; i < numberOfTransformThreads; i++) { + NodeStateEntryWriter entryWriter = new NodeStateEntryWriter(blobStore); + @SuppressWarnings("unchecked") + Future future = ecs.submit(new PipelinedTransformTask( + docStore, + documentNodeStore, + rootRevision, + this.getPathPredicate(), + entryWriter, + mongoDocQueue, + emptyBatchesQueue, + nonEmptyBatchesQueue, + transformStageStatistics + )); + transformFutures.add(future); + } + + @SuppressWarnings("unchecked") + Future sortBatchFuture = ecs.submit(new PipelinedTreeStoreTask( + treeStore, + emptyBatchesQueue, + nonEmptyBatchesQueue, + statisticsProvider, + indexingReporter + )); + + try { + LOG.info("Waiting for tasks to complete"); + int tasksFinished = 0; + int transformTasksFinished = 0; + boolean monitorQueues = true; + while (tasksFinished < numberOfThreads) { + // Wait with a timeout to print statistics periodically + Future completedTask = ecs.poll(30, TimeUnit.SECONDS); + if (completedTask == null) { + // Timeout waiting for a task to complete + if (monitorQueues) { + try { + printStatistics(mongoDocQueue, emptyBatchesQueue, nonEmptyBatchesQueue, transformStageStatistics, false); + } catch (Exception e) { + LOG.warn("Error while logging queue sizes", e); + } + } + } else { + try { + Object result = completedTask.get(); + if (result instanceof PipelinedMongoDownloadTask.Result) { + PipelinedMongoDownloadTask.Result downloadResult = (PipelinedMongoDownloadTask.Result) result; + LOG.info("Download finished. Documents downloaded: {}", downloadResult.getDocumentsDownloaded()); + downloadFuture = null; + + } else if (result instanceof PipelinedTransformTask.Result) { + PipelinedTransformTask.Result transformResult = (PipelinedTransformTask.Result) result; + transformTasksFinished++; + nodeStateEntriesExtracted += transformResult.getEntryCount(); + LOG.info("Transform task {} finished. Entries processed: {}", + transformResult.getThreadId(), transformResult.getEntryCount()); + if (transformTasksFinished == numberOfTransformThreads) { + LOG.info("All transform tasks finished. Total entries processed: {}", nodeStateEntriesExtracted); + // No need to keep monitoring the queues, the download and transform threads are done. + monitorQueues = false; + // Terminate the sort thread. + nonEmptyBatchesQueue.put(PipelinedStrategy.SENTINEL_NSE_BUFFER); + transformStageStatistics.publishStatistics(statisticsProvider, indexingReporter); + transformFutures.clear(); + } + + } else if (result instanceof PipelinedSortBatchTask.Result) { + PipelinedSortBatchTask.Result sortTaskResult = (PipelinedSortBatchTask.Result) result; + LOG.info("Sort batch task finished. Entries processed: {}", sortTaskResult.getTotalEntries()); + // The buffers between transform and merge sort tasks are no longer needed, so remove them + // from the queues so they can be garbage collected. + // These buffers can be very large, so this is important to avoid running out of memory in + // the merge-sort phase + if (!nonEmptyBatchesQueue.isEmpty()) { + LOG.warn("emptyBatchesQueue is not empty. Size: {}", emptyBatchesQueue.size()); + } + emptyBatchesQueue.clear(); + printStatistics(mongoDocQueue, emptyBatchesQueue, nonEmptyBatchesQueue, transformStageStatistics, true); + sortBatchFuture = null; + + } else { + throw new RuntimeException("Unknown result type: " + result); + } + tasksFinished++; + } catch (ExecutionException ex) { + throw new RuntimeException(ex.getCause()); + } catch (Throwable ex) { + throw new RuntimeException(ex); + } + } + } + long elapsedSeconds = start.elapsed(TimeUnit.SECONDS); + INDEXING_PHASE_LOGGER.info("[TASK:PIPELINED-DUMP:END] Metrics: {}", MetricsFormatter.newBuilder() + .add("duration", FormattingUtils.formatToSeconds(elapsedSeconds)) + .add("durationSeconds", elapsedSeconds) + .add("nodeStateEntriesExtracted", nodeStateEntriesExtracted) + .build()); + indexingReporter.addTiming("Build TreeStore (Dump+Merge)", FormattingUtils.formatToSeconds(elapsedSeconds)); + + LOG.info("[INDEXING_REPORT:BUILD_TREE_STORE]\n{}", indexingReporter.generateReport()); + } catch (Throwable e) { + INDEXING_PHASE_LOGGER.info("[TASK:PIPELINED-DUMP:FAIL] Metrics: {}, Error: {}", + MetricsFormatter.createMetricsWithDurationOnly(start), e.toString() + ); + LOG.warn("Error dumping from MongoDB. Cancelling all tasks. Error: {}", e.toString()); + // Cancel in order + cancelFuture(downloadFuture); + for (Future transformTask : transformFutures) { + cancelFuture(transformTask); + } + cancelFuture(sortBatchFuture); + throw new RuntimeException(e); + } + treeStore.close(); + return resultDir; + } finally { + LOG.info("Shutting down build FFS thread pool"); + new ExecutorCloser(threadPool).close(); + } + } + + private void cancelFuture(Future future) { + if (future != null) { + LOG.info("Cancelling future: {}", future); + future.cancel(true); + } + } + + @Override + public long getEntryCount() { + return nodeStateEntriesExtracted; + } +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java new file mode 100644 index 00000000000..257852aaf0d --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCountBin; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.SENTINEL_NSE_BUFFER; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedUtils.formatAsPercentage; +import static org.apache.jackrabbit.oak.plugins.index.IndexUtils.INDEXING_PHASE_LOGGER; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Locale; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import org.apache.jackrabbit.guava.common.base.Stopwatch; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedSortBatchTask.Result; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.apache.jackrabbit.oak.plugins.index.IndexingReporter; +import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter; +import org.apache.jackrabbit.oak.plugins.index.MetricsUtils; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Receives batches of node state entries, sorts then in memory, and finally writes them to a tree store. + */ +public class PipelinedTreeStoreTask implements Callable { + + private static final Logger LOG = LoggerFactory.getLogger(PipelinedTreeStoreTask.class); + private static final String THREAD_NAME = "tree-store-task"; + + private static final int MERGE_BATCH = 10; + private static final boolean SKIP_FINAL_MERGE = false; + + private final TreeStore treeStore; + private final BlockingQueue emptyBuffersQueue; + private final BlockingQueue nonEmptyBuffersQueue; + private final StatisticsProvider statisticsProvider; + private final IndexingReporter reporter; + + private long entriesProcessed = 0; + private long batchesProcessed = 0; + private long timeCreatingSortArrayMillis = 0; + private long timeSortingMillis = 0; + private long timeWritingMillis = 0; + private int unmergedRoots; + + public PipelinedTreeStoreTask(TreeStore treeStore, + ArrayBlockingQueue emptyBuffersQueue, + ArrayBlockingQueue nonEmptyBuffersQueue, + StatisticsProvider statisticsProvider, + IndexingReporter reporter) { + this.treeStore = treeStore; + this.emptyBuffersQueue = emptyBuffersQueue; + this.nonEmptyBuffersQueue = nonEmptyBuffersQueue; + this.statisticsProvider = statisticsProvider; + this.reporter = reporter; + } + + @Override + public Result call() throws Exception { + Stopwatch taskStartTime = Stopwatch.createStarted(); + String originalName = Thread.currentThread().getName(); + Thread.currentThread().setName(THREAD_NAME); + INDEXING_PHASE_LOGGER.info("[TASK:{}:START] Starting sort-and-save task", THREAD_NAME.toUpperCase(Locale.ROOT)); + try { + while (true) { + NodeStateEntryBatch nseBuffer = nonEmptyBuffersQueue.take(); + if (nseBuffer == SENTINEL_NSE_BUFFER) { + synchronized (treeStore) { + TreeSession session = treeStore.getSession(); + Stopwatch start = Stopwatch.createStarted(); + while (session.getRootCount() > MERGE_BATCH) { + LOG.info("Merging {} roots; there are {} roots", + MERGE_BATCH, session.getRootCount()); + session.mergeRoots(MERGE_BATCH); + session.runGC(); + } + if (SKIP_FINAL_MERGE) { + LOG.info("Final merge is skipped"); + } else { + LOG.info("Final merge; {} roots", session.getRootCount()); + session.mergeRoots(Integer.MAX_VALUE); + session.runGC(); + } + long durationSeconds = start.elapsed(TimeUnit.SECONDS); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, durationSeconds); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, 0); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, 0); + MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, 0); + MetricsUtils.addMetricByteSize(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, 0); + LOG.info("Final merge done, {} roots", session.getRootCount()); + } + long totalTimeMillis = taskStartTime.elapsed().toMillis(); + String timeCreatingSortArrayPercentage = formatAsPercentage(timeCreatingSortArrayMillis, totalTimeMillis); + String timeSortingPercentage = formatAsPercentage(timeSortingMillis, totalTimeMillis); + String timeWritingPercentage = formatAsPercentage(timeWritingMillis, totalTimeMillis); + String metrics = MetricsFormatter.newBuilder() + .add("batchesProcessed", batchesProcessed) + .add("entriesProcessed", entriesProcessed) + .add("timeCreatingSortArrayMillis", timeCreatingSortArrayMillis) + .add("timeCreatingSortArrayPercentage", timeCreatingSortArrayPercentage) + .add("timeSortingMillis", timeSortingMillis) + .add("timeSortingPercentage", timeSortingPercentage) + .add("timeWritingMillis", timeWritingMillis) + .add("timeWritingPercentage", timeWritingPercentage) + .add("totalTimeSeconds", totalTimeMillis / 1000) + .build(); + INDEXING_PHASE_LOGGER.info("[TASK:{}:END] Metrics: {}", THREAD_NAME.toUpperCase(Locale.ROOT), metrics); + MetricsUtils.addMetric(statisticsProvider, reporter, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE, + PipelinedUtils.toPercentage(timeCreatingSortArrayMillis, totalTimeMillis)); + MetricsUtils.addMetric(statisticsProvider, reporter, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE, + PipelinedUtils.toPercentage(timeSortingMillis, totalTimeMillis)); + MetricsUtils.addMetric(statisticsProvider, reporter, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE, + PipelinedUtils.toPercentage(timeWritingMillis, totalTimeMillis)); + return new Result(entriesProcessed); + } + sortAndSaveBatch(nseBuffer); + nseBuffer.reset(); + emptyBuffersQueue.put(nseBuffer); + } + } catch (Throwable t) { + INDEXING_PHASE_LOGGER.info("[TASK:{}:FAIL] Metrics: {}, Error: {}", + THREAD_NAME.toUpperCase(Locale.ROOT), + MetricsFormatter.createMetricsWithDurationOnly(taskStartTime), + t.toString()); + LOG.warn("Thread terminating with exception", t); + throw t; + } finally { + Thread.currentThread().setName(originalName); + } + } + + private ArrayList buildSortArray(NodeStateEntryBatch nseb) { + Stopwatch startTime = Stopwatch.createStarted(); + ByteBuffer buffer = nseb.getBuffer(); + int totalPathSize = 0; + ArrayList sortBuffer = new ArrayList<>(nseb.numberOfEntries()); + while (buffer.hasRemaining()) { + // Read the next key from the buffer + int pathLength = buffer.getInt(); + totalPathSize += pathLength; + // Create the String directly from the buffer without creating an intermediate byte[] + String path = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), pathLength, StandardCharsets.UTF_8); + buffer.position(buffer.position() + pathLength); + int valuePosition = buffer.position(); + // Skip the json + int entryLength = buffer.getInt(); + buffer.position(buffer.position() + entryLength); + SortKeyPath entry = new SortKeyPath(path, valuePosition); + sortBuffer.add(entry); + } + timeCreatingSortArrayMillis += startTime.elapsed().toMillis(); + LOG.info("Built sort array in {}. Entries: {}, Total size of path strings: {}", + startTime, sortBuffer.size(), humanReadableByteCountBin(totalPathSize)); + return sortBuffer; + } + + private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { + ByteBuffer buffer = nseb.getBuffer(); + LOG.info("Going to sort batch in memory. Entries: {}, Size: {}", + nseb.numberOfEntries(), humanReadableByteCountBin(nseb.sizeOfEntriesBytes())); + ArrayList sortBuffer = buildSortArray(nseb); + if (sortBuffer.isEmpty()) { + return; + } + Stopwatch sortClock = Stopwatch.createStarted(); + Collections.sort(sortBuffer); + timeSortingMillis += sortClock.elapsed().toMillis(); + LOG.info("Sorted batch in {}. Saving.", sortClock); + Stopwatch saveClock = Stopwatch.createStarted(); + long textSize = 0; + batchesProcessed++; + synchronized (treeStore) { + TreeSession session = treeStore.getSession(); + for (SortKeyPath entry : sortBuffer) { + entriesProcessed++; + // Retrieve the entry from the buffer + int posInBuffer = entry.getBufferPos(); + buffer.position(posInBuffer); + int valueLength = buffer.getInt(); + String value = new String(buffer.array(), buffer.arrayOffset() + buffer.position(), valueLength, StandardCharsets.UTF_8); + textSize += entry.getPath().length() + value.length() + 2; + treeStore.putNode(entry.getPath(), value); + } + session.checkpoint(); + unmergedRoots++; + LOG.info("Root count is {}, we have {} small unmerged roots", + session.getRootCount(), unmergedRoots); + if (unmergedRoots == MERGE_BATCH) { + session.mergeRoots(MERGE_BATCH); + session.runGC(); + LOG.info("Merged {} roots, root count is now {}", + unmergedRoots, session.getRootCount()); + unmergedRoots = 0; + } + timeWritingMillis += saveClock.elapsed().toMillis(); + batchesProcessed++; + LOG.info("Wrote batch of size {} (uncompressed) in {} at {}", + humanReadableByteCountBin(textSize), + saveClock, + PipelinedUtils.formatAsTransferSpeedMBs(textSize, saveClock.elapsed().toMillis()) + ); + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/SortKeyPath.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/SortKeyPath.java new file mode 100644 index 00000000000..0edf84d9d24 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/SortKeyPath.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +public class SortKeyPath implements Comparable { + + private final String path; + private final int valuePosition; + + public SortKeyPath(String path, int valuePosition) { + this.path = path; + this.valuePosition = valuePosition; + } + + public int getBufferPos() { + return valuePosition; + } + + public String getPath() { + return path; + } + + @Override + public int compareTo(SortKeyPath o) { + return path.compareTo(o.path); + } +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java index 13907ab6ea5..03565746e60 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java @@ -74,7 +74,9 @@ public enum IncrementalSortStrategyType { * Incremental store having nodes updated between initial and final checkpoint */ - INCREMENTAL_FFS_STORE + INCREMENTAL_FFS_STORE, + + INCREMENTAL_TREE_STORE } public IncrementalStoreBuilder(File workDir, IndexHelper indexHelper, @@ -109,8 +111,9 @@ public IncrementalStoreBuilder withBlobStore(BlobStore blobStore) { public IndexStore build() throws IOException, CompositeException { logFlags(); File dir = createStoreDir(); - - if (Objects.requireNonNull(sortStrategyType) == IncrementalSortStrategyType.INCREMENTAL_FFS_STORE) { + Objects.requireNonNull(sortStrategyType); + if (sortStrategyType == IncrementalSortStrategyType.INCREMENTAL_FFS_STORE || + sortStrategyType == IncrementalSortStrategyType.INCREMENTAL_TREE_STORE) { IncrementalFlatFileStoreNodeStateEntryWriter entryWriter = new IncrementalFlatFileStoreNodeStateEntryWriter(blobStore); IncrementalIndexStoreSortStrategy strategy = new IncrementalFlatFileStoreStrategy( indexHelper.getNodeStore(), diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java new file mode 100644 index 00000000000..2e87d49e727 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.incrementalstore; + +import static org.apache.jackrabbit.guava.common.base.Preconditions.checkState; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreMetadata; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreMetadataOperatorImpl; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class MergeIncrementalTreeStore implements MergeIncrementalStore { + + private static final String MERGE_BASE_AND_INCREMENTAL_TREE_STORE = "MergeBaseAndIncrementalTreeStore"; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final Logger LOG = LoggerFactory.getLogger(MergeIncrementalTreeStore.class); + + private final File baseFile; + private final File incrementalFile; + private final File mergedFile; + private final Compression algorithm; + + private final static Map OPERATION_MAP = Arrays.stream(IncrementalStoreOperand.values()) + .collect(Collectors.toUnmodifiableMap(IncrementalStoreOperand::toString, k -> IncrementalStoreOperand.valueOf(k.name()))); + + public MergeIncrementalTreeStore(File baseFile, File incrementalFile, File mergedFile, Compression algorithm) throws IOException { + this.baseFile = baseFile; + this.incrementalFile = incrementalFile; + this.mergedFile = mergedFile; + this.algorithm = algorithm; + } + + @Override + public void doMerge() throws IOException { + LOG.info("Merging {} and {}", baseFile.getAbsolutePath(), incrementalFile.getAbsolutePath()); + File baseDir = new File(baseFile.getAbsolutePath() + ".files"); + LOG.info("Unpacking to {}", baseDir.getAbsolutePath()); + FilePacker.unpack(baseFile, baseDir, true); + File mergedDir = new File(mergedFile.getAbsolutePath() + ".files"); + LOG.info("Merging to {}", mergedDir.getAbsolutePath()); + mergeMetadataFiles(); + mergeIndexStore(baseDir, mergedDir); + LOG.info("Packing to {}", mergedFile.getAbsolutePath()); + FilePacker.pack(mergedDir, TreeSession.getFileNameRegex(), mergedFile, true); + LOG.info("Completed"); + } + + @Override + public String getStrategyName() { + return MERGE_BASE_AND_INCREMENTAL_TREE_STORE; + } + + /** + * Merges multiple index store files. + * + * This method is a little verbose, but I think this is fine + * as we are not getting consistent data from checkpoint diff + * and we need to handle cases differently. + */ + private void mergeIndexStore(File baseDir, File mergedDir) throws IOException { + TreeStore baseStore = new TreeStore("base", baseDir, new NodeStateEntryReader(null), 10); + TreeStore mergedStore = new TreeStore("merged", mergedDir, new NodeStateEntryReader(null), 10); + mergedStore.getSession().init(); + Iterator> baseIt = baseStore.getSession().iterator(); + try (BufferedReader incrementalReader = IndexStoreUtils.createReader(incrementalFile, algorithm)) { + StoreEntry base = StoreEntry.readFromTreeStore(baseIt); + StoreEntry increment = StoreEntry.readFromReader(incrementalReader); + while (base != null || increment != null) { + // which one to advance at the end of the loop + boolean advanceBase, advanceIncrement; + // the entry to write (or null, in case of a delete) + StoreEntry write; + if (base == null) { + // base EOF: we expect ADD + if (increment.operation != IncrementalStoreOperand.ADD) { + LOG.warn( + "Expected ADD but got {} for incremental path {} value {}. " + + "Merging will proceed, but this is unexpected.", + increment.operation, increment.path, increment.value); + } + write = increment; + advanceBase = false; + advanceIncrement = true; + } else if (increment == null) { + // increment EOF: copy from base + write = base; + advanceBase = true; + advanceIncrement = false; + } else { + // both base and increment (normal case) + int compare = base.path.compareTo(increment.path); + if (compare < 0) { + // base path is smaller + write = base; + advanceBase = true; + advanceIncrement = false; + } else if (compare > 0) { + // increment path is smaller: we expect ADD + if (increment.operation != IncrementalStoreOperand.ADD) { + LOG.warn("Expected ADD but got {} for incremental path {} value {}. " + + "Merging will proceed, but this is unexpected.", + increment.operation, increment.path, increment.value); + } + write = increment; + advanceBase = false; + advanceIncrement = true; + } else { + // both paths are the same: we expect modify or delete + write = increment; + advanceBase = true; + advanceIncrement = true; + switch (increment.operation) { + case ADD: + LOG.warn("Expected MODIFY/DELETE but got {} for incremental path {} value {}. " + + "Merging will proceed, but this is unexpected.", + increment.operation, increment.path, increment.value); + break; + case MODIFY: + break; + case DELETE: + write = null; + } + } + } + if (write != null) { + mergedStore.putNode(write.path, write.value); + } + if (advanceBase) { + base = StoreEntry.readFromTreeStore(baseIt); + } + if (advanceIncrement) { + increment = StoreEntry.readFromReader(incrementalReader); + } + } + } + baseStore.close(); + mergedStore.getSession().flush(); + mergedStore.close(); + } + + static class StoreEntry { + final String path; + final String value; + final IncrementalStoreOperand operation; + + StoreEntry(String path, String value, IncrementalStoreOperand operation) { + this.path = path; + this.value = value; + this.operation = operation; + } + + static StoreEntry readFromTreeStore(Iterator> it) { + while (it.hasNext()) { + Map.Entry e = it.next(); + if (!e.getValue().isEmpty()) { + return new StoreEntry(e.getKey(), e.getValue(), null); + } + } + return null; + } + + static StoreEntry readFromReader(BufferedReader reader) throws IOException { + String line = reader.readLine(); + if (line == null) { + return null; + } + String[] parts = IncrementalFlatFileStoreNodeStateEntryWriter.getParts(line); + return new StoreEntry(parts[0], parts[1], OPERATION_MAP.get(parts[3])); + } + } + + + private IndexStoreMetadata getIndexStoreMetadataForMergedFile() throws IOException { + File baseFFSMetadataFile = IndexStoreUtils.getMetadataFile(baseFile, algorithm); + File incrementalMetadataFile = IndexStoreUtils.getMetadataFile(incrementalFile, algorithm); + + if (baseFFSMetadataFile.exists() && incrementalMetadataFile.exists()) { + IndexStoreMetadata indexStoreMetadata = new IndexStoreMetadataOperatorImpl() + .getIndexStoreMetadata(baseFFSMetadataFile, algorithm, new TypeReference<>() { + }); + IncrementalIndexStoreMetadata incrementalIndexStoreMetadata = new IndexStoreMetadataOperatorImpl() + .getIndexStoreMetadata(incrementalMetadataFile, algorithm, new TypeReference<>() { + }); + return mergeIndexStores(indexStoreMetadata, incrementalIndexStoreMetadata); + } else { + throw new RuntimeException("either one or both metadataFiles don't exist at path: " + + baseFFSMetadataFile.getAbsolutePath() + ", " + incrementalMetadataFile.getAbsolutePath()); + } + } + + private void mergeMetadataFiles() throws IOException { + try (BufferedWriter writer = IndexStoreUtils.createWriter(IndexStoreUtils.getMetadataFile(mergedFile, algorithm), algorithm)) { + JSON_MAPPER.writeValue(writer, getIndexStoreMetadataForMergedFile()); + } + } + + /** + * We only merge indexStore and incrementalStore if: + * 1. IndexStore's checkpoint equals incrementalStore's before checkpoint. + * 2. IndexStore's preferredPaths equals incrementalStore's preferredPaths. + */ + private IndexStoreMetadata mergeIndexStores(IndexStoreMetadata indexStoreMetadata, + IncrementalIndexStoreMetadata incrementalIndexStoreMetadata) { + checkState(indexStoreMetadata.getCheckpoint().equals(incrementalIndexStoreMetadata.getBeforeCheckpoint())); + return new IndexStoreMetadata(incrementalIndexStoreMetadata.getAfterCheckpoint(), indexStoreMetadata.getStoreType(), + getStrategyName(), Collections.emptySet()); + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java index cdb3c1f78d3..5886cef32db 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/indexstore/IndexStoreUtils.java @@ -128,7 +128,11 @@ public static File getMetadataFile(File indexStoreFile, Compression algorithm) { } private static String getCompressionSuffix(File file) { - return file.getName().substring(file.getName().lastIndexOf(".")); + int lastDot = file.getName().lastIndexOf("."); + if (lastDot < 0) { + return ""; + } + return file.getName().substring(lastDot); } /** diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java new file mode 100644 index 00000000000..8cf3bafc0f3 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilter.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; + +/** + * A utility class that allows skipping nodes that are not included in the index + * definition. + * + * The use case is to speed up indexing by only traversing over the nodes that + * are included in the set of indexes. + */ +public class PathIteratorFilter { + + private final boolean includeAll; + private final TreeSet includedPaths; + + private String cachedMatchingPrefix; + + PathIteratorFilter(SortedSet includedPaths) { + this.includedPaths = new TreeSet<>(includedPaths); + this.includeAll = includedPaths.contains(PathUtils.ROOT_PATH); + } + + public PathIteratorFilter() { + this.includedPaths = new TreeSet<>(); + this.includeAll = true; + } + + /** + * Extract all the path filters from a set of index definitions. + * + * @param indexDefs the index definitions + * @return the list of path filters + */ + public static List extractPathFilters(Set indexDefs) { + return indexDefs.stream().map(IndexDefinition::getPathFilter).collect(Collectors.toList()); + } + + /** + * Extract a list of included paths from a path filter. Only the top-most + * entries are retained. Excluded path are ignored. + * + * @param pathFilters the path filters + * @return the set of included path, sorted by path + */ + public static SortedSet getAllIncludedPaths(List pathFilters) { + TreeSet set = new TreeSet<>(); + // convert to a flat set + for (PathFilter f : pathFilters) { + for (String p : f.getIncludedPaths()) { + set.add(p); + } + } + // only keep entries where the parent isn't in the set + TreeSet result = new TreeSet<>(); + for (String path : set) { + boolean parentExists = false; + String p = path; + while (!PathUtils.denotesRoot(p)) { + p = PathUtils.getParentPath(p); + if (set.contains(p)) { + parentExists = true; + break; + } + } + if (!parentExists) { + result.add(path); + } + } + return result; + } + + public boolean includes(String path) { + if (includeAll) { + return true; + } + String cache = cachedMatchingPrefix; + if (cache != null && path.startsWith(cache)) { + return true; + } + String p = path; + while (!PathUtils.denotesRoot(p)) { + if (includedPaths.contains(p)) { + // add a final slash, so that we only accept children + String newCache = p; + if (!newCache.endsWith("/")) { + newCache += "/"; + } + cachedMatchingPrefix = newCache; + return true; + } + p = PathUtils.getParentPath(p); + } + return false; + } + + /** + * Get the next higher included path, or null if none. + * + * @param path the path + * @return the next included path, or null + */ + public String nextIncludedPath(String path) { + if (includeAll) { + return null; + } + return includedPaths.higher(path); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java new file mode 100644 index 00000000000..779227244bc --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.utils.Hash; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.utils.HyperLogLog; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The prefetcher, in a separate threads, reads ahead of indexing, such that the + * nodestore cache and datastore cache is filled. + * + * There are 3 threads (in addition to the threads that read from the datastore): + * - TRACK_INDEXING - tries to be at the same position as the indexing + * - DATASTORE_PREFETCH - reads ahead about 1 GB, such that the datastore cache is filled + * - NODESTORE_CACHE_FILLER - reads ahead about 32'000 entries, such that the node store cache is filled + */ +public class Prefetcher { + + private static final Logger LOG = LoggerFactory.getLogger(Prefetcher.class); + + private static final int PRETCH_THREADS = 16; + + private final TreeStore prefetchStore; + private final TreeStore indexStore; + private final ExecutorService executorService; + private final AtomicLong downloadMax = new AtomicLong(); + private final AtomicLong iterateCount = new AtomicLong(); + private final Semaphore semaphore = new Semaphore(PRETCH_THREADS); + + private String blobSuffix; + + private volatile long blobReadAheadSize = 4 * 1024 * 1024 * 1024L; + private volatile long nodeReadAheadCount = 64 * 1024; + private volatile long maxBlobSize; + + public Prefetcher(TreeStore prefetchStore, TreeStore indexStore) { + this.prefetchStore = prefetchStore; + this.indexStore = indexStore; + this.executorService = Executors.newFixedThreadPool(3 + PRETCH_THREADS, new ThreadFactory() { + private final AtomicInteger threadNumber = new AtomicInteger(1); + + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setName("BlobPrefetcher-" + threadNumber.getAndIncrement()); + return t; + } + }); + } + + public void setBlobSuffix(String blobSuffix) { + this.blobSuffix = blobSuffix; + } + + public void setBlobReadAheadSize(long blobReadAheadSize) { + this.blobReadAheadSize = blobReadAheadSize; + } + + public void setNodeReadAheadCount(long nodeReadAheadCount) { + this.nodeReadAheadCount = nodeReadAheadCount; + } + + public boolean shutdown() throws InterruptedException { + executorService.shutdown(); + return executorService.awaitTermination(1, TimeUnit.SECONDS); + } + + public void startPrefetch() { + LOG.info("Prefetch suffix '{}', prefetch {}, index {}", + blobSuffix, prefetchStore, indexStore); + executorService.submit( + iterator(PrefetchType.TRACK_INDEXING)); + executorService.submit( + iterator(PrefetchType.NODESTORE_CACHE_FILLER)); + if (!blobSuffix.isEmpty()) { + executorService.submit( + iterator(PrefetchType.BLOB_PREFETCH)); + } + } + + public void sleep(String status) throws InterruptedException { + Thread.sleep(10); + } + + Runnable iterator(PrefetchType prefetchType) { + return () -> { + Iterator it = prefetchStore.iteratorOverPaths(); + HyperLogLog estimatedUniqueBlobCount = new HyperLogLog(1024, 0); + AtomicLong prefetched = new AtomicLong(); + long count = 0; + try { + long totalBlobCount = 0; + long inlinedBlobCount = 0; + long totalBlobSize = 0; + while (it.hasNext()) { + String path = it.next(); + if (++count % 1_000_000 == 0) { + int available = semaphore.availablePermits(); + LOG.info("Iterated {} type {} inlinedCount {} totalCount {} " + + "totalSize {} maxSize {} max {} availableThreads {} " + + "indexing {} prefetch {} path {}", + count, prefetchType, inlinedBlobCount, totalBlobCount, + totalBlobSize, maxBlobSize, downloadMax.get(), available, + indexStore.toString(), prefetchStore.toString(), path); + } + if (prefetchType == PrefetchType.TRACK_INDEXING) { + iterateCount.set(count); + while (true) { + String indexingPath = indexStore.getHighestReadKey(); + if (indexingPath.compareTo(path) >= 0) { + break; + } + sleep("wait for indexing to progress"); + } + } + if (prefetchType == PrefetchType.NODESTORE_CACHE_FILLER) { + while (count - nodeReadAheadCount > iterateCount.get()) { + sleep("wait in node cache fillter"); + } + // this will fill the page cache of the index store + String value = indexStore.getSession().get(path); + // this will not cause a cache miss + TreeStoreNodeState entry = indexStore.buildNodeState(path, value); + indexStore.prefillCache(path, entry); + continue; + } + if (!path.endsWith(blobSuffix)) { + continue; + } + NodeStateEntry nse = prefetchStore.getNodeStateEntry(path); + PropertyState p = nse.getNodeState().getProperty("jcr:data"); + if (p == null || p.isArray() || p.getType() != Type.BINARY) { + continue; + } + Blob blob = p.getValue(Type.BINARY); + if (blob.isInlined()) { + inlinedBlobCount++; + continue; + } + estimatedUniqueBlobCount.add(longHash(blob)); + totalBlobCount++; + totalBlobSize += blob.length(); + maxBlobSize = Math.max(maxBlobSize, blob.length()); + if (prefetchType == PrefetchType.TRACK_INDEXING) { + downloadMax.set(totalBlobSize); + continue; + } + if (prefetchType != PrefetchType.BLOB_PREFETCH) { + throw new IllegalStateException("Incorrect type: " + prefetchType); + } + String indexingPos = indexStore.getHighestReadKey(); + if (indexingPos.compareTo(path) >= 0) { + // we are behind indexing! + // do not download, in order to catch up + LOG.info("Indexing is ahead; ignoring {}", path); + continue; + } + while (totalBlobSize - blobReadAheadSize > downloadMax.get()) { + sleep("wait in downloader"); + } + semaphore.acquire(); + executorService.submit(() -> { + try { + LOG.debug("Prefetching {} took {} ms", path); + InputStream in = blob.getNewStream(); + // read one byte only, in order to prefetch + in.read(); + in.close(); + } catch (IOException e) { + LOG.warn("Prefetching failed", path, e); + } + semaphore.release(); + prefetched.incrementAndGet(); + }); + } + } catch (Exception e) { + LOG.warn("Prefetch error", e); + } finally { + LOG.info("Completed after {} nodes, {} prefetched, {} unique", + count, prefetched.get(), + estimatedUniqueBlobCount.estimate()); + } + }; + } + + private static long longHash(Blob blob) { + // the String.hashCode is only 32 bit + // because of that, we mix it with the length + // and then we use a secondary hash + // otherwise the estimation is way off + int h = blob.getContentIdentity().hashCode(); + return Hash.hash64(h | (blob.length() << 32)); + } + + static enum PrefetchType { + TRACK_INDEXING, + BLOB_PREFETCH, + NODESTORE_CACHE_FILLER + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java new file mode 100644 index 00000000000..c3b4f63e509 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedSet; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry.NodeStateEntryBuilder; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Compression; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.StoreBuilder; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SieveCache; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The tree store is similar to the flat file store, but instead of storing all + * key-value pairs in a single file, it stores the entries in multiple files + * (except if there are very few nodes). + */ +public class TreeStore implements IndexStore { + + private static final Logger LOG = LoggerFactory.getLogger(TreeStore.class); + + public static final String DIRECTORY_NAME = "tree"; + + private static final String STORE_TYPE = "TreeStore"; + private static final String TREE_STORE_CONFIG = "oak.treeStoreConfig"; + + public static final long CACHE_SIZE_NODE_MB = 64; + private static final long CACHE_SIZE_TREE_STORE_MB = 64; + + private static final long MAX_FILE_SIZE_MB = 4; + private static final long MB = 1024 * 1024; + + private final String name; + private final Store store; + private final long cacheSizeTreeStoreMB; + private final File directory; + private final TreeSession session; + private final NodeStateEntryReader entryReader; + private final SieveCache nodeStateCache; + private long entryCount; + private volatile String highestReadKey = ""; + private final AtomicLong nodeCacheHits = new AtomicLong(); + private final AtomicLong nodeCacheMisses = new AtomicLong(); + private final AtomicLong nodeCacheFills = new AtomicLong(); + private int iterationCount; + private PathIteratorFilter filter = new PathIteratorFilter(); + + public TreeStore(String name, File directory, NodeStateEntryReader entryReader, long cacheSizeFactor) { + this.name = name; + this.directory = directory; + this.entryReader = entryReader; + long cacheSizeNodeMB = cacheSizeFactor * CACHE_SIZE_NODE_MB; + long cacheSizeTreeStoreMB = cacheSizeFactor * CACHE_SIZE_TREE_STORE_MB; + this.cacheSizeTreeStoreMB = cacheSizeTreeStoreMB; + nodeStateCache = new SieveCache<>(cacheSizeFactor * cacheSizeNodeMB * MB); + String storeConfig = System.getProperty(TREE_STORE_CONFIG, + "type=file\n" + + TreeSession.CACHE_SIZE_MB + "=" + cacheSizeTreeStoreMB + "\n" + + Store.MAX_FILE_SIZE_BYTES + "=" + MAX_FILE_SIZE_MB * MB + "\n" + + "dir=" + directory.getAbsolutePath()); + this.store = StoreBuilder.build(storeConfig); + store.setWriteCompression(Compression.LZ4); + this.session = new TreeSession(store); + // we don not want to merge too early during the download + session.setMaxRoots(1000); + LOG.info("Open " + toString()); + } + + public void init() { + session.init(); + } + + public void setIndexDefinitions(Set indexDefs) { + if (indexDefs == null) { + return; + } + List pathFilters = PathIteratorFilter.extractPathFilters(indexDefs); + SortedSet includedPaths = PathIteratorFilter.getAllIncludedPaths(pathFilters); + LOG.info("Included paths {}", includedPaths.toString()); + filter = new PathIteratorFilter(includedPaths); + } + + @Override + public String toString() { + return name + + " cache " + cacheSizeTreeStoreMB + + " at " + highestReadKey + + " cache-hits " + nodeCacheHits.get() + + " cache-misses " + nodeCacheMisses.get() + + " cache-fills " + nodeCacheFills.get(); + } + + public Iterator iteratorOverPaths() { + final Iterator> firstIterator = session.iterator(); + return new Iterator() { + + Iterator> it = firstIterator; + String current; + + { + fetch(); + } + + private void fetch() { + while (it.hasNext()) { + Entry e = it.next(); + String key = e.getKey(); + String value = e.getValue(); + // if the value is empty (not null!) this is a child node reference, + // without node data + if (value.isEmpty()) { + continue; + } + if (!filter.includes(key)) { + // if the path is not, see if there is a next included path + String next = filter.nextIncludedPath(key); + if (next == null) { + // it was the last one + break; + } + // this node itself could be a match + key = next; + value = session.get(key); + // for the next fetch operation, we use the new iterator + it = session.iterator(next); + if (value == null || value.isEmpty()) { + // no such node, or it's a child node reference + continue; + } + } + if (++iterationCount % 1_000_000 == 0) { + LOG.info("Fetching {} in {}", iterationCount, TreeStore.this.toString()); + } + current = key; + if (current.compareTo(highestReadKey) > 0) { + highestReadKey = current; + } + return; + } + current = null; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public String next() { + String result = current; + fetch(); + return result; + } + + }; + } + + @Override + public Iterator iterator() { + Iterator it = iteratorOverPaths(); + return new Iterator() { + + NodeStateEntry current; + + { + fetch(); + } + + private void fetch() { + current = it.hasNext() ? getNodeStateEntry(it.next()) : null; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public NodeStateEntry next() { + NodeStateEntry result = current; + fetch(); + return result; + } + + }; + } + + @Override + public void close() throws IOException { + session.flush(); + store.close(); + } + + public String getHighestReadKey() { + return highestReadKey; + } + + public NodeStateEntry getNodeStateEntry(String path) { + return new NodeStateEntryBuilder(getNodeState(path), path).build(); + } + + NodeStateEntry getNodeStateEntry(String path, String value) { + return new NodeStateEntryBuilder(getNodeState(path, value), path).build(); + } + + NodeState getNodeState(String path) { + TreeStoreNodeState result = nodeStateCache.get(path); + if (result != null) { + nodeCacheHits.incrementAndGet(); + return result; + } + nodeCacheMisses.incrementAndGet(); + String value = session.get(path); + if (value == null || value.isEmpty()) { + result = new TreeStoreNodeState(EmptyNodeState.MISSING_NODE, path, this, path.length() * 2); + } else { + result = getNodeState(path, value); + } + if (path.compareTo(highestReadKey) > 0) { + highestReadKey = path; + } + nodeStateCache.put(path, result); + return result; + } + + TreeStoreNodeState getNodeState(String path, String value) { + TreeStoreNodeState result = nodeStateCache.get(path); + if (result != null) { + nodeCacheHits.incrementAndGet(); + return result; + } + nodeCacheMisses.incrementAndGet(); + result = buildNodeState(path, value); + if (path.compareTo(highestReadKey) > 0) { + highestReadKey = path; + } + nodeStateCache.put(path, result); + return result; + } + + TreeStoreNodeState buildNodeState(String path, String value) { + String line = path + "|" + value; + NodeStateEntry entry = entryReader.read(line); + return new TreeStoreNodeState(entry.getNodeState(), path, this, path.length() * 2 + line.length() * 10); + } + + public void prefillCache(String path, TreeStoreNodeState nse) { + TreeStoreNodeState old = nodeStateCache.put(path, nse); + if (old == null) { + nodeCacheFills.incrementAndGet(); + } + } + + /** + * The child node entry for the given path. + * + * @param path the path, e.g. /hello/world + * @return the child node entry, e.g. /helloworld + */ + public static String toChildNodeEntry(String path) { + if (path.equals("/")) { + return "\t"; + } + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + return parentPath + "\t" + nodeName; + } + + /** + * Convert a child node entry to parent and node name. + * This method is used for tooling and testing only. + * It does the reverse of toChildNodeEntry(parentPath, childName) + * + * @param child node entry, e.g. /helloworld + * @return the parent path and the child node name, e.g. ["/hello" "world"] + * @throws IllegalArgumentException if this is not a child node entry + */ + public static String[] toParentAndChildNodeName(String key) { + int index = key.lastIndexOf('\t'); + if (index < 0) { + throw new IllegalArgumentException("Not a child node entry: " + key); + } + return new String[] { key.substring(0, index), key.substring(index + 1) }; + } + + /** + * The child node entry for the given parent and child. + * + * @param path the parentPath, e.g. /hello + * @param childName the name of the child node, e.g. world + * @return the child node entry, e.g. /helloworld + */ + public static String toChildNodeEntry(String parentPath, String childName) { + return parentPath + "\t" + childName; + } + + public void putNode(String path, String json) { + session.put(path, json); + if (!path.equals("/")) { + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + session.put(parentPath + "\t" + nodeName, ""); + } + } + + public TreeSession getSession() { + return session; + } + + public Store getStore() { + return store; + } + + @Override + public String getStorePath() { + return directory.getAbsolutePath(); + } + + @Override + public long getEntryCount() { + return entryCount; + } + + public void setEntryCount(long entryCount) { + this.entryCount = entryCount; + } + + @Override + public String getIndexStoreType() { + return STORE_TYPE; + } + + @Override + public boolean isIncremental() { + return false; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java new file mode 100644 index 00000000000..72da39575ed --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreNodeState.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.apache.jackrabbit.guava.common.collect.Iterators.transform; + +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryObject; +import org.apache.jackrabbit.oak.plugins.memory.MemoryChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.AbstractNodeState; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * A node state of an Oak node that is stored in a tree store. + * + * This is mostly a wrapper. It allows iterating over the children and reading + * children directly. + */ +public class TreeStoreNodeState implements NodeState, MemoryObject { + + private final NodeState delegate; + private final String path; + private final TreeStore treeStore; + private final long estimatedMemory; + + public TreeStoreNodeState(NodeState delegate, String path, TreeStore treeStore, long estimatedMemory) { + this.delegate = delegate; + this.path = path; + this.treeStore = treeStore; + this.estimatedMemory = estimatedMemory; + } + + @Override + public long estimatedMemory() { + return estimatedMemory; + } + + @Override + public boolean exists() { + return delegate.exists(); + } + + @Override + public boolean hasProperty(@NotNull String name) { + return delegate.hasProperty(name); + } + + @Nullable + @Override + public PropertyState getProperty(@NotNull String name) { + return delegate.getProperty(name); + } + + @Override + public boolean getBoolean(@NotNull String name) { + return delegate.getBoolean(name); + } + + @Override + public long getLong(String name) { + return delegate.getLong(name); + } + + @Nullable + @Override + public String getString(String name) { + return delegate.getString(name); + } + + @NotNull + @Override + public Iterable getStrings(@NotNull String name) { + return delegate.getStrings(name); + } + + @Nullable + @Override + public String getName(@NotNull String name) { + return delegate.getName(name); + } + + @NotNull + @Override + public Iterable getNames(@NotNull String name) { + return delegate.getNames(name); + } + + @Override + public long getPropertyCount() { + return delegate.getPropertyCount(); + } + + @NotNull + @Override + public Iterable getProperties() { + return delegate.getProperties(); + } + + @NotNull + @Override + public NodeBuilder builder() { + return delegate.builder(); + } + + @Override + public boolean compareAgainstBaseState(NodeState base, NodeStateDiff diff) { + return AbstractNodeState.compareAgainstBaseState(this, base, diff); + } + + // ~-------------------------------< child node access > + + @Override + public boolean hasChildNode(@NotNull String name) { + return treeStore.getNodeState(PathUtils.concat(path, name)).exists(); + } + + @NotNull + @Override + public NodeState getChildNode(@NotNull String name) throws IllegalArgumentException { + return treeStore.getNodeState(PathUtils.concat(path, name)); + } + + @Override + public long getChildNodeCount(long max) { + long result = 0; + Iterator it = getChildNodeNamesIterator(); + while (it.hasNext()) { + result++; + if (result > max) { + return Long.MAX_VALUE; + } + it.next(); + } + return result; + } + + @Override + public Iterable getChildNodeNames() { + return new Iterable() { + public Iterator iterator() { + return getChildNodeNamesIterator(); + } + }; + } + + @NotNull + @Override + public Iterable getChildNodeEntries() { + return () -> transform(getChildNodeIterator(), + s -> new MemoryChildNodeEntry(PathUtils.getName(s.getPath()), s.getNodeState())); + } + + private Iterator getChildNodeIterator() { + return transform(getChildNodeNamesIterator(), + s -> treeStore.getNodeStateEntry(PathUtils.concat(path, s))); + } + + Iterator getChildNodeNamesIterator() { + Iterator> it = treeStore.getSession().iterator(path); + return new Iterator() { + String current; + { + fetch(); + } + + private void fetch() { + if (!it.hasNext()) { + current = null; + } else { + Entry e = it.next(); + if (!e.getValue().isEmpty()) { + current = null; + } else { + String key = e.getKey(); + int index = key.lastIndexOf('\t'); + if (index < 0) { + throw new IllegalArgumentException(key); + } + current = key.substring(index + 1); + } + } + } + + public boolean hasNext() { + return current != null; + } + + public String next() { + String result = current; + if (result == null) { + throw new IllegalStateException(); + } + fetch(); + return result; + } + }; + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java new file mode 100644 index 00000000000..f2eb586b622 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import java.util.Iterator; + +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.Store; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; + +/** + * A command line utility for the tree store. + */ +public class TreeStoreUtils { + + public static void main(String... args) throws IOException { + String dir = args[0]; + MemoryBlobStore blobStore = new MemoryBlobStore(); + NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); + try (TreeStore treeStore = new TreeStore("utils", new File(dir), entryReader, 16)) { + TreeSession session = treeStore.getSession(); + Store store = treeStore.getStore(); + if (store.keySet().isEmpty()) { + session.init(); + String fileName = args[1]; + try (BufferedReader lineReader = new BufferedReader( + new FileReader(fileName, StandardCharsets.UTF_8))) { + int count = 0; + long start = System.nanoTime(); + while (true) { + String line = lineReader.readLine(); + if (line == null) { + break; + } + count++; + if (count % 1000000 == 0) { + long time = System.nanoTime() - start; + System.out.println(count + " " + (time / count) + " ns/entry"); + } + int index = line.indexOf('|'); + if (index < 0) { + throw new IllegalArgumentException("| is missing: " + line); + } + String path = line.substring(0, index); + String value = line.substring(index + 1); + session.put(path, value); + + if (!path.equals("/")) { + String nodeName = PathUtils.getName(path); + String parentPath = PathUtils.getParentPath(path); + session.put(parentPath + "\t" + nodeName, ""); + } + + } + } + session.flush(); + store.close(); + } + Iterator it = treeStore.iterator(); + long nodeCount = 0; + long childNodeCount = 0; + long start = System.nanoTime(); + while (it.hasNext()) { + NodeStateEntry e = it.next(); + childNodeCount += e.getNodeState().getChildNodeCount(Long.MAX_VALUE); + nodeCount++; + if (nodeCount % 1000000 == 0) { + long time = System.nanoTime() - start; + System.out.println("Node count: " + nodeCount + + " child node count: " + childNodeCount + + " speed: " + (time / nodeCount) + " ns/entry"); + } + } + System.out.println("Node count: " + nodeCount + " Child node count: " + childNodeCount); + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java new file mode 100644 index 00000000000..5d922e6d9c2 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Compression.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Arrays; + +import net.jpountz.lz4.LZ4Compressor; +import net.jpountz.lz4.LZ4Factory; +import net.jpountz.lz4.LZ4FastDecompressor; + +/** + * The enum allows to disable or enable compression of the storage files. + */ +public enum Compression { + NO { + @Override + byte[] compress(byte[] data) { + return data; + } + + @Override + byte[] expand(byte[] data) { + return data; + } + }, + LZ4 { + private final LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor(); + private final LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor(); + + private byte[] compressBuffer = new byte[1024 * 1024]; + + @Override + byte[] compress(byte[] data) { + // synchronization is needed because we share the buffer + synchronized (compressor) { + byte[] buffer = compressBuffer; + if (buffer.length < 2 * data.length) { + // increase the size + buffer = new byte[2 * data.length]; + compressBuffer = buffer; + } + buffer[0] = '4'; + writeInt(buffer, 1, data.length); + int len = 5 + compressor.compress(data, 0, data.length, buffer, 5, buffer.length - 5); + return Arrays.copyOf(buffer, len); + } + } + + @Override + byte[] expand(byte[] data) { + int len = readInt(data, 1); + byte[] target = new byte[len]; + decompressor.decompress(data, 5, target, 0, len); + return target; + } + }; + + abstract byte[] compress(byte[] data); + abstract byte[] expand(byte[] data); + + + public static void writeInt(byte[] buff, int pos, int x) { + buff[pos++] = (byte) (x >> 24); + buff[pos++] = (byte) (x >> 16); + buff[pos++] = (byte) (x >> 8); + buff[pos] = (byte) x; + } + + public static int readInt(byte[] buff, int pos) { + return (buff[pos++] << 24) + ((buff[pos++] & 0xff) << 16) + ((buff[pos++] & 0xff) << 8) + (buff[pos] & 0xff); + } + + public static Compression getCompressionFromData(byte data) { + switch (data) { + case '4': + return Compression.LZ4; + default: + return Compression.NO; + } + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java new file mode 100644 index 00000000000..5894f589267 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.TimeUuid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A storage backend for the tree store that stores files on the local file + * system. + * + * This is the main (read-write) storage backend. + */ +public class FileStore implements Store { + + private static final Logger LOG = LoggerFactory.getLogger(FileStore.class); + + private final Properties config; + private final String directory; + private Compression compression = Compression.NO; + private long writeCount, readCount; + private final long maxFileSizeBytes; + + public String toString() { + return "file(" + directory + ")"; + } + + public FileStore(Properties config) { + this.config = config; + this.directory = config.getProperty("dir"); + this.maxFileSizeBytes = Long.parseLong(config.getProperty( + Store.MAX_FILE_SIZE_BYTES, "" + Store.DEFAULT_MAX_FILE_SIZE_BYTES)); + LOG.info("Max file size {} bytes", maxFileSizeBytes); + new File(directory).mkdirs(); + } + + @Override + public void close() { + } + + @Override + public void setWriteCompression(Compression compression) { + this.compression = compression; + } + + @Override + public PageFile getIfExists(String key) { + readCount++; + File f = getFile(key); + if (!f.exists()) { + return null; + } + try (RandomAccessFile file = new RandomAccessFile(f, "r")) { + long length = file.length(); + if (length == 0) { + // deleted in the meantime + return null; + } + byte[] data = new byte[(int) length]; + file.readFully(data); + Compression c = Compression.getCompressionFromData(data[0]); + data = c.expand(data); + return PageFile.fromBytes(data, maxFileSizeBytes); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public void put(String key, PageFile value) { + writeCount++; + writeFile(key, value.toBytes()); + } + + @Override + public boolean supportsByteOperations() { + return true; + } + + @Override + public byte[] getBytes(String key) { + File f = getFile(key); + if (!f.exists()) { + return null; + } + try { + readCount++; + try (RandomAccessFile file = new RandomAccessFile(f, "r")) { + long length = file.length(); + if (length == 0) { + // deleted in the meantime + return null; + } + byte[] data = new byte[(int) length]; + file.readFully(data); + return data; + } + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public void putBytes(String key, byte[] data) { + try (FileOutputStream out = new FileOutputStream(getFile(key))) { + out.write(data); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + + private void writeFile(String key, byte[] data) { + data = compression.compress(data); + putBytes(key, data); + } + + private File getFile(String key) { + return new File(directory, key); + } + + @Override + public String newFileName() { + return TimeUuid.newUuid().toShortString(); + } + + @Override + public Set keySet() { + File dir = new File(directory); + if (!dir.exists()) { + return Collections.emptySet(); + } + String[] list = dir.list(new FilenameFilter() { + + @Override + public boolean accept(File dir, String name) { + return new File(dir, name).isFile(); + } + + }); + return new HashSet<>(Arrays.asList(list)); + } + + @Override + public void remove(Set set) { + for (String key : set) { + writeCount++; + getFile(key).delete(); + } + } + + @Override + public void removeAll() { + File dir = new File(directory); + for(File f: dir.listFiles()) { + f.delete(); + } + } + + @Override + public long getWriteCount() { + return writeCount; + } + + @Override + public long getReadCount() { + return readCount; + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public long getMaxFileSizeBytes() { + return maxFileSizeBytes; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java new file mode 100644 index 00000000000..a1ec6c3a608 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/GarbageCollection.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.HashSet; +import java.util.List; + +/** + * Remove unreferenced files from the store. + * + * Only root files and inner nodes are read. This is possible because leaf pages + * do not have references to other files. + */ +public class GarbageCollection { + + private final Store store; + + public GarbageCollection(Store store) { + this.store = store; + } + + /** + * Run garbage collection. + * + * @param rootFiles + * @return the result + */ + public GarbageCollectionResult run(List rootFiles) { + HashSet used = mark(rootFiles); + return sweep(used); + } + + HashSet mark(List rootFiles) { + HashSet used = new HashSet<>(); + for(String root : rootFiles) { + mark(root, used); + } + return used; + } + + private void mark(String fileName, HashSet used) { + used.add(fileName); + if (fileName.startsWith(TreeSession.LEAF_PREFIX)) { + return; + } + // root or inner node + PageFile file = store.get(fileName); + if (file.isInnerNode()) { + for (int i = 0; i < file.getValueCount(); i++) { + mark(file.getChildValue(i), used); + } + } + } + + private GarbageCollectionResult sweep(HashSet used) { + GarbageCollectionResult result = new GarbageCollectionResult(); + HashSet removeSet = new HashSet(); + for (String key: new HashSet<>(store.keySet())) { + if (!used.contains(key)) { + removeSet.add(key); + result.countRemoved++; + } else { + result.countKept++; + } + } + store.remove(removeSet); + return result; + } + + /** + * Garbage collection results. + */ + public static class GarbageCollectionResult { + public long sizeInBytes; + public long countRemoved; + public long countKept; + + public String toString() { + return "removed: " + countRemoved + " kept: " + countKept + " size: " + sizeInBytes; + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java new file mode 100644 index 00000000000..8224b23bea5 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/LogStore.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Arrays; +import java.util.Properties; +import java.util.Set; + +/** + * A wrapper for storage backends that allows to log store and read operations. + */ +public class LogStore implements Store { + + private final Properties config; + private final Store backend; + + public String toString() { + return "log(" + backend + ")"; + } + + LogStore(Store backend) { + this.config = backend.getConfig(); + this.backend = backend; + } + + private void log(String message, Object... args) { + System.out.println(backend + "." + message + " " + Arrays.toString(args)); + } + + @Override + public PageFile getIfExists(String key) { + log("getIfExists", key); + return backend.getIfExists(key); + } + + @Override + public void put(String key, PageFile value) { + log("put", key); + backend.put(key, value); + } + + @Override + public String newFileName() { + return backend.newFileName(); + } + + @Override + public Set keySet() { + log("keySet"); + return backend.keySet(); + } + + @Override + public void remove(Set set) { + log("remove", set); + backend.remove(set); + } + + @Override + public void removeAll() { + log("removeAll"); + backend.removeAll(); + } + + @Override + public long getWriteCount() { + return backend.getWriteCount(); + } + + @Override + public long getReadCount() { + return backend.getReadCount(); + } + + @Override + public void setWriteCompression(Compression compression) { + log("setWriteCompression", compression); + backend.setWriteCompression(compression); + } + + @Override + public void close() { + log("close"); + backend.close(); + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public boolean supportsByteOperations() { + return backend.supportsByteOperations(); + } + + @Override + public byte[] getBytes(String key) { + log("getBytes", key); + return backend.getBytes(key); + } + + @Override + public void putBytes(String key, byte[] data) { + log("putBytes", key, data.length); + backend.putBytes(key, data); + } + + @Override + public long getMaxFileSizeBytes() { + return backend.getMaxFileSizeBytes(); + } + +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java new file mode 100644 index 00000000000..9e7c238cab3 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MemoryStore.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.HashMap; +import java.util.Properties; +import java.util.Set; + +/** + * An in-memory storage backend for the tree store. + */ +public class MemoryStore implements Store { + + private final Properties config; + private final HashMap map = new HashMap<>(); + private long nextFileName; + private long writeCount, readCount; + private final long maxFileSizeBytes; + + public MemoryStore() { + this(new Properties()); + } + + public MemoryStore(Properties config) { + this.config = config; + this.maxFileSizeBytes = Long.parseLong(config.getProperty( + Store.MAX_FILE_SIZE_BYTES, "" + Store.DEFAULT_MAX_FILE_SIZE_BYTES)); + } + + @Override + public void setWriteCompression(Compression compression) { + // ignore + } + + public PageFile getIfExists(String key) { + readCount++; + return map.get(key); + } + + public void put(String key, PageFile file) { + writeCount++; + map.put(key, file); + } + + public String toString() { + return "files: " + map.size(); + } + + public String newFileName() { + return "f" + nextFileName++; + } + + public Set keySet() { + return map.keySet(); + } + + public void remove(Set set) { + for (String key : set) { + writeCount++; + map.remove(key); + } + } + + @Override + public void removeAll() { + map.clear(); + nextFileName = 0; + } + + @Override + public long getWriteCount() { + return writeCount; + } + + @Override + public long getReadCount() { + return readCount; + } + + @Override + public void close() { + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public long getMaxFileSizeBytes() { + return maxFileSizeBytes; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java new file mode 100644 index 00000000000..8cc5f01b22d --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFile.java @@ -0,0 +1,393 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.MemoryObject; + +/** + * A B-tree page (leaf, or inner node). + * An inner node contains one more value than keys. + * A leaf page has the same number of keys and values. + */ +public class PageFile implements MemoryObject { + + private static final boolean VERIFY_SIZE = false; + private static final int INITIAL_SIZE_IN_BYTES = 24; + + private String fileName; + private final long maxFileSizeBytes; + + private final boolean innerNode; + + private static ByteBuffer REUSED_BUFFER = ByteBuffer.allocate(1024 * 1024); + + private ArrayList keys = new ArrayList<>(); + private ArrayList values = new ArrayList<>(); + private long update; + private String nextRoot; + private int sizeInBytes = INITIAL_SIZE_IN_BYTES; + + // -1: beginning; 0: middle; 1: end + private int lastSearchIndex; + + // contains unwritten modifications + private boolean modified; + + public PageFile(boolean innerNode, long maxFileSizeBytes) { + this.innerNode = innerNode; + this.maxFileSizeBytes = maxFileSizeBytes; + } + + @Override + public long estimatedMemory() { + return maxFileSizeBytes; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getFileName() { + return fileName; + } + + public void setUpdate(long update) { + modified = true; + this.update = update; + } + + public static PageFile fromBytes(byte[] data, long maxFileSizeBytes) { + ByteBuffer buff = ByteBuffer.wrap(data); + int type = buff.get(); + String nextRoot = readString(buff); + long update = buff.getLong(); + String prefix = readString(buff); + int len = buff.getInt(); + PageFile result; + if (type == 0) { + result = new PageFile(true, maxFileSizeBytes); + for (int i = 0; i < len; i++) { + result.appendRecord(prefix + readString(buff), readString(buff)); + } + result.values.add(readString(buff)); + } else { + result = new PageFile(false, maxFileSizeBytes); + for (int i = 0; i < len; i++) { + result.appendRecord(prefix + readString(buff), readString(buff)); + } + } + if (!nextRoot.isEmpty()) { + result.setNextRoot(nextRoot); + } + result.setUpdate(update); + result.modified = false; + return result; + } + + public byte[] toBytes() { + // synchronization is needed because we share the buffer + synchronized (PageFile.class) { + ByteBuffer buff = REUSED_BUFFER; + if (buff.capacity() < sizeInBytes * 2) { + buff = REUSED_BUFFER = ByteBuffer.allocate(sizeInBytes * 2); + } + buff.rewind(); + // first byte may not be '4', as that is used for LZ4 compression + buff.put((byte) (innerNode ? 0 : 1)); + writeString(buff, nextRoot == null ? "" : nextRoot); + buff.putLong(update); + String prefix = keys.size() < 2 ? "" : commonPrefix(keys.get(0), keys.get(keys.size() - 1)); + writeString(buff, prefix); + buff.putInt(keys.size()); + if (innerNode) { + for (int i = 0; i < keys.size(); i++) { + writeString(buff, keys.get(i).substring(prefix.length())); + writeString(buff, values.get(i)); + } + writeString(buff, values.get(values.size() - 1)); + } else { + for (int i = 0; i < keys.size(); i++) { + writeString(buff, keys.get(i).substring(prefix.length())); + writeString(buff, values.get(i)); + } + } + buff.flip(); + buff.rewind(); + byte[] array = new byte[buff.remaining()]; + buff.get(array); + // reset the limit + REUSED_BUFFER = ByteBuffer.wrap(buff.array()); + return array; + } + } + + private void writeString(ByteBuffer buff, String s) { + if (s == null) { + buff.putShort((short) -2); + return; + } + byte[] data = s.getBytes(StandardCharsets.UTF_8); + if (data.length < Short.MAX_VALUE) { + // could get a bit larger, but some negative values are reserved + buff.putShort((short) data.length); + } else { + buff.putShort((short) -1); + buff.putInt(data.length); + } + buff.put(data); + } + + private static String readString(ByteBuffer buff) { + int len = buff.getShort(); + if (len == -2) { + return null; + } else if (len == -1) { + len = buff.getInt(); + int pos = buff.position(); + buff.position(buff.position() + len); + return new String(buff.array(), pos + buff.arrayOffset(), len, StandardCharsets.UTF_8); + } else { + int pos = buff.position(); + buff.position(buff.position() + len); + return new String(buff.array(), pos + buff.arrayOffset(), len, StandardCharsets.UTF_8); + } + } + + private static String commonPrefix(String prefix, String x) { + if (prefix == null) { + return x; + } + int i = 0; + for (; i < prefix.length() && i < x.length(); i++) { + if (prefix.charAt(i) != x.charAt(i)) { + break; + } + } + return prefix.substring(0, i); + } + + public String toString() { + return keys + "" + values; + } + + public PageFile copy() { + PageFile result = new PageFile(innerNode, maxFileSizeBytes); + result.modified = modified; + result.keys = new ArrayList<>(keys); + result.values = new ArrayList<>(values); + result.sizeInBytes = sizeInBytes; + result.nextRoot = nextRoot; + return result; + } + + public void addChild(int index, String childKey, String newChildFileName) { + modified = true; + if (index > 0) { + keys.add(index - 1, childKey); + sizeInBytes += childKey.length(); + } + values.add(index, newChildFileName); + sizeInBytes += 4; + sizeInBytes += newChildFileName.length(); + } + + public void setValue(int index, String value) { + modified = true; + sizeInBytes -= sizeInBytes(values.get(index)); + sizeInBytes += sizeInBytes(value); + values.set(index, value); + } + + private long sizeInBytes(String obj) { + if (obj == null) { + return 5; + } else if (obj instanceof String) { + return ((String) obj).length() + 2; + } else { + throw new IllegalStateException(); + } + } + + public void removeRecord(int index) { + modified = true; + String key = keys.remove(index); + String value = values.remove(index); + sizeInBytes -= 4; + sizeInBytes -= key.length(); + sizeInBytes -= sizeInBytes(value); + } + + public void appendRecord(String k, String v) { + modified = true; + keys.add(k); + values.add(v); + sizeInBytes += 4; + sizeInBytes += k.length(); + sizeInBytes += sizeInBytes(v); + } + + public void insertRecord(int index, String key, String value) { + modified = true; + keys.add(index, key); + values.add(index, value); + sizeInBytes += 4; + sizeInBytes += key.length(); + sizeInBytes += sizeInBytes(value); + } + + public long getUpdate() { + return update; + } + + public int sizeInBytes() { + if (VERIFY_SIZE) { + int size = 24; + for (String p : keys) { + size += p.length(); + size += 4; + } + for (String o : values) { + size += sizeInBytes(o); + } + if (size != sizeInBytes) { + throw new AssertionError(); + } + } + return sizeInBytes; + } + + public boolean canSplit() { + if (innerNode) { + return keys.size() > 2; + } else { + return keys.size() > 1; + } + } + + public List getKeys() { + return keys; + } + + public int getKeyIndex(String key) { + int index; + if (keys.isEmpty()) { + return -1; + } + if (lastSearchIndex == 1) { + if (key.compareTo(keys.get(keys.size() - 1)) > 0) { + index = -(keys.size() + 1); + } else { + index = Collections.binarySearch(keys, key); + } + } else if (lastSearchIndex == -1) { + if (key.compareTo(keys.get(0)) < 0) { + index = -1; + } else { + index = Collections.binarySearch(keys, key); + } + } else { + index = Collections.binarySearch(keys, key); + } + if (index == -(keys.size() + 1)) { + lastSearchIndex = 1; + } else if (index == -1) { + lastSearchIndex = -1; + } else { + lastSearchIndex = 0; + } + return index; + } + + public String getValue(int index) { + return values.get(index); + } + + public String getChildValue(int index) { + return (String) values.get(index); + } + + public String getNextKey(String largerThan) { + int index; + if (largerThan == null) { + index = 0; + } else { + index = getKeyIndex(largerThan); + if (index < 0) { + index = -index - 1; + } else { + index++; + } + } + if (index < 0 || index >= keys.size()) { + return null; + } + return keys.get(index); + } + + public String getNextRoot() { + return nextRoot; + } + + public void setNextRoot(String nextRoot) { + modified = true; + this.nextRoot = nextRoot; + } + + public void removeKey(int index) { + modified = true; + String key = keys.get(index); + sizeInBytes -= key.length(); + sizeInBytes -= 4; + keys.remove(index); + } + + public void removeValue(int index) { + modified = true; + String x = (String) values.get(index); + sizeInBytes -= x.length(); + values.remove(index); + } + + public boolean isInnerNode() { + return innerNode; + } + + public int getValueCount() { + return values.size(); + } + + public String getKey(int index) { + return keys.get(index); + } + + public void setModified(boolean modified) { + this.modified = modified; + } + + public boolean isModified() { + return modified; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java new file mode 100644 index 00000000000..28a0b69d0c9 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SlowStore.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Properties; +import java.util.Set; + +/** + * A wrapper store to simulate a slow backend store. It can be used for + * simulations. It is not intended to be used for real-world situations. + */ +public class SlowStore implements Store { + + private final Properties config; + private final Store backend; + private final long mbOverhead; + private final long mbPerSecond; + + public String toString() { + return "slow(" + backend + ")"; + } + + SlowStore(Store backend) { + this.config = backend.getConfig(); + this.backend = backend; + this.mbOverhead = Integer.parseInt(config.getProperty("mbOverhead", "3")); + this.mbPerSecond = Integer.parseInt(config.getProperty("mbPerSecond", "70")); + } + + private void delay(long nanos, int sizeInBytes) { + long bytes = sizeInBytes + 1_000_000 * mbOverhead; + long nanosRequired = 1_000 * bytes / mbPerSecond; + long delayNanos = nanosRequired - nanos; + long delayMillis = delayNanos / 1_000_000; + if (delayMillis > 0) { + try { + Thread.sleep(delayMillis); + } catch (InterruptedException e) { + // ignore + } + } + } + + @Override + public PageFile getIfExists(String key) { + long start = System.nanoTime(); + PageFile result = backend.getIfExists(key); + long time = System.nanoTime() - start; + delay(time, result == null ? 0 : result.sizeInBytes()); + return result; + } + + @Override + public void put(String key, PageFile value) { + long start = System.nanoTime(); + backend.put(key, value); + long time = System.nanoTime() - start; + delay(time, value.sizeInBytes()); + } + + @Override + public String newFileName() { + return backend.newFileName(); + } + + @Override + public Set keySet() { + return backend.keySet(); + } + + @Override + public void remove(Set set) { + backend.remove(set); + } + + @Override + public void removeAll() { + backend.removeAll(); + } + + @Override + public long getWriteCount() { + return backend.getWriteCount(); + } + + @Override + public long getReadCount() { + return backend.getReadCount(); + } + + @Override + public void setWriteCompression(Compression compression) { + backend.setWriteCompression(compression); + } + + @Override + public void close() { + backend.close(); + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public boolean supportsByteOperations() { + return backend.supportsByteOperations(); + } + + @Override + public byte[] getBytes(String key) { + long start = System.nanoTime(); + byte[] result = backend.getBytes(key); + long time = System.nanoTime() - start; + delay(time, result.length); + return result; + } + + @Override + public void putBytes(String key, byte[] data) { + long start = System.nanoTime(); + backend.putBytes(key, data); + long time = System.nanoTime() - start; + delay(time, data.length); + } + + @Override + public long getMaxFileSizeBytes() { + return backend.getMaxFileSizeBytes(); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java new file mode 100644 index 00000000000..8f375efe8b3 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StatsStore.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A wrapper store that allows capturing performance counters for a storage + * backend. + */ +public class StatsStore implements Store { + + private final Properties config; + private final Store backend; + private long lastLog; + private AtomicLong pending = new AtomicLong(); + + private final ConcurrentHashMap map = new ConcurrentHashMap<>(); + + public String toString() { + return "stats(" + backend + ")"; + } + + StatsStore(Store backend) { + this.config = backend.getConfig(); + this.backend = backend; + } + + @Override + public PageFile getIfExists(String key) { + long start = start(); + long sizeInBytes = 0; + try { + PageFile result = backend.getIfExists(key); + sizeInBytes = result == null ? 0 : result.sizeInBytes(); + return result; + } finally { + add("getIfExists", start, sizeInBytes); + } + } + + private long start() { + pending.incrementAndGet(); + return System.nanoTime(); + } + + private void add(String key, long start, long bytes) { + long now = System.nanoTime(); + pending.decrementAndGet(); + long nanos = now - start; + Stats stats = map.computeIfAbsent(key, s -> new Stats(key)); + stats.count++; + stats.nanosMax = Math.max(stats.nanosMax, nanos); + stats.nanosMin = Math.min(stats.nanosMin, nanos); + stats.nanosTotal += nanos; + stats.bytesMax = Math.max(stats.bytesMax, bytes); + stats.bytesMin = Math.min(stats.bytesMin, nanos); + stats.bytesTotal += bytes; + if (lastLog == 0) { + lastLog = start; + } + if (now - lastLog > 10_000_000_000L) { + TreeMap sorted = new TreeMap<>(map); + System.out.print(backend.toString()); + System.out.println(sorted.values().toString() + " pending " + pending); + lastLog = now; + } + } + + @Override + public void put(String key, PageFile value) { + long start = start(); + try { + backend.put(key, value); + } finally { + add("put", start, value.sizeInBytes()); + } + } + + @Override + public String newFileName() { + return backend.newFileName(); + } + + @Override + public Set keySet() { + return backend.keySet(); + } + + @Override + public void remove(Set set) { + backend.remove(set); + } + + @Override + public void removeAll() { + backend.removeAll(); + } + + @Override + public long getWriteCount() { + return backend.getWriteCount(); + } + + @Override + public long getReadCount() { + return backend.getReadCount(); + } + + @Override + public void setWriteCompression(Compression compression) { + backend.setWriteCompression(compression); + } + + @Override + public void close() { + backend.close(); + } + + @Override + public Properties getConfig() { + return config; + } + + @Override + public boolean supportsByteOperations() { + return backend.supportsByteOperations(); + } + + @Override + public byte[] getBytes(String key) { + long start = start(); + long len = 0; + try { + byte[] result = backend.getBytes(key); + len = result.length; + return result; + } finally { + add("getBytes", start, len); + } + } + + @Override + public void putBytes(String key, byte[] data) { + long start = start(); + try { + backend.putBytes(key, data); + } finally { + add("putBytes", start, data.length); + } + } + + @Override + public long getMaxFileSizeBytes() { + return backend.getMaxFileSizeBytes(); + } + + static class Stats { + final String key; + long count; + long bytesMin; + long bytesMax; + long bytesTotal; + long nanosMin; + long nanosMax; + long nanosTotal; + + public Stats(String key) { + this.key = key; + } + + public String toString() { + if (count == 0) { + return ""; + } + String result = key; + result += " " + count + " calls"; + if (bytesTotal > 0 && nanosTotal > 0) { + result += " " + (bytesTotal / count / 1_000_000) + " avgMB" + + (bytesTotal == 0 ? "" : (" " + ((bytesTotal * 1_000) / nanosTotal) + " MB/s")); + } + return result; + } + + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java new file mode 100644 index 00000000000..101a706c40e --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/Store.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.Properties; +import java.util.Set; + +/** + * Storage for files in a tree store. + */ +public interface Store { + + public static final String MAX_FILE_SIZE_BYTES = "maxFileSizeBytes"; + public static final int DEFAULT_MAX_FILE_SIZE_BYTES = 8 * 1024 * 1024; + + /** + * Get a file + * + * @param key the file name + * @return the file + */ + default PageFile get(String key) { + PageFile result = getIfExists(key); + if (result == null) { + throw new IllegalStateException("Not found: " + key); + } + return result; + } + + /** + * Get a file if it exists + * + * @param key the file name + * @return the file, or null + */ + PageFile getIfExists(String key); + + /** + * Storage a file. + * + * @param key the file name + * @param value the file + */ + void put(String key, PageFile value); + + /** + * Generate a new file name. + * + * @return + */ + String newFileName(); + + /** + * Get the list of files. + * + * @return the result + */ + Set keySet(); + + /** + * Remove a number of files. + * + * @param set the result + */ + void remove(Set set); + + /** + * Remove all files. + */ + void removeAll(); + + /** + * Get the number of files written. + * + * @return the result + */ + long getWriteCount(); + + /** + * Get the number of files read. + * + * @return the result + */ + long getReadCount(); + + /** + * Set the compression algorithm used for writing from now on. + * + * @param compression the compression algorithm + */ + void setWriteCompression(Compression compression); + + /** + * Close the store + */ + void close(); + + Properties getConfig(); + + /** + * Get the maximum file size configured. + * + * @return the file size, in bytes + */ + long getMaxFileSizeBytes(); + + default boolean supportsByteOperations() { + return false; + } + + default byte[] getBytes(String key) { + throw new UnsupportedOperationException(); + } + + default void putBytes(String key, byte[] data) { + throw new UnsupportedOperationException(); + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java new file mode 100644 index 00000000000..0aaada8d266 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/StoreBuilder.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Properties; + +/** + * A helper class to build storage backends for a tree store. + */ +public class StoreBuilder { + + /** + * Build a store. The configuration options are passed as a list of properties. + * + * - empty string or null: in-memory. + * - "type=memory" + * - "type=file": file system, with "dir" directory + * - "type=stats.another": statistics wrapper around another + * - "type=slow.another": slow wrapper around another (to simulate slowness) + * - "type=log.another": log wrapper around another (to analyze problems) + * - "maxFileSizeBytes=16000000": the maximum file size in bytes + * + * @param config the config + * @return a store + * @throws IllegalArgumentException + */ + public static Store build(String config) throws IllegalArgumentException { + if (config == null || config.isEmpty()) { + return new MemoryStore(new Properties()); + } + config = config.replace(' ', '\n'); + Properties prop = new Properties(); + try { + prop.load(new StringReader(config)); + } catch (IOException e) { + throw new IllegalArgumentException(config, e); + } + return build(prop); + } + + public static Store build(Properties config) { + String type = config.getProperty("type"); + switch (type) { + case "memory": + return new MemoryStore(config); + case "file": + return new FileStore(config); + } + if (type.startsWith("stats.")) { + config.put("type", type.substring("stats.".length())); + return new StatsStore(build(config)); + } else if (type.startsWith("slow.")) { + config.put("type", type.substring("slow.".length())); + return new SlowStore(build(config)); + } else if (type.startsWith("log.")) { + config.put("type", type.substring("log.".length())); + return new LogStore(build(config)); + } + throw new IllegalArgumentException(config.toString()); + } + + public static Properties subProperties(Properties p, String prefix) { + Properties p2 = new Properties(); + for (Object k : p.keySet()) { + if (k.toString().startsWith(prefix)) { + p2.put(k.toString().substring(prefix.length()), p.get(k)); + } + } + return p2; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java new file mode 100644 index 00000000000..a1ecef2bcc6 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java @@ -0,0 +1,910 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.NoSuchElementException; +import java.util.PriorityQueue; +import java.util.Properties; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.ConcurrentLRUCache; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Position; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SortedStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A session that allows reading and writing keys and values in a tree store. + */ +public class TreeSession { + + private static final Logger LOG = LoggerFactory.getLogger(TreeSession.class); + + public static final String CACHE_SIZE_MB = "cacheSizeMB"; + private static final int DEFAULT_CACHE_SIZE_MB = 256; + private static final int DEFAULT_MAX_ROOTS = 10; + + public static final String ROOT_NAME = "root"; + public static final String LEAF_PREFIX = "data_"; + static final String INNER_NODE_PREFIX = "node_"; + static final String DELETED = new String("DELETED"); + + static final boolean MULTI_ROOT = true; + + private final Store store; + private final ConcurrentLRUCache cache; + private long updateId; + private int maxRoots = DEFAULT_MAX_ROOTS; + private long fileReadCount; + + public TreeSession() { + this(new MemoryStore(new Properties())); + } + + public static String getFileNameRegex() { + return "(" + ROOT_NAME + ".*|" + + LEAF_PREFIX + ".*|" + + INNER_NODE_PREFIX + ".*)"; + } + + public TreeSession(Store store) { + this.store = store; + long cacheSizeMB = Long.parseLong(store.getConfig().getProperty( + CACHE_SIZE_MB, "" + DEFAULT_CACHE_SIZE_MB)); + long cacheSizeBytes = cacheSizeMB * 1024 * 1024; + LOG.info("Cache size {} bytes", cacheSizeBytes); + this.cache = new ConcurrentLRUCache<>(cacheSizeBytes) { + + private static final long serialVersionUID = 1L; + + @Override + public void entryWasRemoved(String key, PageFile value) { + if (value.isModified()) { + store.put(key, value); + // not strictly needed as it's no longer referenced + value.setModified(false); + } + } + }; + } + + /** + * Set the maximum number of roots. + * + * @param maxRoots the new value + */ + public void setMaxRoots(int maxRoots) { + this.maxRoots = maxRoots; + } + + public int getMaxRoots() { + return maxRoots; + } + + /** + * Get the number of files read from the cache or storage. + * + * @return the result + */ + public long getFileReadCount() { + return fileReadCount; + } + + private List getRootFileNames() { + LinkedHashSet result = new LinkedHashSet<>(); + String nextRoot = ROOT_NAME; + do { + boolean isNew = result.add(nextRoot); + if (!isNew) { + throw new IllegalStateException("Linked list contains a loop"); + } + PageFile root = getFile(nextRoot); + nextRoot = root.getNextRoot(); + } while (nextRoot != null); + return new ArrayList<>(result); + } + + private void mergeRootsIfNeeded() { + List roots = getRootFileNames(); + if (roots.size() > maxRoots) { + mergeRoots(Integer.MAX_VALUE); + } + } + + /** + * Initialize the storage, creating a new root if needed. + */ + public void init() { + PageFile root = store.getIfExists(ROOT_NAME); + if (root == null) { + root = newPageFile(false); + putFile(ROOT_NAME, root); + } + } + + private PageFile copyPageFile(PageFile old) { + PageFile result = old.copy(); + result.setUpdate(updateId); + return result; + } + + private PageFile newPageFile(boolean isInternalNode) { + PageFile result = new PageFile(isInternalNode, store.getMaxFileSizeBytes()); + result.setUpdate(updateId); + return result; + } + + /** + * Get an entry. + * + * @param key the key + * @return the value, or null + */ + public String get(String key) { + if (key == null) { + throw new NullPointerException(); + } + String fileName = ROOT_NAME; + do { + PageFile file = getFile(fileName); + String nextRoot = file.getNextRoot(); + String result = get(file, key); + if (result != null) { + return result == DELETED ? null : result; + } + fileName = nextRoot; + } while (fileName != null); + return null; + } + + /** + * Get the entry if it exists. + * + * @param root the root file + * @param k the key + * @return null if not found, DELETED if removed, or the value + */ + private String get(PageFile root, String k) { + while (true) { + if (!root.isInnerNode()) { + int index = root.getKeyIndex(k); + if (index >= 0) { + String result = root.getValue(index); + return result == null ? DELETED : result; + } + return null; + } + int index = root.getKeyIndex(k); + if (index < 0) { + index = -index - 2; + } + index++; + String fileName = root.getChildValue(index); + root = getFile(fileName); + // continue with the new file + } + } + + /** + * Put a value. + * To remove an entry, the value needs to be null. + * + * @param key the key + * @param value the value + */ + public void put(String key, String value) { + if (key == null) { + throw new NullPointerException(); + } + if (value == null) { + PageFile root = getFile(ROOT_NAME); + if (root.getNextRoot() != null) { + value = DELETED; + } + } + put(ROOT_NAME, key, value); + } + + /** + * Put a value. + * + * @param rootFileName + * @param key + * @param value (DELETED if we need to put that, or null to remove the entry) + * @return the file name of the root (different than the passed file name, if the file was copied) + */ + private void put(String rootFileName, String key, String value) { + String fileName = rootFileName; + PageFile file = getFile(fileName); + if (file.getUpdate() < updateId) { + fileName = store.newFileName(); + file = copyPageFile(file); + putFile(fileName, file); + } + ArrayList parents = new ArrayList<>(); + String k = key; + while (true) { + int index = file.getKeyIndex(k); + if (!file.isInnerNode()) { + if (index >= 0) { + if (value == null) { + file.removeRecord(index); + } else { + file.setValue(index, value == DELETED ? null : value); + } + } else { + // not found + if (value == null) { + // nothing to do + return; + } + file.insertRecord(-index - 1, k, value == DELETED ? null : value); + } + break; + } + parents.add(fileName); + if (index < 0) { + index = -index - 2; + } + index++; + fileName = file.getChildValue(index); + file = getFile(fileName); + // continue with the new file + } + putFile(fileName, file); + splitOrMerge(fileName, file, parents); + } + + private void splitOrMerge(String fileName, PageFile file, ArrayList parents) { + int size = file.sizeInBytes(); + if (size > store.getMaxFileSizeBytes() && file.canSplit()) { + split(fileName, file, parents); + } else if (file.getKeys().size() == 0) { + merge(fileName, file, parents); + } + } + + private void merge(String fileName, PageFile file, ArrayList parents) { + if (file.getValueCount() > 0) { + return; + } + if (parents.isEmpty()) { + // root: ignore + return; + } + String parentFileName = parents.remove(parents.size() - 1); + PageFile parentFile = getFile(parentFileName); + for (int i = 0; i < parentFile.getValueCount(); i++) { + String pf = parentFile.getChildValue(i); + if (pf.equals(fileName)) { + if (parentFile.getValueCount() == 1) { + parentFile = newPageFile(false); + if (!parentFileName.startsWith(ROOT_NAME)) { + String newParentFileName = LEAF_PREFIX + parentFileName.substring(INNER_NODE_PREFIX.length()); + putFile(newParentFileName, parentFile); + updateChildFileName(parents.get(parents.size() - 1), parentFileName, newParentFileName); + parentFileName = newParentFileName; + } + } else if (i == parentFile.getValueCount() - 1) { + // remove the last entry + parentFile.removeKey(i - 1); + parentFile.removeValue(i); + } else { + parentFile.removeKey(i); + parentFile.removeValue(i); + } + putFile(parentFileName, parentFile); + merge(parentFileName, parentFile, parents); + break; + } + } + } + + private void updateChildFileName(String fileName, String oldChild, String newChild) { + PageFile file = getFile(fileName); + for (int i = 0; i < file.getValueCount(); i++) { + if (file.getChildValue(i).equals(oldChild)) { + file.setValue(i, newChild); + putFile(fileName, file); + return; + } + } + } + + private void split(String fileName, PageFile file, ArrayList parents) { + List keys = new ArrayList<>(file.getKeys()); + String parentFileName, newFileName1, newFileName2; + PageFile parentFile, newFile1, newFile2; + boolean isInternalNode = file.isInnerNode(); + if (parents.isEmpty()) { + // new root + parentFileName = fileName; + parentFile = newPageFile(true); + parentFile.setNextRoot(file.getNextRoot()); + newFileName1 = (isInternalNode ? INNER_NODE_PREFIX : LEAF_PREFIX) + + store.newFileName(); + parentFile.addChild(0, null, newFileName1); + } else { + parentFileName = parents.remove(parents.size() - 1); + parentFile = getFile(parentFileName); + newFileName1 = fileName; + } + newFile1 = newPageFile(isInternalNode); + newFileName2 = (isInternalNode ? INNER_NODE_PREFIX : LEAF_PREFIX) + + store.newFileName(); + newFile2 = newPageFile(isInternalNode); + int sentinelIndex = keys.size() / 2; + String sentinel = keys.get(sentinelIndex); + // shorten the sentinel if possible + String beforeSentinal = keys.get(sentinelIndex - 1); + while (sentinel.length() > 0 && !isInternalNode) { + // for internal nodes, there might be other keys on the left side + // that might be shoter than the entry before the sentinel + String oneShorter = sentinel.substring(0, sentinel.length() - 1); + if (beforeSentinal.compareTo(oneShorter) >= 0) { + break; + } + sentinel = oneShorter; + } + if (!isInternalNode) { + // leaf + for (int i = 0; i < keys.size() / 2; i++) { + String k = keys.get(i); + String v = file.getValue(i); + newFile1.appendRecord(k, v); + } + for (int i = keys.size() / 2; i < keys.size(); i++) { + String k = keys.get(i); + String v = file.getValue(i); + newFile2.appendRecord(k, v); + } + } else { + // inner node + newFile1.addChild(0, null, file.getChildValue(0)); + for (int i = 1; i <= keys.size() / 2; i++) { + String p = keys.get(i - 1); + newFile1.appendRecord(p, file.getChildValue(i)); + } + newFile2.addChild(0, null, file.getChildValue(keys.size() / 2 + 1)); + for (int i = keys.size() / 2 + 2; i <= keys.size(); i++) { + String p = keys.get(i - 1); + newFile2.appendRecord(p, file.getChildValue(i)); + } + } + // insert sentinel into parent + int index = parentFile.getKeyIndex(sentinel); + parentFile.addChild(-index, sentinel, newFileName2); + putFile(newFileName1, newFile1); + putFile(newFileName2, newFile2); + putFile(parentFileName, parentFile); + splitOrMerge(parentFileName, parentFile, parents); + } + + private void putFile(String fileName, PageFile file) { + if (!file.isModified()) { + throw new AssertionError(); + } + file.setFileName(fileName); + cache.put(fileName, file); + } + + private PageFile getFile(String key) { + fileReadCount++; + PageFile result = cache.get(key); + if (result == null) { + result = store.get(key); + result.setFileName(key); + cache.put(key, result); + } + return result; + } + + /** + * Merge a number of roots. Merging will create a new root, which contains the + * data of the previous roots. If there are less roots, this method returns without merging. + * + * @param max the number of roots to merge (Integer.MAX_VALUE for all) + */ + public void mergeRoots(int max) { + List list = getRootFileNames(); + if (list.size() <= 1 || (max != Integer.MAX_VALUE && list.size() < max)) { + return; + } + PageFile root = getFile(ROOT_NAME); + String rootFileCopy = ROOT_NAME + "_" + updateId; + root = copyPageFile(root); + root.setModified(true); + putFile(rootFileCopy, root); + Iterator> it = iterator(null, max); + PageFile newRoot = newPageFile(false); + newRoot.setNextRoot(rootFileCopy); + putFile(ROOT_NAME, newRoot); + while (it.hasNext()) { + Entry e = it.next(); + put(e.getKey(), e.getValue()); + // we can remove files that are processed + } + newRoot = getFile(ROOT_NAME); + if (max < list.size()) { + newRoot.setNextRoot(list.get(max)); + } else { + newRoot.setNextRoot(null); + } + flush(); + } + + /** + * Get the number of roots. + * + * @return the number of roots + */ + public int getRootCount() { + return getRootFileNames().size(); + } + + /** + * Make the current tree read-only and switch to a new root. + * If there are already too many roots, then they will be merged. + * All changes are flushed to storage. + */ + public void checkpoint() { + flush(); + mergeRootsIfNeeded(); + List roots = getRootFileNames(); + if (roots.size() > 1) { + // get the last root + for (String s : roots) { + int index = s.lastIndexOf('_'); + if (index >= 0) { + updateId = Math.max(updateId, Long.parseLong(s.substring(index + 1))); + } + } + updateId++; + } + PageFile root = getFile(ROOT_NAME); + // cache.remove(ROOT_NAME); + String rootFileCopy = ROOT_NAME + "_" + updateId; + root = copyPageFile(root); + root.setFileName(rootFileCopy); + putFile(rootFileCopy, root); + updateId++; + if (MULTI_ROOT) { + root = newPageFile(false); + root.setNextRoot(rootFileCopy); + putFile(ROOT_NAME, root); + // need to flush here + // so that GC does not collect rootFileCopy + flush(); + root = copyPageFile(root); + putFile(ROOT_NAME, root); + } else { + flush(); + root = copyPageFile(root); + putFile(ROOT_NAME, root); + } + } + + /** + * Flush all changes to storage. + */ + public void flush() { + // we store all the pages except for the root, and the root at the very end + // this is to get a smaller chance that the root is stored, + // and points to a page that doesn't exist yet - + // but we don't try to ensure this completely; + // stored inner nodes might point to pages that are not stored yet + PageFile changedRoot = null; + for(String k : cache.keys()) { + PageFile v = cache.get(k); + if (v == null || !v.isModified()) { + continue; + } + if (k.equals(ROOT_NAME)) { + // don't store the changed root yet + changedRoot = v; + } else { + store.put(k, v); + // here we have to reset the flag + v.setModified(false); + } + } + // now store the changed root + if (changedRoot != null) { + store.put(ROOT_NAME, changedRoot); + // here we have to reset the flag + changedRoot.setModified(false); + } + } + + // =============================================================== + // iteration over entries + // this is fast: internally, a stack of Position object is kept + + /** + * Get all entries. Do not add or move entries while + * iterating. + * + * @return the result + */ + public Iterator> iterator() { + return iterator(null); + } + + /** + * Get all entries. Do not add or move entries while iterating. + * + * @param largerThan all returned keys are larger than this; null to start at + * the beginning + * @return the result + */ + public Iterator> iterator(String largerThan) { + return iterator(largerThan, Integer.MAX_VALUE); + } + + public Iterable> entrySet() { + return new Iterable>() { + + @Override + public Iterator> iterator() { + return TreeSession.this.iterator(); + } + + }; + } + + private Iterator> iterator(String largerThan, int maxRootCount) { + ArrayList streams = new ArrayList<>(); + String next = ROOT_NAME; + for (int i = 0; i < maxRootCount; i++) { + streams.add(new SortedStream(i, next, immutableRootIterator(next, largerThan))); + next = getFile(next).getNextRoot(); + if (next == null) { + break; + } + } + PriorityQueue pq = new PriorityQueue<>(streams); + return new Iterator>() { + + Entry current; + String lastKey; + + { + fetchNext(); + } + + private void fetchNext() { + while (pq.size() > 0) { + SortedStream s = pq.poll(); + String key = s.currentKeyOrNull(); + if (key == null) { + // if this is null, it must be the last stream + break; + } + if (key.equals(lastKey)) { + s.next(); + pq.add(s); + continue; + } + String value = s.currentValue(); + s.next(); + pq.add(s); + if (value == DELETED) { + continue; + } + lastKey = key; + current = new Entry<>() { + + @Override + public String getKey() { + return key; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String setValue(String value) { + throw new UnsupportedOperationException(); + } + + }; + return; + } + current = null; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public Entry next() { + Entry result = current; + fetchNext(); + return result; + } + + }; + } + + private Iterator immutableRootIterator(String rootFileName, String largerThan) { + + return new Iterator() { + private final ArrayList stack = new ArrayList<>(); + private Position current; + + { + current = new Position(); + current.file = getFile(rootFileName); + current.valuePos = index(current.file, largerThan); + down(largerThan); + if (current.valuePos >= current.file.getValueCount()) { + next(); + } + } + + private int index(PageFile file, String largerThan) { + if (largerThan == null) { + return 0; + } + int index = file.getKeyIndex(largerThan); + if (file.isInnerNode()) { + if (index < 0) { + index = -index - 2; + } + index++; + } else { + if (index < 0) { + index = -index - 1; + } else { + index++; + } + } + return index; + } + + @Override + public String toString() { + return stack + " " + current; + } + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public Position next() { + if (current == null) { + throw new NoSuchElementException(); + } + Position result = current; + current = new Position(); + current.file = result.file; + current.valuePos = result.valuePos + 1; + while (true) { + if (!current.file.isInnerNode() && current.valuePos < result.file.getValueCount()) { + break; + } + if (stack.size() == 0) { + current = null; + break; + } + current = stack.remove(stack.size() - 1); + current.valuePos++; + if (current.valuePos < current.file.getValueCount()) { + down(null); + break; + } + } + return result; + } + + private void down(String largerThan) { + while (current.file.isInnerNode()) { + stack.add(current); + Position pos = new Position(); + PageFile file = getFile(current.file.getChildValue(current.valuePos)); + pos.file = file; + pos.valuePos = index(pos.file, largerThan); + current = pos; + } + } + }; + } + + // =============================================================== + // iteration over keys, over all roots + // this is a bit slow: internally, *for each key* + // it will traverse from all roots down to the leaf + + /** + * Return all keys in sorted order. Roots don't need to be merged. + * + * @return all keys + */ + public Iterable keys() { + return keys(null); + } + + /** + * Return all keys in sorted order. + * + * @param largerThan all returned keys are larger than this; null to start at + * the beginning + * @return the keys + */ + public Iterable keys(String largerThan) { + final String next = getNextKey(largerThan); + return new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + private String current = next; + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public String next() { + if (current == null) { + throw new NoSuchElementException(); + } + String result = current; + current = getNextKey(current); + return result; + } + + }; + } + + }; + } + + private String getNextKey(String largerThan) { + if (MULTI_ROOT) { + String fileName = ROOT_NAME; + String result = null; + do { + String next = getNextKey(largerThan, fileName); + if (result == null) { + result = next; + } else if (next != null && next.compareTo(result) < 0) { + result = next; + } + PageFile file = getFile(fileName); + fileName = file.getNextRoot(); + } while (fileName != null); + return result; + } else { + return getNextKey(largerThan, ROOT_NAME); + } + } + + private String getNextKey(String largerThan, String fileName) { + PageFile file = getFile(fileName); + if (!file.isInnerNode()) { + String nextKey = file.getNextKey(largerThan); + if (nextKey != null) { + return nextKey; + } + return null; + } + int index; + index = largerThan == null ? -1 : file.getKeyIndex(largerThan); + if (index < 0) { + index = -index - 2; + } + index++; + for (; index < file.getValueCount(); index++) { + fileName = file.getChildValue(index); + String result = getNextKey(largerThan, fileName); + if (result != null) { + return result; + } + } + return null; + } + + // =============================================================== + // garbage collection + + public void runGC() { + flush(); + new GarbageCollection(store).run(getRootFileNames()); + } + + // =============================================================== + // info + + public String getInfo() { + StringBuilder buff = new StringBuilder(); + GarbageCollection gc = new GarbageCollection(store); + int rootId = 0; + for (String r : getRootFileNames()) { + if (store.getIfExists(r) == null) { + // not yet stored - ignore + continue; + } + buff.append(String.format("root #%d contains %d files (file name %s)\n", + rootId, gc.mark(Collections.singletonList(r)).size(), r)); + rootId++; + } + buff.append(cache.toString()); + return buff.toString(); + } + + // =============================================================== + // partitioning + + public String getMinKey() { + return getNextKey(null); + } + + public String getMaxKey() { + if (getFile(ROOT_NAME).getNextRoot() != null) { + throw new UnsupportedOperationException("Not fully merged"); + } + String fileName = ROOT_NAME; + while (true) { + PageFile file = getFile(fileName); + if (!file.isInnerNode()) { + return file.getKey(file.getKeys().size() - 1); + } + fileName = file.getChildValue(file.getValueCount() - 1); + } + } + + public String getApproximateMedianKey(String low, String high) { + if (getFile(ROOT_NAME).getNextRoot() != null) { + throw new UnsupportedOperationException("Not fully merged"); + } + String fileName = ROOT_NAME; + while (true) { + PageFile file = getFile(fileName); + if (!file.isInnerNode()) { + return file.getKey(0); + } + int i1 = file.getKeyIndex(low); + int i2 = file.getKeyIndex(high); + if (i1 < 0) { + i1 = -i1 - 1; + } + if (i2 < 0) { + i2 = -i2 - 1; + } + if (i2 != i1) { + int middle = (i1 + i2) / 2; + return file.getKey(middle); + } + fileName = file.getChildValue(i1); + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java new file mode 100644 index 00000000000..9883d9b621d --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentLRUCache.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A synchronized LRU cache. The cache size is limited by the amount of memory + * (and not number of entries). + * + * @param the key type + * @param the value type + */ +public class ConcurrentLRUCache + extends LinkedHashMap { + + private static final long serialVersionUID = 1L; + private volatile long maxMemoryBytes; + private AtomicLong memoryUsed = new AtomicLong(); + + public ConcurrentLRUCache(long maxMemoryBytes) { + super(16, 0.75f, true); + this.maxMemoryBytes = maxMemoryBytes; + } + + public String toString() { + return "cache entries:" + size() + " max:" + maxMemoryBytes + " used:" + memoryUsed; + } + + public void setSize(int maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + } + + @Override + public synchronized V get(Object key) { + return super.get(key); + } + + public synchronized Set keys() { + return new HashSet<>(super.keySet()); + } + + @Override + public synchronized V put(K key, V value) { + V old = super.put(key, value); + if (old != null) { + memoryUsed.addAndGet(-old.estimatedMemory()); + } + if (value != null) { + memoryUsed.addAndGet(value.estimatedMemory()); + } + return old; + } + + @Override + public synchronized void putAll(Map map) { + for(Map.Entry e : map.entrySet()) { + put(e.getKey(), e.getValue()); + } + } + + @Override + public synchronized V remove(Object key ) { + V old = super.remove(key); + if (old != null) { + memoryUsed.addAndGet(-old.estimatedMemory()); + } + return old; + } + + @Override + public synchronized void clear() { + super.clear(); + memoryUsed.set(0); + } + + public void entryWasRemoved(K key, V value) { + // nothing to do + } + + @Override + public synchronized boolean removeEldestEntry(Map.Entry eldest) { + boolean removeEldest = memoryUsed.get() > maxMemoryBytes; + if (removeEldest) { + memoryUsed.addAndGet(-eldest.getValue().estimatedMemory()); + entryWasRemoved(eldest.getKey(), eldest.getValue()); + } + return removeEldest; + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java new file mode 100644 index 00000000000..36c2038b668 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePacker.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * A utility class that allows converting the files of a tree store into one + * file (pack the files), and back from a file to a list of files (unpack the + * files). This is a bit similar to a zip file, however + * + * - each entry is already compressed, so no additional compression is needed; + * - only files in the same directory can be processed; + * - the pack file starts with a header that contains the list of files; + * - while packing, the files are (optionally) deleted, so that this doesn't require twice the disk space; + * - while unpacking, the pack file is (optionally) truncated, also to conserve disk space. + */ +public class FilePacker { + + /** + * The header of pack files ("PACK"). + */ + public static final String PACK_HEADER = "PACK"; + + public static void main(String... args) throws IOException { + if (args.length <= 2) { + System.out.println("Usage:\n" + + " java -jar target/oak-run-commons-*.jar " + + FilePacker.class.getCanonicalName() + " -d \n" + + " expands a file into the contained files"); + return; + } + if (args[0].equals("-d")) { + File packFile = new File(args[1]); + unpack(packFile, packFile.getParentFile(), false); + } + } + + /** + * Check whether the file starts with the magic header. + * @param file + * @return + * @throws IOException + */ + public static boolean isPackFile(File file) throws IOException { + RandomAccessFile source = new RandomAccessFile(file, "rw"); + byte[] magic = new byte[4]; + source.readFully(magic); + source.close(); + return PACK_HEADER.equals(new String(magic, StandardCharsets.UTF_8)); + } + + /** + * Packs all the files in the source directory into a target file. + * + * @param sourceDirectory the source directory + * @param fileNameRegex the file name regular expression + * @param deleteSource whether the source files are deleted while copying + */ + public static void pack(File sourceDirectory, String fileNameRegex, File targetFile, boolean deleteSource) throws IOException { + if (!sourceDirectory.exists() || !sourceDirectory.isDirectory()) { + throw new IOException("Source directory doesn't exist or is a file: " + sourceDirectory.getAbsolutePath()); + } + List list = Files.list(sourceDirectory.toPath()). + map(p -> p.toFile()). + filter(f -> f.isFile()). + filter(f -> f.getName().matches(fileNameRegex)). + map(f -> new FileEntry(f)). + collect(Collectors.toList()); + RandomAccessFile target = new RandomAccessFile(targetFile, "rw"); + target.write(PACK_HEADER.getBytes(StandardCharsets.UTF_8)); + for (FileEntry f : list) { + target.write(1); + byte[] name = f.fileName.getBytes(StandardCharsets.UTF_8); + target.writeInt(name.length); + target.write(name); + target.writeLong(f.length); + } + target.write(0); + for (FileEntry f : list) { + File file = new File(sourceDirectory, f.fileName); + FileInputStream in = new FileInputStream(file); + in.getChannel().transferTo(0, f.length, target.getChannel()); + in.close(); + if (deleteSource) { + // delete after copying to avoid using twice the space + file.delete(); + } + } + target.close(); + } + + /** + * Unpack a target file target file. The target directory is created if needed. + * Existing files are overwritten. + * + * @param sourceFile the pack file + * @param targetDirectory the target directory + * @param deleteSource whether the source file is truncated while copying, + * and finally deleted. + */ + public static void unpack(File sourceFile, File targetDirectory, boolean deleteSource) throws IOException { + if (!sourceFile.exists() || !sourceFile.isFile()) { + throw new IOException("Source file doesn't exist or is not a file: " + sourceFile.getAbsolutePath()); + } + if (targetDirectory.exists()) { + if (targetDirectory.isFile()) { + throw new IOException("Target file exists: " + targetDirectory.getAbsolutePath()); + } + } else { + targetDirectory.mkdirs(); + } + RandomAccessFile source = new RandomAccessFile(sourceFile, "rw"); + byte[] magic = new byte[4]; + source.readFully(magic); + if (!PACK_HEADER.equals(new String(magic, StandardCharsets.UTF_8))) { + source.close(); + throw new IOException("File header is not '" + PACK_HEADER + "': " + sourceFile.getAbsolutePath()); + } + ArrayList list = new ArrayList<>(); + long offset = 0; + while (source.read() > 0) { + byte[] name = new byte[source.readInt()]; + source.readFully(name); + long length = source.readLong(); + list.add(new FileEntry(new String(name, StandardCharsets.UTF_8), length, offset)); + offset += length; + } + long start = source.getFilePointer(); + for (int i = list.size() - 1; i >= 0; i--) { + FileEntry f = list.get(i); + source.seek(start + f.offset); + FileOutputStream out = new FileOutputStream(new File(targetDirectory, f.fileName)); + source.getChannel().transferTo(source.getFilePointer(), f.length, out.getChannel()); + out.close(); + if (deleteSource) { + // truncate the source to avoid using twice the space + source.setLength(start + f.offset); + } + } + source.close(); + if (deleteSource) { + sourceFile.delete(); + } + } + + static class FileEntry { + final String fileName; + final long length; + long offset; + FileEntry(String fileName, long length, long offset) { + this.fileName = fileName; + this.length = length; + this.offset = offset; + } + FileEntry(File f) { + this.fileName = f.getName(); + this.length = f.length(); + } + } +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java new file mode 100644 index 00000000000..99ed6cb8ea6 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/MemoryObject.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +/** + * A interface for memory-bound cache objects. + */ +public interface MemoryObject { + + /** + * Get the estimate memory size. The value must not change afterwards, otherwise + * the memory calculation is wrong. + * + * @return the memory in bytes + */ + long estimatedMemory(); +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java new file mode 100644 index 00000000000..f50cb060c90 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/Position.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.PageFile; + +/** + * A position of an entry in a page file. + * + * This class is used to iterate over entries in a page file. + */ +public class Position { + public PageFile file; + public int valuePos; + + public String toString() { + return (file.isInnerNode() ? "internal" : "leaf") + " " + valuePos + " " + file.getKeys() + "\n"; + } +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SieveCache.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SieveCache.java new file mode 100644 index 00000000000..5aba5003085 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SieveCache.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A Sieve cache. It is faster and has a higher hit-rate than LRU. + * See https://cachemon.github.io/SIEVE-website/ + * + * @param the key type + * @param the value type + */ +public class SieveCache { + + private final ConcurrentHashMap> map = new ConcurrentHashMap<>(); + private final ConcurrentLinkedDeque> queue = new ConcurrentLinkedDeque<>(); + private final AtomicReference>> hand = new AtomicReference<>(); + + private volatile long maxMemoryBytes; + private AtomicLong memoryUsed = new AtomicLong(); + + public SieveCache(long maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + Iterator> it = queue.iterator(); + hand.set(it); + } + + public void setSize(int maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + } + + public V get(K key) { + Value v = map.get(key); + if (v == null) { + return null; + } + v.visited = true; + return v.value; + } + + public Set keys() { + return new HashSet<>(map.keySet()); + } + + public V put(K key, V value) { + Value v = new Value<>(key, value); + memoryUsed.addAndGet(value.estimatedMemory()); + Value old = map.put(key, v); + V result = null; + if (old == null) { + queue.add(v); + } else { + memoryUsed.addAndGet(-old.value.estimatedMemory()); + result = old.value; + } + while (memoryUsed.get() > maxMemoryBytes) { + Value evict; + synchronized (hand) { + if (memoryUsed.get() < maxMemoryBytes || map.isEmpty()) { + break; + } + Iterator> it = hand.get(); + while (true) { + if (it.hasNext()) { + evict = it.next(); + if (!evict.visited) { + break; + } + evict.visited = false; + } else { + Iterator> it2 = queue.iterator(); + it = hand.compareAndExchange(it, it2); + } + } + it.remove(); + evict = map.remove(evict.key); + } + if (evict != null) { + memoryUsed.addAndGet(-evict.value.estimatedMemory()); + entryWasRemoved(evict.key, evict.value); + } + } + return result; + } + + public void entryWasRemoved(K key, V value) { + // nothing to do + } + + public String toString() { + return "cache entries:" + map.size() + + " queue:" + queue.size() + + " max:" + maxMemoryBytes + + " used:" + memoryUsed; + } + + public int size() { + return map.size(); + } + + private static class Value { + private final K key; + private final V value; + volatile boolean visited; + + public Value(K key, V value) { + this.key = key; + this.value = value; + } + + @Override + public String toString() { + return "(" + key + ":" + value + ")"; + } + } + +} diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java new file mode 100644 index 00000000000..84bfac2aefa --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/SortedStream.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.util.Iterator; + +/** + * A helper class to iterate over key-value pairs in a tree store, in ascending + * key order. The class helps merging multiple streams of key-value pairs. + * + * Internally, it is backed by an iterator over positions in the key-value pair. + */ +public class SortedStream implements Comparable { + + private final int priority; + private final String rootFileName; + private Iterator it; + private String currentKey; + private String currentValue; + + public SortedStream(int priority, String rootFileName, Iterator it) { + this.priority = priority; + this.rootFileName = rootFileName; + this.it = it; + next(); + } + + public String toString() { + return "priority " + priority + " file " + rootFileName + " key " + currentKey + " value " + currentValue; + } + + public String currentKeyOrNull() { + return currentKey; + } + + public String currentValue() { + return currentValue; + } + + public void next() { + if (it.hasNext()) { + Position pos = it.next(); + currentKey = pos.file.getKey(pos.valuePos); + currentValue = pos.file.getValue(pos.valuePos); + } else { + currentKey = null; + currentValue = null; + } + } + + @Override + public int compareTo(SortedStream o) { + if (currentKey == null) { + if (o.currentKey == null) { + return Integer.compare(priority, o.priority); + } + return 1; + } else if (o.currentKey == null) { + return -1; + } + int comp = currentKey.compareTo(o.currentKey); + if (comp == 0) { + return Integer.compare(priority, o.priority); + } + return comp; + } +} \ No newline at end of file diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java new file mode 100644 index 00000000000..4614c32a127 --- /dev/null +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuid.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import java.security.SecureRandom; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A UUID implementation. + * + * It supports creating version 7 UUIDs, which are time-ordered. + * See also draft-ietf-uuidrev-rfc4122bis-00 + * + * Unlike java.util.UUID, the comparison is unsigned, + * so that the string comparison yields the same result. + */ +public class TimeUuid implements Comparable { + + private static final AtomicLong UUID_LAST_MILLIS_AND_COUNT = new AtomicLong(0); + + private static final SecureRandom RANDOM = new SecureRandom(); + + // most significant bits + private final long msb; + + // least significant bits + private final long lsb; + + public TimeUuid(long msb, long lsb) { + this.msb = msb; + this.lsb = lsb; + } + + @Override + public String toString() { + return String.format("%08x-%04x-%04x-%04x-%012x", + msb >>> 32, (msb >>> 16) & 0xffff, msb & 0xffff, + (lsb >>> 48) & 0xffff, lsb & 0xffffffffffffL); + } + + public String toShortString() { + return String.format("%012x%03x%016x", + getTimestampPart(), getCounterPart(), getRandomPart()); + } + + public String toHumanReadableString() { + Instant instant = Instant.ofEpochMilli(getTimestampPart()); + return String.format("%s %03x %016x", + instant.toString(), getCounterPart(), getRandomPart()); + } + + /** + * Get the timestamp part (48 bits). + * + * @return the timestamp part + */ + public long getTimestampPart() { + return msb >>> 16; + } + + /** + * Get the counter part (12 bits). + * + * @return counter part + */ + public long getCounterPart() { + return msb & ((1L << 12) - 1); + } + + /** + * Get the random part (62 bits). + * The first 2 bits are fixed. + * + * @return the random part + */ + public long getRandomPart() { + return lsb; + } + + /** + * Unlike java.util.UUID, the comparison is unsigned, + * so that the string comparison yields the same result. + */ + @Override + public int compareTo(TimeUuid o) { + if (o.msb != msb) { + return Long.compareUnsigned(msb, o.msb); + } + return Long.compareUnsigned(lsb, o.lsb); + } + + @Override + public int hashCode() { + long x = lsb ^ msb; + return (int) ((x >>> 32) ^ x); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + TimeUuid other = (TimeUuid) obj; + return lsb == other.lsb && msb == other.msb; + } + + /** + * Get the next timestamp (in milliseconds) and counter. + * The lowest 12 bits of the returned value is the counter. + * The milliseconds are shifted by 12 bits. + * + * @param now the milliseconds + * @param lastMillisAndCount the last returned value + * @return the new value + */ + public static long getMillisAndCountIncreasing(long now, AtomicLong lastMillisAndCount) { + long result = now << 12; + while (true) { + long last = lastMillisAndCount.get(); + if (result <= last) { + // ensure it is non-decrementing + result = last + 1; + } + long got = lastMillisAndCount.compareAndExchange(last, result); + if (got == last) { + return result; + } + } + } + + static TimeUuid newUuid(long millisAndCount, + long random) { + long millis = millisAndCount >>> 12; + long counter = millisAndCount & ((1L << 12) - 1); + long version = 7; + long variant = 2; + long msb = (millis << 16) | (version << 12) | counter; + long lsb = (variant << 62) | (random & ((1L << 62) - 1)); + return new TimeUuid(msb, lsb); + } + + public static TimeUuid newUuid() { + long millisAndCount = getMillisAndCountIncreasing( + System.currentTimeMillis(), + UUID_LAST_MILLIS_AND_COUNT); + long random = RANDOM.nextLong(); + return newUuid(millisAndCount, random); + } + + public long getMostSignificantBits() { + return msb; + } + + public long getLeastSignificantBits() { + return lsb; + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java new file mode 100644 index 00000000000..9ccb49804ae --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/MergeIncrementalTreeStoreTest.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile; + +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.index.indexer.document.incrementalstore.MergeIncrementalTreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; + +public class MergeIncrementalTreeStoreTest { + private static final String BUILD_TARGET_FOLDER = "target"; + private static final Compression algorithm = Compression.GZIP; + @Rule + public TemporaryFolder folder = new TemporaryFolder(new File(BUILD_TARGET_FOLDER)); + + @Test + public void merge() throws IOException { + File baseDir = folder.newFolder("base"); + File baseFile = folder.newFile("base.gz"); + File baseMetadata = folder.newFile("base.metadata.gz"); + + TreeStore base = new TreeStore("base", baseDir, null, 1); + base.init(); + base.putNode("/tmp", "{prop1=\"foo\"}"); + base.putNode("/tmp/a", "{prop2=\"foo\"}"); + base.putNode("/tmp/a/b", "{prop3=\"foo\"}"); + base.putNode("/tmp/b", "{prop1=\"foo\"}"); + base.putNode("/tmp/b/c", "{prop2=\"foo\"}"); + base.putNode("/tmp/c", "{prop3=\"foo\"}"); + base.close(); + + FilePacker.pack(baseDir, TreeSession.getFileNameRegex(), baseFile, true); + + File increment = folder.newFile("inc.gz"); + File incrementMetadata = folder.newFile("inc.metadata.gz"); + File mergedFile = folder.newFile("merged.gz"); + File mergedDir = folder.newFolder("merged"); + File mergedMetadata = folder.newFile("merged.metadata.gz"); + + try (BufferedWriter baseBW = IndexStoreUtils.createWriter(baseMetadata, algorithm)) { + baseBW.write("{\"checkpoint\":\"" + "r0" + "\",\"storeType\":\"TreeStore\"," + + "\"strategy\":\"" + "BaseFFSCreationStrategy" + "\"}"); + baseBW.newLine(); + } + + try (BufferedWriter baseInc = IndexStoreUtils.createWriter(increment, algorithm)) { + baseInc.write("/tmp/a|{prop2=\"fooModified\"}|r1|M"); + baseInc.newLine(); + baseInc.write("/tmp/b|{prop1=\"foo\"}|r1|D"); + baseInc.newLine(); + baseInc.write("/tmp/b/c/d|{prop2=\"fooNew\"}|r1|A"); + baseInc.newLine(); + baseInc.write("/tmp/c|{prop3=\"fooModified\"}|r1|M"); + baseInc.newLine(); + baseInc.write("/tmp/d|{prop3=\"bar\"}|r1|A"); + baseInc.newLine(); + baseInc.write("/tmp/e|{prop3=\"bar\"}|r1|A"); + } + try (BufferedWriter baseInc = IndexStoreUtils.createWriter(incrementMetadata, algorithm)) { + baseInc.write("{\"beforeCheckpoint\":\"" + "r0" + "\",\"afterCheckpoint\":\"" + "r1" + "\"," + + "\"storeType\":\"" + "IncrementalFFSType" + "\"," + + "\"strategy\":\"" + "pipelineStrategy" + "\"," + + "\"preferredPaths\":[]}"); + baseInc.newLine(); + } + + List expectedMergedList = new LinkedList<>(); + + expectedMergedList.add("/tmp|{prop1=\"foo\"}"); + expectedMergedList.add("/tmp/a|{prop2=\"fooModified\"}"); + expectedMergedList.add("/tmp/a/b|{prop3=\"foo\"}"); + expectedMergedList.add("/tmp/b/c|{prop2=\"foo\"}"); + expectedMergedList.add("/tmp/b/c/d|{prop2=\"fooNew\"}"); + expectedMergedList.add("/tmp/c|{prop3=\"fooModified\"}"); + expectedMergedList.add("/tmp/d|{prop3=\"bar\"}"); + expectedMergedList.add("/tmp/e|{prop3=\"bar\"}"); + + MergeIncrementalTreeStore merge = new MergeIncrementalTreeStore( + baseFile, increment, mergedFile, algorithm); + merge.doMerge(); + List expectedMergedMetadataList = new LinkedList<>(); + expectedMergedMetadataList.add("{\"checkpoint\":\"" + "r1" + "\",\"storeType\":\"TreeStore\"," + + "\"strategy\":\"" + merge.getStrategyName() + "\",\"preferredPaths\":[]}"); + + FilePacker.unpack(mergedFile, mergedDir, true); + TreeStore merged = new TreeStore("merged", mergedDir, null, 1); + HashSet paths = new HashSet<>(); + int i = 0; + for (Entry e : merged.getSession().entrySet()) { + String key = e.getKey(); + if (e.getValue().isEmpty()) { + String[] parts = TreeStore.toParentAndChildNodeName(key); + String childEntry = TreeStore.toChildNodeEntry(parts[0], parts[1]); + assertTrue(paths.add(childEntry)); + continue; + } + String expected = expectedMergedList.get(i++); + String actual = key + "|" + e.getValue(); + Assert.assertEquals(expected, actual); + } + assertEquals(expectedMergedList.size(), i); + assertEquals(expectedMergedList.size(), paths.size()); + merged.close(); + + try (BufferedReader br = IndexStoreUtils.createReader(mergedMetadata, algorithm)) { + for (String line : expectedMergedMetadataList) { + String actual = br.readLine(); + System.out.println(actual); + Assert.assertEquals(line, actual); + } + Assert.assertNull(br.readLine()); + } + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java new file mode 100644 index 00000000000..3318cc77c28 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreIT.java @@ -0,0 +1,681 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; + +import static java.lang.management.ManagementFactory.getPlatformMBeanServer; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelineITUtil.assertMetrics; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelineITUtil.contentDamPathFilter; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_PARALLEL_DUMP; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING; +import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMongoDownloadTask.OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider; +import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; +import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory; +import org.apache.jackrabbit.oak.plugins.document.MongoUtils; +import org.apache.jackrabbit.oak.plugins.document.RevisionVector; +import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection; +import org.apache.jackrabbit.oak.plugins.document.util.Utils; +import org.apache.jackrabbit.oak.plugins.index.ConsoleIndexingReporter; +import org.apache.jackrabbit.oak.plugins.metric.MetricStatisticsProvider; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.contrib.java.lang.system.RestoreSystemProperties; +import org.junit.rules.TemporaryFolder; + +public class PipelinedTreeStoreIT { + private static ScheduledExecutorService executorService; + @Rule + public final MongoConnectionFactory connectionFactory = new MongoConnectionFactory(); + @Rule + public final DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); + @Rule + public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties(); + @Rule + public final TemporaryFolder sortFolder = new TemporaryFolder(); + + + private MetricStatisticsProvider statsProvider; + private ConsoleIndexingReporter indexingReporter; + + @BeforeClass + public static void setup() throws IOException { + Assume.assumeTrue(MongoUtils.isAvailable()); + executorService = Executors.newSingleThreadScheduledExecutor(); + } + + @AfterClass + public static void teardown() { + if (executorService != null) { + executorService.shutdown(); + } + } + + @Before + public void before() { + MongoConnection c = connectionFactory.getConnection(); + if (c != null) { + c.getDatabase().drop(); + } + statsProvider = new MetricStatisticsProvider(getPlatformMBeanServer(), executorService); + indexingReporter = new ConsoleIndexingReporter(); + } + + @After + public void tear() { + MongoConnection c = connectionFactory.getConnection(); + if (c != null) { + c.getDatabase().drop(); + } + statsProvider.close(); + statsProvider = null; + indexingReporter = null; + } + + @Test + public void createFFS_mongoFiltering_include_excludes() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/content/dam/2023"), List.of("/content/dam/2023/02"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/2023|{\"p2\":\"v2023\"}", + "/content/dam/2023/01|{\"p1\":\"v202301\"}", + "/content/dam/2023/02|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_include_excludes2() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + // NOTE: If a path /a/b is in the excluded paths, the descendants of /a/b will not be downloaded but /a/b will + // be downloaded. This is an intentional limitation of the logic to compute the Mongo filter which was done + // to avoid the extra complexity of also filtering the root of the excluded tree. The transform stage would anyway + // filter out these additional documents. + List pathFilters = List.of(new PathFilter(List.of("/content/dam/1000", "/content/dam/2022"), List.of("/content/dam/2022/02", "/content/dam/2022/04"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/1000|{}", + "/content/dam/1000/12|{\"p1\":\"v100012\"}", + "/content/dam/2022|{}", + "/content/dam/2022/01|{\"p1\":\"v202201\"}", + "/content/dam/2022/01/01|{\"p1\":\"v20220101\"}", + "/content/dam/2022/02|{\"p1\":\"v202202\"}", + "/content/dam/2022/03|{\"p1\":\"v202203\"}", + "/content/dam/2022/04|{\"p1\":\"v202204\"}" + ), true); + } + + + @Test + public void createFFS_mongoFiltering_include_excludes3() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/etc", "/home", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_include_excludes_retryOnConnectionErrors() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/etc", "/home", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_include_excludes4() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + + List pathFilters = List.of( + new PathFilter(List.of("/content/dam/1000"), List.of()), + new PathFilter(List.of("/content/dam/2022"), List.of("/content/dam/2022/01")) + ); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/1000|{}", + "/content/dam/1000/12|{\"p1\":\"v100012\"}", + "/content/dam/2022|{}", + "/content/dam/2022/01|{\"p1\":\"v202201\"}", + "/content/dam/2022/02|{\"p1\":\"v202202\"}", + "/content/dam/2022/02/01|{\"p1\":\"v20220201\"}", + "/content/dam/2022/02/02|{\"p1\":\"v20220202\"}", + "/content/dam/2022/02/03|{\"p1\":\"v20220203\"}", + "/content/dam/2022/02/04|{\"p1\":\"v20220204\"}", + "/content/dam/2022/03|{\"p1\":\"v202203\"}", + "/content/dam/2022/04|{\"p1\":\"v202204\"}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_multipleIndexes() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + Predicate pathPredicate = s -> true; + PathFilter pathFilter = new PathFilter(List.of("/content/dam/1000", "/content/dam/2023", "/content/dam/2023/01"), List.of()); + List pathFilters = List.of(pathFilter); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/1000|{}", + "/content/dam/1000/12|{\"p1\":\"v100012\"}", + "/content/dam/2023|{\"p2\":\"v2023\"}", + "/content/dam/2023/01|{\"p1\":\"v202301\"}", + "/content/dam/2023/02|{}", + "/content/dam/2023/02/28|{\"p1\":\"v20230228\"}" + ), true); + } + + @Test + public void createFFS_filter_long_paths() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + + // Create a filter on the node with the longest path + String longestLine = PipelineITUtil.EXPECTED_FFS.stream().max(Comparator.comparingInt(String::length)).get(); + String longestPath = longestLine.substring(0, longestLine.lastIndexOf("|")); + String parent = PathUtils.getParentPath(longestPath); + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of(parent), List.of())); + + // The results should contain all the parents of the node with the longest path + ArrayList expected = new ArrayList<>(); + expected.add(longestPath + "|{}"); + while (true) { + expected.add(parent + "|{}"); + if (parent.equals("/")) { + break; + } + parent = PathUtils.getParentPath(parent); + } + // The list above has the longest paths first, reverse it to match the order in the FFS + Collections.reverse(expected); + + testSuccessfulDownload(pathPredicate, pathFilters, expected, false); + } + + + @Test + public void createFFSCustomExcludePathsRegexRetryOnConnectionErrors() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "true", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "false" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + null, + excludedPathsRegexTestExpected); + } + + @Test + public void createFFSCustomExcludePathsRegexNoRetryOnConnectionError() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "false" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + null, + excludedPathsRegexTestExpected); + } + + @Test + public void createFFSCustomExcludePathsRegexRetryOnConnectionErrorsRegexFiltering() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "true", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + List.of(contentDamPathFilter), + excludedPathsRegexTestExpected); + } + + @Test + public void createFFSCustomExcludePathsRegexNoRetryOnConnectionErrorRegexFiltering() throws Exception { + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + testPipelinedStrategy(Map.of( + // Filter all nodes ending in /metadata.xml or having a path section with ".*.jpg" + OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDE_ENTRIES_REGEX, "/metadata.xml$|/.*.jpg/.*", + OAK_INDEXER_PIPELINED_RETRY_ON_CONNECTION_ERRORS, "false", + OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true" + ), + this::buildNodeStoreForExcludedRegexTest, + pathPredicate, + List.of(contentDamPathFilter), + excludedPathsRegexTestExpected); + } + + private void buildNodeStoreForExcludedRegexTest(DocumentNodeStore rwNodeStore) { + @NotNull NodeBuilder rootBuilder = rwNodeStore.getRoot().builder(); + @NotNull NodeBuilder contentDamBuilder = rootBuilder.child("content").child("dam"); + contentDamBuilder.child("a.jpg").child("jcr:content").child("metadata.xml"); + contentDamBuilder.child("a.jpg").child("jcr:content").child("metadata.text"); + contentDamBuilder.child("image_a.png").child("jcr:content").child("metadata.text"); + contentDamBuilder.child("image_a.png").child("jcr:content").child("metadata.xml"); + try { + rwNodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } catch (CommitFailedException e) { + throw new RuntimeException(e); + } + } + + private final List excludedPathsRegexTestExpected = List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/a.jpg|{}", + "/content/dam/image_a.png|{}", + "/content/dam/image_a.png/jcr:content|{}", + "/content/dam/image_a.png/jcr:content/metadata.text|{}" + ); + + private void testPipelinedStrategy(Map settings, + Consumer contentBuilder, + Predicate pathPredicate, + List pathFilters, + List expected) throws IOException { + settings.forEach(System::setProperty); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + DocumentNodeStore rwNodeStore = rwStore.documentNodeStore; + contentBuilder.accept(rwNodeStore); + MongoTestBackend roStore = createNodeStore(true); + + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, pathFilters); + File file = pipelinedStrategy.createSortedStoreFile(); + + assertTrue(file.exists()); + assertEquals(expected, readAllEntries(file)); + assertMetrics(statsProvider); + } + } + + private void testSuccessfulDownload(Predicate pathPredicate, List pathFilters) + throws CommitFailedException, IOException { + testSuccessfulDownload(pathPredicate, pathFilters, PipelineITUtil.EXPECTED_FFS, false); + } + + private void testSuccessfulDownload(Predicate pathPredicate, List mongoRegexPathFilter, List expected, boolean ignoreLongPaths) + throws CommitFailedException, IOException { + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, mongoRegexPathFilter); + File file = pipelinedStrategy.createSortedStoreFile(); + assertTrue(file.exists()); + List result = readAllEntries(file); + if (ignoreLongPaths) { + // Remove the long paths from the result. The filter on Mongo is best-effort, it will download long path + // documents, even if they do not match the includedPaths. + result = result.stream() + .filter(s -> { + String name = s.split("\\|")[0]; + return name.length() < Utils.PATH_LONG; + }) + .collect(Collectors.toList()); + + } + assertEquals(expected, result); + assertMetrics(statsProvider); + } + } + + @Test + public void createFFS_pathPredicateDoesNotMatch() throws Exception { + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + Predicate pathPredicate = s -> s.startsWith("/content/dam/does-not-exist"); + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, null); + + File file = pipelinedStrategy.createSortedStoreFile(); + + assertTrue(file.exists()); + assertEquals("[]", readAllEntries(file).toString()); + } + } + + @Test + public void createFFS_badNumberOfTransformThreads() throws CommitFailedException, IOException { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, "0"); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + assertThrows("Invalid value for property " + PipelinedStrategy.OAK_INDEXER_PIPELINED_TRANSFORM_THREADS + ": 0. Must be > 0", + IllegalArgumentException.class, + () -> createStrategy(roStore) + ); + } + } + + @Test + public void createFFS_badWorkingMemorySetting() throws CommitFailedException, IOException { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, "-1"); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + assertThrows("Invalid value for property " + PipelinedStrategy.OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB + ": -1. Must be >= 0", + IllegalArgumentException.class, + () -> createStrategy(roStore) + ); + } + } + + @Test + public void createFFS_smallNumberOfDocsPerBatch() throws Exception { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_NUMBER_OF_DOCUMENTS, "2"); + + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + List pathFilters = null; + + testSuccessfulDownload(pathPredicate, pathFilters); + } + + @Test + public void createFFS_largeMongoDocuments() throws Exception { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_MONGO_DOC_BATCH_MAX_SIZE_MB, "1"); + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_MONGO_DOC_QUEUE_RESERVED_MEMORY_MB, "32"); + + Predicate pathPredicate = s -> contentDamPathFilter.filter(s) != PathFilter.Result.EXCLUDE; + List pathFilters = null; + + MongoTestBackend rwStore = createNodeStore(false); + @NotNull NodeBuilder rootBuilder = rwStore.documentNodeStore.getRoot().builder(); + // This property does not fit in the reserved memory, but must still be processed without errors + String longString = RandomStringUtils.random((int) (10 * FileUtils.ONE_MB), true, true); + @NotNull NodeBuilder contentDamBuilder = rootBuilder.child("content").child("dam"); + contentDamBuilder.child("2021").child("01").setProperty("p1", "v202101"); + contentDamBuilder.child("2022").child("01").setProperty("p1", longString); + contentDamBuilder.child("2023").child("01").setProperty("p1", "v202301"); + rwStore.documentNodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + + List expected = List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/content/dam/2021|{}", + "/content/dam/2021/01|{\"p1\":\"v202101\"}", + "/content/dam/2022|{}", + "/content/dam/2022/01|{\"p1\":\"" + longString + "\"}", + "/content/dam/2023|{}", + "/content/dam/2023/01|{\"p1\":\"v202301\"}" + ); + + MongoTestBackend roStore = createNodeStore(true); + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, pathFilters); + + File file = pipelinedStrategy.createSortedStoreFile(); + assertTrue(file.exists()); + assertArrayEquals(expected.toArray(new String[0]), readAllEntriesArray(file)); + assertMetrics(statsProvider); + } + + static String[] readAllEntriesArray(File dir) throws IOException { + return readAllEntries(dir).toArray(new String[0]); + } + + static List readAllEntries(File dir) throws IOException { + TreeStore treeStore = new TreeStore("test", dir, + new NodeStateEntryReader(new MemoryBlobStore()), 1); + ArrayList list = new ArrayList<>(); + TreeSession session = treeStore.getSession(); + for (String k : session.keys()) { + String v = session.get(k); + if (!v.isEmpty()) { + list.add(k + "|" + v); + } + } + treeStore.close(); + return list; + } + + @Test + public void createFFS_mongoFiltering_custom_excluded_paths_1() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/etc", "/home", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_custom_excluded_paths_2() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of("/content/dam", "/jcr:system"))); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFS_mongoFiltering_custom_excluded_paths_3() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home,/content/dam,/jcr:system"); + + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of())); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + @Test + public void createFFSNoMatches() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_PARALLEL_DUMP, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/home,/content/dam,/jcr:system"); + + Predicate pathPredicate = t -> true; + List mongoRegexPathFilters = List.of(new PathFilter(List.of("/doesnotexist"), List.of())); + + // For an included path of /foo, the / should not be included. But the mongo regex filter is only best effort, + // and it will download the parents of all the included paths, even if they are empty. This is not a problem, + // because the filter at the transform stage will remove these paths. This test has no filter at the transform + // stage (pathPredicate is always true), so the / will be included in the result. + testSuccessfulDownload(pathPredicate, mongoRegexPathFilters, List.of("/|{}"), true); + } + + @Test(expected = IllegalArgumentException.class) + public void createFFS_mongoFiltering_custom_excluded_paths_cannot_exclude_root() throws Exception { + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_REGEX_PATH_FILTERING, "true"); + System.setProperty(OAK_INDEXER_PIPELINED_MONGO_CUSTOM_EXCLUDED_PATHS, "/etc,/"); + Predicate pathPredicate = s -> true; + List pathFilters = List.of(new PathFilter(List.of("/"), List.of())); + + testSuccessfulDownload(pathPredicate, pathFilters, List.of( + "/|{}", + "/content|{}", + "/content/dam|{}", + "/etc|{}", + "/home|{}", + "/jcr:system|{}" + ), true); + } + + + @Ignore("This test is for manual execution only. It allocates two byte buffers of 2GB each, which might exceed the memory available in the CI") + public void createFFSWithPipelinedStrategy_veryLargeWorkingMemorySetting() throws Exception { + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_TRANSFORM_THREADS, "1"); + System.setProperty(PipelinedStrategy.OAK_INDEXER_PIPELINED_WORKING_MEMORY_MB, "8000"); + + try (MongoTestBackend rwStore = createNodeStore(false)) { + PipelineITUtil.createContent(rwStore.documentNodeStore); + } + + try (MongoTestBackend roStore = createNodeStore(true)) { + Predicate pathPredicate = s -> s.startsWith("/content/dam"); + PipelinedTreeStoreStrategy pipelinedStrategy = createStrategy(roStore, pathPredicate, null); + pipelinedStrategy.createSortedStoreFile(); + } + } + + private MongoTestBackend createNodeStore(boolean b) { + return PipelineITUtil.createNodeStore(b, connectionFactory, builderProvider); + } + + private PipelinedTreeStoreStrategy createStrategy(MongoTestBackend roStore) { + return createStrategy(roStore, s -> true, null); + } + + private PipelinedTreeStoreStrategy createStrategy(MongoTestBackend backend, Predicate pathPredicate, List mongoRegexPathFilter) { + Set preferredPathElements = Set.of(); + RevisionVector rootRevision = backend.documentNodeStore.getRoot().getRootRevision(); + indexingReporter.setIndexNames(List.of("testIndex")); + return new PipelinedTreeStoreStrategy( + backend.mongoClientURI, + backend.mongoDocumentStore, + backend.documentNodeStore, + rootRevision, + preferredPathElements, + new MemoryBlobStore(), + sortFolder.getRoot(), + Compression.NONE, + pathPredicate, + mongoRegexPathFilter, + null, + statsProvider, + indexingReporter); + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilterTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilterTest.java new file mode 100644 index 00000000000..940672a5759 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/PathIteratorFilterTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.junit.Test; +import org.apache.jackrabbit.oak.spi.filter.PathFilter; + +public class PathIteratorFilterTest { + + @Test + public void convert() { + List list = new ArrayList<>(); + // no includes + assertEquals("[]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // root + list.add(new PathFilter(List.of("/"), Collections.emptyList())); + assertEquals("[/]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // root is higher than /content, so we only need to retain root + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + assertEquals("[/]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + list.clear(); + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + assertEquals("[/content]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // /content is higher than /content/abc, so we only keep /content + list.add(new PathFilter(List.of("/content/abc"), Collections.emptyList())); + assertEquals("[/content]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // /libs is new + list.add(new PathFilter(List.of("/lib"), Collections.emptyList())); + assertEquals("[/content, /lib]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + + // root overrides everything + list.add(new PathFilter(List.of("/"), Collections.emptyList())); + assertEquals("[/]", PathIteratorFilter.getAllIncludedPaths(list).toString()); + } + + @Test + public void emptySet() { + List list = new ArrayList<>(); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertFalse(filter.includes("/")); + assertNull(filter.nextIncludedPath("/")); + } + + @Test + public void all() { + List list = new ArrayList<>(); + list.add(new PathFilter(List.of("/"), Collections.emptyList())); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertTrue(filter.includes("/")); + assertTrue(filter.includes("/content")); + assertTrue(filter.includes("/var")); + assertNull(filter.nextIncludedPath("/")); + } + + @Test + public void content() { + List list = new ArrayList<>(); + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertFalse(filter.includes("/")); + assertEquals("/content", filter.nextIncludedPath("/")); + assertTrue(filter.includes("/content")); + assertTrue(filter.includes("/content/abc")); + assertTrue(filter.includes("/content/def")); + assertFalse(filter.includes("/var")); + assertNull(filter.nextIncludedPath("/var")); + } + + @Test + public void contentAndEtc() { + List list = new ArrayList<>(); + list.add(new PathFilter(List.of("/content"), Collections.emptyList())); + list.add(new PathFilter(List.of("/etc"), Collections.emptyList())); + PathIteratorFilter filter = new PathIteratorFilter(PathIteratorFilter.getAllIncludedPaths(list)); + assertFalse(filter.includes("/")); + assertEquals("/content", filter.nextIncludedPath("/")); + assertTrue(filter.includes("/content")); + assertTrue(filter.includes("/content/abc")); + assertTrue(filter.includes("/content/def")); + assertFalse(filter.includes("/content1")); + assertEquals("/etc", filter.nextIncludedPath("/content1")); + assertTrue(filter.includes("/etc")); + assertTrue(filter.includes("/etc/test")); + assertFalse(filter.includes("/tmp")); + assertNull(filter.nextIncludedPath("/tmp")); + + assertEquals("/content", filter.nextIncludedPath("/")); + assertEquals("/etc", filter.nextIncludedPath("/content1")); + assertNull(filter.nextIncludedPath("/etc")); + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/StorePrefetcherTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/StorePrefetcherTest.java new file mode 100644 index 00000000000..69c3b4c21ab --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/StorePrefetcherTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; + +import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.NodeStateEntryReader; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class StorePrefetcherTest { + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + @Test + public void test() throws IOException, InterruptedException { + BlobStore blobStore = null; + NodeStateEntryReader entryReader = new NodeStateEntryReader(blobStore); + File folder = temporaryFolder.newFolder(); + TreeStore indexStore = new TreeStore("index", + folder, entryReader, 1); + indexStore.getSession().init(); + indexStore.getSession().put("/", "{}"); + indexStore.getSession().put("/content", "{}"); + indexStore.getSession().put("/test.txt", "{}"); + indexStore.getSession().put("/test.txt/jcr:content", "{}"); + indexStore.getSession().checkpoint(); + TreeStore prefetchStore = new TreeStore( + "prefetch", folder, entryReader, 1); + Prefetcher prefetcher = new Prefetcher(prefetchStore, indexStore) { + @Override + public void sleep(String status) throws InterruptedException { + Thread.sleep(0, 1); + } + }; + prefetcher.setBlobReadAheadSize(1); + prefetcher.setNodeReadAheadCount(1); + prefetcher.setBlobSuffix("/test.txt"); + prefetcher.startPrefetch(); + Iterator it = indexStore.iterator(); + while (it.hasNext()) { + Thread.sleep(100); + NodeStateEntry e = it.next(); + assertTrue(e.getPath().compareTo(indexStore.getHighestReadKey()) <= 0); + } + prefetcher.shutdown(); + System.out.println(indexStore.toString()); + assertTrue("Expected shutdown after 1 second", prefetcher.shutdown()); + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java new file mode 100644 index 00000000000..8009cfda932 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreTest.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.OakInitializer; +import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.plugins.name.NamespaceEditorProvider; +import org.apache.jackrabbit.oak.plugins.nodetype.TypeEditorProvider; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfo; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfoProvider; +import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + +public class TreeStoreTest { + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + @Test + public void convertPathTest() { + assertEquals("\t", TreeStore.toChildNodeEntry("/")); + assertEquals("/\tabc", TreeStore.toChildNodeEntry("/abc")); + assertEquals("/hello\tworld", TreeStore.toChildNodeEntry("/hello/world")); + + assertEquals("/\tabc", TreeStore.toChildNodeEntry("/", "abc")); + assertEquals("/hello\tworld", TreeStore.toChildNodeEntry("/hello", "world")); + } + + @Test + public void convertPathAndReverseTest() { + // normal case + String path = "/hello"; + String childNodeName = "world"; + String key = TreeStore.toChildNodeEntry(path, childNodeName); + assertEquals("/hello\tworld", key); + String[] parts = TreeStore.toParentAndChildNodeName(key); + assertEquals(path, parts[0]); + assertEquals(childNodeName, parts[1]); + + // root node + path = "/"; + childNodeName = "test"; + key = TreeStore.toChildNodeEntry(path, childNodeName); + parts = TreeStore.toParentAndChildNodeName(key); + assertEquals(path, parts[0]); + assertEquals(childNodeName, parts[1]); + + // failure case + key = "/"; + try { + parts = TreeStore.toParentAndChildNodeName(key); + fail(); + } catch (IllegalArgumentException e) { + // expected + } + } + + @Test + public void buildAndIterateTest() throws IOException { + File testFolder = temporaryFolder.newFolder(); + TreeStore store = new TreeStore("test", testFolder, null, 1); + try { + store.getSession().init(); + store.putNode("/", "{}"); + store.putNode("/content", "{}"); + Iterator it = store.iteratorOverPaths(); + assertEquals("/", it.next()); + assertEquals("/content", it.next()); + assertFalse(it.hasNext()); + } finally { + store.close(); + } + } + + @Test + public void includedPathTest() throws IOException { + File testFolder = temporaryFolder.newFolder(); + TreeStore store = new TreeStore("test", testFolder, null, 1); + try { + store.getSession().init(); + store.putNode("/", "{}"); + store.putNode("/content", "{}"); + store.putNode("/content/abc", "{}"); + store.putNode("/jcr:system", "{}"); + store.putNode("/jcr:system/jcr:version", "{}"); + store.putNode("/var/abc", "{}"); + store.putNode("/var/def", "{}"); + store.putNode("/zero", "{}"); + + Set defs = inMemoryIndexDefinitions("/content", "/var", "/tmp"); + store.setIndexDefinitions(defs); + + Iterator it = store.iteratorOverPaths(); + assertEquals("/content", it.next()); + assertEquals("/content/abc", it.next()); + assertEquals("/var/abc", it.next()); + assertEquals("/var/def", it.next()); + assertFalse(it.hasNext()); + } finally { + store.close(); + } + } + + private static Set inMemoryIndexDefinitions(String... includedPaths) { + NodeStore store = new MemoryNodeStore(); + EditorHook hook = new EditorHook( + new CompositeEditorProvider(new NamespaceEditorProvider(), new TypeEditorProvider())); + OakInitializer.initialize(store, new InitialContent(), hook); + + Set defns = new HashSet<>(); + IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder(); + defnBuilder.includedPaths(includedPaths); + defnBuilder.indexRule("dam:Asset"); + defnBuilder.aggregateRule("dam:Asset"); + IndexDefinition defn = IndexDefinition.newBuilder(store.getRoot(), defnBuilder.build(), "/foo").build(); + defns.add(defn); + + NodeTypeInfoProvider mockNodeTypeInfoProvider = Mockito.mock(NodeTypeInfoProvider.class); + NodeTypeInfo mockNodeTypeInfo = Mockito.mock(NodeTypeInfo.class, "dam:Asset"); + Mockito.when(mockNodeTypeInfo.getNodeTypeName()).thenReturn("dam:Asset"); + Mockito.when(mockNodeTypeInfoProvider.getNodeTypeInfo("dam:Asset")).thenReturn(mockNodeTypeInfo); + + return defns; + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CompressionTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CompressionTest.java new file mode 100644 index 00000000000..42406eac875 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CompressionTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Random; + +import org.junit.Test; + +public class CompressionTest { + + @Test + public void randomized() { + Random r = new Random(); + for (int i = 0; i < 2000; i++) { + byte[] data = new byte[r.nextInt(1000)]; + for (int j = 0; j < data.length; j++) { + // less random first, and then more random + data[j] = (byte) r.nextInt(1 + (i / 10)); + } + byte[] comp = Compression.LZ4.compress(data); + byte[] test = Compression.LZ4.expand(comp); + assertEquals(data.length, test.length); + assertTrue(Arrays.equals(data, test)); + } + + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java new file mode 100644 index 00000000000..4c7b7a30a39 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/CrudMultiRootTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.junit.Test; + +/** + * Create, read, update, delete test. + */ +public class CrudMultiRootTest { + + @Test + public void memory() { + Store store = StoreBuilder.build(""); + test(store); + testAdd(store); + } + + @Test + public void file() { + Store store = StoreBuilder.build( + "type=file\n" + + "dir=target/files\n" + + "cacheSizeMB=1\n" + + "maxFileSizeBytes=100"); + store.setWriteCompression(Compression.NO); + test(store); + testAdd(store); + } + + @Test + public void fileLZ4() { + Store store = StoreBuilder.build( + "type=file\n" + + "dir=target/files"); + store.setWriteCompression(Compression.LZ4); + test(store); + testAdd(store); + } + + public void testAdd(Store store) { + store.removeAll(); + TreeSession session = new TreeSession(store); + session.init(); + int count = 100; + TreeMap map = new TreeMap<>(); + for (int i = 0; i < count; i++) { + map.put("key" + i, "v" + i); + session.put("key" + i, "v" + i); + session.flush(); + if (count % 10 == 0) { + session.checkpoint(); + session.mergeRoots(10); + } + } + session.mergeRoots(Integer.MAX_VALUE); + session.flush(); + + session = new TreeSession(store); + TreeMap map2 = new TreeMap<>(); + for(String k : session.keys()) { + map2.put(k, session.get(k)); + } + assertEquals(map, map2); + } + + public void test(Store store) { + store.removeAll(); + TreeSession session = new TreeSession(store); + session.init(); + int count = 100; + TreeMap verify = new TreeMap<>(); + for (int i = 0; i < count; i++) { + session.put("hello" + i, "world" + i); + verify.put("hello" + i, "world" + i); + } + session.checkpoint(); + for (int i = 0; i < count; i++) { + if (i % 3 == 0) { + session.put("hello" + i, null); + verify.put("hello" + i, null); + } else if (i % 3 == 1) { + session.put("hello" + i, "World" + i); + verify.put("hello" + i, "World" + i); + } + } + + Iterator> it = verify.entrySet().iterator(); + Iterator it2 = session.keys().iterator(); + String previous = null; + for (int i = 0; i < count; i++) { + Entry e = it.next(); + String k = it2.next(); + assertEquals(e.getKey(), k); + assertEquals(e.getValue(), session.get(k)); + + Iterator it3 = session.keys(previous).iterator(); + assertTrue(it3.hasNext()); + assertEquals("previous: " + previous, k, it3.next()); + Iterator> it4 = session.iterator(previous); + assertTrue("previous: " + previous, it4.hasNext()); + Entry e4 = it4.next(); + assertEquals("previous " + previous, k, e4.getKey()); + assertEquals(e.getValue(), e4.getValue()); + if (it4.hasNext()) { + Entry e4b = it4.next(); + assertFalse("key returned twice " + e4b.getKey(), e4b.getKey().equals(k)); + } + + previous = k; + } + + session = new TreeSession(store); + for (int i = 0; i < count; i++) { + String key = "hello" + i; + assertEquals("world" + i, session.get(key)); + session.put(key, "World " + i); + } + session.flush(); + + session = new TreeSession(store); + for (int i = 0; i < count; i++) { + String key = "hello" + i; + assertEquals("World " + i, session.get(key)); + session.put(key, null); + } + session.flush(); + + session = new TreeSession(store); + for (int i = 0; i < count; i++) { + String key = "hello" + i; + assertEquals(null, session.get(key)); + } + + for (int i = 0; i < 20; i++) { + session.checkpoint(); + } + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java new file mode 100644 index 00000000000..a7295f19125 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/MergeRootsTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; + +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeMap; + +import org.junit.Test; + +public class MergeRootsTest { + + @Test + public void gcTest() { + Store store = StoreBuilder.build(""); + TreeSession session = new TreeSession(store); + session.init(); + session.runGC(); + } + + @Test + public void simpleTest() { + Store store = StoreBuilder.build(""); + TreeSession session = new TreeSession(store); + session.init(); + for (int i = 0; i < 10; i++) { + session.put("x" + i, "y" + i); + session.checkpoint(); + assertEquals(i + 2, session.getRootCount()); + } + assertEquals(11, session.getRootCount()); + session.mergeRoots(2); + assertEquals(10, session.getRootCount()); + for (int i = 0; i < 10; i++) { + assertEquals("y" + i, session.get("x" + i)); + } + } + + @Test + public void multipleRootAppendTest() { + Store store = StoreBuilder.build(""); + TreeSession session = new TreeSession(store); + session.init(); + for(int j = 1; j <= 3; j++) { + for (int i = 0; i < 10 * j; i++) { + session.put("x" + i, "y" + i + "j" + j); + } + session.checkpoint(); + } + for (int i = 0; i < 10 * 3; i++) { + if (!("y" + i + "j3").equals(session.get("x" + i))) { + assertEquals("y" + i + "j3", session.get("x" + i)); + } + } + session.mergeRoots(Integer.MAX_VALUE); + for (int i = 0; i < 10 * 3; i++) { + if (!("y" + i + "j3").equals(session.get("x" + i))) { + assertEquals("y" + i + "j3", session.get("x" + i)); + } + } + } + + @Test + public void multipleRootRandomOverwriteTest() { + Store store = StoreBuilder.build(""); + TreeSession session = new TreeSession(store); + session.init(); + TreeMap map = new TreeMap<>(); + Random r = new Random(42); + for(int j = 1; j <= 3; j++) { + for (int i = 0; i < 10; i++) { + String k = "x" + r.nextInt(30); + String v = "y" + i + "j" + j; + session.put(k, v); + map.put(k, v); + } + session.checkpoint(); + } + session.mergeRoots(Integer.MAX_VALUE); + for(Entry e : map.entrySet()) { + assertEquals(e.getValue(), session.get(e.getKey())); + } + TreeMap map2 = new TreeMap<>(); + + for(Entry e : map.entrySet()) { + map2.put(e.getKey(), e.getValue()); + } + assertEquals(map2, map); + } + + @Test + public void logStructuredMerge() { + Store store = StoreBuilder.build( + "type=memory\n" + + "maxFileSizeBytes=10000"); + TreeSession session = new TreeSession(store); + session.init(); + for (int batch = 1; batch <= 200; batch++) { + // System.out.println("batch " + batch); + if (batch % 10 == 0) { + session.mergeRoots(10); + // System.out.println("merged 10 roots; new root count: " + session.getRootCount()); + } + if (batch % 100 == 0) { + session.mergeRoots(20); + // System.out.println("merged 20 roots; new root count: " + session.getRootCount()); + // System.out.println(session.getInfo()); + } + session.flush(); + // System.out.println("info:"); + // System.out.println(session.getInfo()); + session.runGC(); + for (int j = 0; j < 100; j++) { + session.put("x" + j + " " + Math.random(), new String(new char[1000])); + } + session.checkpoint(); + } + assertEquals(3, session.getRootCount()); + session.mergeRoots(Integer.MAX_VALUE); + assertEquals(1, session.getRootCount()); + } + + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFileTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFileTest.java new file mode 100644 index 00000000000..57122da06f8 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/PageFileTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class PageFileTest { + + @Test + public void serializeLeafNode() { + PageFile f = new PageFile(false, 1_000_000); + f.appendRecord("test", null); + f.appendRecord("", ""); + f.appendRecord(new String(new char[8000]) + "x", new String(new char[80000]) + "y"); + f.appendRecord("a", "b"); + f.setUpdate(-3); + byte[] data = f.toBytes(); + PageFile f2 = PageFile.fromBytes(data, 1_000_000); + assertTrue(f.isInnerNode() == f2.isInnerNode()); + for (int i = 0; i < f.getKeys().size(); i++) { + assertEquals(f.getKey(i), f2.getKey(i)); + assertEquals(f.getValue(i), f2.getValue(i)); + } + assertEquals(f.getUpdate(), f2.getUpdate()); + } + + @Test + public void serializeInnerNode() { + PageFile f = new PageFile(true, 1_000_000); + f.appendRecord("test", null); + f.appendRecord("", ""); + f.appendRecord(new String(new char[8000]) + "x", new String(new char[80000]) + "y"); + f.appendRecord("a", "b"); + f.setUpdate(-3); + byte[] data = f.toBytes(); + PageFile f2 = PageFile.fromBytes(data, 1_000_000); + assertTrue(f.isInnerNode() == f2.isInnerNode()); + for (int i = 0; i < f.getKeys().size(); i++) { + assertEquals(f.getKey(i), f2.getKey(i)); + assertEquals(f.getValue(i), f2.getValue(i)); + } + assertEquals(f.getUpdate(), f2.getUpdate()); + } + +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java new file mode 100644 index 00000000000..67e92056aa4 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/RandomizedTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Random; + +import org.junit.Test; + +public class RandomizedTest { + + @Test + public void test() { + Store store = StoreBuilder.build( + "type=memory\n" + + "cacheSizeMB=1\n" + + "maxFileSizeBytes=100"); + TreeSession session = new TreeSession(store); + session.init(); + HashMap verify = new HashMap<>(); + Random r = new Random(1); + boolean verifyAllAlways = false; + + int count = 100000; + int size = 100; + for (int i = 0; i < count; i++) { + String key = "" + r.nextInt(size); + String value; + boolean remove = r.nextBoolean(); + if (remove) { + value = null; + } else { + value = "x" + r.nextInt(size); + } + verify(verify, session, key); + log("#" + i + " put " + key + "=" + (value == null ? "null" : value.toString().replace('\n', ' '))); + verify.put(key, value); + session.put(key, value); + verify(verify, session, key); + if (r.nextInt(size) == 0) { + log("flush"); + session.flush(); + } + if (verifyAllAlways) { + for(String k : verify.keySet()) { + verify(verify, session, k); + } + } + } + + String min = session.getMinKey(); + assertEquals("0", min); + String max = session.getMaxKey(); + assertEquals("99", max); + String median = session.getApproximateMedianKey(min, max); + assertEquals("53", median); + } + + private void verify(HashMap verify, TreeSession session, String key) { + String a = verify.get(key); + String b = session.get(key); + if (a == null || b == null) { + if (a != null) { + fail(key + " a: " + a + " b: " + b); + } + } else { + assertEquals( + key + " a: " + a + " b: " + b, + a.toString(), b.toString()); + } + } + + private void log(String msg) { + // System.out.println(msg); + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java new file mode 100644 index 00000000000..95e4303a4ec --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/SessionCacheTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class SessionCacheTest { + + @Test + public void test() { + Store store = StoreBuilder.build("type:memory\n" + + Store.MAX_FILE_SIZE_BYTES + "=1000\n" + + TreeSession.CACHE_SIZE_MB + "=1"); + TreeSession s = new TreeSession(store); + s.init(); + for (int i = 0; i < 50_000; i++) { + s.put("k" + i, "v" + i); + } + s.flush(); + assertEquals("root #0 contains 1756 files (file name root)\n" + + "cache entries:1049 max:1048576 used:1049000", s.getInfo()); + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java new file mode 100644 index 00000000000..ba13518be8c --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/CacheTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.Test; + +public class CacheTest { + + @Test + public void memoryBoundCacheTest() { + ConcurrentLRUCache cache = new ConcurrentLRUCache<>(1000); + for (int i = 0; i < 200; i++) { + cache.put("k" + i, new MemoryValue("v" + i, 10)); + } + assertEquals(101, cache.size()); + } + + @Test + public void sieveCacheTest() { + AtomicLong removed = new AtomicLong(); + SieveCache cache = new SieveCache<>(1000) { + @Override + public void entryWasRemoved(String key, MemoryValue value) { + removed.incrementAndGet(); + } + }; + for (int i = 0; i < 200; i++) { + cache.put("k" + i, new MemoryValue("v" + i, 10)); + } + assertEquals(100, removed.get()); + assertEquals(100, cache.size()); + for (int j = 0; j < 10; j++) { + for (int i = 0; i < 50; i++) { + cache.put("k" + i, new MemoryValue("v" + i, 10)); + } + } + assertEquals(150, removed.get()); + assertEquals(100, cache.size()); + } + + static class MemoryValue implements MemoryObject { + + final String value; + final int memory; + + MemoryValue(String value, int memory) { + this.value = value; + this.memory = memory; + } + + @Override + public long estimatedMemory() { + return memory; + } + + public String toString() { + return value; + } + + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentCacheTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentCacheTest.java new file mode 100644 index 00000000000..35e91dcaefd --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/ConcurrentCacheTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertTrue; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.Test; + +public class ConcurrentCacheTest { + + @Test + public void testRandomOperations() throws Exception { + Random r = new Random(1); + int maxMemory = 1000; +// final MemoryBoundCache cache = +// new MemoryBoundCache<>(maxMemory); + final SieveCache cache = + new SieveCache<>(maxMemory); + final Exception[] ex = new Exception[1]; + int size = 3; + Thread[] threads = new Thread[size]; + final AtomicBoolean stop = new AtomicBoolean(); + for (int i = 0; i < size; i++) { + Thread t = new Thread() { + @Override + public void run() { + while (!stop.get()) { + try { + cache.get(r.nextInt(maxMemory)); + cache.keys(); + cache.put(r.nextInt(maxMemory), new MemoryValue("1", 1 + r.nextInt(10))); + cache.size(); + } catch (Exception e) { + ex[0] = e; + } + } + } + }; + t.start(); + threads[i] = t; + } + try { + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() < start + 1000) { + for (int i = 0; i < 100000 && ex[0] == null; i++) { + cache.put(r.nextInt(maxMemory), new MemoryValue("1", 1 + r.nextInt(10))); + } + } + } finally { + stop.set(true); + for (Thread t : threads) { + t.join(); + } + } + if (ex[0] != null) { + throw ex[0]; + } + int memorySum = 0; + for(Integer k : cache.keys()) { + memorySum += cache.get(k).memory; + } + System.out.println(cache.toString()); + assertTrue(memorySum >= 0); + assertTrue("totalMemory: " + memorySum, memorySum <= maxMemory); + assertTrue(cache.size() >= memorySum / 10); + assertTrue(cache.size() < memorySum); + } + + static class MemoryValue implements MemoryObject { + + final String value; + final int memory; + + MemoryValue(String value, int memory) { + this.value = value; + this.memory = memory; + } + + @Override + public long estimatedMemory() { + return memory; + } + + public String toString() { + return value; + } + + } +} + diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java new file mode 100644 index 00000000000..25fdc19b67e --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/FilePackerTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.ArrayList; +import java.util.Random; + +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.TreeSession; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; +import java.util.zip.CheckedOutputStream; + +public class FilePackerTest { + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + @Test + public void headerMismatch() throws IOException { + File pack = temporaryFolder.newFile("pack"); + File dir = temporaryFolder.newFolder("sourceDir"); + RandomAccessFile f = new RandomAccessFile(pack, "rw"); + f.writeUTF("test"); + try { + FilePacker.unpack(pack, dir, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("File header is not 'PACK'")); + } + f.close(); + pack.delete(); + } + + @Test + public void sourceIsDirectory() throws IOException { + File dir = temporaryFolder.newFolder("source"); + try { + FilePacker.unpack(dir, dir, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("Source file doesn't exist or is not a file")); + } + dir.delete(); + } + + @Test + public void sourceMissing() throws IOException { + File dir = temporaryFolder.newFolder("source"); + try { + FilePacker.unpack(new File(dir, "missing"), dir, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("Source file doesn't exist or is not a file")); + } + dir.delete(); + } + + @Test + public void targetDirectoryIsFile() throws IOException { + File target = temporaryFolder.newFile(); + try { + FilePacker.unpack(target, target, false); + fail(); + } catch (IOException e) { + assertTrue(e.getMessage().startsWith("Target file exists")); + } + target.delete(); + } + + @Test + public void packUnpack() throws IOException { + packUnpack(false); + packUnpack(true); + } + + public void packUnpack(boolean delete) throws IOException { + File dir = temporaryFolder.newFolder("sourceDir"); + File pack = temporaryFolder.newFile("pack"); + ArrayList list = new ArrayList<>(); + Random r = new Random(1); + for (int i = 0; i < 5; i++) { + FileEntry f = new FileEntry(); + f.file = File.createTempFile("root_", ".txt", dir); + list.add(f); + CRC32 crc = new CRC32(); + CheckedOutputStream out = new CheckedOutputStream( + new BufferedOutputStream(new FileOutputStream(f.file)), + crc); + // at most ~3 MB + int s = i * i * 50; + f.length = i * (s * s + r.nextInt(100000)); + for (int j = 0; j < f.length; j++) { + out.write(r.nextInt()); + } + f.contentHash = crc.getValue(); + out.close(); + } + // for debugging + // System.out.println(pack.getAbsolutePath()); + // System.out.println(dir.getAbsolutePath()); + FilePacker.pack(dir, TreeSession.getFileNameRegex(), pack, delete); + + for (FileEntry f : list) { + if (delete) { + assertFalse(f.file.exists()); + } else { + f.file.delete(); + } + } + dir.delete(); + + FilePacker.unpack(pack, dir, delete); + if (delete) { + assertFalse(pack.exists()); + } else { + assertTrue(pack.exists()); + } + for (FileEntry f : list) { + if (!f.file.exists()) { + fail(); + } + CRC32 crc = new CRC32(); + CheckedInputStream in = new CheckedInputStream( + new BufferedInputStream(new FileInputStream(f.file)), crc); + while (in.read() >= 0) + ; + in.close(); + assertEquals(f.contentHash, crc.getValue()); + } + // cleanup + for (FileEntry f : list) { + f.file.delete(); + } + dir.delete(); + pack.delete(); + } + + static class FileEntry { + File file; + long length; + long contentHash; + } +} diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuidTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuidTest.java new file mode 100644 index 00000000000..623cc93f883 --- /dev/null +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/utils/TimeUuidTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.Test; + +public class TimeUuidTest { + + @Test + public void stringRepresentation() { + TimeUuid u = new TimeUuid(112989318798340096L, -5844840652695650548L); + assertEquals("2024-08-19T15:09:41.859Z 000 aee2f464c7503f0c", u.toHumanReadableString()); + + assertEquals("01916b2fd263000aee2f464c7503f0c", u.toShortString()); + assertEquals(0x1916b2fd263L, u.getTimestampPart()); + assertEquals(0L, u.getCounterPart()); + assertEquals(0xaee2f464c7503f0cL, u.getRandomPart()); + } + + @Test + public void convert() { + Random r = new Random(1); + for (int i = 0; i < 1000; i++) { + long msb = r.nextLong(); + long lsb = r.nextLong(); + TimeUuid a = TimeUuid.newUuid(msb, lsb); + TimeUuid b = TimeUuid.newUuid(msb, lsb); + assertEquals(a.toString(), b.toString()); + } + } + + @Test + public void compare() { + Random r = new Random(1); + TimeUuid lastY = TimeUuid.newUuid(0, 0); + for (int i = 0; i < 1000; i++) { + long a = r.nextBoolean() ? r.nextInt(10) : r.nextLong(); + long b = r.nextBoolean() ? r.nextInt(10) : r.nextLong(); + TimeUuid y = TimeUuid.newUuid(a, b); + assertEquals((int) Math.signum(y.compareTo(lastY)), + (int) Math.signum(y.toString().compareTo(lastY.toString()))); + if (y.compareTo(lastY) == 0) { + assertEquals(y.hashCode(), lastY.hashCode()); + assertTrue(y.equals(lastY)); + } else { + assertFalse(y.equals(lastY)); + } + lastY = y; + } + } + + @Test + public void versionAndVariant() { + TimeUuid x = TimeUuid.newUuid(); + UUID y = new UUID(x.getMostSignificantBits(), x.getLeastSignificantBits()); + assertEquals(7, y.version()); + assertEquals(2, y.variant()); + } + + @Test + public void incremental() { + TimeUuid last = TimeUuid.newUuid(); + for (int i = 0; i < 1000; i++) { + TimeUuid x = TimeUuid.newUuid(); + assertTrue(x.compareTo(last) >= 0); + last = x; + } + } + + @Test + public void getMillisIncreasing() { + AtomicLong lastMillis = new AtomicLong(); + assertEquals((10 << 12) + 0, TimeUuid.getMillisAndCountIncreasing(10, lastMillis)); + assertEquals((10 << 12) + 0, lastMillis.get()); + assertEquals((10 << 12) + 1, TimeUuid.getMillisAndCountIncreasing(9, lastMillis)); + assertEquals((10 << 12) + 1, lastMillis.get()); + assertEquals((11 << 12) + 0, TimeUuid.getMillisAndCountIncreasing(11, lastMillis)); + assertEquals((11 << 12) + 0, lastMillis.get()); + } + + @Test + public void fastGeneration() { + int size = 1 << 14; + ArrayList list = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + list.add(TimeUuid.newUuid()); + } + for (int i = 1; i < size; i++) { + TimeUuid a = list.get(i - 1); + TimeUuid b = list.get(i); + assertFalse(a.equals(b)); + assertTrue(a.compareTo(b) < 0); + } + } + +} diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java index e8818bcd690..0d6c8d88ae6 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java @@ -30,7 +30,6 @@ import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.index.async.AsyncIndexerLucene; import org.apache.jackrabbit.oak.index.indexer.document.DocumentStoreIndexer; -import org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStore; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStore; import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater; import org.apache.jackrabbit.oak.run.cli.CommonOptions; @@ -253,14 +252,9 @@ private File reindex(IndexOptions idxOpts, ExtendedIndexHelper extendedIndexHelp log.info("Using Document order traversal to perform reindexing"); try (DocumentStoreIndexer indexer = new DocumentStoreIndexer(extendedIndexHelper, indexerSupport)) { if (idxOpts.buildFlatFileStoreSeparately()) { - IndexStore indexStore = indexer.buildFlatFileStore(); - if (indexStore instanceof FlatFileStore) { - FlatFileStore ffs = (FlatFileStore) indexStore; - String pathToFFS = ffs.getFlatFileStorePath(); - System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToFFS); - } else { - throw new IllegalArgumentException("Store is not FlatFileStore, cannot cannot use option to build flat file store separately."); - } + IndexStore store = indexer.buildStore(); + String pathToStore = store.getStorePath(); + System.setProperty(OAK_INDEXER_SORTED_FILE_PATH, pathToStore); } indexer.reindex(); } diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexStoreCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexStoreCommand.java new file mode 100644 index 00000000000..c28a0e8a6d3 --- /dev/null +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/index/merge/IndexStoreCommand.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.index.merge; + +import static java.util.Arrays.asList; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; + +import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.index.indexer.document.flatfile.LZ4Compression; +import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; +import org.apache.jackrabbit.oak.index.indexer.document.tree.TreeStore; +import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.FilePacker; +import org.apache.jackrabbit.oak.run.commons.Command; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; + +public class IndexStoreCommand implements Command { + + public final static String INDEX_STORE = "index-store"; + + @SuppressWarnings("unchecked") + @Override + public void execute(String... args) throws IOException { + OptionParser parser = new OptionParser(); + OptionSpec helpSpec = parser.acceptsAll( + asList("h", "?", "help"), "show help").forHelp(); + OptionSet options = parser.parse(args); + parser.nonOptions( + "An index store file").ofType(File.class); + if (options.has(helpSpec) + || options.nonOptionArguments().isEmpty()) { + System.out.println("Mode: " + INDEX_STORE); + System.out.println(); + parser.printHelpOn(System.out); + return; + } + for (String fileName : ((List) options.nonOptionArguments())) { + File file = new File(fileName); + if (!file.exists()) { + System.out.println("File not found: " + fileName); + return; + } + if (FilePacker.isPackFile(file)) { + File treeFile = new File(file.getAbsoluteFile().getParent(), "tree"); + treeFile.mkdirs(); + FilePacker.unpack(file, treeFile, false); + file = treeFile; + } + if (file.isDirectory()) { + System.out.println("Tree store " + fileName); + listTreeStore(file); + } else if (file.isFile()) { + System.out.println("Flat file " + fileName); + listFlatFile(file); + } + } + } + + public void listFlatFile(File file) throws IOException { + BufferedReader reader; + if (file.getName().endsWith(".lz4")) { + reader = IndexStoreUtils.createReader(file, new LZ4Compression()); + } else if (file.getName().endsWith(".gz")) { + reader = IndexStoreUtils.createReader(file, Compression.GZIP); + } else { + reader = IndexStoreUtils.createReader(file, Compression.NONE); + } + while (true) { + String line = reader.readLine(); + if (line == null) { + break; + } + System.out.println(line); + } + reader.close(); + } + + public static void listTreeStore(File directory) throws IOException { + TreeStore treeStore = new TreeStore("tree", directory, null, 1); + Iterator it = treeStore.iteratorOverPaths(); + while (it.hasNext()) { + String path = it.next(); + String node = treeStore.getSession().get(path); + System.out.println(path + "|" + node); + } + treeStore.close(); + } + +} diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java index 6c723cb93f1..cb6f6e34fde 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java @@ -23,6 +23,7 @@ import org.apache.jackrabbit.oak.exporter.NodeStateExportCommand; import org.apache.jackrabbit.oak.index.IndexCommand; import org.apache.jackrabbit.oak.index.merge.IndexDiffCommand; +import org.apache.jackrabbit.oak.index.merge.IndexStoreCommand; import org.apache.jackrabbit.oak.run.commons.Command; import org.apache.jackrabbit.oak.run.commons.Modes; @@ -51,8 +52,9 @@ public final class AvailableModes { .put("garbage", new GarbageCommand()) .put("help", new HelpCommand()) .put("history", new HistoryCommand()) - .put("index-merge", new IndexMergeCommand()) .put("index-diff", new IndexDiffCommand()) + .put("index-merge", new IndexMergeCommand()) + .put(IndexStoreCommand.INDEX_STORE, new IndexStoreCommand()) .put(IndexCommand.NAME, new IndexCommand()) .put(IOTraceCommand.NAME, new IOTraceCommand()) .put(JsonIndexCommand.INDEX, new JsonIndexCommand())