chore: build

src/data-index/index.d.ts

@@ -72,7 +72,7 @@ export declare class DataIndex {
     constructor(storage: Storage, path: string, key: string, options?: DataIndexOptions);
     get allMetadataKeys(): string[];
     setCacheTimeout(seconds: number, sliding?: boolean): void;
-    cache(op: string, param: unknown, results?: IndexQueryResults): IndexQueryResults;
+    cache(op: string, param: unknown, results?: any): any;
     delete(): Promise<void>;
     close(): Promise<void>;
     /**
@@ -95,21 +95,21 @@ export declare class DataIndex {
     _processUpdateQueue(): Promise<void>;
     handleRecordUpdate(path: string, oldValue: unknown, newValue: unknown, indexMetadata?: IndexMetaData): Promise<void>;
     _lock(mode?: string, timeout?: number): Promise<import("../thread-safe").ThreadSafeLock>;
-    count(op: string, val: IndexableValueOrArray): Promise<number | IndexQueryResults>;
-    take(skip: number, take: number, ascending: boolean): Promise<IndexQueryResults>;
+    count(op: string, val: IndexableValueOrArray): Promise<any>;
+    take(skip: number, take: number, ascending: boolean): Promise<any>;
     static get validOperators(): string[];
     get validOperators(): string[];
     query(op: BlacklistingSearchOperator): Promise<IndexQueryResults>;
     query(op: string, val: IndexableValueOrArray, options?: {
         /** previous results to filter upon */
         filter?: IndexQueryResults;
     }): Promise<IndexQueryResults>;
-    build(options: {
-        addCallback?: (callback: (key: string, recordPointer: IndexRecordPointer, metadata: IndexMetaData) => void, value: unknown, // unknown on purpose
+    build(options?: {
+        addCallback?: (callback: (value: IndexableValue, recordPointer: IndexRecordPointer, metadata: IndexMetaData) => void, value: unknown, // unknown on purpose
         recordPointer: IndexRecordPointer, metadata?: IndexMetaData, env?: {
             path: string;
             wildcards: string[];
-            key: string;
+            key: string | number;
             locale: string;
         }) => void;
         valueTypes?: number[];
@@ -126,22 +126,20 @@ export declare class DataIndex {
         release(): void;
     }>;
 }
-declare class IndexQueryResult {
+export declare class IndexQueryResult {
     key: string | number;
     path: string;
     value: IndexableValue;
     metadata: IndexMetaData;
     values: BPlusTreeLeafEntryValue[];
     constructor(key: string | number, path: string, value: IndexableValue, metadata: IndexMetaData);
 }
-declare class IndexQueryResults extends Array<IndexQueryResult> {
-    static from(results: IndexQueryResults | IndexQueryResult[], filterKey: string): IndexQueryResults;
-    values: BPlusTreeLeafEntryValue[];
+export declare class IndexQueryResults extends Array<IndexQueryResult> {
+    static fromResults(results: IndexQueryResults | IndexQueryResult[], filterKey: string): IndexQueryResults;
+    entryValues: BPlusTreeLeafEntryValue[];
     hints: IndexQueryHint[];
     stats: IndexQueryStats;
-    private _filterKey;
-    set filterKey(key: string);
-    get filterKey(): string;
+    filterKey: string;
     filterMetadata(key: string, op: string, compare: IndexableValueOrArray): IndexQueryResults;
     constructor(length: number);
     constructor(...results: IndexQueryResult[]);
@@ -172,14 +170,121 @@ export declare class ArrayIndex extends DataIndex {
     build(): Promise<this>;
     static get validOperators(): string[];
     get validOperators(): string[];
+    query(op: BlacklistingSearchOperator): Promise<IndexQueryResults>;
+    query(op: string, val: IndexableValueOrArray, options?: {
+        filter?: IndexQueryResults;
+    }): Promise<IndexQueryResults>;
+}
+declare class WordInfo {
+    word: string;
+    indexes: number[];
+    sourceIndexes: number[];
+    constructor(word: string, indexes: number[], sourceIndexes: number[]);
+    get occurs(): number;
+}
+declare class TextInfo {
+    static get locales(): {
+        default: {
+            pattern: string;
+            flags: string;
+        };
+        en: {
+            stoplist: string[];
+        };
+        get(locale: string): {
+            pattern?: string;
+            flags?: string;
+            stoplist?: string[];
+        };
+    };
+    locale: string;
+    words: Map<string, WordInfo>;
+    ignored: string[];
+    getWordInfo(word: string): WordInfo;
     /**
-     * @param op "contains" or "!contains"
-     * @param val value to search for
+     * Reconstructs an array of words in the order they were encountered
      */
-    query(op: 'contains' | '!contains', val: IndexableValueOrArray): Promise<IndexQueryResults>;
+    toSequence(): string[];
+    /**
+     * Returns all unique words in an array
+     */
+    toArray(): string[];
+    get uniqueWordCount(): number;
+    get wordCount(): number;
+    constructor(text: string, options?: {
+        /**
+         * Set the text locale to accurately convert words to lowercase
+         * @default "en"
+         */
+        locale?: string;
+        /**
+         * Overrides the default RegExp pattern used
+         * @default "[\w']+"
+         */
+        pattern?: RegExp | string;
+        /**
+         * Add characters to the word detection regular expression. Useful to keep wildcards such as * and ? in query texts
+         */
+        includeChars?: string;
+        /**
+         * Overrides the default RegExp flags (`gmi`) used
+         * @default "gmi"
+         */
+        flags?: string;
+        /**
+         * Optional callback functions that pre-processes the value before performing word splitting.
+         */
+        prepare?: (value: any, locale: string, keepChars: string) => string;
+        /**
+         * Optional callback function that is able to perform word stemming. Will be executed before performing criteria checks
+         */
+        stemming?: (word: string, locale: string) => string;
+        /**
+         * Minimum length of words to include
+         * @default 1
+         */
+        minLength?: number;
+        /**
+         * Maximum length of words to include, should be increased if you expect words in your texts
+         * like "antidisestablishmentarianism" (28), "floccinaucinihilipilification" (29) or "pneumonoultramicroscopicsilicovolcanoconiosis" (45)
+         * @default 25
+         */
+        maxLength?: number;
+        /**
+         * Words to ignore. You can use a default stoplist from TextInfo.locales
+         */
+        blacklist?: string[];
+        /**
+         * Words to include even if they do not meet the min & maxLength criteria
+         */
+        whitelist?: string[];
+        /**
+         * Whether to use a default stoplist to blacklist words (if available for locale)
+         * @default false
+         */
+        useStoplist?: boolean;
+    });
 }
 export interface FullTextIndexOptions extends DataIndexOptions {
+    /**
+     * FullText configuration settings.
+     * NOTE: these settings are not stored in the index file because they contain callback functions
+     * that might not work after a (de)serializion cycle. Besides this, it is also better for security
+     * reasons not to store executable code in index files!
+     *
+     * That means that in order to keep fulltext indexes working as intended, you will have to:
+     *  - call `db.indexes.create` for fulltext indexes each time your app starts, even if the index exists already
+     *  - rebuild the index if you change this config. (pass `rebuild: true` in the index options)
+     */
     config?: {
+        /**
+         * callback function that prepares a text value for indexing.
+         * Useful to perform any actions on the text before it is split into words, such as:
+         *  - transforming compressed / encrypted data to strings
+         *  - perform custom word stemming: allows you to replace strings like `I've` to `I have`
+         * Important: do not remove any of the characters passed in `keepChars` (`"*?`)!
+         */
+        prepare?: (value: any, locale: string, keepChars?: string) => string;
         /**
          * callback function that transforms (or filters) words being indexed
          */
@@ -200,6 +305,7 @@ export interface FullTextIndexOptions extends DataIndexOptions {
         /**
          * Uses the value of a specific key as locale. Allows different languages to be indexed correctly,
          * overrides options.textLocale
+         * @deprecated move to options.textLocaleKey
          */
         localeKey?: string;
         /**
@@ -212,6 +318,23 @@ export interface FullTextIndexOptions extends DataIndexOptions {
         maxLength?: number;
     };
 }
+export interface FullTextContainsQueryOptions {
+    /**
+     * Locale to use for the words in the query. When omitted, the default index locale is used
+     */
+    locale?: string;
+    /**
+     *  Used internally: treats the words in val as a phrase, eg: "word1 word2 word3": words need to occur in this exact order
+     */
+    phrase?: boolean;
+    /**
+     * Sets minimum amount of characters that have to be used for wildcard (sub)queries such as "a%" to guard the
+     * system against extremely large result sets. Length does not include the wildcard characters itself. Default
+     * value is 2 (allows "an*" but blocks "a*")
+     * @default 2
+     */
+    minimumWildcardWordLength?: number;
+}
 /**
  * A full text index allows all words in text nodes to be indexed and searched.
  * Eg: "Every word in this text must be indexed." will be indexed with every word
@@ -224,33 +347,19 @@ export declare class FullTextIndex extends DataIndex {
     config: FullTextIndexOptions['config'];
     constructor(storage: Storage, path: string, key: string, options: FullTextIndexOptions);
     get type(): KnownIndexType;
+    getTextInfo(val: string, locale?: string): TextInfo;
     test(obj: any, op: 'fulltext:contains' | 'fulltext:!contains', val: string): boolean;
-    handleRecordUpdate(path: string, oldValue: unknown, newValue: unknown): Promise<void>;
+    handleRecordUpdate(path: string, oldValue: any, newValue: any): Promise<void>;
     build(): Promise<this>;
     static get validOperators(): string[];
     get validOperators(): string[];
+    query(op: string | BlacklistingSearchOperator, val?: string, options?: any): Promise<IndexQueryResults>;
     /**
      *
      * @param op Operator to use, can be either "fulltext:contains" or "fulltext:!contains"
      * @param val Text to search for. Can include * and ? wildcards, OR's for combined searches, and "quotes" for phrase searches
      */
-    query(op: string, val: string, options?: {
-        /**
-         * Locale to use for the words in the query. When omitted, the default index locale is used
-         */
-        locale?: string;
-        /**
-         *  Used internally: treats the words in val as a phrase, eg: "word1 word2 word3": words need to occur in this exact order
-         */
-        phrase?: boolean;
-        /**
-         * Sets minimum amount of characters that have to be used for wildcard (sub)queries such as "a%" to guard the
-         * system against extremely large result sets. Length does not include the wildcard characters itself. Default
-         * value is 2 (allows "an*" but blocks "a*")
-         * @default 2
-         */
-        minimumWildcardWordLength?: number;
-    }): Promise<IndexQueryResults>;
+    contains(op: 'fulltext:contains' | 'fulltext:!contains', val: string, options?: FullTextContainsQueryOptions): Promise<IndexQueryResults>;
 }
 export declare class GeoIndex extends DataIndex {
     constructor(storage: Storage, path: string, key: string, options: DataIndexOptions);
@@ -264,10 +373,13 @@ export declare class GeoIndex extends DataIndex {
         long: number;
         radius: number;
     }): boolean;
+    query(op: string | BlacklistingSearchOperator, val?: IndexableValueOrArray, options?: {
+        filter?: IndexQueryResults;
+    }): Promise<IndexQueryResults>;
     /**
      * @param op Only 'geo:nearby' is supported at the moment
      */
-    query(op: 'geo:nearby', val: {
+    nearby(val: {
         /**
          * nearby query center latitude
          */