From f96453f217c68107eaa517a2322740675e97f99f Mon Sep 17 00:00:00 2001 From: Tom Andersen Date: Tue, 26 Nov 2024 11:49:32 -0500 Subject: [PATCH 1/4] new stages --- dev/src/expression.ts | 32 ---- dev/src/index.ts | 1 - dev/src/pipeline.ts | 386 ++++++++++++++++++++++++++++++++---------- dev/src/serializer.ts | 15 ++ dev/src/stage.ts | 104 +++++++++++- types/firestore.d.ts | 192 +++++++++++++++++++-- 6 files changed, 585 insertions(+), 145 deletions(-) diff --git a/dev/src/expression.ts b/dev/src/expression.ts index 5b5f5678d..e4be1f707 100644 --- a/dev/src/expression.ts +++ b/dev/src/expression.ts @@ -1908,38 +1908,6 @@ export class Field extends Expr implements Selectable { } } -/** - * @beta - */ -export class Fields extends Expr implements Selectable { - exprType: ExprType = 'Field'; - selectable = true as const; - - private constructor(private fields: Field[]) { - super(); - } - - static of(name: string, ...others: string[]): Fields { - return new Fields([Field.of(name), ...others.map(Field.of)]); - } - - static ofAll(): Fields { - return new Fields([]); - } - - fieldList(): Field[] { - return this.fields.map(f => f); - } - - _toProto(serializer: Serializer): api.IValue { - return { - arrayValue: { - values: this.fields.map(f => f._toProto(serializer)), - }, - }; - } -} - /** * @beta * diff --git a/dev/src/index.ts b/dev/src/index.ts index 0e0274bc6..8c8342ced 100644 --- a/dev/src/index.ts +++ b/dev/src/index.ts @@ -137,7 +137,6 @@ export { Expr, ExprWithAlias, Field, - Fields, Constant, Function, Ordering, diff --git a/dev/src/pipeline.ts b/dev/src/pipeline.ts index 4fc32d402..c3851558d 100644 --- a/dev/src/pipeline.ts +++ b/dev/src/pipeline.ts @@ -21,7 +21,6 @@ import { Expr, ExprWithAlias, Field, - Fields, FilterCondition, Function, Ordering, @@ -51,6 +50,12 @@ import { Stage, Distinct, RemoveFields, + Replace, + Sample, + SampleOptions, + Union, + Unnest, + UnnestOptions, } from './stage'; import {ApiMapValue, defaultPipelineConverter} from './types'; import * as protos from '../protos/firestore_v1_proto_api'; @@ -107,21 +112,21 @@ export class PipelineSource implements firestore.PipelineSource { * * // Example 1: Select specific fields and rename 'rating' to 'bookRating' * const results1 = await db.pipeline() - * .collection("books") - * .select("title", "author", Field.of("rating").as("bookRating")) + * .collection('books') + * .select('title', 'author', Field.of('rating').as('bookRating')) * .execute(); * - * // Example 2: Filter documents where 'genre' is "Science Fiction" and 'published' is after 1950 + * // Example 2: Filter documents where 'genre' is 'Science Fiction' and 'published' is after 1950 * const results2 = await db.pipeline() - * .collection("books") - * .where(and(Field.of("genre").eq("Science Fiction"), Field.of("published").gt(1950))) + * .collection('books') + * .where(and(Field.of('genre').eq('Science Fiction'), Field.of('published').gt(1950))) * .execute(); * * // Example 3: Calculate the average rating of books published after 1980 * const results3 = await db.pipeline() - * .collection("books") - * .where(Field.of("published").gt(1980)) - * .aggregate(avg(Field.of("rating")).as("averageRating")) + * .collection('books') + * .where(Field.of('published').gt(1980)) + * .aggregate(avg(Field.of('rating')).as('averageRating')) * .execute(); * ``` */ @@ -134,6 +139,12 @@ export class Pipeline private converter: firestore.FirestorePipelineConverter = defaultPipelineConverter() ) {} + private _addStage(stage: Stage): Pipeline { + const copy = this.stages.map(s => s); + copy.push(stage); + return new Pipeline(this.db, copy, this.converter); + } + /** * Adds new fields to outputs from previous stages. * @@ -150,10 +161,10 @@ export class Pipeline * Example: * * ```typescript - * firestore.pipeline().collection("books") + * firestore.pipeline().collection('books') * .addFields( - * Field.of("rating").as("bookRating"), // Rename 'rating' to 'bookRating' - * add(5, Field.of("quantity")).as("totalCost") // Calculate 'totalCost' + * Field.of('rating').as('bookRating'), // Rename 'rating' to 'bookRating' + * add(5, Field.of('quantity')).as('totalCost') // Calculate 'totalCost' * ); * ``` * @@ -161,9 +172,7 @@ export class Pipeline * @return A new Pipeline object with this stage appended to the stage list. */ addFields(...fields: firestore.Selectable[]): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new AddFields(this.selectablesToMap(fields))); - return new Pipeline(this.db, copy, this.converter); + return this._addStage(new AddFields(this.selectablesToMap(fields))); } /** @@ -172,11 +181,11 @@ export class Pipeline * Example: * * ```typescript - * firestore.pipeline().collection("books") + * firestore.pipeline().collection('books') * // removes field 'rating' and 'cost' from the previous stage outputs. * .removeFields( - * Field.of("rating"), - * "cost" + * Field.of('rating'), + * 'cost' * ); * ``` * @@ -186,13 +195,11 @@ export class Pipeline removeFields( ...fields: (firestore.Field | string)[] ): Pipeline { - const copy = this.stages.map(s => s); - copy.push( + return this._addStage( new RemoveFields( fields.map(f => (typeof f === 'string' ? Field.of(f) : (f as Field))) ) ); - return new Pipeline(this.db, copy, this.converter); } /** @@ -214,11 +221,11 @@ export class Pipeline *

Example: * * ```typescript - * firestore.pipeline().collection("books") + * firestore.pipeline().collection('books') * .select( - * "firstName", - * Field.of("lastName"), - * Field.of("address").toUppercase().as("upperAddress"), + * 'firstName', + * Field.of('lastName'), + * Field.of('address').toUppercase().as('upperAddress'), * ); * ``` * @@ -229,9 +236,7 @@ export class Pipeline select( ...selections: (firestore.Selectable | string)[] ): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new Select(this.selectablesToMap(selections))); - return new Pipeline(this.db, copy, this.converter); + return this._addStage(new Select(this.selectablesToMap(selections))); } private selectablesToMap( @@ -243,14 +248,11 @@ export class Pipeline result.set(selectable as string, Field.of(selectable)); } else if (selectable instanceof Field) { result.set((selectable as Field).fieldName(), selectable); - } else if (selectable instanceof Fields) { - const fields = selectable as Fields; - for (const field of fields.fieldList()) { - result.set(field.fieldName(), field); - } } else if (selectable instanceof ExprWithAlias) { const expr = selectable as ExprWithAlias; result.set(expr.alias, expr.expr); + } else { + throw new Error('unexpected selectable: ' + selectable); } } return result; @@ -260,7 +262,7 @@ export class Pipeline * Filters the documents from previous stages to only include those matching the specified {@link * FilterCondition}. * - *

This stage allows you to apply conditions to the data, similar to a "WHERE" clause in SQL. + *

This stage allows you to apply conditions to the data, similar to a 'WHERE' clause in SQL. * You can filter documents based on their field values, using implementations of {@link * FilterCondition}, typically including but not limited to: * @@ -275,11 +277,11 @@ export class Pipeline *

Example: * * ```typescript - * firestore.pipeline().collection("books") + * firestore.pipeline().collection('books') * .where( * and( - * gt(Field.of("rating"), 4.0), // Filter for ratings greater than 4.0 - * Field.of("genre").eq("Science Fiction") // Equivalent to gt("genre", "Science Fiction") + * gt(Field.of('rating'), 4.0), // Filter for ratings greater than 4.0 + * Field.of('genre').eq('Science Fiction') // Equivalent to gt('genre', 'Science Fiction') * ) * ); * ``` @@ -288,9 +290,7 @@ export class Pipeline * @return A new Pipeline object with this stage appended to the stage list. */ where(condition: FilterCondition & firestore.Expr): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new Where(condition)); - return new Pipeline(this.db, copy, this.converter); + return this._addStage(new Where(condition)); } /** @@ -304,8 +304,8 @@ export class Pipeline * * ```typescript * // Retrieve the second page of 20 results - * firestore.pipeline().collection("books") - * .sort(Field.of("published").descending()) + * firestore.pipeline().collection('books') + * .sort(Field.of('published').descending()) * .offset(20) // Skip the first 20 results * .limit(20); // Take the next 20 results * ``` @@ -314,9 +314,7 @@ export class Pipeline * @return A new Pipeline object with this stage appended to the stage list. */ offset(offset: number): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new Offset(offset)); - return new Pipeline(this.db, copy, this.converter); + return this._addStage(new Offset(offset)); } /** @@ -336,8 +334,8 @@ export class Pipeline * * ```typescript * // Limit the results to the top 10 highest-rated books - * firestore.pipeline().collection("books") - * .sort(Field.of("rating").descending()) + * firestore.pipeline().collection('books') + * .sort(Field.of('rating').descending()) * .limit(10); * ``` * @@ -345,9 +343,7 @@ export class Pipeline * @return A new Pipeline object with this stage appended to the stage list. */ limit(limit: number): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new Limit(limit)); - return new Pipeline(this.db, copy, this.converter); + return this._addStage(new Limit(limit)); } /** @@ -369,21 +365,19 @@ export class Pipeline * * ```typescript * // Get a list of unique author names in uppercase and genre combinations. - * firestore.pipeline().collection("books") - * .distinct(toUppercase(Field.of("author")).as("authorName"), Field.of("genre"), "publishedAt") - * .select("authorName"); + * firestore.pipeline().collection('books') + * .distinct(toUppercase(Field.of('author')).as('authorName'), Field.of('genre'), 'publishedAt') + * .select('authorName'); * ``` * - * @param selectables The {@link Selectable} expressions to consider when determining distinct + * @param groups The {@link Selectable} expressions to consider when determining distinct * value combinations or {@code string}s representing field names. * @return A new {@code Pipeline} object with this stage appended to the stage list. */ distinct( ...groups: (string | firestore.Selectable)[] ): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new Distinct(this.selectablesToMap(groups || []))); - return new Pipeline(this.db, copy, this.converter); + return this._addStage(new Distinct(this.selectablesToMap(groups || []))); } /** @@ -397,10 +391,10 @@ export class Pipeline * * ```typescript * // Calculate the average rating and the total number of books - * firestore.pipeline().collection("books") + * firestore.pipeline().collection('books') * .aggregate( - * Field.of("rating").avg().as("averageRating"), - * countAll().as("totalBooks") + * Field.of('rating').avg().as('averageRating'), + * countAll().as('totalBooks') * ); * ``` * @@ -432,10 +426,10 @@ export class Pipeline * * ```typescript * // Calculate the average rating for each genre. - * firestore.pipeline().collection("books") + * firestore.pipeline().collection('books') * .aggregate({ - * accumulators: [avg(Field.of("rating")).as("avg_rating")] - * groups: ["genre"] + * accumulators: [avg(Field.of('rating')).as('avg_rating')], + * groups: ['genre'] * }); * ``` * @@ -456,9 +450,8 @@ export class Pipeline }, ...rest: firestore.AccumulatorTarget[] ): Pipeline { - const copy = this.stages.map(s => s); if ('accumulators' in optionsOrTarget) { - copy.push( + return this._addStage( new Aggregate( new Map( optionsOrTarget.accumulators.map( @@ -472,7 +465,7 @@ export class Pipeline ) ); } else { - copy.push( + return this._addStage( new Aggregate( new Map( [optionsOrTarget, ...rest].map(target => [ @@ -484,14 +477,226 @@ export class Pipeline ) ); } - return new Pipeline(this.db, copy, this.converter); } - findNearest(options: firestore.FindNearestOptions): Pipeline; findNearest(options: firestore.FindNearestOptions): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new FindNearest(options)); - return new Pipeline(this.db, copy); + return this._addStage(new FindNearest(options)); + } + + /** + * Fully overwrites all fields in a document with those coming from a nested map. + * + *

This stage allows you to emit a map value as a document. Each key of the map becomes a field + * on the document that contains the corresponding value. + * + *

Example: + * + * 

{@code
+   * // Input.
+   * // {
+   * //  'name': 'John Doe Jr.',
+   * //  'parents': {
+   * //    'father': 'John Doe Sr.',
+   * //    'mother': 'Jane Doe'
+   * // }
+   *
+   * // Emit parents as document.
+   * firestore.pipeline().collection('people').replace(Field.of('parents'));
+   *
+   * // Output
+   * // {
+   * //  'father': 'John Doe Sr.',
+   * //  'mother': 'Jane Doe'
+   * // }
+   * }
+ * + * @param field The {@link Selectable} field containing the nested map. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + replace( + field: firestore.Selectable | string + ): FirebaseFirestore.Pipeline { + return this._addStage( + new Replace(this.selectableToExpr(field), 'full_replace') + ); + } + + private selectableToExpr( + selectable: FirebaseFirestore.Selectable | string + ): Expr { + if (typeof selectable === 'string') { + return Field.of(selectable); + } else if (selectable instanceof Field) { + return selectable; + } else if (selectable instanceof ExprWithAlias) { + return selectable.expr; + } else { + throw new Error('unexpected selectable: ' + selectable); + } + } + + /** + * Performs a pseudo-random sampling of the documents from the previous stage. + * + *

This stage will filter documents pseudo-randomly. The parameter specifies how number of + * documents to be returned. + * + *

Examples: + * + * 

{@code
+   * // Sample 25 books, if available.
+   * firestore.pipeline().collection('books')
+   *     .sample(25);
+   * }
+   *
+ * + * @param documents The number of documents to sample.. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + sample(documents: number): FirebaseFirestore.Pipeline; + + /** + * Performs a pseudo-random sampling of the documents from the previous stage. + * + *

This stage will filter documents pseudo-randomly. The 'options' parameter specifies how + * sampling will be performed. See {@code SampleOptions} for more information. + * + *

Examples: + * + * // Sample 10 books, if available. + * firestore.pipeline().collection("books") + * .sample({ documents: 10 }); + * + * // Sample 50% of books. + * firestore.pipeline().collection("books") + * .sample({ percentage: 0.5 }); + * } + * + * + * @param options The {@code SampleOptions} specifies how sampling is performed. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + sample( + options: {percentage: number} | {documents: number} + ): FirebaseFirestore.Pipeline; + sample( + documentsOrOptions: number | {percentage: number} | {documents: number} + ): FirebaseFirestore.Pipeline { + if (typeof documentsOrOptions === 'number') { + return this._addStage( + new Sample({limit: documentsOrOptions, mode: 'documents'}) + ); + } else if ('percentage' in documentsOrOptions) { + return this._addStage( + new Sample({limit: documentsOrOptions.percentage, mode: 'percent'}) + ); + } else { + return this._addStage( + new Sample({limit: documentsOrOptions.documents, mode: 'documents'}) + ); + } + } + + /** + * Performs union of all documents from two pipelines, including duplicates. + * + *

This stage will pass through documents from previous stage, and also pass through documents + * from previous stage of the `other` {@code Pipeline} given in parameter. The order of documents + * emitted from this stage is undefined. + * + *

Example: + * + * 

{@code
+   * // Emit documents from books collection and magazines collection.
+   * firestore.pipeline().collection('books')
+   *     .union(firestore.pipeline().collection('magazines'));
+   * }
+ * + * @param other The other {@code Pipeline} that is part of union. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + union( + other: FirebaseFirestore.Pipeline + ): FirebaseFirestore.Pipeline { + return this._addStage(new Union(other)); + } + + /** + * Produces a document for each element in array found in previous stage document. + * + *

For each previous stage document, this stage will emit zero or more augmented documents. The + * input array found in the previous stage document field specified by the `fieldName` parameter, + * will for each input array element produce an augmented document. The input array element will + * augment the previous stage document by replacing the field specified by `fieldName` parameter + * with the element value. + * + *

In other words, the field containing the input array will be removed from the augmented + * document and replaced by the corresponding array element. + * + *

Example: + * + * 

{@code
+   * // Input:
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tags': [ 'comedy', 'space', 'adventure' ], ... }
+   *
+   * // Emit a book document for each tag of the book.
+   * firestore.pipeline().collection('books')
+   *     .unnest('tags');
+   *
+   * // Output:
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tags': 'comedy', ... }
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tags': 'space', ... }
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tags': 'adventure', ... }
+   * }
+ * + * @param field The name of the field containing the array. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + unnest( + field: FirebaseFirestore.Selectable | string + ): FirebaseFirestore.Pipeline; + + /** + * Produces a document for each element in array found in previous stage document. + * + *

For each previous stage document, this stage will emit zero or more augmented documents. The + * input array found in the previous stage document field specified by the `fieldName` parameter, + * will for each input array element produce an augmented document. The input array element will + * augment the previous stage document by replacing the field specified by `fieldName` parameter + * with the element value. + * + *

In other words, the field containing the input array will be removed from the augmented + * document and replaced by the corresponding array element. + * + *

Example: + * + * 

{@code
+   * // Input:
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tags': [ 'comedy', 'space', 'adventure' ], ... }
+   *
+   * // Emit a book document for each tag of the book.
+   * firestore.pipeline().collection('books')
+   *     .unnest({ field: 'tags', UnnestOptions.indexField('tagIndex'));
+   *
+   * // Output:
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tagIndex': 0, 'tags': 'comedy', ... }
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tagIndex': 1, 'tags': 'space', ... }
+   * // { 'title': 'The Hitchhiker's Guide to the Galaxy', 'tagIndex': 2, 'tags': 'adventure', ... }
+   * }
+ * + * @param fieldName The name of the field containing the array. + * @param options The {@code UnnestOptions} options. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + unnest(options: UnnestOptions): FirebaseFirestore.Pipeline; + unnest( + fieldOrOptions: FirebaseFirestore.Selectable | string | UnnestOptions + ): FirebaseFirestore.Pipeline { + if (typeof fieldOrOptions === 'string' || 'selectable' in fieldOrOptions) { + return this._addStage(new Unnest({field: fieldOrOptions})); + } else { + return this._addStage(new Unnest(fieldOrOptions)); + } } /** @@ -508,10 +713,10 @@ export class Pipeline * ```typescript * // Sort books by rating in descending order, and then by title in ascending order for books * // with the same rating - * firestore.pipeline().collection("books") + * firestore.pipeline().collection('books') * .sort( - * Field.of("rating").descending(), - * Field.of("title").ascending() + * Field.of('rating').descending(), + * Field.of('title').ascending() * ); * ``` * @@ -528,16 +733,13 @@ export class Pipeline }, ...rest: Ordering[] ): Pipeline { - const copy = this.stages.map(s => s); // Option object if ('orderings' in optionsOrOrderings) { - copy.push(new Sort(optionsOrOrderings.orderings)); + return this._addStage(new Sort(optionsOrOrderings.orderings)); } else { // Ordering object - copy.push(new Sort([optionsOrOrderings, ...rest])); + return this._addStage(new Sort([optionsOrOrderings, ...rest])); } - - return new Pipeline(this.db, copy, this.converter); } /** @@ -547,13 +749,13 @@ export class Pipeline * stages. Each generic stage is defined by a unique `name` and a set of `params` that control its * behavior. * - *

Example (Assuming there is no "where" stage available in SDK): + *

Example (Assuming there is no 'where' stage available in SDK): * * ```typescript - * // Assume we don't have a built-in "where" stage - * firestore.pipeline().collection("books") - * .genericStage("where", [Field.of("published").lt(1900)]) // Custom "where" stage - * .select("title", "author"); + * // Assume we don't have a built-in 'where' stage + * firestore.pipeline().collection('books') + * .genericStage('where', [Field.of('published').lt(1900)]) // Custom 'where' stage + * .select('title', 'author'); * ``` * * @param name The unique name of the generic stage to add. @@ -561,9 +763,7 @@ export class Pipeline * @return A new {@code Pipeline} object with this stage appended to the stage list. */ genericStage(name: string, params: any[]): Pipeline { - const copy = this.stages.map(s => s); - copy.push(new GenericStage(name, params)); - return new Pipeline(this.db, copy, this.converter); + return this._addStage(new GenericStage(name, params)); } withConverter(converter: null): Pipeline; @@ -654,9 +854,9 @@ export class Pipeline *

Example: * * ```typescript - * const futureResults = await firestore.pipeline().collection("books") - * .where(gt(Field.of("rating"), 4.5)) - * .select("title", "author", "rating") + * const futureResults = await firestore.pipeline().collection('books') + * .where(gt(Field.of('rating'), 4.5)) + * .select('title', 'author', 'rating') * .execute(); * ``` * @@ -688,9 +888,9 @@ export class Pipeline * * @example * ```typescript - * firestore.pipeline().collection("books") - * .where(gt(Field.of("rating"), 4.5)) - * .select("title", "author", "rating") + * firestore.pipeline().collection('books') + * .where(gt(Field.of('rating'), 4.5)) + * .select('title', 'author', 'rating') * .stream() * .on('data', (pipelineResult) => {}) * .on('end', () => {}); diff --git a/dev/src/serializer.ts b/dev/src/serializer.ts index a66bba1b4..bbb703e5a 100644 --- a/dev/src/serializer.ts +++ b/dev/src/serializer.ts @@ -105,6 +105,21 @@ export class Serializer { * @param val The object to encode * @returns The Firestore Proto or null if we are deleting a field. */ + encodeValue(val: FieldTransform | undefined): null; + encodeValue( + val: + | string + | boolean + | number + | bigint + | Date + | null + | Buffer + | Uint8Array + | VectorValue + | Map + ): api.IValue; + encodeValue(val: unknown): api.IValue | null; encodeValue(val: unknown): api.IValue | null { if (val instanceof FieldTransform) { return null; diff --git a/dev/src/stage.ts b/dev/src/stage.ts index 5be9be513..8ed76400e 100644 --- a/dev/src/stage.ts +++ b/dev/src/stage.ts @@ -196,7 +196,7 @@ export class Where implements Stage { * @beta */ export interface FindNearestOptions { - field: firestore.Field; + field: firestore.Field | string; vectorValue: firestore.VectorValue | number[]; distanceMeasure: 'euclidean' | 'cosine' | 'dot_product'; limit?: number; @@ -224,7 +224,10 @@ export class FindNearest implements Stage { return { name: this.name, args: [ - (this._options.field as unknown as Field)._toProto(serializer), + (typeof this._options.field === 'string' + ? Field.of(this._options.field) + : (this._options.field as unknown as Field) + )._toProto(serializer), this._options.vectorValue instanceof VectorValue ? serializer.encodeValue(this._options.vectorValue)! : serializer.encodeVector(this._options.vectorValue as number[]), @@ -235,6 +238,78 @@ export class FindNearest implements Stage { } } +/** + * @beta + */ +export interface SampleOptions { + limit: number; + mode: 'documents' | 'percent'; +} + +/** + * @beta + */ +export class Sample implements Stage { + name = 'sample'; + + constructor(private _options: SampleOptions) {} + + _toProto(serializer: Serializer): api.Pipeline.IStage { + return { + name: this.name, + args: [ + serializer.encodeValue(this._options.limit)!, + serializer.encodeValue(this._options.mode)!, + ], + }; + } +} + +/** + * @beta + */ +export class Union implements Stage { + name = 'union'; + + constructor(private _other: FirebaseFirestore.Pipeline) {} + + _toProto(serializer: Serializer): api.Pipeline.IStage { + return { + name: this.name, + args: [serializer.encodeValue(this._other)!], + }; + } +} + +/** + * @beta + */ +export interface UnnestOptions { + field: firestore.Selectable | string; + indexField?: string; +} + +/** + * @beta + */ +export class Unnest implements Stage { + name = 'unnest'; + + constructor(private options: UnnestOptions) {} + + _toProto(serializer: Serializer): api.Pipeline.IStage { + const args = [serializer.encodeValue(this.options.field)!]; + const indexField = this.options.indexField; + if (indexField) { + args.push(serializer.encodeValue(indexField)); + } + return { + name: this.name, + args: args, + }; + } +} + /** * @beta */ @@ -267,6 +342,31 @@ export class Offset implements Stage { } } +/** + * @beta + */ +export class Replace implements Stage { + name = 'replace'; + + constructor( + private field: Expr, + private mode: + | 'full_replace' + | 'merge_prefer_nest' + | 'merge_prefer_parent' = 'full_replace' + ) {} + + _toProto(serializer: Serializer): api.Pipeline.IStage { + return { + name: this.name, + args: [ + serializer.encodeValue(this.field)!, + serializer.encodeValue(this.mode), + ], + }; + } +} + /** * @beta */ diff --git a/types/firestore.d.ts b/types/firestore.d.ts index fce440cd2..57c50d81e 100644 --- a/types/firestore.d.ts +++ b/types/firestore.d.ts @@ -4733,22 +4733,6 @@ declare namespace FirebaseFirestore { fieldName(): string; } - /** - * @beta - */ - export class Fields extends Expr implements Selectable { - exprType: ExprType; - selectable: true; - static of(name: string, ...others: string[]): Fields; - static ofAll(): Fields; - /** - * Returns the list of fields. - * - * @return The list of fields. - */ - fieldList(): Field[]; - } - /** * @beta * @@ -8832,7 +8816,7 @@ declare namespace FirebaseFirestore { * @beta */ export interface FindNearestOptions { - field: Field; + field: Field | string; vectorValue: VectorValue | number[]; distanceMeasure: 'euclidean' | 'cosine' | 'dot_product'; limit?: number; @@ -8860,6 +8844,28 @@ declare namespace FirebaseFirestore { name: string; } + /** + * @beta + */ + export class Replace implements Stage { + name: string; + } + + /** + * @beta + */ + export class Sample implements Stage { + name: string; + } + + /** + * @beta + */ + export interface SampleOptions { + limit: number; + mode: 'documents' | 'percent'; + } + /** * @beta */ @@ -8874,6 +8880,28 @@ declare namespace FirebaseFirestore { name: string; } + /** + * @beta + */ + export class Union implements Stage { + name: string; + } + + /** + * @beta + */ + export class Unnest implements Stage { + name: string; + } + + /** + * @beta + */ + export interface UnnestOptions { + field: Selectable | string; + indexField?: string; + } + /** * @beta */ @@ -9215,6 +9243,83 @@ declare namespace FirebaseFirestore { findNearest(options: FindNearestOptions): Pipeline; + /** + * Fully overwrites all fields in a document with those coming from a nested map. + * + *

This stage allows you to emit a map value as a document. Each key of the map becomes a field + * on the document that contains the corresponding value. + * + *

Example: + * + * 

{@code
+     * // Input.
+     * // {
+     * //  "name": "John Doe Jr.",
+     * //  "parents": {
+     * //    "father": "John Doe Sr.",
+     * //    "mother": "Jane Doe"
+     * // }
+     *
+     * // Emit parents as document.
+     * firestore.pipeline().collection("people").replace(Field.of("parents"));
+     *
+     * // Output
+     * // {
+     * //  "father": "John Doe Sr.",
+     * //  "mother": "Jane Doe"
+     * // }
+     * }
+ * + * @param field The {@link Selectable} field containing the nested map. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + replace(field: Selectable | string): Pipeline; + + /** + * Performs a pseudo-random sampling of the documents from the previous stage. + * + *

This stage will filter documents pseudo-randomly. The parameter specifies how number of + * documents to be returned. + * + *

Examples: + * + * 

{@code
+     * // Sample 25 books, if available.
+     * firestore.pipeline().collection("books")
+     *     .sample(25);
+     * }
+     *
+ * + * @param documents The number of documents to sample. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + sample(documents: number): Pipeline; + + /** + * Performs a pseudo-random sampling of the documents from the previous stage. + * + *

This stage will filter documents pseudo-randomly. The 'options' parameter specifies how + * sampling will be performed. See {@code SampleOptions} for more information. + * + *

Examples: + * + * // Sample 10 books, if available. + * firestore.pipeline().collection("books") + * .sample({ documents: 10 }); + * + * // Sample 50% of books. + * firestore.pipeline().collection("books") + * .sample({ percentage: 0.5 }); + * } + * + * + * @param options The {@code SampleOptions} specifies how sampling is performed. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + sample( + options: {percentage: number} | {documents: number} + ): Pipeline; + /** * Sorts the documents from previous stages based on one or more {@link Ordering} criteria. * @@ -9242,6 +9347,59 @@ declare namespace FirebaseFirestore { sort(...orderings: Ordering[]): Pipeline; sort(options: {orderings: Ordering[]}): Pipeline; + /** + * Performs union of all documents from two pipelines, including duplicates. + * + *

This stage will pass through documents from previous stage, and also pass through documents + * from previous stage of the `other` {@code Pipeline} given in parameter. The order of documents + * emitted from this stage is undefined. + * + *

Example: + * + * 

{@code
+     * // Emit documents from books collection and magazines collection.
+     * firestore.pipeline().collection("books")
+     *     .union(firestore.pipeline().collection("magazines"));
+     * }
+ * + * @param other The other {@code Pipeline} that is part of union. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + union(other: Pipeline): Pipeline; + + /** + * Produces a document for each element in array found in previous stage document. + * + *

For each previous stage document, this stage will emit zero or more augmented documents. The + * input array found in the previous stage document field specified by the `fieldName` parameter, + * will for each input array element produce an augmented document. The input array element will + * augment the previous stage document by replacing the field specified by `fieldName` parameter + * with the element value. + * + *

In other words, the field containing the input array will be removed from the augmented + * document and replaced by the corresponding array element. + * + *

Example: + * + * 

{@code
+     * // Input:
+     * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": [ "comedy", "space", "adventure" ], ... }
+     *
+     * // Emit a book document for each tag of the book.
+     * firestore.pipeline().collection("books")
+     *     .unnest("tags");
+     *
+     * // Output:
+     * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": "comedy", ... }
+     * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": "space", ... }
+     * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": "adventure", ... }
+     * }
+ * + * @param field The name of the field containing the array. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + unnest(field: Selectable | string): Pipeline; + /** * Adds a generic stage to the pipeline. * From 9036489404256d6e6ff953c7db9c2b557b22e24b Mon Sep 17 00:00:00 2001 From: Tom Andersen Date: Tue, 26 Nov 2024 13:19:27 -0500 Subject: [PATCH 2/4] add tests --- dev/system-test/pipeline.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/dev/system-test/pipeline.ts b/dev/system-test/pipeline.ts index 82d380b11..63827d269 100644 --- a/dev/system-test/pipeline.ts +++ b/dev/system-test/pipeline.ts @@ -864,4 +864,39 @@ describe.only('Pipeline class', () => { .execute(); expectResults(result, {title: "The Hitchhiker's Guide to the Galaxy"}); }); + + it('run pipeline with sample limit of 3', async () => { + const results = await randomCol.pipeline().sample(3).execute(); + expect(results.length).to.equal(3); + }); + + it('run pipeline with sample limit of {documents: 3}', async () => { + const results = await randomCol.pipeline().sample({documents: 3}).execute(); + expect(results.length).to.equal(3); + }); + + it('run pipeline with sample limit of {percentage: 0.6}', async () => { + const results = await randomCol + .pipeline() + .sample({percentage: 0.6}) + .execute(); + expect(results.length).to.equal(6); + }); + + it('run pipeline with union', async () => { + const results = await randomCol + .pipeline() + .union(randomCol.pipeline()) + .execute(); + expect(results.length).to.equal(20); + }); + + it('run pipeline with unnest', async () => { + const results = await randomCol + .pipeline() + .where(eq('title', "The Hitchhiker's Guide to the Galaxy")) + .unnest('tags') + .execute(); + expect(results.length).to.equal(3); + }); }); From 47bf5e7e1d548c8e56533c89e963bd28fbece8e0 Mon Sep 17 00:00:00 2001 From: Tom Andersen Date: Wed, 27 Nov 2024 12:24:26 -0500 Subject: [PATCH 3/4] fix --- dev/src/pipeline-util.ts | 1 - types/firestore.d.ts | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dev/src/pipeline-util.ts b/dev/src/pipeline-util.ts index 1bb77b482..d668862a3 100644 --- a/dev/src/pipeline-util.ts +++ b/dev/src/pipeline-util.ts @@ -156,7 +156,6 @@ export class ExecutionUtil { enc, callback ) => { - console.log(`Pipeline response: ${JSON.stringify(proto, null, 2)}`); if (proto === NOOP_MESSAGE) { callback(undefined); return; diff --git a/types/firestore.d.ts b/types/firestore.d.ts index 71448e5fd..83fab89c2 100644 --- a/types/firestore.d.ts +++ b/types/firestore.d.ts @@ -5018,6 +5018,13 @@ declare namespace FirebaseFirestore { filterable: true; } + /** + * @beta + */ + export class NotEqAny extends Function implements FilterCondition { + filterable: true; + } + /** * @beta */ From ffb57cbe090ab7d446ccd041fa8f8c5856c2c2a3 Mon Sep 17 00:00:00 2001 From: Tom Andersen Date: Wed, 27 Nov 2024 12:41:04 -0500 Subject: [PATCH 4/4] fix --- dev/src/pipeline-util.ts | 4 ---- dev/system-test/firestore.ts | 1 - dev/system-test/pipeline.ts | 2 +- dev/system-test/query.ts | 4 ++-- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/dev/src/pipeline-util.ts b/dev/src/pipeline-util.ts index d668862a3..3200eba5d 100644 --- a/dev/src/pipeline-util.ts +++ b/dev/src/pipeline-util.ts @@ -219,10 +219,6 @@ export class ExecutionUtil { explainOptions ); - console.log( - `Executing pipeline: \n ${JSON.stringify(request, null, 2)}` - ); - let streamActive: Deferred; do { streamActive = new Deferred(); diff --git a/dev/system-test/firestore.ts b/dev/system-test/firestore.ts index cf040220b..7181d9fdb 100644 --- a/dev/system-test/firestore.ts +++ b/dev/system-test/firestore.ts @@ -19,7 +19,6 @@ import { QuerySnapshot, SetOptions, Settings, - VectorValue, WithFieldValue, } from '@google-cloud/firestore'; diff --git a/dev/system-test/pipeline.ts b/dev/system-test/pipeline.ts index d5c58b277..1d0181d81 100644 --- a/dev/system-test/pipeline.ts +++ b/dev/system-test/pipeline.ts @@ -64,7 +64,7 @@ import {PipelineResult} from '../src/pipeline'; import {verifyInstance} from '../test/util/helpers'; import {DeferredPromise, getTestRoot} from './firestore'; -describe.only('Pipeline class', () => { +describe('Pipeline class', () => { let firestore: Firestore; let randomCol: CollectionReference; diff --git a/dev/system-test/query.ts b/dev/system-test/query.ts index 84853f79b..e7d800382 100644 --- a/dev/system-test/query.ts +++ b/dev/system-test/query.ts @@ -1803,8 +1803,8 @@ describe('Query class', () => { }); (process.env.FIRESTORE_EMULATOR_HOST === undefined - ? describe - : describe.only)('multiple inequality', () => { + ? describe.skip + : describe)('multiple inequality', () => { it('supports multiple inequality queries', async () => { const collection = await testCollectionWithDocs({ doc1: {key: 'a', sort: 0, v: 0},