feat(analytics): add new granularities with support for FF [MA-2149]

- Add new granularity values and definitions
- Add support for passing one feature flag to query time and `allowedGranularities`
- Add new timeframe -> granularity mappings
- Support providing data granularity to query time; prepare to deprecate passing it via timeframe
- Narrow the type of Druid granularity to what we actually use

packages/analytics/analytics-utilities/src/granularity.ts

@@ -5,8 +5,15 @@ import { getTimezoneOffset } from 'date-fns-tz'
 // Units are milliseconds, which are what Druid expects.
 export const Granularities = {
   secondly: 1000,
+  tenSecondly: 10 * 1000,
+  thirtySecondly: 30 * 1000,
   minutely: 60 * 1000,
+  fiveMinutely: 5 * 60 * 1000,
+  tenMinutely: 10 * 60 * 1000,
+  thirtyMinutely: 30 * 60 * 1000,
   hourly: 60 * 60 * 1000,
+  twoHourly: 2 * 60 * 60 * 1000,
+  twelveHourly: 12 * 60 * 60 * 1000,
   daily: 60 * 60 * 24 * 1000,
   weekly: 60 * 60 * 24 * 7 * 1000,
   trend: 0,
@@ -23,18 +30,14 @@ export function granularitiesToOptions(
 }
 
 export function granularityMsToQuery(
-  granularity: number | null,
-  origin?: string,
-): DruidGranularity | null {
-  if (granularity) {
-    return {
-      duration: granularity,
-      type: 'duration',
-      origin,
-    }
+  granularity: number,
+  origin: string,
+): DruidGranularity {
+  return {
+    duration: granularity,
+    type: 'duration',
+    origin,
   }
-
-  return null
 }
 
 export function msToGranularity(ms?: number): GranularityValues | null {

packages/analytics/analytics-utilities/src/queryTime.spec.tz.ts

@@ -1,10 +1,9 @@
-import { afterEach, describe, expect, beforeEach, it, vi } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { add, setDate, startOfDay, startOfMonth, startOfWeek, subMilliseconds } from 'date-fns'
 
 import { TimeframeKeys } from './types'
 import { DeltaQueryTime, TimeseriesQueryTime, UnaryQueryTime } from './queryTime'
-import { Timeframe } from './timeframes'
-import { datePickerSelectionToTimeframe, TimePeriods } from './timeframes'
+import { datePickerSelectionToTimeframe, Timeframe, TimePeriods } from './timeframes'
 import { formatInTimeZone } from 'date-fns-tz'
 import { runUtcTest } from './specUtils'
 
@@ -39,6 +38,18 @@ describe('granularity enforcement', () => {
 
     expect(tsQuery.granularitySeconds()).toBe(24 * 60 * 60)
   })
+
+  it('handles fine granularity in default responses', () => {
+    // Should permit allowed finer grains if requested.
+    expect(new TimeseriesQueryTime(getTimePeriod(TimeframeKeys.ONE_DAY), 'tenMinutely', undefined, undefined, true).granularitySeconds()).toBe(10 * 60)
+
+    // Should not permit finer grains outside the allowed finer grains.
+    expect(new TimeseriesQueryTime(getTimePeriod(TimeframeKeys.ONE_DAY), 'tenSecondly', undefined, undefined, true).granularitySeconds()).toBe(5 * 60)
+
+    // Should pick an appropriate default response granularity.
+    expect(new TimeseriesQueryTime(getTimePeriod(TimeframeKeys.ONE_DAY), undefined, undefined, undefined, true).granularitySeconds()).toBe(5 * 60)
+  })
+
 })
 
 describe('timeframe start/end times', () => {
@@ -535,6 +546,47 @@ runDstTest('daylight savings time: fall', () => {
   })
 })
 
+runUtcTest('UTC: fine granularity', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+    const fakeNow = new Date('2023-11-09T01:17:32Z')
+    standardizeTimezone(fakeNow)
+    vi.setSystemTime(fakeNow)
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+  })
+
+  it('handles fine granularity in rounding - timeseries, 5 minutely', () => {
+    const tsQuery = new TimeseriesQueryTime(getTimePeriod(TimeframeKeys.ONE_DAY), undefined, undefined, undefined,true)
+
+    expect(tsQuery.endDate()).toEqual(new Date('2023-11-09T01:20:00Z'))
+    expect(tsQuery.startDate()).toEqual(new Date('2023-11-08T01:20:00Z'))
+  })
+
+  it('handles fine granularity in rounding - timeseries, hourly', () => {
+    const tsQuery = new TimeseriesQueryTime(getTimePeriod(TimeframeKeys.THIRTY_DAY), undefined, undefined, undefined,true)
+
+    expect(tsQuery.endDate()).toEqual(new Date('2023-11-09T02:00:00Z'))
+    expect(tsQuery.startDate()).toEqual(new Date('2023-10-10T02:00:00Z'))
+  })
+
+  it('handles data granularity overrides - unary, daily', () => {
+    const unaryQuery = new UnaryQueryTime(getTimePeriod(TimeframeKeys.SEVEN_DAY), undefined, 'daily')
+
+    expect(unaryQuery.endDate()).toEqual(new Date('2023-11-10T00:00:00Z'))
+    expect(unaryQuery.startDate()).toEqual(new Date('2023-11-03T00:00:00Z'))
+  })
+
+  it('handles data granularity overrides - delta, daily', () => {
+    const deltaQuery = new DeltaQueryTime(getTimePeriod(TimeframeKeys.SEVEN_DAY), undefined, 'daily')
+
+    expect(deltaQuery.endDate()).toEqual(new Date('2023-11-10T00:00:00Z'))
+    expect(deltaQuery.startDate()).toEqual(new Date('2023-10-27T00:00:00Z'))
+  })
+})
+
 runUtcTest('UTC: timezone handling', () => {
   beforeEach(() => {
     vi.useFakeTimers()

packages/analytics/analytics-utilities/src/queryTime.ts

@@ -12,15 +12,17 @@ import type { Timeframe } from './timeframes'
 abstract class BaseQueryTime implements QueryTime {
   protected readonly timeframe: Timeframe
   protected readonly tz?: string
+  protected readonly dataGranularity: GranularityValues
 
-  constructor(timeframe: Timeframe, tz?: string) {
+  constructor(timeframe: Timeframe, tz?: string, dataGranularity?: GranularityValues) {
     // This is an abstract class.
     if (this.constructor === BaseQueryTime) {
       throw new Error('BaseQueryTime is not meant to be used directly.')
     }
 
     this.timeframe = timeframe
     this.tz = tz
+    this.dataGranularity = dataGranularity ?? timeframe.dataGranularity
   }
 
   abstract startDate(): Date
@@ -50,7 +52,7 @@ abstract class BaseQueryTime implements QueryTime {
     return Math.floor(this.granularityMs() / 1000)
   }
 
-  granularityDruid(): DruidGranularity | null {
+  granularityDruid(): DruidGranularity {
     return granularityMsToQuery(this.granularityMs(), this.startDate().toISOString())
   }
 
@@ -79,11 +81,16 @@ abstract class BaseQueryTime implements QueryTime {
 export class TimeseriesQueryTime extends BaseQueryTime {
   private readonly granularity: GranularityValues
 
-  constructor(timeframe: Timeframe, granularity?: GranularityValues, tz?: string) {
-    super(timeframe, tz)
+  constructor(timeframe: Timeframe, granularity?: GranularityValues, tz?: string, dataGranularity?: GranularityValues, fineGrain?: boolean) {
+    super(timeframe, tz, dataGranularity)
 
-    if (granularity && timeframe.allowedGranularities().has(granularity)) {
+    if (granularity && timeframe.allowedGranularities(fineGrain).has(granularity)) {
       this.granularity = granularity
+    } else if (fineGrain) {
+      // TODO: when removing the feature flag, consider redefining `defaultResponseGranularity`
+      // in the timeframes constructor: it should probably handle this calculation on its own.
+      const finestGranularity = timeframe.allowedGranularities(fineGrain).keys().next().value
+      this.granularity = finestGranularity ?? timeframe.defaultResponseGranularity
     } else {
       this.granularity = timeframe.defaultResponseGranularity
     }
@@ -105,11 +112,11 @@ export class TimeseriesQueryTime extends BaseQueryTime {
 // We expect to get back 1 value, such that we can just show a big number without any trend information.
 export class UnaryQueryTime extends BaseQueryTime {
   startDate(): Date {
-    return this.calculateStartDate(this.timeframe.isRelative, this.timeframe.dataGranularity)
+    return this.calculateStartDate(this.timeframe.isRelative, this.dataGranularity)
   }
 
   endDate(): Date {
-    return ceilToNearestTimeGrain(this.timeframe.rawEnd(this.tz), this.timeframe.dataGranularity, this.tz)
+    return ceilToNearestTimeGrain(this.timeframe.rawEnd(this.tz), this.dataGranularity, this.tz)
   }
 
   granularityMs(): number {
@@ -122,7 +129,7 @@ export class UnaryQueryTime extends BaseQueryTime {
 // timeframe to calculate a trend.
 export class DeltaQueryTime extends UnaryQueryTime {
   startDate(): Date {
-    return this.calculateStartDate(this.timeframe.isRelative, this.timeframe.dataGranularity, 2)
+    return this.calculateStartDate(this.timeframe.isRelative, this.dataGranularity, 2)
   }
 
   granularityMs(): number {

packages/analytics/analytics-utilities/src/timeframes.spec.ts

@@ -61,6 +61,14 @@ describe('allowedGranularities', () => {
     expect(TimePeriods.get(TimeframeKeys.PREVIOUS_MONTH)?.allowedGranularities())
       .toEqual(new Set(['daily', 'weekly']))
   })
+
+  it('meets new specs for standard timeframes with flag', () => {
+    expect(TimePeriods.get(TimeframeKeys.FIFTEEN_MIN)?.allowedGranularities(true))
+      .toEqual(new Set(['tenSecondly', 'thirtySecondly', 'minutely']))
+
+    expect(TimePeriods.get(TimeframeKeys.ONE_DAY)?.allowedGranularities(true))
+      .toEqual(new Set(['fiveMinutely', 'tenMinutely', 'thirtyMinutely', 'hourly']))
+  })
 })
 
 describe('cacheKey', () => {

packages/analytics/analytics-utilities/src/timeframes.ts

@@ -59,6 +59,8 @@ export class Timeframe implements ITimeframe {
 
   private _endCustom?: Date
 
+  private _allowedGranularitiesOverride?: GranularityValues[]
+
   constructor(opts: TimeframeOptions) {
     this.display = opts.display
     this.timeframeText = opts.timeframeText
@@ -70,6 +72,7 @@ export class Timeframe implements ITimeframe {
     this.isRelative = opts.isRelative
     this._startCustom = opts.startCustom
     this._endCustom = opts.endCustom
+    this._allowedGranularitiesOverride = opts.allowedGranularitiesOverride
   }
 
   // rawEnd does not consider granularity and should not be used directly in queries.
@@ -95,7 +98,12 @@ export class Timeframe implements ITimeframe {
     return this.timeframeLength()
   }
 
-  allowedGranularities() {
+  allowedGranularities(fineGrain?: boolean) {
+    if (this._allowedGranularitiesOverride && fineGrain) {
+      // Note: queryTime's granularity determination currently expects this to be sorted from fine to coarse.
+      return new Set(this._allowedGranularitiesOverride)
+    }
+
     const allowedValues: Set<GranularityValues> = new Set()
     const hours = this.maximumTimeframeLength() / 3600
 
@@ -265,6 +273,7 @@ export const TimePeriods = new Map<string, Timeframe>([
       dataGranularity: 'minutely',
       isRelative: true,
       allowedTiers: ['free', 'trial', 'plus', 'enterprise'],
+      allowedGranularitiesOverride: ['tenSecondly', 'thirtySecondly', 'minutely'],
     }),
   ],
   [
@@ -278,6 +287,7 @@ export const TimePeriods = new Map<string, Timeframe>([
       dataGranularity: 'minutely',
       isRelative: true,
       allowedTiers: ['free', 'trial', 'plus', 'enterprise'],
+      allowedGranularitiesOverride: ['tenSecondly', 'thirtySecondly', 'minutely', 'fiveMinutely', 'tenMinutely'],
     }),
   ],
   [
@@ -291,6 +301,7 @@ export const TimePeriods = new Map<string, Timeframe>([
       dataGranularity: 'hourly',
       isRelative: true,
       allowedTiers: ['free', 'trial', 'plus', 'enterprise'],
+      allowedGranularitiesOverride: ['thirtySecondly', 'minutely', 'fiveMinutely', 'tenMinutely', 'thirtyMinutely'],
     }),
   ],
   [
@@ -304,6 +315,7 @@ export const TimePeriods = new Map<string, Timeframe>([
       dataGranularity: 'hourly',
       isRelative: true,
       allowedTiers: ['free', 'trial', 'plus', 'enterprise'],
+      allowedGranularitiesOverride: ['minutely', 'fiveMinutely', 'tenMinutely', 'thirtyMinutely', 'hourly'],
     }),
   ],
   [
@@ -317,6 +329,7 @@ export const TimePeriods = new Map<string, Timeframe>([
       dataGranularity: 'hourly',
       isRelative: true,
       allowedTiers: ['free', 'trial', 'plus', 'enterprise'],
+      allowedGranularitiesOverride: ['fiveMinutely', 'tenMinutely', 'thirtyMinutely', 'hourly'],
     }),
   ],
   [
@@ -330,6 +343,7 @@ export const TimePeriods = new Map<string, Timeframe>([
       dataGranularity: 'daily',
       isRelative: true,
       allowedTiers: ['trial', 'plus', 'enterprise'],
+      allowedGranularitiesOverride: ['thirtyMinutely', 'hourly', 'twoHourly', 'twelveHourly', 'daily'],
     }),
   ],
   [
@@ -343,6 +357,7 @@ export const TimePeriods = new Map<string, Timeframe>([
       dataGranularity: 'daily',
       isRelative: true,
       allowedTiers: ['trial', 'plus', 'enterprise'],
+      allowedGranularitiesOverride: ['hourly', 'twoHourly', 'twelveHourly', 'daily', 'weekly'],
     }),
   ],
   [

packages/analytics/analytics-utilities/src/types/druid-granularity.ts

@@ -1,7 +1,5 @@
 export interface DruidGranularity {
   type: string
-  duration?: number
-  period?: string
-  timeZone?: string
-  origin?: number | string
+  duration: number
+  origin: string
 }

packages/analytics/analytics-utilities/src/types/explore/common.ts

@@ -63,8 +63,15 @@ export type TimeRangeV4 = AbsoluteTimeRangeV4 | RelativeTimeRangeV4
 
 export const granularityValues = [
   'secondly',
+  'tenSecondly',
+  'thirtySecondly',
   'minutely',
+  'fiveMinutely',
+  'tenMinutely',
+  'thirtyMinutely',
   'hourly',
+  'twoHourly',
+  'twelveHourly',
   'daily',
   'weekly',
   'trend',

packages/analytics/analytics-utilities/src/types/query-time.ts

@@ -2,7 +2,7 @@ import type { DruidGranularity } from '.'
 
 export interface QueryTime {
   granularitySeconds(): number
-  granularityDruid(): DruidGranularity | null
+  granularityDruid(): DruidGranularity
   startSeconds(): number
   endSeconds(): number
   startMs(): number

packages/analytics/analytics-utilities/src/types/timeframe-options.ts

@@ -11,4 +11,5 @@ export interface TimeframeOptions {
   allowedTiers: Array<string>
   startCustom?: Date
   endCustom?: Date
+  allowedGranularitiesOverride?: GranularityValues[]
 }

packages/analytics/analytics-utilities/src/types/timeframe.ts

@@ -13,5 +13,5 @@ export interface ITimeframe {
   rawStart(_tz?: string): Date;
   timeframeLengthMs(): number;
   maximumTimeframeLength(): number;
-  allowedGranularities(): Set<GranularityValues>;
+  allowedGranularities(fineGrain?: boolean): Set<GranularityValues>;
 }