From 3cecaba8e4e74026ab639e402a321b51cfaf2705 Mon Sep 17 00:00:00 2001 From: Victor Berchet Date: Mon, 15 Jul 2024 17:33:07 +0200 Subject: [PATCH 1/2] Add an option to get 48h of live data history --- apps/fetcher/src/app/trackers/flymaster.ts | 4 +- apps/fetcher/src/app/trackers/inreach.ts | 4 +- apps/fetcher/src/app/trackers/refresh.ts | 70 ++++++++----- apps/fetcher/src/app/trackers/skylines.ts | 4 +- apps/fetcher/src/app/trackers/spot.ts | 4 +- apps/fetcher/src/app/trackers/tracker.ts | 4 +- apps/fetcher/src/app/trackers/xcontest.ts | 4 +- apps/fetcher/src/app/ufos/refresh.ts | 16 +-- apps/fetcher/src/fetcher.ts | 97 +++++++++++-------- .../src/app/components/ui/main-menu.ts | 3 +- .../src/app/components/ui/supporter-modal.ts | 4 +- apps/fxc-front/src/app/logic/live-track.ts | 4 +- apps/fxc-front/src/app/pages/admin.ts | 4 +- .../fxc-front/src/app/pages/privacy-policy.ts | 2 +- .../src/app/redux/live-track-slice.ts | 3 +- apps/fxc-front/src/app/workers/live-track.ts | 4 +- apps/fxc-server/src/app/routes/admin.ts | 4 +- apps/fxc-server/src/app/routes/live-track.ts | 49 ++++++---- libs/common/src/lib/live-track.ts | 34 +++++-- libs/common/src/lib/redis-keys.ts | 18 ++-- 20 files changed, 206 insertions(+), 130 deletions(-) diff --git a/apps/fetcher/src/app/trackers/flymaster.ts b/apps/fetcher/src/app/trackers/flymaster.ts index a50cff0f..34db4445 100644 --- a/apps/fetcher/src/app/trackers/flymaster.ts +++ b/apps/fetcher/src/app/trackers/flymaster.ts @@ -6,7 +6,7 @@ import type { protos, TrackerNames } from '@flyxc/common'; import { fetchResponse, formatReqError, - LIVE_MINIMAL_INTERVAL_SEC, + LiveDataIntervalSec, removeBeforeFromLiveTrack, simplifyLiveTrack, validateFlymasterAccount, @@ -79,7 +79,7 @@ export class FlymasterFetcher extends TrackerFetcher { const points = parse(flight); let track = makeLiveTrack(points); track = removeBeforeFromLiveTrack(track, fetchSecond - 300); - simplifyLiveTrack(track, LIVE_MINIMAL_INTERVAL_SEC); + simplifyLiveTrack(track, LiveDataIntervalSec.Recent); updates.trackerDeltas.set(dsId, track); }); diff --git a/apps/fetcher/src/app/trackers/inreach.ts b/apps/fetcher/src/app/trackers/inreach.ts index 15268ac0..95420f90 100644 --- a/apps/fetcher/src/app/trackers/inreach.ts +++ b/apps/fetcher/src/app/trackers/inreach.ts @@ -9,7 +9,7 @@ import { fetchResponse, formatReqError, Keys, - LIVE_MINIMAL_INTERVAL_SEC, + LiveDataIntervalSec, parallelTasksWithTimeout, parseRetryAfterS, protos, @@ -91,7 +91,7 @@ export class InreachFetcher extends TrackerFetcher { try { const points = parse(await response.text()); const track = makeLiveTrack(points); - simplifyLiveTrack(track, LIVE_MINIMAL_INTERVAL_SEC); + simplifyLiveTrack(track, LiveDataIntervalSec.Recent); if (track.timeSec.length > 0) { updates.trackerDeltas.set(id, track); } diff --git a/apps/fetcher/src/app/trackers/refresh.ts b/apps/fetcher/src/app/trackers/refresh.ts index baa34915..ae7195a7 100644 --- a/apps/fetcher/src/app/trackers/refresh.ts +++ b/apps/fetcher/src/app/trackers/refresh.ts @@ -1,11 +1,9 @@ import type { protos } from '@flyxc/common'; import { Keys, - LIVE_AGE_OLD_SEC, LIVE_FETCH_TIMEOUT_SEC, - LIVE_MINIMAL_INTERVAL_SEC, - LIVE_OLD_INTERVAL_SEC, - LIVE_TRACKER_RETENTION_SEC, + LiveDataIntervalSec, + LiveDataRetentionSec, mergeLiveTracks, removeBeforeFromLiveTrack, simplifyLiveTrack, @@ -34,6 +32,19 @@ export function disconnectOgnClient() { ognClient.disconnect(); } +/** + * Refreshes all the trackers. + * + * Process: + * - fetch live data + * - update tracks (remove outdated point, decimates point according to their age), + * - add elevation information. + * + * @param pipeline - The ChainableCommander instance for executing commands. + * @param state - The FetcherState object containing current state information. + * @param redis - The Redis client for caching data. + * @param datastore - The Datastore instance for storing data. + */ export async function resfreshTrackers( pipeline: ChainableCommander, state: protos.FetcherState, @@ -51,22 +62,23 @@ export async function resfreshTrackers( new XcontestFetcher(state, pipeline), ]; - const updatePromises = await Promise.allSettled(fetchers.map((f) => f.refresh(LIVE_FETCH_TIMEOUT_SEC))); + const fetchResults = await Promise.allSettled(fetchers.map((f) => f.refresh(LIVE_FETCH_TIMEOUT_SEC))); const trackerUpdates: TrackerUpdates[] = []; - for (const p of updatePromises) { - if (p.status === 'fulfilled') { - const updates = p.value; + for (const result of fetchResults) { + if (result.status === 'fulfilled') { + const updates = result.value; trackerUpdates.push(updates); addTrackerLogs(pipeline, updates, state); } else { - console.error(`Tracker update error: ${p.reason}`); + console.error(`Tracker update error: ${result.reason}`); } } + // Drop points older than the max retention. const nowSec = Math.round(Date.now() / 1000); - const fullStartSec = nowSec - LIVE_TRACKER_RETENTION_SEC; + const dropBeforeSec = nowSec - LiveDataRetentionSec.Max; // Apply the updates. for (const [idStr, pilot] of Object.entries(state.pilots)) { @@ -74,15 +86,28 @@ export async function resfreshTrackers( // Merge updates for (const updates of trackerUpdates) { if (updates.trackerDeltas.has(id)) { - pilot.track = mergeLiveTracks(pilot.track!, updates.trackerDeltas.get(id)!); + pilot.track = mergeLiveTracks(pilot.track, updates.trackerDeltas.get(id)); } } - // Trim and simplify - pilot.track = removeBeforeFromLiveTrack(pilot.track!, fullStartSec); - simplifyLiveTrack(pilot.track, LIVE_MINIMAL_INTERVAL_SEC); - // Reduce precision for old point. - simplifyLiveTrack(pilot.track, LIVE_OLD_INTERVAL_SEC, { toSec: nowSec - LIVE_AGE_OLD_SEC }); + // Remove outdated points + pilot.track = removeBeforeFromLiveTrack(pilot.track, dropBeforeSec); + + // Decimates points according to their age. + simplifyLiveTrack(pilot.track, LiveDataIntervalSec.AfterH24, { + toSec: nowSec - 24 * 3600, + }); + simplifyLiveTrack(pilot.track, LiveDataIntervalSec.H12ToH24, { + fromSec: nowSec - 24 * 3600, + toSec: nowSec - 12 * 3600, + }); + simplifyLiveTrack(pilot.track, LiveDataIntervalSec.H6ToH12, { + fromSec: nowSec - 12 * 3600, + toSec: nowSec - 6 * 3600, + }); + simplifyLiveTrack(pilot.track, LiveDataIntervalSec.Recent, { + fromSec: nowSec - 6 * 3600, + }); } // Add the elevation for the last fix of every tracks when not present. @@ -96,19 +121,18 @@ export function addTrackerLogs( updates: TrackerUpdates, state: protos.FetcherState, ): void { - const name = updates.name; - const time = updates.startFetchSec; + const { name, startFetchSec } = updates; pushListCap( pipeline, Keys.trackerErrorsByType.replace('{name}', name), - updates.errors.map((e) => `[${time}] ${e}`), + updates.errors.map((e) => `[${startFetchSec}] ${e}`), 20, ); pushListCap( pipeline, Keys.trackerErrorsById.replace('{name}', name), - [...updates.trackerErrors.entries()].map(([id, e]) => `[${time}] id=${id} ${e}`), + [...updates.trackerErrors.entries()].map(([id, e]) => `[${startFetchSec}] id=${id} ${e}`), 20, ); pushListCap(pipeline, Keys.trackerNumFetches.replace('{name}', name), [updates.fetchedTracker.size], 20); @@ -127,16 +151,16 @@ export function addTrackerLogs( pushListCap( pipeline, Keys.trackerConsecutiveErrorsById.replace('{name}', name), - [`[${time}] id=${id} ${error}`], + [`[${startFetchSec}] id=${id} ${error}`], 20, ); } - const numErrors = state.pilots[id][name].numErrors; + const { numErrors } = state.pilots[id][name]; if (numErrors > 300) { pushListCap( pipeline, Keys.trackerManyErrorsById.replace('{name}', name), - [`[${time}] id=${id} ${numErrors} errors`], + [`[${startFetchSec}] id=${id} ${numErrors} errors`], 20, ); } diff --git a/apps/fetcher/src/app/trackers/skylines.ts b/apps/fetcher/src/app/trackers/skylines.ts index 3c7e8636..6666cfcc 100644 --- a/apps/fetcher/src/app/trackers/skylines.ts +++ b/apps/fetcher/src/app/trackers/skylines.ts @@ -7,7 +7,7 @@ import { decodeDeltas, fetchResponse, formatReqError, - LIVE_MINIMAL_INTERVAL_SEC, + LiveDataIntervalSec, removeBeforeFromLiveTrack, simplifyLiveTrack, validateSkylinesAccount, @@ -61,7 +61,7 @@ export class SkylinesFetcher extends TrackerFetcher { const points = parse(flight); let track = makeLiveTrack(points); track = removeBeforeFromLiveTrack(track, keepFromSec); - simplifyLiveTrack(track, LIVE_MINIMAL_INTERVAL_SEC); + simplifyLiveTrack(track, LiveDataIntervalSec.Recent); updates.trackerDeltas.set(dsId, track); }); [...sklIdToDsId.values()].forEach((id) => updates.fetchedTracker.add(id)); diff --git a/apps/fetcher/src/app/trackers/spot.ts b/apps/fetcher/src/app/trackers/spot.ts index b3cd929f..c8ad9594 100644 --- a/apps/fetcher/src/app/trackers/spot.ts +++ b/apps/fetcher/src/app/trackers/spot.ts @@ -6,7 +6,7 @@ import type { protos, TrackerNames } from '@flyxc/common'; import { fetchResponse, formatReqError, - LIVE_MINIMAL_INTERVAL_SEC, + LiveDataIntervalSec, simplifyLiveTrack, validateSpotAccount, } from '@flyxc/common'; @@ -43,7 +43,7 @@ export class SpotFetcher extends TrackerFetcher { try { const points = parse(await response.text()); const track = makeLiveTrack(points); - simplifyLiveTrack(track, LIVE_MINIMAL_INTERVAL_SEC); + simplifyLiveTrack(track, LiveDataIntervalSec.Recent); if (track.timeSec.length > 0) { updates.trackerDeltas.set(id, track); } diff --git a/apps/fetcher/src/app/trackers/tracker.ts b/apps/fetcher/src/app/trackers/tracker.ts index 928f415b..b6346589 100644 --- a/apps/fetcher/src/app/trackers/tracker.ts +++ b/apps/fetcher/src/app/trackers/tracker.ts @@ -1,7 +1,7 @@ // Base class for fetching tracker updates. import type { protos, TrackerNames } from '@flyxc/common'; -import { LIVE_REFRESH_SEC, LIVE_TRACKER_RETENTION_SEC } from '@flyxc/common'; +import { LIVE_REFRESH_SEC, TRACKERS_MAX_FETCH_DURATION_SEC } from '@flyxc/common'; import type { ChainableCommander } from 'ioredis'; // Updates for a tick of a tracker type (InReach, Spot, ...). @@ -123,7 +123,7 @@ export class TrackerFetcher { const tracker = this.getTracker(id); const nowSec = Math.round(Date.now() / 1000); const lastFetchSec = tracker ? tracker.lastFetchSec : nowSec; - return Math.max(startSec - LIVE_TRACKER_RETENTION_SEC, lastFetchSec - paddingSec); + return Math.max(startSec - TRACKERS_MAX_FETCH_DURATION_SEC, lastFetchSec - paddingSec); } // Counts the number of requests and errors. diff --git a/apps/fetcher/src/app/trackers/xcontest.ts b/apps/fetcher/src/app/trackers/xcontest.ts index 050834f7..460c98d1 100644 --- a/apps/fetcher/src/app/trackers/xcontest.ts +++ b/apps/fetcher/src/app/trackers/xcontest.ts @@ -6,8 +6,8 @@ import type { protos, TrackerNames } from '@flyxc/common'; import { fetchResponse, formatReqError, - LIVE_TRACKER_RETENTION_SEC, parallelTasksWithTimeout, + TRACKERS_MAX_FETCH_DURATION_SEC, validateXContestAccount, } from '@flyxc/common'; import { Secrets } from '@flyxc/secrets'; @@ -41,7 +41,7 @@ export class XcontestFetcher extends TrackerFetcher { const xcontestIdToLastFlight = new Map(); // Get all the users for the retention period - const openTimeMs = new Date().getTime() - LIVE_TRACKER_RETENTION_SEC * 1000; + const openTimeMs = new Date().getTime() - TRACKERS_MAX_FETCH_DURATION_SEC * 1000; const liveUserUrl = `https://api.xcontest.org/livedata/users?entity=group:flyxc&source=live&opentime={openTimeISO}`.replace( diff --git a/apps/fetcher/src/app/ufos/refresh.ts b/apps/fetcher/src/app/ufos/refresh.ts index 68f17fed..82fa96e2 100644 --- a/apps/fetcher/src/app/ufos/refresh.ts +++ b/apps/fetcher/src/app/ufos/refresh.ts @@ -2,8 +2,8 @@ import type { protos } from '@flyxc/common'; import { Keys, LIVE_FETCH_TIMEOUT_SEC, - LIVE_MINIMAL_INTERVAL_SEC, - LIVE_UFO_RETENTION_SEC, + LiveDataIntervalSec, + LiveDataRetentionSec, mergeLiveTracks, removeBeforeFromLiveTrack, simplifyLiveTrack, @@ -32,23 +32,23 @@ export async function resfreshUfoFleets(pipeline: ChainableCommander, state: pro } const nowSec = Math.round(Date.now() / 1000); - const ufoStartSec = nowSec - LIVE_UFO_RETENTION_SEC; + const ufoStartSec = nowSec - LiveDataRetentionSec.Ufo; for (const fleetUpdate of fleetUpdates) { - const fleetName = fleetUpdate.fleetName; + const { fleetName } = fleetUpdate; const ufoTracks = state.ufoFleets[fleetName].ufos ?? {}; - for (const [id, livetrack] of fleetUpdate.deltas.entries()) { + for (const [id, track] of fleetUpdate.deltas.entries()) { if (ufoTracks[id] != null) { - ufoTracks[id] = mergeLiveTracks(ufoTracks[id], livetrack); + ufoTracks[id] = mergeLiveTracks(ufoTracks[id], track); } else { - ufoTracks[id] = livetrack; + ufoTracks[id] = track; } } // eslint-disable-next-line prefer-const for (let [id, track] of Object.entries(ufoTracks)) { - simplifyLiveTrack(track, LIVE_MINIMAL_INTERVAL_SEC); + simplifyLiveTrack(track, LiveDataIntervalSec.Recent); track = removeBeforeFromLiveTrack(track, ufoStartSec); if (track.timeSec.length == 0) { delete ufoTracks[id]; diff --git a/apps/fetcher/src/fetcher.ts b/apps/fetcher/src/fetcher.ts index 6b0fc447..429e716d 100644 --- a/apps/fetcher/src/fetcher.ts +++ b/apps/fetcher/src/fetcher.ts @@ -2,14 +2,12 @@ import process from 'node:process'; import { differentialEncodeLiveTrack, - EXPORT_UPDATE_SEC, Keys, LIVE_REFRESH_SEC, - LONG_INCREMENTAL_UPDATE_SEC, + LiveDataRetentionSec, protos, removeBeforeFromLiveTrack, removeDeviceFromLiveTrack, - SHORT_INCREMENTAL_UPDATE_SEC, } from '@flyxc/common'; import { getDatastore, getRedisClient, pushListCap } from '@flyxc/common-node'; import { Secrets } from '@flyxc/secrets'; @@ -161,66 +159,65 @@ async function updateAll(pipeline: ChainableCommander, state: protos.FetcherStat await Promise.allSettled([resfreshTrackers(pipeline, state, redis, datastore), resfreshUfoFleets(pipeline, state)]); const nowSec = Math.round(Date.now() / 1000); - const longIncrementalStartSec = nowSec - LONG_INCREMENTAL_UPDATE_SEC; - const shortIncrementalStartSec = nowSec - SHORT_INCREMENTAL_UPDATE_SEC; - const exportStartSec = nowSec - EXPORT_UPDATE_SEC; // Create the binary proto output. - const fullTracks = protos.LiveDifferentialTrackGroup.create(); + const fullTracksH12 = protos.LiveDifferentialTrackGroup.create(); + const fullTracksH24 = protos.LiveDifferentialTrackGroup.create(); + const fullTracksH48 = protos.LiveDifferentialTrackGroup.create(); const longIncTracks = protos.LiveDifferentialTrackGroup.create({ incremental: true }); const shortIncTracks = protos.LiveDifferentialTrackGroup.create({ incremental: true }); const flymeTracks = protos.LiveDifferentialTrackGroup.create(); + // Add pilots. for (const [idStr, pilot] of Object.entries(state.pilots)) { const id = Number(idStr); // Add to group const name = pilot.name || 'unknown'; - if (pilot.track!.timeSec.length > 0) { - fullTracks.tracks.push(differentialEncodeLiveTrack(pilot.track!, id, name)); - - const longIncTrack = removeBeforeFromLiveTrack(pilot.track!, longIncrementalStartSec); - if (longIncTrack.timeSec.length > 0) { - longIncTracks.tracks.push(differentialEncodeLiveTrack(longIncTrack, id, name)); - const shortIncTrack = removeBeforeFromLiveTrack(longIncTrack, shortIncrementalStartSec); - if (shortIncTrack.timeSec.length > 0) { - shortIncTracks.tracks.push(differentialEncodeLiveTrack(shortIncTrack, id, name)); - } - } + if (pilot.track.timeSec.length > 0) { + fullTracksH48.tracks.push(differentialEncodeLiveTrack(pilot.track, id, name)); + const fullH24 = maybePushTrack(fullTracksH24, pilot.track, nowSec - LiveDataRetentionSec.FullH24, idStr, name); + const fullH12 = maybePushTrack(fullTracksH12, fullH24, nowSec - LiveDataRetentionSec.FullH12, idStr, name); + + const longInc = maybePushTrack( + longIncTracks, + fullH12, + nowSec - LiveDataRetentionSec.IncrementalLong, + idStr, + name, + ); + maybePushTrack(shortIncTracks, longInc, nowSec - LiveDataRetentionSec.IncrementalShort, idStr, name); if (pilot.share) { - const exportTrack = removeBeforeFromLiveTrack(longIncTrack, exportStartSec); - if (exportTrack.timeSec.length > 0) { - const flymeTrack = removeDeviceFromLiveTrack(exportTrack, 'flyme'); - if (flymeTrack.timeSec.length > 0) { - flymeTracks.tracks.push(differentialEncodeLiveTrack(flymeTrack, id, name)); - flymeTracks.remoteId.push(pilot.flyme?.account ?? ''); - } - } + const flymeTrack = removeDeviceFromLiveTrack(fullH12, 'flyme'); + maybePushTrack(flymeTracks, flymeTrack, nowSec - LiveDataRetentionSec.ExportToPartners, idStr, name); } } } + // Add UFOs. for (const [name, fleet] of Object.entries(state.ufoFleets)) { for (const [idStr, track] of Object.entries(fleet.ufos)) { const id = `${name}-${idStr}`; - fullTracks.tracks.push(differentialEncodeLiveTrack(track, id)); - const longIncTrack = removeBeforeFromLiveTrack(track, longIncrementalStartSec); - if (longIncTrack.timeSec.length > 0) { - longIncTracks.tracks.push(differentialEncodeLiveTrack(longIncTrack, id)); - const shortIncTrack = removeBeforeFromLiveTrack(longIncTrack, shortIncrementalStartSec); - if (shortIncTrack.timeSec.length > 0) { - shortIncTracks.tracks.push(differentialEncodeLiveTrack(shortIncTrack, id)); - } - } + fullTracksH48.tracks.push(differentialEncodeLiveTrack(track, id)); + + const fullH24 = maybePushTrack(fullTracksH24, track, nowSec - LiveDataRetentionSec.FullH24, id); + const fullH12 = maybePushTrack(fullTracksH12, fullH24, nowSec - LiveDataRetentionSec.FullH12, id); + + const longInc = maybePushTrack(longIncTracks, fullH12, nowSec - LiveDataRetentionSec.IncrementalLong, id); + maybePushTrack(shortIncTracks, longInc, nowSec - LiveDataRetentionSec.IncrementalShort, id); } } pipeline - .set(Keys.fetcherFullProto, Buffer.from(protos.LiveDifferentialTrackGroup.toBinary(fullTracks))) - .set(Keys.fetcherFullNumTracks, fullTracks.tracks.length) + .set(Keys.fetcherFullProtoH12, Buffer.from(protos.LiveDifferentialTrackGroup.toBinary(fullTracksH12))) + .set(Keys.fetcherFullProtoH24, Buffer.from(protos.LiveDifferentialTrackGroup.toBinary(fullTracksH24))) + .set(Keys.fetcherFullProtoH48, Buffer.from(protos.LiveDifferentialTrackGroup.toBinary(fullTracksH48))) + .set(Keys.fetcherFullNumTracksH12, fullTracksH12.tracks.length) + .set(Keys.fetcherFullNumTracksH24, fullTracksH24.tracks.length) + .set(Keys.fetcherFullNumTracksH48, fullTracksH48.tracks.length) .set(Keys.fetcherLongIncrementalProto, Buffer.from(protos.LiveDifferentialTrackGroup.toBinary(longIncTracks))) .set(Keys.fetcherShortIncrementalProto, Buffer.from(protos.LiveDifferentialTrackGroup.toBinary(shortIncTracks))) - .set(Keys.fetcherLongIncrementalNumTracks, longIncTracks.tracks.length) + .set(Keys.fetcherIncrementalNumTracksLong, longIncTracks.tracks.length) .set(Keys.fetcherExportFlymeProto, Buffer.from(protos.LiveDifferentialTrackGroup.toBinary(flymeTracks))); } catch (e) { console.log(`tick error ${e}`); @@ -243,3 +240,27 @@ async function shutdown(state: protos.FetcherState) { console.log('Exit...'); process.exit(); } + +/** + * Conditionally adds a live track to a track group after removing outdated points. + * + * @param dstTracks - The destination track group to potentially add the track to. + * @param track - The live track to be processed and potentially added. + * @param dropBeforeSec - The timestamp in seconds to drop points before. + * @param id - The identifier for the track. + * @param name - The name associated with the track. + * @returns The processed live track after removing outdated points. + */ +function maybePushTrack( + dstTracks: protos.LiveDifferentialTrackGroup, + track: protos.LiveTrack, + dropBeforeSec: number, + id: string, + name?: string, +) { + track = removeBeforeFromLiveTrack(track, dropBeforeSec); + if (track.timeSec.length > 0) { + dstTracks.tracks.push(differentialEncodeLiveTrack(track, id, name)); + } + return track; +} diff --git a/apps/fxc-front/src/app/components/ui/main-menu.ts b/apps/fxc-front/src/app/components/ui/main-menu.ts index 44cc4946..94966329 100644 --- a/apps/fxc-front/src/app/components/ui/main-menu.ts +++ b/apps/fxc-front/src/app/components/ui/main-menu.ts @@ -723,9 +723,10 @@ export class LiveTrackItems extends connect(store)(LitElement) { value=${this.historyMin} @ionChange=${this.handleHistory} > - 40mn + 40min 12h 24h + 48h `; } diff --git a/apps/fxc-front/src/app/components/ui/supporter-modal.ts b/apps/fxc-front/src/app/components/ui/supporter-modal.ts index bd8fbe9d..35205dec 100644 --- a/apps/fxc-front/src/app/components/ui/supporter-modal.ts +++ b/apps/fxc-front/src/app/components/ui/supporter-modal.ts @@ -33,7 +33,9 @@ export class SupporterModal extends LitElement { flyXC (formerly VisuGps) has always been open-source and free to use since 2007.

-

The development effort represents hundreds of hours per year and hosting flyXC costs about $10 a month.

+

+ The development effort represents hundreds of hours per year and hosting flyXC costs around $10 a month. +

If you can afford it, please consider supporting flyXC with a small donation by clicking the button below. It will help keep me motivated to maintain and improve it. diff --git a/apps/fxc-front/src/app/logic/live-track.ts b/apps/fxc-front/src/app/logic/live-track.ts index 08e4bcb9..4dd45b7a 100644 --- a/apps/fxc-front/src/app/logic/live-track.ts +++ b/apps/fxc-front/src/app/logic/live-track.ts @@ -6,7 +6,7 @@ import { isEmergencyTrack, IsSimplifiableFix, isUfo, - LIVE_MINIMAL_INTERVAL_SEC, + LiveDataIntervalSec, mergeLiveTracks, simplifyLiveTrack, } from '@flyxc/common'; @@ -210,7 +210,7 @@ export function updateLiveTracks( } if (id in updatedTracks) { track = mergeLiveTracks(track, updatedTracks[id]); - simplifyLiveTrack(track, LIVE_MINIMAL_INTERVAL_SEC); + simplifyLiveTrack(track, LiveDataIntervalSec.Recent); } updatedTracks[id] = track; } diff --git a/apps/fxc-front/src/app/pages/admin.ts b/apps/fxc-front/src/app/pages/admin.ts index 5deaaf23..94f5c1b5 100644 --- a/apps/fxc-front/src/app/pages/admin.ts +++ b/apps/fxc-front/src/app/pages/admin.ts @@ -178,7 +178,7 @@ export class DashSummary extends LitElement { } render(): TemplateResult { - const trackerM20 = this.values[common.Keys.fetcherLongIncrementalNumTracks]; + const trackerM20 = this.values[common.Keys.fetcherIncrementalNumTracksLong]; const nextStopSec = this.values[common.Keys.fetcherNextStopSec]; const lastStopSec = this.values[common.Keys.fetcherStoppedSec]; @@ -198,7 +198,7 @@ export class DashSummary extends LitElement { this.values[common.Keys.trackerNum], 'https://console.cloud.google.com/datastore/entities;kind=LiveTrack;ns=__$DEFAULT$__;sortCol=created;sortDir=DESCENDING/query/kind?project=fly-xc', )} - ${singleLineItem('Trackers h24', this.values[common.Keys.fetcherFullNumTracks])} + ${singleLineItem('Trackers h24', this.values[common.Keys.fetcherFullNumTracksH24])} ${singleLineItem('Trackers m20', trackerM20)} ${singleLineItem( 'Last device updated', diff --git a/apps/fxc-front/src/app/pages/privacy-policy.ts b/apps/fxc-front/src/app/pages/privacy-policy.ts index 93c29939..b5af1107 100644 --- a/apps/fxc-front/src/app/pages/privacy-policy.ts +++ b/apps/fxc-front/src/app/pages/privacy-policy.ts @@ -72,7 +72,7 @@ export class PrivacyPolicyPage extends LitElement {

We collect your registered tracker positions continuously and use that to display your position on the map - for up to 24 hours. By signing up for flyXC and registering your tracker devices, you acknowledge and agree + for up to 48 hours. By signing up for flyXC and registering your tracker devices, you acknowledge and agree that your tracker devices location will be displayed and viewable via our services.

diff --git a/apps/fxc-front/src/app/redux/live-track-slice.ts b/apps/fxc-front/src/app/redux/live-track-slice.ts index a87263ed..e4d5e4cb 100644 --- a/apps/fxc-front/src/app/redux/live-track-slice.ts +++ b/apps/fxc-front/src/app/redux/live-track-slice.ts @@ -99,7 +99,8 @@ export const updateTrackers = createAsyncThunk('liveTrack/fetch', async (_: unde try { const state = (api.getState() as RootState).liveTrack; const timeSec = Math.round((state.fetchMillis ?? 0) / 1000); - const response = await fetch(`${import.meta.env.VITE_API_SERVER}/api/live/tracks.pbf?s=${timeSec}`); + const fetchMin = Math.round(state.historyMin); + const response = await fetch(`${import.meta.env.VITE_API_SERVER}/api/live/tracks.pbf?s=${timeSec}&fm=${fetchMin}`); if (response.status === 200) { const tracks = state.tracks.entities; trackWorker.postMessage({ diff --git a/apps/fxc-front/src/app/workers/live-track.ts b/apps/fxc-front/src/app/workers/live-track.ts index 53ce5ccb..d79925fc 100644 --- a/apps/fxc-front/src/app/workers/live-track.ts +++ b/apps/fxc-front/src/app/workers/live-track.ts @@ -1,6 +1,6 @@ // Process the live tracking info from the server. -import { isUfo, LIVE_UFO_RETENTION_SEC, protos, removeBeforeFromLiveTrack, TRACK_GAP_MIN } from '@flyxc/common'; +import { isUfo, LiveDataRetentionSec, protos, removeBeforeFromLiveTrack, TRACK_GAP_MIN } from '@flyxc/common'; import { trackToFeatures, updateLiveTracks } from '../logic/live-track'; @@ -29,7 +29,7 @@ w.addEventListener('message', (message: MessageEvent) => { const now = Math.round(Date.now() / 1000); for (let track of updatedTracks) { - const dropBeforeSec = now - (isUfo(track.flags[0]) ? LIVE_UFO_RETENTION_SEC : historySec); + const dropBeforeSec = now - (isUfo(track.flags[0]) ? LiveDataRetentionSec.Ufo : historySec); track = removeBeforeFromLiveTrack(track, dropBeforeSec); if (track.timeSec.length > 0) { diff --git a/apps/fxc-server/src/app/routes/admin.ts b/apps/fxc-server/src/app/routes/admin.ts index 0adb334b..d78395ea 100644 --- a/apps/fxc-server/src/app/routes/admin.ts +++ b/apps/fxc-server/src/app/routes/admin.ts @@ -144,8 +144,8 @@ async function getDashboardValues(redis: Redis, datastore: Datastore): Promise { res.set('Cache-Control', 'no-store'); const token = req.header('token'); - if (!token) { - // Pick the incremental proto if last request was recent. - const lastUpdateSec = Number(req.query.s ?? 0); - const nowSec = Math.round(Date.now() / 1000); - const deltaSec = nowSec - lastUpdateSec; - let key = Keys.fetcherFullProto; - if (deltaSec < SHORT_INCREMENTAL_UPDATE_SEC) { - key = Keys.fetcherShortIncrementalProto; - } else if (deltaSec < LONG_INCREMENTAL_UPDATE_SEC) { - key = Keys.fetcherLongIncrementalProto; - } - res.set('Content-Type', 'application/x-protobuf'); - res.send(await redis.getBuffer(key)); - } else { + if (token) { switch (token) { case Secrets.FLYME_TOKEN: { const groupProto = await redis.getBuffer(Keys.fetcherExportFlymeProto); @@ -63,6 +44,32 @@ export function getTrackerRouter(redis: Redis, datastore: Datastore): Router { default: res.sendStatus(400); } + } else { + let key: Keys; + const lastUpdateSec = Number(req.query.s ?? 0); + const nowSec = Math.round(Date.now() / 1000); + const deltaSec = nowSec - lastUpdateSec; + // Pick the incremental proto if last request was recent. + if (deltaSec < LiveDataRetentionSec.IncrementalShort) { + key = Keys.fetcherShortIncrementalProto; + } else if (deltaSec < LiveDataRetentionSec.IncrementalLong) { + key = Keys.fetcherLongIncrementalProto; + } else { + switch (Number(req.query.fm ?? 0)) { + case 24 * 60: + key = Keys.fetcherFullProtoH24; + break; + + case 48 * 60: + key = Keys.fetcherFullNumTracksH48; + break; + + default: + key = Keys.fetcherFullProtoH12; + } + } + res.set('Content-Type', 'application/x-protobuf'); + res.send(await redis.getBuffer(key)); } }); diff --git a/libs/common/src/lib/live-track.ts b/libs/common/src/lib/live-track.ts index 124401c0..0bef7392 100644 --- a/libs/common/src/lib/live-track.ts +++ b/libs/common/src/lib/live-track.ts @@ -9,18 +9,35 @@ const DEVICE_TYPE_NUM_BITS = 5; const MAX_NUM_DEVICES = 2 ** DEVICE_TYPE_NUM_BITS - 1; const DEVICE_TYPE_BITMASK = 2 ** DEVICE_TYPE_NUM_BITS - 1; -// How long to retain live tracking positions. -export const LIVE_TRACKER_RETENTION_SEC = 24 * 3600; -export const LIVE_FLIGHT_MODE_RETENTION_SEC = 40 * 60; -// How long to keep ufo positions -export const LIVE_UFO_RETENTION_SEC = 1 * 3600; +export enum LiveDataRetentionSec { + // Incremental updates + IncrementalShort = 5 * 60, + IncrementalLong = 20 * 60, + // Full updates + FullH12 = 12 * 3600, + FullH24 = 24 * 3600, + FullH48 = 48 * 3600, + Max = FullH48, + // UFO updates + Ufo = 3600, + // Export to partners (max H12) + ExportToPartners = 5 * 60, +} + +// Minium track point intervals. +export enum LiveDataIntervalSec { + Recent = 5, + H6ToH12 = 60, + H12ToH24 = 3 * 60, + AfterH24 = 6 * 60, +} + +export const TRACKERS_MAX_FETCH_DURATION_SEC = 24 * 3600; // Age for a point to be considered old. export const LIVE_AGE_OLD_SEC = 6 * 3600; // Minimum interval for old points points. export const LIVE_OLD_INTERVAL_SEC = 3 * 60; -// Minimum interval between points. -export const LIVE_MINIMAL_INTERVAL_SEC = 5; // Refresh interval (how often one update is triggered) export const LIVE_REFRESH_SEC = 60; @@ -30,9 +47,6 @@ export const LIVE_FETCH_TIMEOUT_SEC = LIVE_REFRESH_SEC - 20; // Break tracks if gap is more than. export const TRACK_GAP_MIN = 60; -// Incremental updates. -export const LONG_INCREMENTAL_UPDATE_SEC = 20 * 60; -export const SHORT_INCREMENTAL_UPDATE_SEC = 5 * 60; // Export to partners. export const EXPORT_UPDATE_SEC = 5 * 60; diff --git a/libs/common/src/lib/redis-keys.ts b/libs/common/src/lib/redis-keys.ts index 1a08bb57..7983ad17 100644 --- a/libs/common/src/lib/redis-keys.ts +++ b/libs/common/src/lib/redis-keys.ts @@ -52,15 +52,21 @@ export enum Keys { // # Live Tracks - // Full tracks. - fetcherFullProto = 'f:live:full:proto', - // Number of full tracks. - fetcherFullNumTracks = 'f:live:full:size', + // Full tracks + fetcherFullProtoH12 = 'f:live:full:proto:h12', + fetcherFullProtoH24 = 'f:live:full:proto', + fetcherFullProtoH48 = 'f:live:full:proto:48', + // Number of pilots in full live tracks + fetcherFullNumTracksH12 = 'f:live:full:size:h12', + fetcherFullNumTracksH24 = 'f:live:full:size', + fetcherFullNumTracksH48 = 'f:live:full:size:h48', + // Incremental tracks. fetcherLongIncrementalProto = 'f:live:inc:long:proto', fetcherShortIncrementalProto = 'f:live:inc:short:proto', - // Number of incremental tracks. - fetcherLongIncrementalNumTracks = 'f:live:inc:size', + // Number of pilots in incremental tracks. + fetcherIncrementalNumTracksLong = 'f:live:inc:size', + // Tracks exported to FlyMe. fetcherExportFlymeProto = 'f:live:export:flyme', From 7a6e093e57008a7c293cd1d0979fbedfa45e7de9 Mon Sep 17 00:00:00 2001 From: Victor Berchet Date: Mon, 15 Jul 2024 18:17:34 +0200 Subject: [PATCH 2/2] fixup --- apps/fxc-server/src/app/routes/live-track.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/fxc-server/src/app/routes/live-track.ts b/apps/fxc-server/src/app/routes/live-track.ts index bf787a73..07ba834d 100644 --- a/apps/fxc-server/src/app/routes/live-track.ts +++ b/apps/fxc-server/src/app/routes/live-track.ts @@ -61,7 +61,7 @@ export function getTrackerRouter(redis: Redis, datastore: Datastore): Router { break; case 48 * 60: - key = Keys.fetcherFullNumTracksH48; + key = Keys.fetcherFullProtoH48; break; default: