Skip to content

Commit

Permalink
Update @journeyapps/powersync-sdk-common to latest 0.1.0. Make `newAt…
Browse files Browse the repository at this point in the history 
…tachmentRecord` async
  • Loading branch information
@Manrich121
Manrich121 committed Nov 1, 2023
1 parent 2032643 commit c20fd44
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 114 deletions.
2 changes: 1 addition & 1 deletion packages/powersync-attachments/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

## 0.0.1

Initial release.
Initial version.
2 changes: 1 addition & 1 deletion packages/powersync-attachments/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"devDependencies": {
},
"dependencies": {
"@journeyapps/powersync-sdk-common": "0.0.2"
"@journeyapps/powersync-sdk-common": "0.1.0"
},
"repository": "https://github.com/journeyapps/powersync-react-native-sdk",
"author": "JOURNEYAPPS",
Expand Down
152 changes: 53 additions & 99 deletions packages/powersync-attachments/src/AbstractAttachmentQueue.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
import {
AbstractPowerSyncDatabase,
Transaction,
} from "@journeyapps/powersync-sdk-common";
import { ATTACHMENT_TABLE, AttachmentRecord, AttachmentState } from "./Schema";
import { EncodingType, StorageAdapter } from "./StorageAdapter";
import { AbstractPowerSyncDatabase, Transaction } from '@journeyapps/powersync-sdk-common';
import { ATTACHMENT_TABLE, AttachmentRecord, AttachmentState } from './Schema';
import { EncodingType, StorageAdapter } from './StorageAdapter';

export interface AttachmentQueueOptions {
powersync: AbstractPowerSyncDatabase;
Expand All @@ -16,24 +13,24 @@ export interface AttachmentQueueOptions {
* How many attachments to keep in the cache
*/
cacheLimit?: number;
/**
* The name of the directory where attachments are stored on the device, not the full path
*/
attachmentDirectoryName?: string;
/**
* Whether to mark the initial watched attachment IDs to be synced
*/
performInitialSync?: boolean;
}

export const DEFAULT_ATTACHMENT_QUEUE_OPTIONS: Partial<AttachmentQueueOptions> =
{
attachmentDirectoryName: "attachments",
syncInterval: 30_000,
cacheLimit: 100,
performInitialSync: true,
};

export abstract class AbstractAttachmentQueue<
T extends AttachmentQueueOptions = AttachmentQueueOptions
> {
export const DEFAULT_ATTACHMENT_QUEUE_OPTIONS: Partial<AttachmentQueueOptions> = {
attachmentDirectoryName: 'attachments',
syncInterval: 30_000,
cacheLimit: 100,
performInitialSync: true
};

export abstract class AbstractAttachmentQueue<T extends AttachmentQueueOptions = AttachmentQueueOptions> {
uploading: boolean;
downloading: boolean;
initialSync: boolean;
Expand All @@ -43,7 +40,7 @@ export abstract class AbstractAttachmentQueue<
constructor(options: T) {
this.options = {
...DEFAULT_ATTACHMENT_QUEUE_OPTIONS,
...options,
...options
};
this.downloadQueue = new Set<string>();
this.uploading = false;
Expand All @@ -65,9 +62,7 @@ export abstract class AbstractAttachmentQueue<
/**
* Create a new AttachmentRecord, this gets called when the attachment id is not found in the database.
*/
abstract newAttachmentRecord(
record?: Partial<AttachmentRecord>
): AttachmentRecord;
abstract newAttachmentRecord(record?: Partial<AttachmentRecord>): Promise<AttachmentRecord>;

protected get powersync() {
return this.options.powersync;
Expand All @@ -82,9 +77,7 @@ export abstract class AbstractAttachmentQueue<
}

get storageDirectory() {
return `${this.storage.getUserStorageDirectory()}${
this.options.attachmentDirectoryName
}`;
return `${this.storage.getUserStorageDirectory()}${this.options.attachmentDirectoryName}`;
}

async init() {
Expand All @@ -110,7 +103,7 @@ export abstract class AbstractAttachmentQueue<

async watchAttachmentIds() {
for await (const ids of this.attachmentIds()) {
const _ids = `${ids.map((id) => `'${id}'`).join(",")}`;
const _ids = `${ids.map((id) => `'${id}'`).join(',')}`;
console.debug(`Queuing for sync, attachment IDs: [${_ids}]`);

if (this.initialSync) {
Expand All @@ -127,55 +120,43 @@ export abstract class AbstractAttachmentQueue<
);
}

const attachmentsInDatabase =
await this.powersync.getAll<AttachmentRecord>(
`SELECT * FROM ${this.table} WHERE state < ${AttachmentState.ARCHIVED}`
);
const attachmentsInDatabase = await this.powersync.getAll<AttachmentRecord>(
`SELECT * FROM ${this.table} WHERE state < ${AttachmentState.ARCHIVED}`
);

for (const id of ids) {
const record = attachmentsInDatabase.find((r) => r.id == id);
// 1. ID is not in the database
if (!record) {
const newRecord = this.newAttachmentRecord({
const newRecord = await this.newAttachmentRecord({
id: id,
state: AttachmentState.QUEUED_SYNC,
state: AttachmentState.QUEUED_SYNC
});
console.debug(
`Attachment (${id}) not found in database, creating new record`
);
console.debug(`Attachment (${id}) not found in database, creating new record`);
await this.saveToQueue(newRecord);
} else if (
record.local_uri == null ||
!(await this.storage.fileExists(record.local_uri))
) {
} else if (record.local_uri == null || !(await this.storage.fileExists(record.local_uri))) {
// 2. Attachment in database but no local file, mark as queued download
console.debug(
`Attachment (${id}) found in database but no local file, marking as queued download`
);
console.debug(`Attachment (${id}) found in database but no local file, marking as queued download`);
await this.update({
...record,
state: AttachmentState.QUEUED_DOWNLOAD,
state: AttachmentState.QUEUED_DOWNLOAD
});
}
}

// 3. Attachment in database and not in AttachmentIds, mark as archived
await this.powersync.execute(
`UPDATE ${this.table} SET state = ${
`UPDATE ${this.table} SET state = ${AttachmentState.ARCHIVED} WHERE state < ${
AttachmentState.ARCHIVED
} WHERE state < ${AttachmentState.ARCHIVED} AND id NOT IN (${ids
.map((id) => `'${id}'`)
.join(",")})`
} AND id NOT IN (${ids.map((id) => `'${id}'`).join(',')})`
);
}
}

async saveToQueue(
record: Omit<AttachmentRecord, "timestamp">
): Promise<AttachmentRecord> {
async saveToQueue(record: Omit<AttachmentRecord, 'timestamp'>): Promise<AttachmentRecord> {
const updatedRecord: AttachmentRecord = {
...record,
timestamp: new Date().getTime(),
timestamp: new Date().getTime()
};

await this.powersync.execute(
Expand All @@ -187,21 +168,18 @@ export abstract class AbstractAttachmentQueue<
updatedRecord.local_uri || null,
updatedRecord.media_type || null,
updatedRecord.size || null,
updatedRecord.state,
updatedRecord.state
]
);

return updatedRecord;
}

async record(id: string): Promise<AttachmentRecord | null> {
return this.powersync.getOptional<AttachmentRecord>(
`SELECT * FROM ${this.table} WHERE id = ?`,
[id]
);
return this.powersync.getOptional<AttachmentRecord>(`SELECT * FROM ${this.table} WHERE id = ?`, [id]);
}

async update(record: Omit<AttachmentRecord, "timestamp">): Promise<void> {
async update(record: Omit<AttachmentRecord, 'timestamp'>): Promise<void> {
const timestamp = new Date().getTime();
await this.powersync.execute(
`UPDATE ${this.table}
Expand All @@ -213,15 +191,7 @@ export abstract class AbstractAttachmentQueue<
media_type = ?,
state = ?
WHERE id = ?`,
[
timestamp,
record.filename,
record.local_uri || null,
record.size,
record.media_type,
record.state,
record.id,
]
[timestamp, record.filename, record.local_uri || null, record.size, record.media_type, record.state, record.id]
);
}

Expand All @@ -246,7 +216,7 @@ export abstract class AbstractAttachmentQueue<
try {
// Delete file on storage
await this.storage.deleteFile(uri, {
filename: record.filename,
filename: record.filename
});
} catch (e) {
console.error(e);
Expand All @@ -269,41 +239,37 @@ export abstract class AbstractAttachmentQueue<

async uploadAttachment(record: AttachmentRecord) {
if (!record.local_uri) {
throw new Error(
`No local_uri for record ${JSON.stringify(record, null, 2)}`
);
throw new Error(`No local_uri for record ${JSON.stringify(record, null, 2)}`);
}
try {
if (!(await this.storage.fileExists(record.local_uri))) {
console.warn(`File for ${record.id} does not exist, skipping upload`);
await this.update({
...record,
state: AttachmentState.QUEUED_DOWNLOAD,
state: AttachmentState.QUEUED_DOWNLOAD
});
return true;
}

const fileBuffer = await this.storage.readFile(record.local_uri, {
encoding: EncodingType.Base64,
mediaType: record.media_type,
mediaType: record.media_type
});

await this.storage.uploadFile(record.filename, fileBuffer, {
mediaType: record.media_type,
mediaType: record.media_type
});
// Mark as uploaded
await this.update({ ...record, state: AttachmentState.SYNCED });
console.debug(`Uploaded attachment "${record.id}" to Cloud Storage`);
return true;
} catch (e: any) {
if (e.error == "Duplicate") {
if (e.error == 'Duplicate') {
console.debug(`File already uploaded, marking ${record.id} as synced`);
await this.update({ ...record, state: AttachmentState.SYNCED });
return false;
}
console.error(
`UploadAttachment error for record ${JSON.stringify(record, null, 2)}`
);
console.error(`UploadAttachment error for record ${JSON.stringify(record, null, 2)}`);
return false;
}
}
Expand All @@ -313,9 +279,7 @@ export abstract class AbstractAttachmentQueue<
record.local_uri = this.getLocalUri(record.filename);
}
if (await this.storage.fileExists(record.local_uri)) {
console.debug(
`Local file already downloaded, marking "${record.id}" as synced`
);
console.debug(`Local file already downloaded, marking "${record.id}" as synced`);
await this.update({ ...record, state: AttachmentState.SYNCED });
return true;
}
Expand All @@ -328,37 +292,28 @@ export abstract class AbstractAttachmentQueue<
const reader = new FileReader();
reader.onloadend = () => {
// remove the header from the result: 'data:*/*;base64,'
resolve(
reader.result?.toString().replace(/^data:.+;base64,/, "") || ""
);
resolve(reader.result?.toString().replace(/^data:.+;base64,/, '') || '');
};
reader.onerror = reject;
reader.readAsDataURL(fileBlob);
});

// Ensure directory exists
await this.storage.makeDir(record.local_uri.replace(record.filename, ""));
await this.storage.makeDir(record.local_uri.replace(record.filename, ''));
// Write the file
await this.storage.writeFile(record.local_uri, base64Data, {
encoding: EncodingType.Base64,
encoding: EncodingType.Base64
});

await this.update({
...record,
media_type: fileBlob.type,
state: AttachmentState.SYNCED,
state: AttachmentState.SYNCED
});
console.debug(`Downloaded attachment "${record.id}"`);
return true;
} catch (e) {
console.error(
`Download attachment error for record ${JSON.stringify(
record,
null,
2
)}`,
e
);
console.error(`Download attachment error for record ${JSON.stringify(record, null, 2)}`, e);
}
return false;
}
Expand Down Expand Up @@ -409,9 +364,9 @@ export abstract class AbstractAttachmentQueue<
}
record = await this.getNextUploadRecord();
}
console.debug("Finished uploading attachments");
console.debug('Finished uploading attachments');
} catch (error) {
console.error("Upload failed:", error);
console.error('Upload failed:', error);
} finally {
this.uploading = false;
}
Expand Down Expand Up @@ -473,9 +428,9 @@ export abstract class AbstractAttachmentQueue<
}
await this.downloadRecord(record);
}
console.debug("Finished downloading attachments");
console.debug('Finished downloading attachments');
} catch (e) {
console.error("Downloads failed:", e);
console.error('Downloads failed:', e);
} finally {
this.downloading = false;
}
Expand All @@ -486,8 +441,7 @@ export abstract class AbstractAttachmentQueue<
}

async expireCache() {
const res = await this.powersync
.getAll<AttachmentRecord>(`SELECT * FROM ${this.table}
const res = await this.powersync.getAll<AttachmentRecord>(`SELECT * FROM ${this.table}
WHERE
state = ${AttachmentState.SYNCED} OR state = ${AttachmentState.ARCHIVED}
ORDER BY
Expand Down
13 changes: 0 additions & 13 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2141,19 +2141,6 @@
dependencies:
"@journeyapps/powersync-sdk-common" "^0.0.1"

"@journeyapps/[email protected]":
version "0.0.2"
resolved "https://registry.yarnpkg.com/@journeyapps/powersync-sdk-common/-/powersync-sdk-common-0.0.2.tgz#2a7073f21491e85b4bb434c46b9833f5401579bd"
integrity sha512-J90yVpv1L+zmVy8PnMCo1KMur9Zq1IwNngyuhais2rmOIMzZzMzTE6sdcUlioq/cphI0wOagUEXcyE8+iLbptA==
dependencies:
async-mutex "^0.4.0"
can-ndjson-stream "^1.0.2"
event-iterator "^2.0.0"
js-logger "^1.6.1"
lodash "^4.17.21"
object-hash "^3.0.0"
uuid "^3.0.0"

"@journeyapps/powersync-sdk-common@^0.0.1", "@journeyapps/powersync-sdk-common@^0.0.1-alpha.0":
version "0.0.1"
resolved "https://registry.yarnpkg.com/@journeyapps/powersync-sdk-common/-/powersync-sdk-common-0.0.1.tgz#8ff1a440a91ef7bbc488a1b3752cc344f4a6ee35"
Expand Down

0 comments on commit c20fd44

Please sign in to comment.