Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

poc: Configurable Web VFS #418

Draft
wants to merge 19 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/kind-suns-float.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@powersync/web': minor
---

Added support for OPFS virtual filesystem.
2 changes: 1 addition & 1 deletion demos/react-supabase-todolist/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"@powersync/web": "workspace:*",
"@emotion/react": "11.11.4",
"@emotion/styled": "11.11.5",
"@journeyapps/wa-sqlite": "^1.0.0",
"@journeyapps/wa-sqlite": "^1.1.1",
"@mui/icons-material": "^5.15.12",
"@mui/material": "^5.15.12",
"@mui/x-data-grid": "^6.19.6",
Expand Down
6 changes: 3 additions & 3 deletions demos/react-supabase-todolist/src/app/index.tsx
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { createRoot } from 'react-dom/client';
import { RouterProvider } from 'react-router-dom';
import { router } from '@/app/router';
import { SystemProvider } from '@/components/providers/SystemProvider';
import { ThemeProviderContainer } from '@/components/providers/ThemeProviderContainer';
import { router } from '@/app/router';
import { createRoot } from 'react-dom/client';
import { RouterProvider } from 'react-router-dom';

const root = createRoot(document.getElementById('app')!);
root.render(<App />);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,16 @@ export const useSupabase = () => React.useContext(SupabaseContext);
export const db = new PowerSyncDatabase({
schema: AppSchema,
database: {
dbFilename: 'example.db'
dbFilename: 's.sqlite'
},
flags: {
enableMultiTabs: typeof SharedWorker !== 'undefined'
}
// database: new WASQLiteOpenFactory({
// dbFilename: 'examplsw1se112.db'
// // vfs: WASQLiteVFS.OPFSCoopSyncVFS
// // vfs: WASQLiteVFS.OPFSCoopSyncVFS //Out of memory errors on iOS Safari
// })
});

export const SystemProvider = ({ children }: { children: React.ReactNode }) => {
Expand Down
1 change: 1 addition & 0 deletions demos/react-supabase-todolist/src/index.html
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
<html lang="en">
<head>
<meta name="theme-color" content="#c44eff" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="apple-touch-icon" href="/icons/icon.png" />
<link rel="stylesheet" href="./app/globals.css" />
<script type="module" src="./app/index.tsx"></script>
Expand Down
14 changes: 10 additions & 4 deletions demos/react-supabase-todolist/vite.config.mts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import wasm from 'vite-plugin-wasm';
import topLevelAwait from 'vite-plugin-top-level-await';
import { fileURLToPath, URL } from 'url';
import topLevelAwait from 'vite-plugin-top-level-await';
import wasm from 'vite-plugin-wasm';

import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react';
import { defineConfig } from 'vite';
import { VitePWA } from 'vite-plugin-pwa';

// https://vitejs.dev/config/
Expand All @@ -25,9 +25,15 @@ export default defineConfig({
// Don't optimize these packages as they contain web workers and WASM files.
// https://github.com/vitejs/vite/issues/11672#issuecomment-1415820673
exclude: ['@journeyapps/wa-sqlite', '@powersync/web'],
include: [],
include: []
// include: ['@powersync/web > js-logger'], // <-- Include `js-logger` when it isn't installed and imported.
},
server: {
headers: {
'Cross-Origin-Opener-Policy': 'same-origin',
'Cross-Origin-Embedder-Policy': 'require-corp'
}
},
plugins: [
wasm(),
topLevelAwait(),
Expand Down
33 changes: 33 additions & 0 deletions packages/web/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,39 @@ Install it in your app with:
npm install @journeyapps/wa-sqlite
```

### Encryption with Multiple Ciphers

To enable encryption you need to specify an encryption key when instantiating the PowerSync database.

> The PowerSync Web SDK uses the ChaCha20 cipher algorithm by [default](https://utelle.github.io/SQLite3MultipleCiphers/docs/ciphers/cipher_chacha20/).

```typescript
export const db = new PowerSyncDatabase({
// The schema you defined
schema: AppSchema,
database: {
// Filename for the SQLite database — it's important to only instantiate one instance per file.
dbFilename: 'example.db'
// Optional. Directory where the database file is located.'
// dbLocation: 'path/to/directory'
},
// Encryption key for the database.
encryptionKey: 'your-encryption-key'
});

// If you are using a custom WASQLiteOpenFactory or WASQLiteDBAdapter, you need specify the encryption key inside the construtor
export const db = new PowerSyncDatabase({
schema: AppSchema,
database: new WASQLiteOpenFactory({
//new WASQLiteDBAdapter
dbFilename: 'example.db',
vfs: WASQLiteVFS.OPFSCoopSyncVFS,
// Encryption key for the database.
encryptionKey: 'your-encryption-key'
})
});
```

## Webpack

See the [example Webpack config](https://github.com/powersync-ja/powersync-js/blob/main/demos/example-webpack/webpack.config.js) for details on polyfills and requirements.
Expand Down
6 changes: 3 additions & 3 deletions packages/web/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,19 @@
"author": "JOURNEYAPPS",
"license": "Apache-2.0",
"peerDependencies": {
"@journeyapps/wa-sqlite": "^1.0.0",
"@journeyapps/wa-sqlite": "^1.1.1",
"@powersync/common": "workspace:^1.22.0"
},
"dependencies": {
"@powersync/common": "workspace:*",
"async-mutex": "^0.4.0",
"bson": "^6.6.0",
"comlink": "^4.4.1",
"comlink": "^4.4.2",
"commander": "^12.1.0",
"js-logger": "^1.6.1"
},
"devDependencies": {
"@journeyapps/wa-sqlite": "^1.0.0",
"@journeyapps/wa-sqlite": "^1.1.1",
"@types/uuid": "^9.0.6",
"@vitest/browser": "^2.1.4",
"crypto-browserify": "^3.12.0",
Expand Down
43 changes: 38 additions & 5 deletions packages/web/src/db/PowerSyncDatabase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import {
AbstractPowerSyncDatabase,
DBAdapter,
DEFAULT_POWERSYNC_CLOSE_OPTIONS,
isDBAdapter,
isSQLOpenFactory,
PowerSyncDatabaseOptions,
PowerSyncDatabaseOptionsWithDBAdapter,
PowerSyncDatabaseOptionsWithOpenFactory,
Expand All @@ -14,21 +16,22 @@ import {
StreamingSyncImplementation
} from '@powersync/common';
import { Mutex } from 'async-mutex';
import { getNavigatorLocks } from '../shared/navigator';
import { WASQLiteOpenFactory } from './adapters/wa-sqlite/WASQLiteOpenFactory';
import {
DEFAULT_WEB_SQL_FLAGS,
ResolvedWebSQLOpenOptions,
resolveWebSQLFlags,
WebSQLFlags
} from './adapters/web-sql-flags';
import { WebDBAdapter } from './adapters/WebDBAdapter';
import { SharedWebStreamingSyncImplementation } from './sync/SharedWebStreamingSyncImplementation';
import { SSRStreamingSyncImplementation } from './sync/SSRWebStreamingSyncImplementation';
import { WebRemote } from './sync/WebRemote';
import {
WebStreamingSyncImplementation,
WebStreamingSyncImplementationOptions
} from './sync/WebStreamingSyncImplementation';
import { getNavigatorLocks } from '../shared/navigator';

export interface WebPowerSyncFlags extends WebSQLFlags {
/**
Expand All @@ -55,14 +58,24 @@ type WithWebSyncOptions<Base> = Base & {
sync?: WebSyncOptions;
};

export interface WebEncryptionOptions {
/**
* Encryption key for the database.
* If set, the database will be encrypted using Multiple Ciphers.
*/
encryptionKey?: string;
}

type WithWebEncryptionOptions<Base> = Base & WebEncryptionOptions;

export type WebPowerSyncDatabaseOptionsWithAdapter = WithWebSyncOptions<
WithWebFlags<PowerSyncDatabaseOptionsWithDBAdapter>
>;
export type WebPowerSyncDatabaseOptionsWithOpenFactory = WithWebSyncOptions<
WithWebFlags<PowerSyncDatabaseOptionsWithOpenFactory>
>;
export type WebPowerSyncDatabaseOptionsWithSettings = WithWebSyncOptions<
WithWebFlags<PowerSyncDatabaseOptionsWithSettings>
WithWebFlags<WithWebEncryptionOptions<PowerSyncDatabaseOptionsWithSettings>>
>;

export type WebPowerSyncDatabaseOptions = WithWebSyncOptions<WithWebFlags<PowerSyncDatabaseOptions>>;
Expand All @@ -72,14 +85,28 @@ export const DEFAULT_POWERSYNC_FLAGS: Required<WebPowerSyncFlags> = {
externallyUnload: false
};

export const resolveWebPowerSyncFlags = (flags?: WebPowerSyncFlags): WebPowerSyncFlags => {
export const resolveWebPowerSyncFlags = (flags?: WebPowerSyncFlags): Required<WebPowerSyncFlags> => {
return {
...DEFAULT_POWERSYNC_FLAGS,
...flags,
...resolveWebSQLFlags(flags)
};
};

/**
* Asserts that the database options are valid for custom database constructors.
*/
function assertValidDatabaseOptions(options: WebPowerSyncDatabaseOptions): void {
if ('database' in options && 'encryptionKey' in options) {
const { database } = options;
if (isSQLOpenFactory(database) || isDBAdapter(database)) {
throw new Error(
`Invalid configuration: 'encryptionKey' should only be included inside the database object when using a custom ${isSQLOpenFactory(database) ? 'WASQLiteOpenFactory' : 'WASQLiteDBAdapter'} constructor.`
);
}
}
}

/**
* A PowerSync database which provides SQLite functionality
* which is automatically synced.
Expand Down Expand Up @@ -107,6 +134,8 @@ export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
constructor(protected options: WebPowerSyncDatabaseOptions) {
super(options);

assertValidDatabaseOptions(options);

this.resolvedFlags = resolveWebPowerSyncFlags(options.flags);

if (this.resolvedFlags.enableMultiTabs && !this.resolvedFlags.externallyUnload) {
Expand All @@ -120,7 +149,8 @@ export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
protected openDBAdapter(options: WebPowerSyncDatabaseOptionsWithSettings): DBAdapter {
const defaultFactory = new WASQLiteOpenFactory({
...options.database,
flags: resolveWebPowerSyncFlags(options.flags)
flags: resolveWebPowerSyncFlags(options.flags),
encryptionKey: options.encryptionKey
});
return defaultFactory.openDB();
}
Expand Down Expand Up @@ -191,7 +221,10 @@ export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
const logger = this.options.logger;
logger ? logger.warn(warning) : console.warn(warning);
}
return new SharedWebStreamingSyncImplementation(syncOptions);
return new SharedWebStreamingSyncImplementation({
...syncOptions,
db: this.database as WebDBAdapter // This should always be the case
});
default:
return new WebStreamingSyncImplementation(syncOptions);
}
Expand Down
3 changes: 3 additions & 0 deletions packages/web/src/db/adapters/AbstractWebSQLOpenFactory.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
import { DBAdapter, SQLOpenFactory } from '@powersync/common';
import Logger, { ILogger } from 'js-logger';
import { SSRDBAdapter } from './SSRDBAdapter';
import { ResolvedWebSQLFlags, WebSQLOpenFactoryOptions, isServerSide, resolveWebSQLFlags } from './web-sql-flags';

export abstract class AbstractWebSQLOpenFactory implements SQLOpenFactory {
protected resolvedFlags: ResolvedWebSQLFlags;
protected logger: ILogger;

constructor(protected options: WebSQLOpenFactoryOptions) {
this.resolvedFlags = resolveWebSQLFlags(options.flags);
this.logger = options.logger ?? Logger.get(`AbstractWebSQLOpenFactory - ${this.options.dbFilename}`);
}

/**
Expand Down
31 changes: 31 additions & 0 deletions packages/web/src/db/adapters/AsyncDatabaseConnection.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import { BatchedUpdateNotification, QueryResult } from '@powersync/common';
import { ResolvedWebSQLOpenOptions } from './web-sql-flags';

/**
* Proxied query result does not contain a function for accessing row values
*/
export type ProxiedQueryResult = Omit<QueryResult, 'rows'> & {
rows: {
_array: any[];
length: number;
};
};
export type OnTableChangeCallback = (event: BatchedUpdateNotification) => void;

/**
* @internal
* An async Database connection which provides basic async SQL methods.
* This is usually a proxied through a web worker.
*/
export interface AsyncDatabaseConnection<Config extends ResolvedWebSQLOpenOptions = ResolvedWebSQLOpenOptions> {
init(): Promise<void>;
close(): Promise<void>;
execute(sql: string, params?: any[]): Promise<ProxiedQueryResult>;
executeBatch(sql: string, params?: any[]): Promise<ProxiedQueryResult>;
registerOnTableChange(callback: OnTableChangeCallback): Promise<() => void>;
getConfig(): Promise<Config>;
}

export type OpenAsyncDatabaseConnection<Config extends ResolvedWebSQLOpenOptions = ResolvedWebSQLOpenOptions> = (
config: Config
) => AsyncDatabaseConnection;
Loading
Loading