From a053ee583db96919d38b96c58f1d8d616b4ab283 Mon Sep 17 00:00:00 2001 From: Coki Date: Tue, 19 Nov 2024 19:03:39 +0800 Subject: [PATCH 1/6] fix: improve policy line parsing This change makes the policy parser more robust and flexible, allowing for --- src/persist/fileAdapter.ts | 6 ++--- src/persist/helper.ts | 47 ++++++++++++++++++++++++++++++-------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/persist/fileAdapter.ts b/src/persist/fileAdapter.ts index e6b7f8c..b4926a3 100644 --- a/src/persist/fileAdapter.ts +++ b/src/persist/fileAdapter.ts @@ -34,11 +34,11 @@ export class FileAdapter implements Adapter { private async loadPolicyFile(model: Model, handler: (line: string, model: Model) => void): Promise { const bodyBuf = await (this.fs ? this.fs : mustGetDefaultFileSystem()).readFileSync(this.filePath); const lines = bodyBuf.toString().split('\n'); - lines.forEach((n: string, index: number) => { - if (!n) { + lines.forEach((line: string) => { + if (!line || line.trim().startsWith('#')) { return; } - handler(n, model); + handler(line, model); }); } diff --git a/src/persist/helper.ts b/src/persist/helper.ts index 141b886..bf6536c 100644 --- a/src/persist/helper.ts +++ b/src/persist/helper.ts @@ -1,5 +1,4 @@ import { Model } from '../model'; -import { parse } from 'csv-parse/sync'; export class Helper { public static loadPolicyLine(line: string, model: Model): void { @@ -7,17 +6,39 @@ export class Helper { return; } - const tokens = parse(line, { - delimiter: ',', - skip_empty_lines: true, - trim: true, - }); + let tokens: string[] = []; + let currentToken = ''; + let inQuotes = false; + let bracketCount = 0; + + for (let i = 0; i < line.length; i++) { + const char = line[i]; + + if (char === '"' && (i === 0 || line[i - 1] !== '\\')) { + inQuotes = !inQuotes; + } else if (char === '(' && !inQuotes) { + bracketCount++; + } else if (char === ')' && !inQuotes) { + bracketCount--; + } + + if (char === ',' && !inQuotes && bracketCount === 0) { + tokens.push(currentToken.trim()); + currentToken = ''; + } else { + currentToken += char; + } + } - if (!tokens || !tokens[0]) { + if (currentToken) { + tokens.push(currentToken.trim()); + } + + if (!tokens || tokens.length === 0) { return; } - const key = tokens[0][0]; + const key = tokens[0]; const sec = key.substring(0, 1); const item = model.model.get(sec); if (!item) { @@ -28,6 +49,14 @@ export class Helper { if (!policy) { return; } - policy.policy.push(tokens[0].slice(1)); + + const values = tokens.slice(1).map((v) => { + if (v.startsWith('"') && v.endsWith('"')) { + v = v.slice(1, -1); + } + return v.replace(/""/g, '"'); + }); + + policy.policy.push(values); } } From 7b7e9ffcf528331f9410e64fa60fdf3b0e32d288 Mon Sep 17 00:00:00 2001 From: Coki Date: Tue, 19 Nov 2024 19:28:20 +0800 Subject: [PATCH 2/6] chore: remove csv-parse dependency from package.json --- package.json | 1 - yarn.lock | 5 ----- 2 files changed, 6 deletions(-) diff --git a/package.json b/package.json index 10c0342..14591f2 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,6 @@ "@casbin/expression-eval": "^5.2.0", "await-lock": "^2.0.1", "buffer": "^6.0.3", - "csv-parse": "^5.3.5", "minimatch": "^7.4.2" }, "files": [ diff --git a/yarn.lock b/yarn.lock index 0c6a1c8..7ad2ef1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2282,11 +2282,6 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" -csv-parse@^5.3.5: - version "5.3.5" - resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.3.5.tgz#9924bbba9f7056122f06b7af18edc1a7f022ce99" - integrity sha512-8O5KTIRtwmtD3+EVfW6BCgbwZqJbhTYsQZry12F1TP5RUp0sD9tp1UnCWic3n0mLOhzeocYaCZNYxOGSg3dmmQ== - cz-conventional-changelog@3.2.0: version "3.2.0" resolved "https://registry.npmjs.org/cz-conventional-changelog/-/cz-conventional-changelog-3.2.0.tgz#6aef1f892d64113343d7e455529089ac9f20e477" From a51195471002f316987bdffcd02f11b5d9289e72 Mon Sep 17 00:00:00 2001 From: Coki Date: Tue, 19 Nov 2024 19:31:52 +0800 Subject: [PATCH 3/6] refactor: use const instead of let for tokens array in Helper class --- src/persist/helper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/persist/helper.ts b/src/persist/helper.ts index bf6536c..45cb5f5 100644 --- a/src/persist/helper.ts +++ b/src/persist/helper.ts @@ -6,7 +6,7 @@ export class Helper { return; } - let tokens: string[] = []; + const tokens: string[] = []; let currentToken = ''; let inQuotes = false; let bracketCount = 0; From 29ca36b559612e71ff8983bcc83a75c67b1e12fc Mon Sep 17 00:00:00 2001 From: Coki Date: Tue, 19 Nov 2024 20:23:11 +0800 Subject: [PATCH 4/6] fix: policy line parsing for nested expressions and quoted values --- src/persist/helper.ts | 31 +++++++++++++++++++++++-------- test/persist/helper.test.ts | 2 +- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/persist/helper.ts b/src/persist/helper.ts index 45cb5f5..82dd749 100644 --- a/src/persist/helper.ts +++ b/src/persist/helper.ts @@ -14,22 +14,32 @@ export class Helper { for (let i = 0; i < line.length; i++) { const char = line[i]; - if (char === '"' && (i === 0 || line[i - 1] !== '\\')) { - inQuotes = !inQuotes; - } else if (char === '(' && !inQuotes) { + if (char === '(') { bracketCount++; - } else if (char === ')' && !inQuotes) { + } else if (char === ')') { bracketCount--; } + if (char === '"' && (i === 0 || line[i - 1] !== '\\')) { + inQuotes = !inQuotes; + currentToken += char; + continue; + } + if (char === ',' && !inQuotes && bracketCount === 0) { - tokens.push(currentToken.trim()); - currentToken = ''; + if (currentToken) { + tokens.push(currentToken.trim()); + currentToken = ''; + } } else { currentToken += char; } } + if (bracketCount !== 0) { + throw new Error(`Unmatched brackets in policy line: ${line}`); + } + if (currentToken) { tokens.push(currentToken.trim()); } @@ -38,7 +48,11 @@ export class Helper { return; } - const key = tokens[0]; + let key = tokens[0].trim(); + if (key.startsWith('"') && key.endsWith('"')) { + key = key.slice(1, -1); + } + const sec = key.substring(0, 1); const item = model.model.get(sec); if (!item) { @@ -51,10 +65,11 @@ export class Helper { } const values = tokens.slice(1).map((v) => { + v = v.trim(); if (v.startsWith('"') && v.endsWith('"')) { v = v.slice(1, -1); } - return v.replace(/""/g, '"'); + return v.replace(/""/g, '"').trim(); }); policy.policy.push(values); diff --git a/test/persist/helper.test.ts b/test/persist/helper.test.ts index aa3217c..cfacaa1 100644 --- a/test/persist/helper.test.ts +++ b/test/persist/helper.test.ts @@ -45,7 +45,7 @@ m = r.sub == p.sub && r.obj == p.obj && r.act == p.act ['admin', '/', 'POST'], ['admin', '/', 'PUT'], ['admin', '/', 'DELETE'], - [' admin', '/ ', 'PATCH'], + ['admin', '/', 'PATCH'], ]; testdata.forEach((n) => { From 6e7ae077f9e1ff4d82e905b565d4d037b618e0d7 Mon Sep 17 00:00:00 2001 From: Coki Date: Wed, 20 Nov 2024 15:06:03 +0800 Subject: [PATCH 5/6] feat: enhance csv-parse for policy --- package.json | 1 + src/persist/helper.ts | 58 +++++++++++++++++++++---------------------- yarn.lock | 5 ++++ 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index 14591f2..4dd5994 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@casbin/expression-eval": "^5.2.0", "await-lock": "^2.0.1", "buffer": "^6.0.3", + "csv-parse": "^5.5.6", "minimatch": "^7.4.2" }, "files": [ diff --git a/src/persist/helper.ts b/src/persist/helper.ts index 82dd749..18e5a1d 100644 --- a/src/persist/helper.ts +++ b/src/persist/helper.ts @@ -1,4 +1,5 @@ import { Model } from '../model'; +import { parse } from 'csv-parse/sync'; export class Helper { public static loadPolicyLine(line: string, model: Model): void { @@ -6,33 +7,37 @@ export class Helper { return; } - const tokens: string[] = []; + const rawTokens = parse(line, { + delimiter: ',', + skip_empty_lines: true, + trim: true, + relax_quotes: true, + }); + + if (!rawTokens || rawTokens.length === 0 || !rawTokens[0]) { + return; + } + + const tokens: string[] = rawTokens[0]; + + const processedTokens: string[] = []; let currentToken = ''; - let inQuotes = false; let bracketCount = 0; - for (let i = 0; i < line.length; i++) { - const char = line[i]; - - if (char === '(') { - bracketCount++; - } else if (char === ')') { - bracketCount--; + for (const token of tokens) { + for (const char of token) { + if (char === '(') { + bracketCount++; + } else if (char === ')') { + bracketCount--; + } } - if (char === '"' && (i === 0 || line[i - 1] !== '\\')) { - inQuotes = !inQuotes; - currentToken += char; - continue; - } + currentToken += (currentToken ? ',' : '') + token; - if (char === ',' && !inQuotes && bracketCount === 0) { - if (currentToken) { - tokens.push(currentToken.trim()); - currentToken = ''; - } - } else { - currentToken += char; + if (bracketCount === 0) { + processedTokens.push(currentToken); + currentToken = ''; } } @@ -40,15 +45,11 @@ export class Helper { throw new Error(`Unmatched brackets in policy line: ${line}`); } - if (currentToken) { - tokens.push(currentToken.trim()); - } - - if (!tokens || tokens.length === 0) { + if (processedTokens.length === 0) { return; } - let key = tokens[0].trim(); + let key = processedTokens[0].trim(); if (key.startsWith('"') && key.endsWith('"')) { key = key.slice(1, -1); } @@ -64,8 +65,7 @@ export class Helper { return; } - const values = tokens.slice(1).map((v) => { - v = v.trim(); + const values = processedTokens.slice(1).map((v) => { if (v.startsWith('"') && v.endsWith('"')) { v = v.slice(1, -1); } diff --git a/yarn.lock b/yarn.lock index 7ad2ef1..1646a31 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2282,6 +2282,11 @@ cssstyle@^2.3.0: dependencies: cssom "~0.3.6" +csv-parse@^5.5.6: + version "5.5.6" + resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-5.5.6.tgz#0d726d58a60416361358eec291a9f93abe0b6b1a" + integrity sha512-uNpm30m/AGSkLxxy7d9yRXpJQFrZzVWLFBkS+6ngPcZkw/5k3L/jjFuj7tVnEpRn+QgmiXr21nDlhCiUK4ij2A== + cz-conventional-changelog@3.2.0: version "3.2.0" resolved "https://registry.npmjs.org/cz-conventional-changelog/-/cz-conventional-changelog-3.2.0.tgz#6aef1f892d64113343d7e455529089ac9f20e477" From b4168121c75dabe5b617c0d089f607f6633203c8 Mon Sep 17 00:00:00 2001 From: Coki Date: Fri, 22 Nov 2024 14:59:40 +0800 Subject: [PATCH 6/6] refactor: reorganize policy parsing and loading logic --- src/persist/helper.ts | 64 ++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/src/persist/helper.ts b/src/persist/helper.ts index 18e5a1d..2ddbb88 100644 --- a/src/persist/helper.ts +++ b/src/persist/helper.ts @@ -1,36 +1,47 @@ import { Model } from '../model'; import { parse } from 'csv-parse/sync'; -export class Helper { - public static loadPolicyLine(line: string, model: Model): void { +export interface IPolicyParser { + parse(line: string): string[][] | null; +} + +export class BasicCsvParser implements IPolicyParser { + parse(line: string): string[][] | null { if (!line || line.trimStart().charAt(0) === '#') { - return; + return null; } - const rawTokens = parse(line, { + return parse(line, { delimiter: ',', skip_empty_lines: true, trim: true, relax_quotes: true, }); + } +} - if (!rawTokens || rawTokens.length === 0 || !rawTokens[0]) { - return; - } +export class BracketAwareCsvParser implements IPolicyParser { + private readonly baseParser: IPolicyParser; - const tokens: string[] = rawTokens[0]; + constructor(baseParser: IPolicyParser = new BasicCsvParser()) { + this.baseParser = baseParser; + } + + parse(line: string): string[][] | null { + const rawTokens = this.baseParser.parse(line); + if (!rawTokens || !rawTokens[0]) { + return null; + } + const tokens = rawTokens[0]; const processedTokens: string[] = []; let currentToken = ''; let bracketCount = 0; for (const token of tokens) { for (const char of token) { - if (char === '(') { - bracketCount++; - } else if (char === ')') { - bracketCount--; - } + if (char === '(') bracketCount++; + else if (char === ')') bracketCount--; } currentToken += (currentToken ? ',' : '') + token; @@ -45,11 +56,24 @@ export class Helper { throw new Error(`Unmatched brackets in policy line: ${line}`); } - if (processedTokens.length === 0) { + return processedTokens.length > 0 ? [processedTokens] : null; + } +} + +export class PolicyLoader { + private readonly parser: IPolicyParser; + + constructor(parser: IPolicyParser = new BracketAwareCsvParser()) { + this.parser = parser; + } + + loadPolicyLine(line: string, model: Model): void { + const tokens = this.parser.parse(line); + if (!tokens || !tokens[0]) { return; } - let key = processedTokens[0].trim(); + let key = tokens[0][0].trim(); if (key.startsWith('"') && key.endsWith('"')) { key = key.slice(1, -1); } @@ -65,7 +89,7 @@ export class Helper { return; } - const values = processedTokens.slice(1).map((v) => { + const values = tokens[0].slice(1).map((v) => { if (v.startsWith('"') && v.endsWith('"')) { v = v.slice(1, -1); } @@ -75,3 +99,11 @@ export class Helper { policy.policy.push(values); } } + +export class Helper { + private static readonly policyLoader = new PolicyLoader(); + + public static loadPolicyLine(line: string, model: Model): void { + Helper.policyLoader.loadPolicyLine(line, model); + } +}