From 7c3ac6138806d9c10b246913390a9abe0351cef3 Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sun, 25 Feb 2024 16:19:46 -0500 Subject: [PATCH 1/8] ci(gha): Upgrade all actions --- .github/workflows/node-release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml index 51560b10e..f064b8939 100644 --- a/.github/workflows/node-release.yml +++ b/.github/workflows/node-release.yml @@ -26,7 +26,7 @@ jobs: runs-on: ${{ matrix.settings.host }} steps: - name: Checkout repository - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Install Rust uses: actions-rs/toolchain@v1 @@ -38,13 +38,13 @@ jobs: run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/ - name: Cache Cargo Registry - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: ~/.cargo/registry key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} - name: Install Node ${{ matrix.node-version }} - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: latest cache: yarn @@ -65,7 +65,7 @@ jobs: with: python-version: 3.x - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: bindings-${{ matrix.settings.target }} path: ${{ env.APP_NAME }}bindings/node/*.node @@ -76,9 +76,9 @@ jobs: needs: - build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: latest check-latest: true @@ -88,7 +88,7 @@ jobs: working-directory: ./bindings/node run: yarn install - name: Download all artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: ./bindings/node/artifacts - name: Move artifacts From 80b1347b32201538fbdf6266d12b467db617d24a Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sun, 25 Feb 2024 16:22:01 -0500 Subject: [PATCH 2/8] build: Add binaries to pack --- bindings/node/package.json | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/bindings/node/package.json b/bindings/node/package.json index cda4c4978..51da36499 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -14,7 +14,32 @@ "description": "Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.", "files": [ "index.d.ts", - "index.js" + "index.js", + "tokenizers.android-arm64.node", + "tokenizers-android-arm64", + "tokenizers.android-arm-eabi.node", + "tokenizers-android-arm-eabi", + "tokenizers.win32-x64-msvc.node", + "tokenizers-win32-x64-msvc", + "tokenizers.win32-ia32-msvc.node", + "tokenizers-win32-ia32-msvc", + "tokenizers.win32-arm64-msvc.node", + "tokenizers-win32-arm64-msvc", + "tokenizers.darwin-universal.node", + "tokenizers-darwin-universal", + "tokenizers.darwin-x64.node", + "tokenizers-darwin-x64", + "tokenizers.darwin-arm64.node", + "tokenizers-darwin-arm64", + "tokenizers.freebsd-x64.node", + "tokenizers-freebsd-x64", + "tokenizers.linux-x64-musl.node", + "tokenizers-linux-x64-musl", + "tokenizers.linux-x64-gnu.node", + "tokenizers-linux-x64-gnu", + "tokenizers.linux-arm64-musl.node", + "tokenizers-linux-arm64-musl", + "tokenizers.linux-arm64-gnu.node" ], "napi": { "name": "tokenizers", From bc84612493d42892106b17943c60195d5b4b893c Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sat, 16 Mar 2024 12:35:26 -0400 Subject: [PATCH 3/8] chore(node): Add volta package config --- bindings/node/package.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bindings/node/package.json b/bindings/node/package.json index 51da36499..fc30d2076 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -134,5 +134,9 @@ "singleQuote": true, "arrowParens": "always" }, - "packageManager": "yarn@3.5.1" + "packageManager": "yarn@3.5.1", + "volta": { + "node": "20.11.1", + "yarn": "3.5.1" + } } From 38a8a54ce417f98eebcabf8dbccbe94c9483db4c Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sat, 16 Mar 2024 12:36:18 -0400 Subject: [PATCH 4/8] ci(mac): Add Mac ARM runner for node --- .github/workflows/node-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml index f064b8939..8fcaa9964 100644 --- a/.github/workflows/node-release.yml +++ b/.github/workflows/node-release.yml @@ -23,6 +23,8 @@ jobs: target: x86_64-pc-windows-msvc - host: ubuntu-latest target: x86_64-unknown-linux-gnu + - host: flyci-macos-large-latest-m2 + target: arm64-apple-darwin runs-on: ${{ matrix.settings.host }} steps: - name: Checkout repository From 12893181e0382d68a431c25540f83c78990e5f9a Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sat, 16 Mar 2024 13:20:09 -0400 Subject: [PATCH 5/8] build(node): Specify node build version --- .github/workflows/node-release.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml index 8fcaa9964..86a2f709f 100644 --- a/.github/workflows/node-release.yml +++ b/.github/workflows/node-release.yml @@ -1,9 +1,7 @@ name: Node Release env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: us-east-1 + NODE_VERSION: 20 on: push: @@ -48,7 +46,7 @@ jobs: - name: Install Node ${{ matrix.node-version }} uses: actions/setup-node@v4 with: - node-version: latest + node-version: ${{ env.NODE_VERSION }} cache: yarn cache-dependency-path: ./bindings/node/ From 0eabb769ac4fdf64fc9ac345359b439960b57f7b Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sat, 16 Mar 2024 13:31:07 -0400 Subject: [PATCH 6/8] build(node): Change target --- bindings/node/index.js | 28 +++++++++---------- bindings/node/package.json | 9 +++--- bindings/node/yarn.lock | 56 +++++++++++++++++++------------------- 3 files changed, 47 insertions(+), 46 deletions(-) diff --git a/bindings/node/index.js b/bindings/node/index.js index 5af833cf2..6db79f7dc 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -37,7 +37,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.android-arm64.node') } else { - nativeBinding = require('tokenizers-android-arm64') + nativeBinding = require('@turingscript/tokenizers-android-arm64') } } catch (e) { loadError = e @@ -49,7 +49,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.android-arm-eabi.node') } else { - nativeBinding = require('tokenizers-android-arm-eabi') + nativeBinding = require('@turingscript/tokenizers-android-arm-eabi') } } catch (e) { loadError = e @@ -67,7 +67,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.win32-x64-msvc.node') } else { - nativeBinding = require('tokenizers-win32-x64-msvc') + nativeBinding = require('@turingscript/tokenizers-win32-x64-msvc') } } catch (e) { loadError = e @@ -79,7 +79,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.win32-ia32-msvc.node') } else { - nativeBinding = require('tokenizers-win32-ia32-msvc') + nativeBinding = require('@turingscript/tokenizers-win32-ia32-msvc') } } catch (e) { loadError = e @@ -91,7 +91,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.win32-arm64-msvc.node') } else { - nativeBinding = require('tokenizers-win32-arm64-msvc') + nativeBinding = require('@turingscript/tokenizers-win32-arm64-msvc') } } catch (e) { loadError = e @@ -107,7 +107,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.darwin-universal.node') } else { - nativeBinding = require('tokenizers-darwin-universal') + nativeBinding = require('@turingscript/tokenizers-darwin-universal') } break } catch {} @@ -118,7 +118,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.darwin-x64.node') } else { - nativeBinding = require('tokenizers-darwin-x64') + nativeBinding = require('@turingscript/tokenizers-darwin-x64') } } catch (e) { loadError = e @@ -130,7 +130,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.darwin-arm64.node') } else { - nativeBinding = require('tokenizers-darwin-arm64') + nativeBinding = require('@turingscript/tokenizers-darwin-arm64') } } catch (e) { loadError = e @@ -149,7 +149,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.freebsd-x64.node') } else { - nativeBinding = require('tokenizers-freebsd-x64') + nativeBinding = require('@turingscript/tokenizers-freebsd-x64') } } catch (e) { loadError = e @@ -164,7 +164,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-x64-musl.node') } else { - nativeBinding = require('tokenizers-linux-x64-musl') + nativeBinding = require('@turingscript/tokenizers-linux-x64-musl') } } catch (e) { loadError = e @@ -175,7 +175,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-x64-gnu.node') } else { - nativeBinding = require('tokenizers-linux-x64-gnu') + nativeBinding = require('@turingscript/tokenizers-linux-x64-gnu') } } catch (e) { loadError = e @@ -189,7 +189,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-arm64-musl.node') } else { - nativeBinding = require('tokenizers-linux-arm64-musl') + nativeBinding = require('@turingscript/tokenizers-linux-arm64-musl') } } catch (e) { loadError = e @@ -200,7 +200,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-arm64-gnu.node') } else { - nativeBinding = require('tokenizers-linux-arm64-gnu') + nativeBinding = require('@turingscript/tokenizers-linux-arm64-gnu') } } catch (e) { loadError = e @@ -213,7 +213,7 @@ switch (platform) { if (localFileExisted) { nativeBinding = require('./tokenizers.linux-arm-gnueabihf.node') } else { - nativeBinding = require('tokenizers-linux-arm-gnueabihf') + nativeBinding = require('@turingscript/tokenizers-linux-arm-gnueabihf') } } catch (e) { loadError = e diff --git a/bindings/node/package.json b/bindings/node/package.json index fc30d2076..1b2b2df87 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,14 +1,14 @@ { - "name": "tokenizers", + "name": "@turingscript/tokenizers", "version": "0.14.0-dev0", "repository": { "type": "git", - "url": "git+https://github.com/huggingface/tokenizers.git" + "url": "git+https://github.com/turingscript/tokenizers.git" }, "bugs": { - "url": "https://github.com/huggingface/tokenizers/issues" + "url": "https://github.com/turingscript/tokenizers/issues" }, - "homepage": "https://github.com/huggingface/tokenizers/tree/master/bindings/node", + "homepage": "https://github.com/turingscript/tokenizers/tree/master/bindings/node", "author": "Anthony MOI ", "license": "Apache-2.0", "description": "Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.", @@ -51,6 +51,7 @@ "i686-pc-windows-msvc", "armv7-unknown-linux-gnueabihf", "aarch64-apple-darwin", + "arm64-apple-darwin", "aarch64-linux-android", "x86_64-unknown-freebsd", "aarch64-unknown-linux-musl", diff --git a/bindings/node/yarn.lock b/bindings/node/yarn.lock index eb56219cc..c06c9c08b 100644 --- a/bindings/node/yarn.lock +++ b/bindings/node/yarn.lock @@ -1153,6 +1153,34 @@ __metadata: languageName: node linkType: hard +"@turingscript/tokenizers@workspace:.": + version: 0.0.0-use.local + resolution: "@turingscript/tokenizers@workspace:." + dependencies: + "@napi-rs/cli": ^2.14.6 + "@swc-node/register": ^1.5.5 + "@swc/core": ^1.3.32 + "@taplo/cli": ^0.5.2 + "@types/jest": ^29.5.1 + "@typescript-eslint/eslint-plugin": ^5.50.0 + "@typescript-eslint/parser": ^5.50.0 + ava: ^5.1.1 + benny: ^3.7.1 + chalk: ^5.2.0 + eslint: ^8.33.0 + eslint-config-prettier: ^8.6.0 + eslint-plugin-import: ^2.27.5 + eslint-plugin-prettier: ^4.2.1 + husky: ^8.0.3 + jest: ^29.5.0 + lint-staged: ^13.1.0 + npm-run-all: ^4.1.5 + prettier: ^2.8.3 + ts-jest: ^29.1.0 + typescript: ^5.0.0 + languageName: unknown + linkType: soft + "@types/babel__core@npm:^7.1.14": version: 7.20.1 resolution: "@types/babel__core@npm:7.20.1" @@ -6371,34 +6399,6 @@ __metadata: languageName: node linkType: hard -"tokenizers@workspace:.": - version: 0.0.0-use.local - resolution: "tokenizers@workspace:." - dependencies: - "@napi-rs/cli": ^2.14.6 - "@swc-node/register": ^1.5.5 - "@swc/core": ^1.3.32 - "@taplo/cli": ^0.5.2 - "@types/jest": ^29.5.1 - "@typescript-eslint/eslint-plugin": ^5.50.0 - "@typescript-eslint/parser": ^5.50.0 - ava: ^5.1.1 - benny: ^3.7.1 - chalk: ^5.2.0 - eslint: ^8.33.0 - eslint-config-prettier: ^8.6.0 - eslint-plugin-import: ^2.27.5 - eslint-plugin-prettier: ^4.2.1 - husky: ^8.0.3 - jest: ^29.5.0 - lint-staged: ^13.1.0 - npm-run-all: ^4.1.5 - prettier: ^2.8.3 - ts-jest: ^29.1.0 - typescript: ^5.0.0 - languageName: unknown - linkType: soft - "ts-jest@npm:^29.1.0": version: 29.1.1 resolution: "ts-jest@npm:29.1.1" From 464e697f2b20d1c67689ad5428e2893da4ff9ee6 Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sat, 16 Mar 2024 13:32:28 -0400 Subject: [PATCH 7/8] ci: Remove github token for now --- .github/workflows/node-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml index 86a2f709f..d9b9aa76b 100644 --- a/.github/workflows/node-release.yml +++ b/.github/workflows/node-release.yml @@ -104,5 +104,4 @@ jobs: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc npm publish --access public --tag next env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} From 23be0b199b724b7abf913db07314d8eb1c77cc46 Mon Sep 17 00:00:00 2001 From: Aaron Long Date: Sun, 17 Mar 2024 10:22:13 -0400 Subject: [PATCH 8/8] refactor(ci): Clean up more of the script --- .github/workflows/node-release.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml index d9b9aa76b..ed4b62783 100644 --- a/.github/workflows/node-release.yml +++ b/.github/workflows/node-release.yml @@ -43,7 +43,7 @@ jobs: path: ~/.cargo/registry key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }} - - name: Install Node ${{ matrix.node-version }} + - name: Install Node ${{ env.NODE_VERSION }} uses: actions/setup-node@v4 with: node-version: ${{ env.NODE_VERSION }} @@ -60,16 +60,13 @@ jobs: yarn build && strip -x *.node - - name: Install Python - uses: actions/setup-python@v1 - with: - python-version: 3.x - name: Upload artifact uses: actions/upload-artifact@v4 with: name: bindings-${{ matrix.settings.target }} - path: ${{ env.APP_NAME }}bindings/node/*.node + path: ./bindings/node/*.node if-no-files-found: error + publish: name: Publish runs-on: ubuntu-latest