Merge pull request #1 from aaronclong/fork-node-packing

build(node): Include binaries in NPM packing
turingscript · Mar 17, 2024 · 4685b8f · 4685b8f
2 parents 29fef1e + 23be0b1
commit 4685b8f
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 66 deletions.
diff --git a/.github/workflows/node-release.yml b/.github/workflows/node-release.yml
@@ -1,9 +1,7 @@
 name: Node Release
 
 env:
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  AWS_DEFAULT_REGION: us-east-1
+  NODE_VERSION: 20
 
 on:
   push:
@@ -23,10 +21,12 @@ jobs:
             target: x86_64-pc-windows-msvc
           - host: ubuntu-latest
             target: x86_64-unknown-linux-gnu
+          - host: flyci-macos-large-latest-m2
+            target: arm64-apple-darwin
     runs-on: ${{ matrix.settings.host }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
 
       - name: Install Rust
         uses: actions-rs/toolchain@v1
@@ -38,15 +38,15 @@ jobs:
         run: sudo chown -R $(whoami):$(id -ng) ~/.cargo/
 
       - name: Cache Cargo Registry
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cargo/registry
           key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
 
-      - name: Install Node ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+      - name: Install Node ${{ env.NODE_VERSION }}
+        uses: actions/setup-node@v4
         with:
-          node-version: latest
+          node-version: ${{ env.NODE_VERSION }}
           cache: yarn
           cache-dependency-path: ./bindings/node/
 
@@ -60,25 +60,22 @@ jobs:
           yarn build &&
           strip -x *.node
 
-      - name: Install Python
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.x
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: bindings-${{ matrix.settings.target }}
-          path: ${{ env.APP_NAME }}bindings/node/*.node
+          path: ./bindings/node/*.node
           if-no-files-found: error
+
   publish:
     name: Publish
     runs-on: ubuntu-latest
     needs:
       - build
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: latest
           check-latest: true
@@ -88,7 +85,7 @@ jobs:
         working-directory: ./bindings/node
         run: yarn install
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: ./bindings/node/artifacts
       - name: Move artifacts
@@ -104,5 +101,4 @@ jobs:
           echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
           npm publish --access public --tag next
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
diff --git a/bindings/node/index.js b/bindings/node/index.js
@@ -37,7 +37,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.android-arm64.node')
           } else {
-            nativeBinding = require('tokenizers-android-arm64')
+            nativeBinding = require('@turingscript/tokenizers-android-arm64')
           }
         } catch (e) {
           loadError = e
@@ -49,7 +49,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.android-arm-eabi.node')
           } else {
-            nativeBinding = require('tokenizers-android-arm-eabi')
+            nativeBinding = require('@turingscript/tokenizers-android-arm-eabi')
           }
         } catch (e) {
           loadError = e
@@ -67,7 +67,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.win32-x64-msvc.node')
           } else {
-            nativeBinding = require('tokenizers-win32-x64-msvc')
+            nativeBinding = require('@turingscript/tokenizers-win32-x64-msvc')
           }
         } catch (e) {
           loadError = e
@@ -79,7 +79,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.win32-ia32-msvc.node')
           } else {
-            nativeBinding = require('tokenizers-win32-ia32-msvc')
+            nativeBinding = require('@turingscript/tokenizers-win32-ia32-msvc')
           }
         } catch (e) {
           loadError = e
@@ -91,7 +91,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.win32-arm64-msvc.node')
           } else {
-            nativeBinding = require('tokenizers-win32-arm64-msvc')
+            nativeBinding = require('@turingscript/tokenizers-win32-arm64-msvc')
           }
         } catch (e) {
           loadError = e
@@ -107,7 +107,7 @@ switch (platform) {
       if (localFileExisted) {
         nativeBinding = require('./tokenizers.darwin-universal.node')
       } else {
-        nativeBinding = require('tokenizers-darwin-universal')
+        nativeBinding = require('@turingscript/tokenizers-darwin-universal')
       }
       break
     } catch {}
@@ -118,7 +118,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.darwin-x64.node')
           } else {
-            nativeBinding = require('tokenizers-darwin-x64')
+            nativeBinding = require('@turingscript/tokenizers-darwin-x64')
           }
         } catch (e) {
           loadError = e
@@ -130,7 +130,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.darwin-arm64.node')
           } else {
-            nativeBinding = require('tokenizers-darwin-arm64')
+            nativeBinding = require('@turingscript/tokenizers-darwin-arm64')
           }
         } catch (e) {
           loadError = e
@@ -149,7 +149,7 @@ switch (platform) {
       if (localFileExisted) {
         nativeBinding = require('./tokenizers.freebsd-x64.node')
       } else {
-        nativeBinding = require('tokenizers-freebsd-x64')
+        nativeBinding = require('@turingscript/tokenizers-freebsd-x64')
       }
     } catch (e) {
       loadError = e
@@ -164,7 +164,7 @@ switch (platform) {
             if (localFileExisted) {
               nativeBinding = require('./tokenizers.linux-x64-musl.node')
             } else {
-              nativeBinding = require('tokenizers-linux-x64-musl')
+              nativeBinding = require('@turingscript/tokenizers-linux-x64-musl')
             }
           } catch (e) {
             loadError = e
@@ -175,7 +175,7 @@ switch (platform) {
             if (localFileExisted) {
               nativeBinding = require('./tokenizers.linux-x64-gnu.node')
             } else {
-              nativeBinding = require('tokenizers-linux-x64-gnu')
+              nativeBinding = require('@turingscript/tokenizers-linux-x64-gnu')
             }
           } catch (e) {
             loadError = e
@@ -189,7 +189,7 @@ switch (platform) {
             if (localFileExisted) {
               nativeBinding = require('./tokenizers.linux-arm64-musl.node')
             } else {
-              nativeBinding = require('tokenizers-linux-arm64-musl')
+              nativeBinding = require('@turingscript/tokenizers-linux-arm64-musl')
             }
           } catch (e) {
             loadError = e
@@ -200,7 +200,7 @@ switch (platform) {
             if (localFileExisted) {
               nativeBinding = require('./tokenizers.linux-arm64-gnu.node')
             } else {
-              nativeBinding = require('tokenizers-linux-arm64-gnu')
+              nativeBinding = require('@turingscript/tokenizers-linux-arm64-gnu')
             }
           } catch (e) {
             loadError = e
@@ -213,7 +213,7 @@ switch (platform) {
           if (localFileExisted) {
             nativeBinding = require('./tokenizers.linux-arm-gnueabihf.node')
           } else {
-            nativeBinding = require('tokenizers-linux-arm-gnueabihf')
+            nativeBinding = require('@turingscript/tokenizers-linux-arm-gnueabihf')
           }
         } catch (e) {
           loadError = e

diff --git a/bindings/node/package.json b/bindings/node/package.json
@@ -1,20 +1,45 @@
 {
-  "name": "tokenizers",
+  "name": "@turingscript/tokenizers",
   "version": "0.14.0-dev0",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/huggingface/tokenizers.git"
+    "url": "git+https://github.com/turingscript/tokenizers.git"
   },
   "bugs": {
-    "url": "https://github.com/huggingface/tokenizers/issues"
+    "url": "https://github.com/turingscript/tokenizers/issues"
   },
-  "homepage": "https://github.com/huggingface/tokenizers/tree/master/bindings/node",
+  "homepage": "https://github.com/turingscript/tokenizers/tree/master/bindings/node",
   "author": "Anthony MOI <[email protected]>",
   "license": "Apache-2.0",
   "description": "Provides an implementation of today's most used tokenizers, with a focus on performances and versatility.",
   "files": [
     "index.d.ts",
-    "index.js"
+    "index.js",
+    "tokenizers.android-arm64.node",
+    "tokenizers-android-arm64",
+    "tokenizers.android-arm-eabi.node",
+    "tokenizers-android-arm-eabi",
+    "tokenizers.win32-x64-msvc.node",
+    "tokenizers-win32-x64-msvc",
+    "tokenizers.win32-ia32-msvc.node",
+    "tokenizers-win32-ia32-msvc",
+    "tokenizers.win32-arm64-msvc.node",
+    "tokenizers-win32-arm64-msvc",
+    "tokenizers.darwin-universal.node",
+    "tokenizers-darwin-universal",
+    "tokenizers.darwin-x64.node",
+    "tokenizers-darwin-x64",
+    "tokenizers.darwin-arm64.node",
+    "tokenizers-darwin-arm64",
+    "tokenizers.freebsd-x64.node",
+    "tokenizers-freebsd-x64",
+    "tokenizers.linux-x64-musl.node",
+    "tokenizers-linux-x64-musl",
+    "tokenizers.linux-x64-gnu.node",
+    "tokenizers-linux-x64-gnu",
+    "tokenizers.linux-arm64-musl.node",
+    "tokenizers-linux-arm64-musl",
+    "tokenizers.linux-arm64-gnu.node"
   ],
   "napi": {
     "name": "tokenizers",
@@ -26,6 +51,7 @@
         "i686-pc-windows-msvc",
         "armv7-unknown-linux-gnueabihf",
         "aarch64-apple-darwin",
+        "arm64-apple-darwin",
         "aarch64-linux-android",
         "x86_64-unknown-freebsd",
         "aarch64-unknown-linux-musl",
@@ -109,5 +135,9 @@
     "singleQuote": true,
     "arrowParens": "always"
   },
-  "packageManager": "[email protected]"
+  "packageManager": "[email protected]",
+  "volta": {
+    "node": "20.11.1",
+    "yarn": "3.5.1"
+  }
 }
diff --git a/bindings/node/yarn.lock b/bindings/node/yarn.lock
@@ -1153,6 +1153,34 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@turingscript/tokenizers@workspace:.":
+  version: 0.0.0-use.local
+  resolution: "@turingscript/tokenizers@workspace:."
+  dependencies:
+    "@napi-rs/cli": ^2.14.6
+    "@swc-node/register": ^1.5.5
+    "@swc/core": ^1.3.32
+    "@taplo/cli": ^0.5.2
+    "@types/jest": ^29.5.1
+    "@typescript-eslint/eslint-plugin": ^5.50.0
+    "@typescript-eslint/parser": ^5.50.0
+    ava: ^5.1.1
+    benny: ^3.7.1
+    chalk: ^5.2.0
+    eslint: ^8.33.0
+    eslint-config-prettier: ^8.6.0
+    eslint-plugin-import: ^2.27.5
+    eslint-plugin-prettier: ^4.2.1
+    husky: ^8.0.3
+    jest: ^29.5.0
+    lint-staged: ^13.1.0
+    npm-run-all: ^4.1.5
+    prettier: ^2.8.3
+    ts-jest: ^29.1.0
+    typescript: ^5.0.0
+  languageName: unknown
+  linkType: soft
+
 "@types/babel__core@npm:^7.1.14":
   version: 7.20.1
   resolution: "@types/babel__core@npm:7.20.1"
@@ -6371,34 +6399,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"tokenizers@workspace:.":
-  version: 0.0.0-use.local
-  resolution: "tokenizers@workspace:."
-  dependencies:
-    "@napi-rs/cli": ^2.14.6
-    "@swc-node/register": ^1.5.5
-    "@swc/core": ^1.3.32
-    "@taplo/cli": ^0.5.2
-    "@types/jest": ^29.5.1
-    "@typescript-eslint/eslint-plugin": ^5.50.0
-    "@typescript-eslint/parser": ^5.50.0
-    ava: ^5.1.1
-    benny: ^3.7.1
-    chalk: ^5.2.0
-    eslint: ^8.33.0
-    eslint-config-prettier: ^8.6.0
-    eslint-plugin-import: ^2.27.5
-    eslint-plugin-prettier: ^4.2.1
-    husky: ^8.0.3
-    jest: ^29.5.0
-    lint-staged: ^13.1.0
-    npm-run-all: ^4.1.5
-    prettier: ^2.8.3
-    ts-jest: ^29.1.0
-    typescript: ^5.0.0
-  languageName: unknown
-  linkType: soft
-
 "ts-jest@npm:^29.1.0":
   version: 29.1.1
   resolution: "ts-jest@npm:29.1.1"