♻️ Make "go to source definition" work (#658)

Sorry for all the codeowners reviews

Make "go to source definition" work by using the package, thanks to
`.d.ts.map` files generated by `tsc`.

Defer type generation to `tsc` instead of `tsup`, since `tsup` is not
able to generate `.d.ts.map`. Also removed `shared` package - since we
don't want to publish it, having "go to source" with it would have been
problematic. Instead, "shared" files are duplicated between
`@huggingface/inference` and `@huggingface/hub`.

Maybe later we can create `@huggingface/webblob` and
`@huggingface/fileblob` if we want to reuse them.

See also: https://www.npmjs.com/package/dts-buddy used by svelte-kit

.github/workflows/hub-publish.yml

@@ -52,6 +52,44 @@ jobs:
           git add ../..
           git commit -m "🔖 @hugginface/hub $BUMPED_VERSION"
           git tag "hub-v$BUMPED_VERSION"
+
+      - name: Make sure that the latest version of @huggingface/tasks is consistent with the local version
+        run: |
+          LOCAL_TASKS_VERSION=$(node -p "require('./package.json').version")
+          REMOTE_TASKS_VERSION=$(npm view @huggingface/tasks version)
+
+          # If the versions are different, error 
+          if [ "$LOCAL_TASKS_VERSION" != "$REMOTE_TASKS_VERSION" ]; then
+            echo "Error: The local @huggingface/tasks package version ($LOCAL_TASKS_VERSION) differs from the remote version ($REMOTE_TASKS_VERSION). Release halted."
+            exit 1
+          fi
+
+          npm pack @huggingface/tasks
+          mv huggingface-tasks-$LOCAL_TASKS_VERSION.tgz tasks-local.tgz
+
+          npm pack @huggingface/tasks@$REMOTE_TASKS_VERSION
+          mv huggingface-tasks-$REMOTE_TASKS_VERSION.tgz tasks-remote.tgz
+
+          # Compute checksum of local tar. We need to extract both tar since the remote compression might be different
+          tar -xf tasks-local.tgz
+          LOCAL_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          echo "Local package checksum: $LOCAL_CHECKSUM"
+
+          rm -Rf package
+
+          tar -xf tasks-remote.tgz
+          REMOTE_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          echo "Remote package checksum: $REMOTE_CHECKSUM"
+
+          rm -Rf package
+
+          if [ "$LOCAL_CHECKSUM" != "$REMOTE_CHECKSUM" ]; then
+            echo "Checksum Verification Failed: The local @huggingface/tasks package differs from the remote version. Release halted. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM"
+            exit 1
+          fi
+          echo "Checksum Verification Successful: The local and remote @huggingface/tasks packages are consistent. Proceeding with the @huggingface/widgets package release. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM."
+        working-directory: packages/tasks
+
       - run: pnpm publish --no-git-checks .
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/test.yml

@@ -6,7 +6,7 @@ on:
       - main
 
 jobs:
-  test:
+  node:
     runs-on: ubuntu-latest
     timeout-minutes: 10
 
@@ -42,11 +42,57 @@ jobs:
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
+  browser:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Could use something like rmacklin/fetch-through-merge-base@v0 instead when the repo gets bigger
+          fetch-depth: 0
+      - name: "Extracting the merge base into 'SINCE'"
+        id: since
+        run: |
+          if [ -z "${{  github.event.pull_request.head.ref }}" ]
+          then
+            echo "SINCE=${{ github.sha }}^1" >> $GITHUB_OUTPUT
+          else
+            echo "SINCE=$(git merge-base origin/${{ github.event.pull_request.base.ref }} ${{ github.sha }})" >> $GITHUB_OUTPUT
+          fi
+
+      - run: corepack enable
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: "**/pnpm-lock.yaml"
+      - run: |
+          pnpm install --frozen-lockfile --filter .
+          pnpm install --frozen-lockfile --filter ...[${{ steps.since.outputs.SINCE }}]...
+          pnpm --filter ...[${{ steps.since.outputs.SINCE }}]... build
+
       - name: Test in browser
         run: VCR_MODE=playback pnpm --filter ...[${{ steps.since.outputs.SINCE }}] test:browser
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
 
+  e2e:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - run: corepack enable
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: "**/pnpm-lock.yaml"
+
       - name: E2E - start mock npm registry
         run: |
           npm i -g verdaccio verdaccio-memory verdaccio-auth-memory
@@ -58,7 +104,7 @@ jobs:
         working-directory: e2e
         run: |
           sleep 3
-          pnpm i --filter inference... --filter hub... --frozen-lockfile
+          pnpm i --filter root --filter inference... --filter hub... --frozen-lockfile
           pnpm --filter inference --filter hub --filter tasks publish --force --no-git-checks --registry http://localhost:4874/
 
       - name: E2E test - test yarn install

e2e/.gitignore

@@ -0,0 +1,2 @@
+# Modified during E2E tests
+.npmrc

package.json

@@ -1,4 +1,6 @@
 {
+	"private": true,
+	"name": "@huggingface/root",
 	"license": "MIT",
 	"packageManager": "[email protected]",
 	"scripts": {

packages/agents/package.json

@@ -31,7 +31,7 @@
 		"format": "prettier --write .",
 		"format:check": "prettier --check .",
 		"prepublishOnly": "pnpm run build",
-		"build": "tsup",
+		"build": "tsup && tsc --emitDeclarationOnly --declaration",
 		"test": "vitest run",
 		"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts",
 		"check": "tsc"
@@ -53,8 +53,7 @@
 	"author": "Hugging Face",
 	"license": "MIT",
 	"devDependencies": {
-		"@types/node": "^18.13.0",
-		"type-fest": "^3.9.0"
+		"@types/node": "^18.13.0"
 	},
 	"dependencies": {
 		"@huggingface/inference": "^2.6.1"

packages/agents/tsconfig.json

@@ -12,8 +12,9 @@
 		"skipLibCheck": true,
 		"noImplicitOverride": true,
 		"outDir": "./dist",
-		"declaration": true
+		"declaration": true,
+		"declarationMap": true
 	},
-	"include": ["src", "test", "index.ts", "../shared/src"],
+	"include": ["src", "test", "index.ts"],
 	"exclude": ["dist"]
 }

packages/agents/tsup.config.ts

@@ -5,9 +5,6 @@ const baseConfig: Options = {
 	format: ["cjs", "esm"],
 	outDir: "dist",
 	clean: true,
-	dts: {
-		resolve: true,
-	},
 };
 
 const nodeConfig: Options = {

packages/gguf/.eslintignore

@@ -0,0 +1 @@
+dist

packages/gguf/package.json

@@ -31,7 +31,7 @@
 		"format": "prettier --write .",
 		"format:check": "prettier --check .",
 		"prepublishOnly": "pnpm run build",
-		"build": "tsup src/index.ts --format cjs,esm --clean --dts",
+		"build": "tsup src/index.ts --format cjs,esm --clean && tsc --emitDeclarationOnly --declaration",
 		"build:llm": "tsx scripts/generate-llm.ts && pnpm run format",
 		"test": "vitest run",
 		"check": "tsc"
@@ -47,8 +47,5 @@
 		"gguf"
 	],
 	"author": "Hugging Face",
-	"license": "MIT",
-	"devDependencies": {
-		"type-fest": "^3.9.0"
-	}
+	"license": "MIT"
 }

packages/gguf/tsconfig.json

@@ -11,7 +11,9 @@
 		"strictNullChecks": true,
 		"skipLibCheck": true,
 		"noImplicitOverride": true,
-		"outDir": "./dist"
+		"outDir": "./dist",
+		"declaration": true,
+		"declarationMap": true
 	},
 	"include": ["src"],
 	"exclude": ["dist"]

packages/gguf/tsup.config.ts

@@ -5,9 +5,6 @@ const baseConfig: Options = {
 	format: ["cjs", "esm"],
 	outDir: "dist",
 	clean: true,
-	dts: {
-		resolve: true,
-	},
 };
 
 const nodeConfig: Options = {

packages/hub/README.md

@@ -122,4 +122,4 @@ Under the hood, `@huggingface/hub` uses a lazy blob implementation to load the f
 ## Dependencies
 
 - `hash-wasm` : Only used in the browser, when committing files over 10 MB. Browsers do not natively support streaming sha256 computations.
-- `type-fest` : Typings only
+- `@huggingface/tasks` : Typings only

packages/hub/package.json

@@ -33,7 +33,7 @@
 		"format": "prettier --write .",
 		"format:check": "prettier --check .",
 		"prepublishOnly": "pnpm run build",
-		"build": "tsup",
+		"build": "tsup && tsc --emitDeclarationOnly --declaration",
 		"prepare": "pnpm run build",
 		"test": "vitest run",
 		"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts",
@@ -56,11 +56,10 @@
 	"author": "Hugging Face",
 	"license": "MIT",
 	"devDependencies": {
-		"@huggingface/tasks": "workspace:^",
-		"@types/node": "^20.11.28",
-		"type-fest": "^3.9.0"
+		"@types/node": "^20.11.28"
 	},
 	"dependencies": {
+		"@huggingface/tasks": "workspace:^",
 		"hash-wasm": "^4.9.0"
 	}
 }

packages/hub/src/error.ts

@@ -1,4 +1,4 @@
-import type { JsonObject } from "type-fest";
+import type { JsonObject } from "./vendor/type-fest/basic";
 
 export async function createApiError(
 	response: Response,

packages/hub/src/lib/commit.spec.ts

@@ -9,7 +9,7 @@ import { deleteRepo } from "./delete-repo";
 import { downloadFile } from "./download-file";
 import { fileDownloadInfo } from "./file-download-info";
 import { insecureRandomString } from "../utils/insecureRandomString";
-import { isFrontend } from "../../../shared";
+import { isFrontend } from "../utils/isFrontend";
 
 const lfsContent = "O123456789".repeat(100_000);
 

packages/hub/src/lib/commit.ts

@@ -1,4 +1,3 @@
-import { isFrontend, base64FromBytes } from "../../../shared";
 import { HUB_URL } from "../consts";
 import { HubApiError, createApiError, InvalidApiResponseFormatError } from "../error";
 import type {
@@ -21,6 +20,8 @@ import { toRepoId } from "../utils/toRepoId";
 import { WebBlob } from "../utils/WebBlob";
 import { createBlob } from "../utils/createBlob";
 import { eventToGenerator } from "../utils/eventToGenerator";
+import { base64FromBytes } from "../utils/base64FromBytes";
+import { isFrontend } from "../utils/isFrontend";
 
 const CONCURRENT_SHAS = 5;
 const CONCURRENT_LFS_UPLOADS = 5;

packages/hub/src/lib/create-repo.ts

@@ -2,7 +2,7 @@ import { HUB_URL } from "../consts";
 import { createApiError } from "../error";
 import type { ApiCreateRepoPayload } from "../types/api/api-create-repo";
 import type { Credentials, RepoDesignation, SpaceSdk } from "../types/public";
-import { base64FromBytes } from "../../../shared";
+import { base64FromBytes } from "../utils/base64FromBytes";
 import { checkCredentials } from "../utils/checkCredentials";
 import { toRepoId } from "../utils/toRepoId";
 

packages/hub/src/lib/oauth-login-url.ts

@@ -1,6 +1,6 @@
-import { base64FromBytes } from "../../../shared/src";
 import { HUB_URL } from "../consts";
 import { createApiError } from "../error";
+import { base64FromBytes } from "../utils/base64FromBytes";
 
 /**
  * Use "Sign in with Hub" to authenticate a user, and get oauth user info / access token.

packages/hub/src/lib/parse-safetensors-metadata.ts

@@ -1,4 +1,3 @@
-import type { SetRequired } from "type-fest";
 import type { Credentials, RepoDesignation } from "../types/public";
 import { checkCredentials } from "../utils/checkCredentials";
 import { omit } from "../utils/omit";
@@ -7,6 +6,7 @@ import { typedEntries } from "../utils/typedEntries";
 import { downloadFile } from "./download-file";
 import { fileExists } from "./file-exists";
 import { promisesQueue } from "../utils/promisesQueue";
+import type { SetRequired } from "../vendor/type-fest/set-required";
 
 export const SAFETENSORS_FILE = "model.safetensors";
 export const SAFETENSORS_INDEX_FILE = "model.safetensors.index.json";

packages/hub/src/types/api/api-commit.d.ts → packages/hub/src/types/api/api-commit.ts

@@ -149,7 +149,7 @@ export interface ApiCommitDeletedEntry {
 	path: string;
 }
 
-interface ApiCommitLfsFile {
+export interface ApiCommitLfsFile {
 	path: string;
 	oldPath?: string;
 	/** Required if {@link oldPath} is not set */

packages/hub/src/types/api/api-create-repo.d.ts → packages/hub/src/types/api/api-create-repo.ts

@@ -1,5 +1,5 @@
-import type { SetRequired } from "type-fest";
-import type { SpaceHardwareFlavor, SpaceSdk } from "../public";
+import type { SetRequired } from "../../vendor/type-fest/set-required";
+import type { RepoType, SpaceHardwareFlavor, SpaceSdk } from "../public";
 import type { ApiCommitFile } from "./api-commit";
 
 export type ApiCreateRepoPayload = {