Skip to content

Commit

Permalink
run prettier everywhere
Browse files Browse the repository at this point in the history
  • Loading branch information
coyotte508 committed Dec 3, 2024
1 parent ae7979b commit de048e0
Show file tree
Hide file tree
Showing 16 changed files with 166 additions and 141 deletions.
1 change: 0 additions & 1 deletion .github/workflows/dduf-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ jobs:
git commit . -m "🔖 @huggingface/dduf $BUMPED_VERSION"
git tag "dduf-v$BUMPED_VERSION"
- name: "Check Deps are published before publishing this package"
run: pnpm -w check-deps blob

Expand Down
5 changes: 1 addition & 4 deletions packages/dduf/src/check-dduf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,7 @@ export interface DDUFFileEntry {
fileHeaderOffset: number;
}

export async function* checkDDUF(
url: Blob | URL,
opts?: { log?: (x: string) => void }
): AsyncGenerator<DDUFFileEntry> {
export async function* checkDDUF(url: Blob | URL, opts?: { log?: (x: string) => void }): AsyncGenerator<DDUFFileEntry> {
const blob = url instanceof Blob ? url : await createBlob(url);

opts?.log?.("File size: " + blob.size);
Expand Down
2 changes: 1 addition & 1 deletion packages/hub/src/lib/cache-management.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ const FILES_TO_IGNORE: string[] = [".DS_Store"];
export const REPO_ID_SEPARATOR: string = "--";

export function getRepoFolderName({ name, type }: RepoId): string {
const parts = [`${type}s`, ...name.split("/")]
const parts = [`${type}s`, ...name.split("/")];
return parts.join(REPO_ID_SEPARATOR);
}

Expand Down
10 changes: 5 additions & 5 deletions packages/hub/src/lib/dataset-info.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ describe("datasetInfo", () => {
});

it("should return the dataset info with author", async () => {
const info: DatasetEntry & Pick<ApiDatasetInfo, 'author'> = await datasetInfo({
const info: DatasetEntry & Pick<ApiDatasetInfo, "author"> = await datasetInfo({
name: "nyu-mll/glue",
additionalFields: ['author'],
additionalFields: ["author"],
});
expect(info).toEqual({
id: "621ffdd236468d709f181e3f",
Expand All @@ -32,12 +32,12 @@ describe("datasetInfo", () => {
updatedAt: expect.any(Date),
likes: expect.any(Number),
private: false,
author: 'nyu-mll'
author: "nyu-mll",
});
});

it("should return the dataset info for a specific revision", async () => {
const info: DatasetEntry & Pick<ApiDatasetInfo, 'sha'> = await datasetInfo({
const info: DatasetEntry & Pick<ApiDatasetInfo, "sha"> = await datasetInfo({
name: "nyu-mll/glue",
revision: "cb2099c76426ff97da7aa591cbd317d91fb5fcb7",
additionalFields: ["sha"],
Expand All @@ -50,7 +50,7 @@ describe("datasetInfo", () => {
updatedAt: expect.any(Date),
likes: expect.any(Number),
private: false,
sha: 'cb2099c76426ff97da7aa591cbd317d91fb5fcb7'
sha: "cb2099c76426ff97da7aa591cbd317d91fb5fcb7",
});
});
});
4 changes: 3 additions & 1 deletion packages/hub/src/lib/dataset-info.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ export async function datasetInfo<
]).toString();

const response = await (params.fetch || fetch)(
`${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent(params.revision ?? "HEAD")}?${search.toString()}`,
`${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent(
params.revision ?? "HEAD"
)}?${search.toString()}`,
{
headers: {
...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}),
Expand Down
152 changes: 83 additions & 69 deletions packages/hub/src/lib/download-file-to-cache-dir.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,90 +8,98 @@ import { getHFHubCachePath, getRepoFolderName } from "./cache-management";
import { toRepoId } from "../utils/toRepoId";
import { downloadFileToCacheDir } from "./download-file-to-cache-dir";

vi.mock('node:fs/promises', () => ({
vi.mock("node:fs/promises", () => ({
writeFile: vi.fn(),
rename: vi.fn(),
symlink: vi.fn(),
lstat: vi.fn(),
mkdir: vi.fn(),
stat: vi.fn()
stat: vi.fn(),
}));

vi.mock('./paths-info', () => ({
vi.mock("./paths-info", () => ({
pathsInfo: vi.fn(),
}));

const DUMMY_REPO: RepoId = {
name: 'hello-world',
type: 'model',
name: "hello-world",
type: "model",
};

const DUMMY_ETAG = "dummy-etag";

// utility test method to get blob file path
function _getBlobFile(params: {
function _getBlobFile(params: {
repo: RepoDesignation;
etag: string;
cacheDir?: string, // default to {@link getHFHubCache}
cacheDir?: string; // default to {@link getHFHubCache}
}) {
return join(params.cacheDir ?? getHFHubCachePath(), getRepoFolderName(toRepoId(params.repo)), "blobs", params.etag);
}

// utility test method to get snapshot file path
function _getSnapshotFile(params: {
function _getSnapshotFile(params: {
repo: RepoDesignation;
path: string;
revision : string;
cacheDir?: string, // default to {@link getHFHubCache}
revision: string;
cacheDir?: string; // default to {@link getHFHubCache}
}) {
return join(params.cacheDir ?? getHFHubCachePath(), getRepoFolderName(toRepoId(params.repo)), "snapshots", params.revision, params.path);
return join(
params.cacheDir ?? getHFHubCachePath(),
getRepoFolderName(toRepoId(params.repo)),
"snapshots",
params.revision,
params.path
);
}

describe('downloadFileToCacheDir', () => {
describe("downloadFileToCacheDir", () => {
const fetchMock: typeof fetch = vi.fn();
beforeEach(() => {
vi.resetAllMocks();
// mock 200 request
vi.mocked(fetchMock).mockResolvedValue({
status: 200,
ok: true,
body: 'dummy-body'
body: "dummy-body",
} as unknown as Response);

// prevent to use caching
vi.mocked(stat).mockRejectedValue(new Error('Do not exists'));
vi.mocked(lstat).mockRejectedValue(new Error('Do not exists'));
vi.mocked(stat).mockRejectedValue(new Error("Do not exists"));
vi.mocked(lstat).mockRejectedValue(new Error("Do not exists"));
});

test('should throw an error if fileDownloadInfo return nothing', async () => {
test("should throw an error if fileDownloadInfo return nothing", async () => {
await expect(async () => {
await downloadFileToCacheDir({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
fetch: fetchMock,
});
}).rejects.toThrowError('cannot get path info for /README.md');
}).rejects.toThrowError("cannot get path info for /README.md");

expect(pathsInfo).toHaveBeenCalledWith(expect.objectContaining({
repo: DUMMY_REPO,
paths: ['/README.md'],
fetch: fetchMock,
}));
expect(pathsInfo).toHaveBeenCalledWith(
expect.objectContaining({
repo: DUMMY_REPO,
paths: ["/README.md"],
fetch: fetchMock,
})
);
});

test('existing symlinked and blob should not re-download it', async () => {
test("existing symlinked and blob should not re-download it", async () => {
// <cache>/<repo>/<revision>/snapshots/README.md
const expectPointer = _getSnapshotFile({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7",
});
// stat ensure a symlink and the pointed file exists
vi.mocked(stat).mockResolvedValue({} as Stats) // prevent default mocked reject
vi.mocked(stat).mockResolvedValue({} as Stats); // prevent default mocked reject

const output = await downloadFileToCacheDir({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
fetch: fetchMock,
revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7",
});
Expand All @@ -100,17 +108,17 @@ describe('downloadFileToCacheDir', () => {
// Get call argument for stat
const starArg = vi.mocked(stat).mock.calls[0][0];

expect(starArg).toBe(expectPointer)
expect(starArg).toBe(expectPointer);
expect(fetchMock).not.toHaveBeenCalledWith();

expect(output).toBe(expectPointer);
});

test('existing blob should only create the symlink', async () => {
test("existing blob should only create the symlink", async () => {
// <cache>/<repo>/<revision>/snapshots/README.md
const expectPointer = _getSnapshotFile({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
revision: "dummy-commit-hash",
});
// <cache>/<repo>/blobs/<etag>
Expand All @@ -122,21 +130,23 @@ describe('downloadFileToCacheDir', () => {
// mock existing blob only no symlink
vi.mocked(lstat).mockResolvedValue({} as Stats);
// mock pathsInfo resolve content
vi.mocked(pathsInfo).mockResolvedValue([{
oid: DUMMY_ETAG,
size: 55,
path: 'README.md',
type: 'file',
lastCommit: {
date: new Date(),
id: 'dummy-commit-hash',
title: 'Commit msg',
vi.mocked(pathsInfo).mockResolvedValue([
{
oid: DUMMY_ETAG,
size: 55,
path: "README.md",
type: "file",
lastCommit: {
date: new Date(),
id: "dummy-commit-hash",
title: "Commit msg",
},
},
}]);
]);

const output = await downloadFileToCacheDir({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
fetch: fetchMock,
});

Expand All @@ -153,11 +163,11 @@ describe('downloadFileToCacheDir', () => {
expect(output).toBe(expectPointer);
});

test('expect resolve value to be the pointer path of downloaded file', async () => {
test("expect resolve value to be the pointer path of downloaded file", async () => {
// <cache>/<repo>/<revision>/snapshots/README.md
const expectPointer = _getSnapshotFile({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
revision: "dummy-commit-hash",
});
// <cache>/<repo>/blobs/<etag>
Expand All @@ -166,21 +176,23 @@ describe('downloadFileToCacheDir', () => {
etag: DUMMY_ETAG,
});

vi.mocked(pathsInfo).mockResolvedValue([{
oid: DUMMY_ETAG,
size: 55,
path: 'README.md',
type: 'file',
lastCommit: {
date: new Date(),
id: 'dummy-commit-hash',
title: 'Commit msg',
vi.mocked(pathsInfo).mockResolvedValue([
{
oid: DUMMY_ETAG,
size: 55,
path: "README.md",
type: "file",
lastCommit: {
date: new Date(),
id: "dummy-commit-hash",
title: "Commit msg",
},
},
}]);
]);

const output = await downloadFileToCacheDir({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
fetch: fetchMock,
});

Expand All @@ -191,11 +203,11 @@ describe('downloadFileToCacheDir', () => {
expect(output).toBe(expectPointer);
});

test('should write fetch response to blob', async () => {
test("should write fetch response to blob", async () => {
// <cache>/<repo>/<revision>/snapshots/README.md
const expectPointer = _getSnapshotFile({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
revision: "dummy-commit-hash",
});
// <cache>/<repo>/blobs/<etag>
Expand All @@ -205,30 +217,32 @@ describe('downloadFileToCacheDir', () => {
});

// mock pathsInfo resolve content
vi.mocked(pathsInfo).mockResolvedValue([{
oid: DUMMY_ETAG,
size: 55,
path: 'README.md',
type: 'file',
lastCommit: {
date: new Date(),
id: 'dummy-commit-hash',
title: 'Commit msg',
vi.mocked(pathsInfo).mockResolvedValue([
{
oid: DUMMY_ETAG,
size: 55,
path: "README.md",
type: "file",
lastCommit: {
date: new Date(),
id: "dummy-commit-hash",
title: "Commit msg",
},
},
}]);
]);

await downloadFileToCacheDir({
repo: DUMMY_REPO,
path: '/README.md',
path: "/README.md",
fetch: fetchMock,
});

const incomplete = `${expectedBlob}.incomplete`;
// 1. should write fetch#response#body to incomplete file
expect(writeFile).toHaveBeenCalledWith(incomplete, 'dummy-body');
expect(writeFile).toHaveBeenCalledWith(incomplete, "dummy-body");
// 2. should rename the incomplete to the blob expected name
expect(rename).toHaveBeenCalledWith(incomplete, expectedBlob);
// 3. should create symlink pointing to blob
expect(symlink).toHaveBeenCalledWith(expectedBlob, expectPointer);
});
});
});
4 changes: 2 additions & 2 deletions packages/hub/src/lib/download-file-to-cache-dir.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ function getFilePointer(storageFolder: string, revision: string, relativeFilenam
*/
async function exists(path: string, followSymlinks?: boolean): Promise<boolean> {
try {
if(followSymlinks) {
if (followSymlinks) {
await stat(path);
} else {
await lstat(path);
Expand Down Expand Up @@ -54,7 +54,7 @@ export async function downloadFileToCacheDir(
*/
revision?: string;
hubUrl?: string;
cacheDir?: string,
cacheDir?: string;
/**
* Custom fetch function to use instead of the default one, for example to use a proxy or edit headers.
*/
Expand Down
Loading

0 comments on commit de048e0

Please sign in to comment.