From 0ddefe85b25e3d5d68ff2baccceaaf8b5061d5f4 Mon Sep 17 00:00:00 2001 From: BarbossHack Date: Sun, 12 May 2024 13:49:59 +0200 Subject: [PATCH 1/3] Add support for .version --- sample/Cargo.toml | 1 + src/core/Item.ts | 2 +- src/toml/parser.ts | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sample/Cargo.toml b/sample/Cargo.toml index 6ed12d9..5e3a22e 100644 --- a/sample/Cargo.toml +++ b/sample/Cargo.toml @@ -33,6 +33,7 @@ Inflector = "0.11.4" block-modes = "0.8.1" # crates: disable-check jpegxl-sys = "0.8.2" tracing = "=0.1.37" +smoldot.version = "0.17.0" [dependencies.clap] version = "3.0.0-beta.2" diff --git a/src/core/Item.ts b/src/core/Item.ts index 2fd531f..0f62a7b 100644 --- a/src/core/Item.ts +++ b/src/core/Item.ts @@ -3,7 +3,7 @@ */ export default class Item { key: string = ""; - values: Array = []; + values: Array = []; value: string | undefined = ""; start: number = -1; end: number = -1; diff --git a/src/toml/parser.ts b/src/toml/parser.ts index 768c7bf..c21606b 100644 --- a/src/toml/parser.ts +++ b/src/toml/parser.ts @@ -70,6 +70,7 @@ function findVersion(item: Item): Item[] { const dependency = findVersionTable(field); if (dependency) dependencies.push(dependency); } else if (field.value != null) { + if (field.key.endsWith(".version")) field.key = field.key.replace(".version", ""); dependencies.push(field) } } @@ -226,7 +227,7 @@ function isCratesDep(i: Item): boolean { for (let value of i.values) { if (value.key === "git" || value.key === "path") { return false; - } else if (value.key === "package") { + } else if (value.key === "package" && value.value !== undefined) { i.key = value.value; } } From ecd4c0bf1eb5f68c400ca71f07ad3114d2b352b4 Mon Sep 17 00:00:00 2001 From: BarbossHack Date: Sun, 12 May 2024 14:08:08 +0200 Subject: [PATCH 2/3] Improve some `path` detection --- sample/Cargo.toml | 8 ++++++++ src/toml/parser.ts | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sample/Cargo.toml b/sample/Cargo.toml index 5e3a22e..27e3959 100644 --- a/sample/Cargo.toml +++ b/sample/Cargo.toml @@ -35,6 +35,14 @@ jpegxl-sys = "0.8.2" tracing = "=0.1.37" smoldot.version = "0.17.0" +[dependencies.depsonpath] +version = "7.0.0-alpha1" +path = "../lib" + +[workspace.dependencies] +my_package.path = "../lib" +my_package2 = { path = "../" } + [dependencies.clap] version = "3.0.0-beta.2" default-features = false diff --git a/src/toml/parser.ts b/src/toml/parser.ts index c21606b..b18e996 100644 --- a/src/toml/parser.ts +++ b/src/toml/parser.ts @@ -65,7 +65,7 @@ export function findCrateAndVersion( function findVersion(item: Item): Item[] { let dependencies: Item[] = []; for (const field of item.values) { - if (field.key.endsWith(".workspace")) continue; + if (field.key.endsWith(".workspace") || field.key.endsWith(".path")) continue; if (field.values.length > 0) { const dependency = findVersionTable(field); if (dependency) dependencies.push(dependency); @@ -81,7 +81,7 @@ function findVersionTable(table: Item): Item | null { let item = null let itemName = null; for (const field of table.values) { - if (field.key === "workspace") return null; + if (field.key === "workspace" || field.key === "path") return null; if (field.key === "version") { item = new Item(field); item.key = table.key; From 917b607a32d9e66c45c8c1635c833876e4a73b0d Mon Sep 17 00:00:00 2001 From: BarbossHack Date: Sat, 18 May 2024 15:20:23 +0200 Subject: [PATCH 3/3] Add support for Alternate Registries --- sample/Cargo.toml | 6 ++++ sample/alternate-registry.sh | 53 +++++++++++++++++++++++++++++ src/api/crates-index-server.ts | 51 ---------------------------- src/api/sparse-index-server.ts | 49 ++++++++++++--------------- src/core/AlternateRegistry.ts | 10 ++++++ src/core/Item.ts | 4 ++- src/core/fetcher.ts | 22 +++++------- src/core/listener.ts | 62 +++++++++++++++++++++++++++++----- src/toml/parser.ts | 39 ++++++++++++++++++++- src/ui/decoration.ts | 24 +++++++------ 10 files changed, 208 insertions(+), 112 deletions(-) create mode 100644 sample/alternate-registry.sh delete mode 100644 src/api/crates-index-server.ts create mode 100644 src/core/AlternateRegistry.ts diff --git a/sample/Cargo.toml b/sample/Cargo.toml index 6ed12d9..e2828ae 100644 --- a/sample/Cargo.toml +++ b/sample/Cargo.toml @@ -33,6 +33,12 @@ Inflector = "0.11.4" block-modes = "0.8.1" # crates: disable-check jpegxl-sys = "0.8.2" tracing = "=0.1.37" +external = { version = "0.1.0", registry = "public-registry" } +external-unknown = { version = "0.1.0", registry = "unknown-registry" } + +[dependencies.external2] +registry = "private-registry" +version = "0.0.2" [dependencies.clap] version = "3.0.0-beta.2" diff --git a/sample/alternate-registry.sh b/sample/alternate-registry.sh new file mode 100644 index 0000000..73d8121 --- /dev/null +++ b/sample/alternate-registry.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Helper script to launch 2 local alternate registries, for testing purpose + +set -eu + +if [ $(command -v podman) ]; then + CONTAINER=podman +elif [ $(command -v docker) ]; then + CONTAINER=docker +else + echo "[-] Please install docker or podman." + exit 1 +fi + +# Run alternate registries +$CONTAINER rm -f public-registry || true +$CONTAINER rm -f private-registry || true +$CONTAINER run --rm -it -d -p 8000:8000 --name public-registry -e KELLNR_REGISTRY__AUTH_REQUIRED=false ghcr.io/kellnr/kellnr:5.0.0 +$CONTAINER run --rm -it -d -p 127.0.0.1:8001:8000 --name private-registry -e KELLNR_REGISTRY__AUTH_REQUIRED=true -e KELLNR_ORIGIN__PORT=8001 ghcr.io/kellnr/kellnr:5.0.0 + +# Push crate to `public-registry` +cd $(mktemp -d) +cargo init external --registry public-registry +cd external +cargo publish --allow-dirty --index "sparse+http://localhost:8000/api/v1/crates/" --token "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD" + +# Push crate to `private-registry` +cd $(mktemp -d) +cargo init external2 --registry private-registry +cd external2 +cargo publish -v --allow-dirty --index "sparse+http://localhost:8001/api/v1/crates/" --token "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD" + +# Check +curl --fail-with-body http://localhost:8000/api/v1/crates/ex/te/external +curl --fail-with-body -H "Authorization: Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD" http://localhost:8001/api/v1/crates/ex/te/external2 + +echo -e "\n\nhttp://localhost:8000" +echo -e "http://localhost:8001" + +echo ' +---------------- +[registry] +global-credential-providers = ["cargo:token"] + +[registries] +public-registry = { index = "sparse+http://localhost:8000/api/v1/crates/" } + +[registries.private-registry] +index = "sparse+http://localhost:8001/api/v1/crates/" +token = "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD" +---------------- +' diff --git a/src/api/crates-index-server.ts b/src/api/crates-index-server.ts deleted file mode 100644 index fdf3046..0000000 --- a/src/api/crates-index-server.ts +++ /dev/null @@ -1,51 +0,0 @@ -import * as https from 'https'; -import { CrateMetadatas } from './crateMetadatas'; -import NodeCache from "node-cache"; - -const cache = new NodeCache({ stdTTL: 60 * 10 }); - -export const versions = (name: string, indexServerURL: string) => { - // clean dirty names - name = name.replace(/"/g, ""); - - return new Promise(function (resolve, reject) { - const cached = cache.get(name); - if (cached) { - resolve(cached); - return; - } - var req = https.get(`${indexServerURL}/index/versions/${name}`, function (res) { - // reject on bad status - if (!res.statusCode) { - reject(new Error('statusCode=' + res.statusCode)); - return; - } - if (res.statusCode < 200 || res.statusCode >= 300) { - return reject(new Error('statusCode=' + res.statusCode)); - } - // cumulate data - var crate_metadatas: CrateMetadatas; - var body: any = []; - res.on('data', function (chunk) { - body.push(chunk); - }); - // resolve on end - res.on('end', function () { - try { - crate_metadatas = JSON.parse(Buffer.concat(body).toString()); - cache.set(name, crate_metadatas); - } catch (e) { - reject(e); - } - resolve(crate_metadatas); - }); - }); - // reject on request error - req.on('error', function (err) { - // This is not a "Second reject", just a different sort of failure - reject(err); - }); - // IMPORTANT - req.end(); - }); -}; diff --git a/src/api/sparse-index-server.ts b/src/api/sparse-index-server.ts index 4ac950e..43e25f8 100644 --- a/src/api/sparse-index-server.ts +++ b/src/api/sparse-index-server.ts @@ -1,11 +1,12 @@ import * as https from 'https'; +import * as http from 'http'; import { CrateMetadatas } from './crateMetadatas'; import NodeCache from "node-cache"; export const sparseIndexServerURL = "https://index.crates.io"; const cache = new NodeCache({ stdTTL: 60 * 10 }); -export const versions = (name: string, indexServerURL: string) => { +export const versions = (name: string, indexServerURL?: string, registryToken?: string) => { // clean dirty names name = name.replace(/"/g, ""); @@ -25,7 +26,26 @@ export const versions = (name: string, indexServerURL: string) => { } else { prefix = lower_name.substring(0, 2) + "/" + lower_name.substring(2, 4); } - var req = https.get(`${indexServerURL}/${prefix}/${lower_name}`, function (res) { + + // This could happen if crate have an alternate registry, but index was not found. + // We should not default on sparse index in this case, juste ignore this crate fetch. + if (indexServerURL === undefined) return; + + // Add a trailing '/', and parse as `URL()` + let indexServerURLParsed: URL = new URL(`${indexServerURL.replace(/\/$/, "")}/`) + let options = { + hostname: indexServerURLParsed.hostname, + port: indexServerURLParsed.port, + path: `${indexServerURLParsed.pathname}${prefix}/${lower_name}`, + headers: {} + } + if (registryToken !== undefined) { + options.headers = { + Authorization: registryToken + } + } + const requests = indexServerURLParsed.protocol == "https:" ? https : http; + var req = requests.get(options, function (res) { // reject on bad status if (!res.statusCode) { reject(new Error('statusCode=' + res.statusCode)); @@ -69,28 +89,3 @@ export const versions = (name: string, indexServerURL: string) => { req.end(); }); }; - -// Check if `config.json` exists at root of `indexServerURL` -export async function isSparseCompatible(indexServerURL: string): Promise { - return new Promise(function (resolve, reject) { - const cached = cache.get(indexServerURL); - if (cached) { - resolve(cached); - return; - } - - var req = https.get(`${indexServerURL}/config.json`, (res) => { - if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300 && res.headers['content-type'] === "application/json") { - cache.set(indexServerURL, true); - resolve(true); - } else { - cache.set(indexServerURL, false); - resolve(false); - } - }).on('error', (e) => { - reject(e); - }); - - req.end(); - }); -} diff --git a/src/core/AlternateRegistry.ts b/src/core/AlternateRegistry.ts new file mode 100644 index 0000000..7692015 --- /dev/null +++ b/src/core/AlternateRegistry.ts @@ -0,0 +1,10 @@ +export class AlternateRegistry { + name: string = ""; + index?: string; + token?: string; + constructor(name: string, index?: string, token?: string) { + this.name = name; + this.index = index; + this.token = token; + } +} diff --git a/src/core/Item.ts b/src/core/Item.ts index 2fd531f..c58512e 100644 --- a/src/core/Item.ts +++ b/src/core/Item.ts @@ -3,8 +3,9 @@ */ export default class Item { key: string = ""; - values: Array = []; + values: Array = []; value: string | undefined = ""; + registry?: string; start: number = -1; end: number = -1; constructor(item?: Item) { @@ -12,6 +13,7 @@ export default class Item { this.key = item.key; this.values = item.values; this.value = item.value; + this.registry = item.registry; this.start = item.start; this.end = item.end; } diff --git a/src/core/fetcher.ts b/src/core/fetcher.ts index 016eb2b..e2fafbe 100644 --- a/src/core/fetcher.ts +++ b/src/core/fetcher.ts @@ -9,34 +9,30 @@ import compareVersions from "../semver/compareVersions"; import { CompletionItem, CompletionItemKind, CompletionList, workspace } from "vscode"; import { sortText } from "../providers/autoCompletion"; import { CrateMetadatas } from "../api/crateMetadatas"; +import { AlternateRegistry } from "./AlternateRegistry"; -export async function fetchCrateVersions(dependencies: Item[]): Promise<[Promise, Map]> { +export async function fetchCrateVersions(dependencies: Item[], alternateRegistries?: AlternateRegistry[]): Promise<[Promise, Map]> { // load config const config = workspace.getConfiguration(""); const shouldListPreRels = !!config.get("crates.listPreReleases"); var indexServerURL = config.get("crates.indexServerURL") ?? sparseIndexServerURL; - var versions; - try { - versions = sparseVersions; - } catch (e) { - console.error(`Could not check index compatibility for url "${indexServerURL}" (using sparse instead) : ${e}`); - indexServerURL = sparseIndexServerURL; - versions = sparseVersions; - } - StatusBar.setText("Loading", "👀 Fetching " + indexServerURL.replace(/^https?:\/\//, '')); - let transformer = transformServerResponse(versions, shouldListPreRels, indexServerURL); + let transformer = transformServerResponse(sparseVersions, shouldListPreRels, indexServerURL, alternateRegistries); let responsesMap: Map = new Map(); const responses = dependencies.map(transformer); return [Promise.all(responses), responsesMap]; } -function transformServerResponse(versions: (name: string, indexServerURL: string) => Promise, shouldListPreRels: boolean, indexServerURL: string): (i: Item) => Promise { +function transformServerResponse(versions: (name: string, indexServerURL?: string, registryToken?: string) => Promise, shouldListPreRels: boolean, indexServerURL: string, alternateRegistries?: AlternateRegistry[]): (i: Item) => Promise { return function (item: Item): Promise { - return versions(item.key, indexServerURL).then((crate: any) => { + // Use the sparse index if (and only if) the crate does not use an alternate registry + const alternateRegistry = alternateRegistries?.find((registry) => item.registry == registry.name); + var thisCrateRegistry = item.registry !== undefined ? alternateRegistry?.index : indexServerURL; + var thisCrateToken = item.registry !== undefined ? alternateRegistry?.token : undefined; + return versions(item.key, thisCrateRegistry, thisCrateToken).then((crate: any) => { const versions = crate.versions.reduce((result: any[], item: string) => { const isPreRelease = !shouldListPreRels && (item.indexOf("-alpha") !== -1 || item.indexOf("-beta") !== -1 || item.indexOf("-rc") !== -1 || item.indexOf("-pre") !== -1); if (!isPreRelease) diff --git a/src/core/listener.ts b/src/core/listener.ts index 713a48f..d8806db 100644 --- a/src/core/listener.ts +++ b/src/core/listener.ts @@ -3,18 +3,27 @@ * Filters active editor files according to the extension. */ import { Position, Range, TextDocument, TextEditor } from "vscode"; -import { parse, filterCrates } from "../toml/parser"; +import { parse, filterCrates, parseAlternateRegistries } from "../toml/parser"; import { StatusBar } from "../ui/status-bar"; import { status } from "../toml/commands"; import Item from "./Item"; import decorate, { decorationHandle } from "../ui/decorator"; import { fetchCrateVersions } from "./fetcher"; import Dependency from "./Dependency"; +import path from "path"; +import { homedir } from "os"; +import { promises as async_fs } from 'fs'; +import fs from 'fs'; +import { AlternateRegistry } from "./AlternateRegistry"; -function parseToml(text: string): Item[] { +function parseToml(cargoTomlContent: string, alternateRegistries?: AlternateRegistry[]): Item[] { console.log("Parsing..."); - const toml = parse(text); - const tomlDependencies = filterCrates(toml.values); + const toml = parse(cargoTomlContent); + var tomlDependencies = filterCrates(toml.values); + // Filter out crates that have an alternate registry we don't know. + tomlDependencies = tomlDependencies.filter((crate) => + crate.registry === undefined || alternateRegistries?.find((registry) => crate.registry == registry.name && registry.index !== undefined) !== undefined + ); console.log("Parsed"); return tomlDependencies; } @@ -47,13 +56,50 @@ export async function parseAndDecorate( _wasSaved: boolean = false, fetchDeps: boolean = true ) { - const text = editor.document.getText(); + + // Parse credentials if any + let credentialTokens = undefined; + try { + const file = path.join(homedir(), '.cargo', 'credentials.toml'); + if (fs.existsSync(file)) { + const credentialsTomlContent = await async_fs.readFile(file, 'utf-8'); + credentialTokens = parseAlternateRegistries(credentialsTomlContent); + } + } catch (error) { + console.error(error); + } + + // Parse alternate registries if any + let alternateRegistries = undefined; + try { + const legacyFile = path.join(homedir(), '.cargo', 'config'); + const file = path.join(homedir(), '.cargo', 'config.toml'); + const confFile = fs.existsSync(file) ? file : legacyFile; + if (fs.existsSync(confFile)) { + const configTomlContent = await async_fs.readFile(confFile, 'utf-8'); + alternateRegistries = parseAlternateRegistries(configTomlContent); + } + } catch (error) { + console.error(error); + } + + // Merge credential tokens into registries + alternateRegistries?.map((registry) => { + if (registry.token === undefined) { + registry.token = credentialTokens?.find((credential) => credential.name == registry.name)?.token; + return registry; + } else { + return registry + } + }); + try { - // Parse + // Parse crates + const cargoTomlContent = editor.document.getText(); StatusBar.setText("Loading", "Parsing Cargo.toml"); - dependencies = parseToml(text); + dependencies = parseToml(cargoTomlContent, alternateRegistries); if (fetchDeps || !fetchedDeps || !fetchedDepsMap) { - const data = await fetchCrateVersions(dependencies); + const data = await fetchCrateVersions(dependencies, alternateRegistries); fetchedDeps = await data[0]; fetchedDepsMap = data[1]; } diff --git a/src/toml/parser.ts b/src/toml/parser.ts index 768c7bf..a0e2bf0 100644 --- a/src/toml/parser.ts +++ b/src/toml/parser.ts @@ -1,5 +1,6 @@ import { TextDocument } from "vscode"; import Item from "../core/Item"; +import { AlternateRegistry } from "../core/AlternateRegistry"; export const RE_VERSION = /^[ \t]*(? key === "index")?.value?.replace("sparse+", ""); + const token = registry.values.find(({ key }) => key === "token")?.value; + alternateRegistries.push(new AlternateRegistry(name, index, token)); +} + /** * * @param data Parse the given document and index all items. @@ -226,7 +263,7 @@ function isCratesDep(i: Item): boolean { for (let value of i.values) { if (value.key === "git" || value.key === "path") { return false; - } else if (value.key === "package") { + } else if (value.key === "package" && value.value !== undefined) { i.key = value.value; } } diff --git a/src/ui/decoration.ts b/src/ui/decoration.ts index 422f37d..641e7d4 100644 --- a/src/ui/decoration.ts +++ b/src/ui/decoration.ts @@ -65,7 +65,9 @@ export default function decoration( contentCss = decorationPreferences.errorDecoratorCss; } else { hoverMessage.appendMarkdown("#### Versions"); - hoverMessage.appendMarkdown(` _( [View Crate](https://crates.io/crates/${item.key.replace(/"/g, "")}) | [Check Reviews](https://web.crev.dev/rust-reviews/crate/${item.key.replace(/"/g, "")}) )_`); + if (item.registry === undefined) { + hoverMessage.appendMarkdown(` _( [View Crate](https://crates.io/crates/${item.key.replace(/"/g, "")}) | [Check Reviews](https://web.crev.dev/rust-reviews/crate/${item.key.replace(/"/g, "")}) )_`); + } hoverMessage.isTrusted = true; if (versions.length > 0) { @@ -86,7 +88,7 @@ export default function decoration( }; const isCurrent = version === maxSatisfying; const encoded = encodeURI(JSON.stringify(replaceData)); - const docs = (i === 0 || isCurrent) ? `[(docs)](https://docs.rs/crate/${item.key.replace(/"/g, "")}/${version})` : ""; + const docs = ((i === 0 || isCurrent) && item.registry === undefined) ? `[(docs)](https://docs.rs/crate/${item.key.replace(/"/g, "")}/${version})` : ""; const command = `${isCurrent ? "**" : ""}[${version}](command:crates.replaceVersion?${encoded})${docs}${isCurrent ? "**" : ""}`; hoverMessage.appendMarkdown("\n * "); hoverMessage.appendMarkdown(command); @@ -124,18 +126,18 @@ export default function decoration( contentCss.after!.contentText = contentCss.after!.contentText!.replace("${version}", versions[0]) } - + const deco = { range: new Range( editor.document.positionAt(start), endofline, - ), - hoverMessage, - renderOptions: {}, - }; - if (version != "?" && contentCss.after!.contentText!.length > 0) { - deco.renderOptions = contentCss; - } - + ), + hoverMessage, + renderOptions: {}, + }; + if (version != "?" && contentCss.after!.contentText!.length > 0) { + deco.renderOptions = contentCss; + } + return deco; }