Add support for Alternate Registries

sample/Cargo.toml

@@ -33,6 +33,21 @@ Inflector = "0.11.4"
 block-modes = "0.8.1" # crates: disable-check
 jpegxl-sys = "0.8.2"
 tracing = "=0.1.37"
+external = { version = "0.1.0", registry = "public-registry" }
+external-unknown = { version = "0.1.0", registry = "unknown-registry" }
+
+[dependencies.external2]
+registry = "private-registry"
+version = "0.0.2"
+smoldot.version = "0.17.0"
+
+[dependencies.depsonpath]
+version = "7.0.0-alpha1"
+path = "../lib"
+
+[workspace.dependencies]
+my_package.path = "../lib"
+my_package2 = { path = "../" }
 
 [dependencies.clap]
 version = "3.0.0-beta.2"

sample/alternate-registry.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Helper script to launch 2 local alternate registries, for testing purpose
+
+set -eu
+
+if [ $(command -v podman) ]; then
+    CONTAINER=podman
+elif [ $(command -v docker) ]; then
+    CONTAINER=docker
+else
+    echo "[-] Please install docker or podman."
+    exit 1
+fi
+
+# Run alternate registries
+$CONTAINER rm -f public-registry || true
+$CONTAINER rm -f private-registry || true
+$CONTAINER run --rm -it -d -p 8000:8000 --name public-registry -e KELLNR_REGISTRY__AUTH_REQUIRED=false ghcr.io/kellnr/kellnr:5.0.0
+$CONTAINER run --rm -it -d -p 127.0.0.1:8001:8000 --name private-registry -e KELLNR_REGISTRY__AUTH_REQUIRED=true -e KELLNR_ORIGIN__PORT=8001 ghcr.io/kellnr/kellnr:5.0.0
+
+# Push crate to `public-registry`
+cd $(mktemp -d)
+cargo init external --registry public-registry
+cd external
+cargo publish --allow-dirty --index "sparse+http://localhost:8000/api/v1/crates/" --token "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD"
+
+# Push crate to `private-registry`
+cd $(mktemp -d)
+cargo init external2 --registry private-registry
+cd external2
+cargo publish -v --allow-dirty --index "sparse+http://localhost:8001/api/v1/crates/" --token "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD"
+
+# Check
+curl --fail-with-body http://localhost:8000/api/v1/crates/ex/te/external
+curl --fail-with-body -H "Authorization: Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD" http://localhost:8001/api/v1/crates/ex/te/external2
+
+echo -e "\n\nhttp://localhost:8000"
+echo -e "http://localhost:8001"
+
+echo '
+----------------
+[registry]
+global-credential-providers = ["cargo:token"]
+
+[registries]
+public-registry = { index = "sparse+http://localhost:8000/api/v1/crates/" }
+
+[registries.private-registry]
+index = "sparse+http://localhost:8001/api/v1/crates/"
+token = "Zy9HhJ02RJmg0GCrgLfaCVfU6IwDfhXD"
+----------------
+'

src/api/sparse-index-server.ts

@@ -1,11 +1,12 @@
 import * as https from 'https';
+import * as http from 'http';
 import { CrateMetadatas } from './crateMetadatas';
 import NodeCache from "node-cache";
 
 export const sparseIndexServerURL = "https://index.crates.io";
 const cache = new NodeCache({ stdTTL: 60 * 10 });
 
-export const versions = (name: string, indexServerURL: string) => {
+export const versions = (name: string, indexServerURL?: string, registryToken?: string) => {
   // clean dirty names
   name = name.replace(/"/g, "");
 
@@ -25,7 +26,26 @@ export const versions = (name: string, indexServerURL: string) => {
     } else {
       prefix = lower_name.substring(0, 2) + "/" + lower_name.substring(2, 4);
     }
-    var req = https.get(`${indexServerURL}/${prefix}/${lower_name}`, function (res) {
+
+    // This could happen if crate have an alternate registry, but index was not found.
+    // We should not default on sparse index in this case, juste ignore this crate fetch.
+    if (indexServerURL === undefined) return;
+
+    // Add a trailing '/', and parse as `URL()`
+    let indexServerURLParsed: URL = new URL(`${indexServerURL.replace(/\/$/, "")}/`)
+    let options = {
+      hostname: indexServerURLParsed.hostname,
+      port: indexServerURLParsed.port,
+      path: `${indexServerURLParsed.pathname}${prefix}/${lower_name}`,
+      headers: {}
+    }
+    if (registryToken !== undefined) {
+      options.headers = {
+        Authorization: registryToken
+      }
+    }
+    const requests = indexServerURLParsed.protocol == "https:" ? https : http;
+    var req = requests.get(options, function (res) {
       // reject on bad status
       if (!res.statusCode) {
         reject(new Error('statusCode=' + res.statusCode));
@@ -69,28 +89,3 @@ export const versions = (name: string, indexServerURL: string) => {
     req.end();
   });
 };
-
-// Check if `config.json` exists at root of `indexServerURL`
-export async function isSparseCompatible(indexServerURL: string): Promise<boolean> {
-  return new Promise<boolean>(function (resolve, reject) {
-    const cached = cache.get<boolean>(indexServerURL);
-    if (cached) {
-      resolve(cached);
-      return;
-    }
-
-    var req = https.get(`${indexServerURL}/config.json`, (res) => {
-      if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300 && res.headers['content-type'] === "application/json") {
-        cache.set(indexServerURL, true);
-        resolve(true);
-      } else {
-        cache.set(indexServerURL, false);
-        resolve(false);
-      }
-    }).on('error', (e) => {
-      reject(e);
-    });
-
-    req.end();
-  });
-}

src/core/AlternateRegistry.ts

@@ -0,0 +1,10 @@
+export class AlternateRegistry {
+    name: string = "";
+    index?: string;
+    token?: string;
+    constructor(name: string, index?: string, token?: string) {
+        this.name = name;
+        this.index = index;
+        this.token = token;
+    }
+}

src/core/Item.ts

@@ -3,15 +3,17 @@
  */
 export default class Item {
   key: string = "";
-  values: Array<any> = [];
+  values: Array<Item> = [];
   value: string | undefined = "";
+  registry?: string;
   start: number = -1;
   end: number = -1;
   constructor(item?: Item) {
     if (item) {
       this.key = item.key;
       this.values = item.values;
       this.value = item.value;
+      this.registry = item.registry;
       this.start = item.start;
       this.end = item.end;
     }

src/core/fetcher.ts

@@ -9,34 +9,30 @@ import compareVersions from "../semver/compareVersions";
 import { CompletionItem, CompletionItemKind, CompletionList, workspace } from "vscode";
 import { sortText } from "../providers/autoCompletion";
 import { CrateMetadatas } from "../api/crateMetadatas";
+import { AlternateRegistry } from "./AlternateRegistry";
 
-export async function fetchCrateVersions(dependencies: Item[]): Promise<[Promise<Dependency[]>, Map<string, Dependency[]>]> {
+export async function fetchCrateVersions(dependencies: Item[], alternateRegistries?: AlternateRegistry[]): Promise<[Promise<Dependency[]>, Map<string, Dependency[]>]> {
   // load config
   const config = workspace.getConfiguration("");
   const shouldListPreRels = !!config.get("crates.listPreReleases");
   var indexServerURL = config.get<string>("crates.indexServerURL") ?? sparseIndexServerURL;
 
-  var versions;
-  try {
-    versions = sparseVersions;
-  } catch (e) {
-    console.error(`Could not check index compatibility for url "${indexServerURL}" (using sparse instead) : ${e}`);
-    indexServerURL = sparseIndexServerURL;
-    versions = sparseVersions;
-  }
-
   StatusBar.setText("Loading", "👀 Fetching " + indexServerURL.replace(/^https?:\/\//, ''));
 
-  let transformer = transformServerResponse(versions, shouldListPreRels, indexServerURL);
+  let transformer = transformServerResponse(sparseVersions, shouldListPreRels, indexServerURL, alternateRegistries);
   let responsesMap: Map<string, Dependency[]> = new Map();
   const responses = dependencies.map(transformer);
   return [Promise.all(responses), responsesMap];
 }
 
 
-function transformServerResponse(versions: (name: string, indexServerURL: string) => Promise<CrateMetadatas>, shouldListPreRels: boolean, indexServerURL: string): (i: Item) => Promise<Dependency> {
+function transformServerResponse(versions: (name: string, indexServerURL?: string, registryToken?: string) => Promise<CrateMetadatas>, shouldListPreRels: boolean, indexServerURL: string, alternateRegistries?: AlternateRegistry[]): (i: Item) => Promise<Dependency> {
   return function (item: Item): Promise<Dependency> {
-    return versions(item.key, indexServerURL).then((crate: any) => {
+    // Use the sparse index if (and only if) the crate does not use an alternate registry
+    const alternateRegistry = alternateRegistries?.find((registry) => item.registry == registry.name);
+    var thisCrateRegistry = item.registry !== undefined ? alternateRegistry?.index : indexServerURL;
+    var thisCrateToken = item.registry !== undefined ? alternateRegistry?.token : undefined;
+    return versions(item.key, thisCrateRegistry, thisCrateToken).then((crate: any) => {
       const versions = crate.versions.reduce((result: any[], item: string) => {
         const isPreRelease = !shouldListPreRels && (item.indexOf("-alpha") !== -1 || item.indexOf("-beta") !== -1 || item.indexOf("-rc") !== -1 || item.indexOf("-pre") !== -1);
         if (!isPreRelease)

src/core/listener.ts

@@ -3,18 +3,27 @@
  * Filters active editor files according to the extension.
  */
 import { Position, Range, TextDocument, TextEditor } from "vscode";
-import { parse, filterCrates } from "../toml/parser";
+import { parse, filterCrates, parseAlternateRegistries } from "../toml/parser";
 import { StatusBar } from "../ui/status-bar";
 import { status } from "../toml/commands";
 import Item from "./Item";
 import decorate, { decorationHandle } from "../ui/decorator";
 import { fetchCrateVersions } from "./fetcher";
 import Dependency from "./Dependency";
+import path from "path";
+import { homedir } from "os";
+import { promises as async_fs } from 'fs';
+import fs from 'fs';
+import { AlternateRegistry } from "./AlternateRegistry";
 
-function parseToml(text: string): Item[] {
+function parseToml(cargoTomlContent: string, alternateRegistries?: AlternateRegistry[]): Item[] {
   console.log("Parsing...");
-  const toml = parse(text);
-  const tomlDependencies = filterCrates(toml.values);
+  const toml = parse(cargoTomlContent);
+  var tomlDependencies = filterCrates(toml.values);
+  // Filter out crates that have an alternate registry we don't know.
+  tomlDependencies = tomlDependencies.filter((crate) =>
+    crate.registry === undefined || alternateRegistries?.find((registry) => crate.registry == registry.name && registry.index !== undefined) !== undefined
+  );
   console.log("Parsed");
   return tomlDependencies;
 }
@@ -47,13 +56,50 @@ export async function parseAndDecorate(
   _wasSaved: boolean = false,
   fetchDeps: boolean = true
 ) {
-  const text = editor.document.getText();
+
+  // Parse credentials if any
+  let credentialTokens = undefined;
+  try {
+    const file = path.join(homedir(), '.cargo', 'credentials.toml');
+    if (fs.existsSync(file)) {
+      const credentialsTomlContent = await async_fs.readFile(file, 'utf-8');
+      credentialTokens = parseAlternateRegistries(credentialsTomlContent);
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  // Parse alternate registries if any
+  let alternateRegistries = undefined;
+  try {
+    const legacyFile = path.join(homedir(), '.cargo', 'config');
+    const file = path.join(homedir(), '.cargo', 'config.toml');
+    const confFile = fs.existsSync(file) ? file : legacyFile;
+    if (fs.existsSync(confFile)) {
+      const configTomlContent = await async_fs.readFile(confFile, 'utf-8');
+      alternateRegistries = parseAlternateRegistries(configTomlContent);
+    }
+  } catch (error) {
+    console.error(error);
+  }
+
+  // Merge credential tokens into registries
+  alternateRegistries?.map((registry) => {
+    if (registry.token === undefined) {
+      registry.token = credentialTokens?.find((credential) => credential.name == registry.name)?.token;
+      return registry;
+    } else {
+      return registry
+    }
+  });
+
   try {
-    // Parse
+    // Parse crates
+    const cargoTomlContent = editor.document.getText();
     StatusBar.setText("Loading", "Parsing Cargo.toml");
-    dependencies = parseToml(text);
+    dependencies = parseToml(cargoTomlContent, alternateRegistries);
     if (fetchDeps || !fetchedDeps || !fetchedDepsMap) {
-      const data = await fetchCrateVersions(dependencies);
+      const data = await fetchCrateVersions(dependencies, alternateRegistries);
       fetchedDeps = await data[0];
       fetchedDepsMap = data[1];
     }