Skip to content

Commit

Permalink
merge pull request #7
Browse files Browse the repository at this point in the history 
Introduce support for by-hash specification
  • Loading branch information
@Nipheris
Nipheris authored Apr 5, 2024
2 parents 051b4aa + f356d3e commit 2e7bd7c
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 56 deletions.
45 changes: 45 additions & 0 deletions src/deb/cas.mts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import * as path from 'node:path';
import { PassThrough, Readable } from 'node:stream';
import { buffer } from 'node:stream/consumers';
import { createGzip } from 'node:zlib';
import { createHash } from 'node:crypto';
import { createWriteStream } from 'node:fs';
import { pipeline } from 'node:stream/promises';

import { createDir } from '../fs.mjs';

export interface ContentAddress {
sha256: Uint8Array;
}

export interface ContentDescription {
size: number;
address: ContentAddress;
}

export async function storeGzippedStream(root: string, stream: NodeJS.ReadableStream): Promise<ContentDescription> {
const gzip = createGzip({ level: 9 });
const gzippedStream = new PassThrough();
await pipeline(stream, gzip, gzippedStream);
return storeStream(root, gzippedStream);
}

// https://wiki.debian.org/DebianRepository/Format#indices_acquisition_via_hashsums_.28by-hash.29
export async function storeStream(root: string, stream: NodeJS.ReadableStream): Promise<ContentDescription> {
const contentBuffer = await buffer(stream);

// compute a digest
const sha256Hash = createHash('sha256');
await pipeline(Readable.from(contentBuffer), sha256Hash);
const sha256 = sha256Hash.read();

// store in a file
const sha256Dir = path.join(root, 'SHA256');
const fileName = path.join(sha256Dir, Buffer.from(sha256).toString('hex'));
createDir(sha256Dir);
await pipeline(Readable.from(contentBuffer), createWriteStream(fileName));
return {
size: contentBuffer.length,
address: { sha256 },
};
}
119 changes: 63 additions & 56 deletions src/deb/deb-builder.mts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import { createGzip } from 'zlib';
import * as path from 'node:path';
import { appendFile, writeFile } from 'node:fs/promises';
import { readdirSync, readFileSync, renameSync, unlinkSync } from 'node:fs';
import { randomBytes } from 'node:crypto';
import { Readable } from 'node:stream';

import * as fs from 'fs';
import * as ini from 'ini';
import * as path from 'path';
import * as tar from 'tar';

import * as cas from './cas.mjs';
import type { Artifact, ArtifactProvider } from '../artifact-provider.mjs';
import { createDir, execToolToFile, removeDir } from '../fs.mjs';
import type { Deployer } from '../deployer.mjs';
Expand All @@ -23,7 +26,9 @@ const ReleaseFileTemplate =
Label: Ubuntu/Debian
Architecture: $ARCH
Component: $COMPONENT
Codename: $DISTRIBUTION\n`;
Acquire-By-Hash: yes
Codename: $DISTRIBUTION
Date: $DATE\n`;

function iterateComponents(repo: DebRepo, callback: (distribution: string, component: string, deb: Artifact[]) => void): void {
const distributions = Object.keys(repo);
Expand Down Expand Up @@ -94,23 +99,26 @@ export class DebBuilder implements Deployer {
return `${this.config.applicationName}-${version}_${arch}.deb`;
}

private async makeReleaseFileAndSign(distribution: string, component: string, arch: string): Promise<void> {
const releaseContent = ReleaseFileTemplate
// eslint-disable-next-line max-params
private async makeReleaseFileAndSign(distribution: string, component: string, arch: string, indices: BinaryPackageIndexDescription[]): Promise<void> {
let releaseContent = ReleaseFileTemplate
.replace('$ORIGIN', this.config.origin)
.replace('$DISTRIBUTION', distribution)
.replace('$ARCH', arch)
.replace('$COMPONENT', component);
.replace('$COMPONENT', component)
.replace('$DATE', new Date().toUTCString());

const distributionPath = path.join(this.distsPath, distribution);
const releaseFilePath = path.join(distributionPath, 'Release');
const releaseGpgFilePath = path.join(distributionPath, 'Release.gpg');
const inReleaseFilePath = path.join(distributionPath, 'InRelease');
releaseContent += 'SHA256:\n';
for (const { address, size, name } of indices) {
releaseContent += ` ${Buffer.from(address.sha256).toString('hex')}\t${size}\t${name}\n`;
}

await fs.promises.writeFile(releaseFilePath, releaseContent);
const tempReleaseFile = path.join(this.tempPath, `Release-${randomBytes(4).toString('hex')}`);
await writeFile(tempReleaseFile, releaseContent);

await execToolToFile('apt-ftparchive', ['release', distributionPath], releaseFilePath, true);
await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '-abs', '-o', releaseGpgFilePath, releaseFilePath]);
await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '--clearsign', '-o', inReleaseFilePath, releaseFilePath]);
const distributionPath = path.join(this.distsPath, distribution);
const inReleaseFilePath = path.join(distributionPath, 'InRelease');
await execToolToFile('gpg', ['--no-tty', '--default-key', this.config.gpgKeyName, '--clearsign', '-o', inReleaseFilePath, tempReleaseFile]);
}

private async dpkgScanpackages(): Promise<void> {
Expand Down Expand Up @@ -144,7 +152,7 @@ export class DebBuilder implements Deployer {
.pipe(tar.extract({ cwd: whereExtract, strip: 1 }, ['./control']))
// eslint-disable-next-line max-statements
.on('finish', () => {
const controlMetaContent = fs.readFileSync(path.join(whereExtract, 'control'), 'utf-8').replaceAll(':', '=');
const controlMetaContent = readFileSync(path.join(whereExtract, 'control'), 'utf-8').replaceAll(':', '=');
const controlMeta = ini.parse(controlMetaContent);
const arch = controlMeta['Architecture'];
const version = controlMeta['Version'];
Expand All @@ -163,7 +171,7 @@ export class DebBuilder implements Deployer {
`binary-${arch}`,
`${this.debFileName(version, arch)}.meta`);
createDir(path.dirname(targetMetaPath));
fs.renameSync(path.join(whereExtract, 'control'), targetMetaPath);
renameSync(path.join(whereExtract, 'control'), targetMetaPath);

removeDir(whereExtract);

Expand All @@ -173,7 +181,7 @@ export class DebBuilder implements Deployer {
this.config.applicationName,
distribution,
this.debFileName(version, arch));
const relativeDebPath = path.relative(this.rootPath, debPath);
const relativeDebPath = path.relative(this.rootPath, debPath).replace(/\\/gu, '/');
const debSize = controlTar.headers['content-range']?.split('/')[1];
const sha1 = controlTar.headers['x-checksum-sha1'];
const sha256 = controlTar.headers['x-checksum-sha256'];
Expand All @@ -185,7 +193,7 @@ export class DebBuilder implements Deployer {

const dataToAppend = `Filename: ${relativeDebPath}\nSize: ${debSize}\nSHA1: ${sha1}\nSHA256: ${sha256}\nMD5Sum: ${md5}\n`;

fs.promises.appendFile(targetMetaPath, dataToAppend).then(() => resolve());
appendFile(targetMetaPath, dataToAppend).then(() => resolve());

const createFileOperation = this.packageCreator(deb.md5, debPath);

Expand All @@ -201,57 +209,56 @@ export class DebBuilder implements Deployer {
}

private async makeRelease(): Promise<{}> {
const compressFile = (filePath: string): Promise<void> => new Promise<void>(resolve => {
const inp = fs.createReadStream(filePath);
const out = fs.createWriteStream(`${filePath}.gz`);

const gzip = createGzip({ level: 9 });

inp.pipe(gzip).pipe(out)
.on('finish', () => {
resolve();
});
});

const compressPromises: Promise<void>[] = [];
const binaryPackageIndices: BinaryPackageIndexDescription[] = [];

const componentPromises: Promise<void[]>[] = [];
iterateComponents(this.config.repo, (distribution, component) => {
const componentRoot = path.join(this.distsPath, distribution, component);
const componentsByArch = fs.readdirSync(componentRoot).map(dist => path.join(componentRoot, dist));

componentsByArch.forEach(dist => {
const targetPackagesFile = path.join(dist, 'Packages');
const metaFiles = fs.readdirSync(dist)
.filter(fileName => fileName.endsWith('.meta'))
.map(metaFile => path.join(dist, metaFile));

let packagesContent = '';

for (const metaFile of metaFiles) {
packagesContent += fs.readFileSync(metaFile);
packagesContent += '\n';
fs.unlinkSync(metaFile);
}

fs.writeFileSync(targetPackagesFile, packagesContent);
componentPromises.push(Promise.all(
readdirSync(componentRoot).map(async binarySubdir => {
const binaryPath = path.join(componentRoot, binarySubdir);
const metaFiles = readdirSync(binaryPath)
.filter(fileName => fileName.endsWith('.meta'))
.map(metaFile => path.join(binaryPath, metaFile));

let packagesContent = '';

for (const metaFile of metaFiles) {
packagesContent += readFileSync(metaFile);
packagesContent += '\n';
unlinkSync(metaFile);
}

compressPromises.push(compressFile(targetPackagesFile));
});
const storePackageIndex = async(
shortName: string,
store: (root: string, stream: NodeJS.ReadableStream) => Promise<cas.ContentDescription>,
) => {
const name = `${component}/${binarySubdir}/${shortName}`;
const { size, address } = await store(path.resolve(binaryPath, 'by-hash'), Readable.from(packagesContent));
binaryPackageIndices.push({ name, size, address });
};
await storePackageIndex('Packages', cas.storeStream);
await storePackageIndex('Packages.gz', cas.storeGzippedStream);
}),
));
});

await Promise.all(compressPromises);
await Promise.all(componentPromises);

const releasesPromises: Promise<void>[] = [];

iterateComponents(this.config.repo, (distribution, component) => {
const archesSet = this.archesByDistComp.get(`${distribution}/${component}`);
if (!archesSet) {
throw new Error('No arch was found for distribution');
}

releasesPromises.push(this.makeReleaseFileAndSign(distribution, component, [...archesSet.values()].join(' ')));
releasesPromises.push(this.makeReleaseFileAndSign(distribution, component, [...archesSet.values()].join(' '), binaryPackageIndices));
});

return Promise.all(releasesPromises);
}
}

interface BinaryPackageIndexDescription {
name: string;
size: number;
address: cas.ContentAddress;
}

0 comments on commit 2e7bd7c

Please sign in to comment.