diff --git a/.changeset/grumpy-hairs-jam.md b/.changeset/grumpy-hairs-jam.md new file mode 100644 index 000000000..4414e41e7 --- /dev/null +++ b/.changeset/grumpy-hairs-jam.md @@ -0,0 +1,5 @@ +--- +'@e2b/cli': patch +--- + +Add workaround for broken Docker auth during template build diff --git a/apps/web/src/app/(docs)/docs/troubleshooting/templates/build-authentication-error/page.mdx b/apps/web/src/app/(docs)/docs/troubleshooting/templates/build-authentication-error/page.mdx new file mode 100644 index 000000000..ea61a1674 --- /dev/null +++ b/apps/web/src/app/(docs)/docs/troubleshooting/templates/build-authentication-error/page.mdx @@ -0,0 +1,70 @@ +# Docker push authentication error + +When the CLI tries to push a Docker image to the registry, you might encounter an authentication error. This error sometimes occurs for users when Docker doesn't send any credentials to the registry. +To resolve this issue, you can use the following steps: + +## MacOS + +1. Open Docker Desktop. +2. Go to Settings. +3. Go to Docker Engine. +4. Add the following line to the json configuration: + +```json +{ + "insecure-registries": ["host.docker.internal:49984"] +} +``` + +It may look like this: + +```json +{ + "builder": { + "gc": { + "defaultKeepStorage": "20GB", + "enabled": true + } + }, + "features": { + "buildkit": true + }, + "insecure-registries": [ + "host.docker.internal:49984" + ] +} +``` +This allows Docker to send requests to local proxy, which handles the authentication. + +5. Click Apply & Restart. + + +## Linux + +1. Edit the Docker configuration file (usually `/etc/docker/daemon.json`) and add the following line, if the file doesn't exist, create it: +```json +{ + "insecure-registries": ["localhost:49984"] +} +``` + +2. Restart Docker: + +```bash +sudo systemctl restart docker +``` + +## Windows + +1. Open Docker Desktop. +2. Go to Settings. +3. Go to Docker Engine. +4. Add the following line to the json configuration: + +```json +{ + "insecure-registries": ["host.docker.internal:49984"] +} +``` +5. Click Apply & Restart. + diff --git a/apps/web/src/components/Navigation/routes.tsx b/apps/web/src/components/Navigation/routes.tsx index 51d65db07..3b2b7ed02 100644 --- a/apps/web/src/components/Navigation/routes.tsx +++ b/apps/web/src/components/Navigation/routes.tsx @@ -1,9 +1,4 @@ -import { - Home, - CheckCircle, - MessagesSquare, - Braces, -} from 'lucide-react' +import { Home, CheckCircle, MessagesSquare, Braces } from 'lucide-react' export interface NavLink { title: string @@ -147,7 +142,7 @@ export const routes: NavGroup[] = [ title: 'Interactive charts', href: '/docs/code-interpreting/create-charts-visualizations/interactive-charts', }, - ] + ], }, { title: 'Streaming', @@ -184,13 +179,13 @@ export const routes: NavGroup[] = [ title: 'Bash', href: '/docs/code-interpreting/supported-languages/bash', }, - ] + ], }, // { // title: '* Parsing code execution results', // href: '/docs/code-interpreting/todo', // }, - ] + ], }, // { // title: 'Guides', // How to's @@ -309,7 +304,7 @@ export const routes: NavGroup[] = [ title: 'Customize CPU & RAM', href: '/docs/sandbox-template/customize-cpu-ram', }, - ] + ], }, { title: 'Filesystem', @@ -334,7 +329,7 @@ export const routes: NavGroup[] = [ title: 'Download data', href: '/docs/filesystem/download', }, - ] + ], }, { title: 'Commands', @@ -351,7 +346,7 @@ export const routes: NavGroup[] = [ title: 'Run commands in background', href: '/docs/commands/background', }, - ] + ], }, // { // title: '* Async Python SDK', @@ -411,7 +406,21 @@ export const routes: NavGroup[] = [ title: 'Shutdown running sandboxes', href: '/docs/cli/shutdown-sandboxes', }, - ] + ], + }, + { + title: 'Troubleshooting', + items: [ + { + title: 'Templates', + links: [ + { + title: 'Build authentication error', + href: '/docs/troubleshooting/templates/build-authentication-error', + }, + ], + }, + ], }, // { // // Maybe move integrations to a separate docs page? @@ -433,4 +442,3 @@ export const routes: NavGroup[] = [ // }, // ...apiRefRoutes, ] - diff --git a/packages/cli/src/commands/template/build.ts b/packages/cli/src/commands/template/build.ts index 43eb9dbc7..b49ffd33e 100644 --- a/packages/cli/src/commands/template/build.ts +++ b/packages/cli/src/commands/template/build.ts @@ -30,6 +30,7 @@ import * as child_process from 'child_process' import { client } from 'src/api' import { handleE2BRequestError } from '../../utils/errors' import { getUserConfig } from 'src/user' +import { buildWithProxy } from './buildWithProxy' const templateCheckInterval = 500 // 0.5 sec @@ -56,7 +57,7 @@ async function getTemplateBuildLogs({ logsOffset, }, }, - }, + } ) handleE2BRequestError(res.error, 'Error getting template build status') @@ -64,7 +65,7 @@ async function getTemplateBuildLogs({ } async function requestTemplateBuild( - args?: e2b.paths['/templates']['post']['requestBody']['content']['application/json'], + args?: e2b.paths['/templates']['post']['requestBody']['content']['application/json'] ) { return await client.api.POST('/templates', { body: args, @@ -73,7 +74,7 @@ async function requestTemplateBuild( async function requestTemplateRebuild( templateID: string, - args?: e2b.paths['/templates/{templateID}']['post']['requestBody']['content']['application/json'], + args?: e2b.paths['/templates/{templateID}']['post']['requestBody']['content']['application/json'] ) { return await client.api.POST('/templates/{templateID}', { body: args, @@ -95,7 +96,7 @@ async function triggerTemplateBuild(templateID: string, buildID: string) { buildID, }, }, - }, + } ) handleE2BRequestError(res.error, 'Error triggering template build') @@ -105,53 +106,53 @@ async function triggerTemplateBuild(templateID: string, buildID: string) { export const buildCommand = new commander.Command('build') .description( `build sandbox template defined by ${asLocalRelative( - defaultDockerfileName, + defaultDockerfileName )} or ${asLocalRelative( - fallbackDockerfileName, + fallbackDockerfileName )} in root directory. By default the root directory is the current working directory. This command also creates ${asLocal( - configName, - )} config.`, + configName + )} config.` ) .argument( '[template]', `specify ${asBold( - '[template]', + '[template]' )} to rebuild it. If you don's specify ${asBold( - '[template]', + '[template]' )} and there is no ${asLocal( - 'e2b.toml', - )} a new sandbox template will be created.`, + 'e2b.toml' + )} a new sandbox template will be created.` ) .addOption(pathOption) .option( '-d, --dockerfile ', `specify path to Dockerfile. By default E2B tries to find ${asLocal( - defaultDockerfileName, - )} or ${asLocal(fallbackDockerfileName)} in root directory.`, + defaultDockerfileName + )} or ${asLocal(fallbackDockerfileName)} in root directory.` ) .option( '-n, --name ', - 'specify sandbox template name. You can use the template name to start the sandbox with SDK. The template name must be lowercase and contain only letters, numbers, dashes and underscores.', + 'specify sandbox template name. You can use the template name to start the sandbox with SDK. The template name must be lowercase and contain only letters, numbers, dashes and underscores.' ) .option( '-c, --cmd ', - 'specify command that will be executed when the sandbox is started.', + 'specify command that will be executed when the sandbox is started.' ) .addOption(teamOption) .addOption(configOption) .option( '--cpu-count ', 'specify the number of CPUs that will be used to run the sandbox. The default value is 2.', - parseInt, + parseInt ) .option( '--memory-mb ', 'specify the amount of memory in megabytes that will be used to run the sandbox. Must be an even number. The default value is 512.', - parseInt, + parseInt ) .option( '--build-arg ', - 'specify additional build arguments for the build command. The format should be =.', + 'specify additional build arguments for the build command. The format should be =.' ) .alias('bd') .action( @@ -167,13 +168,13 @@ export const buildCommand = new commander.Command('build') cpuCount?: number memoryMb?: number buildArg?: [string] - }, + } ) => { try { const dockerInstalled = commandExists.sync('docker') if (!dockerInstalled) { console.error( - 'Docker is required to build and push the sandbox template. Please install Docker and try again.', + 'Docker is required to build and push the sandbox template. Please install Docker and try again.' ) process.exit(1) } @@ -193,8 +194,8 @@ export const buildCommand = new commander.Command('build') if (newName && !/[a-z0-9-_]+/.test(newName)) { console.error( `Name ${asLocal( - newName, - )} is not valid. Name can only contain lowercase letters, numbers, dashes and underscores.`, + newName + )} is not valid. Name can only contain lowercase letters, numbers, dashes and underscores.` ) process.exit(1) } @@ -223,8 +224,8 @@ export const buildCommand = new commander.Command('build') ? [config.template_name] : undefined, }, - relativeConfigPath, - )}`, + relativeConfigPath + )}` ) templateID = config.template_id dockerfile = opts.dockerfile || config.dockerfile @@ -242,7 +243,7 @@ export const buildCommand = new commander.Command('build') if (config && templateID && config.template_id !== templateID) { // error: you can't specify different ID than the one in config console.error( - "You can't specify different ID than the one in config. If you want to build a new sandbox template remove the config file.", + "You can't specify different ID than the one in config. If you want to build a new sandbox template remove the config file." ) process.exit(1) } @@ -254,21 +255,21 @@ export const buildCommand = new commander.Command('build') ) { console.log( `The sandbox template name will be changed from ${asLocal( - config.template_name, - )} to ${asLocal(newName)}.`, + config.template_name + )} to ${asLocal(newName)}.` ) } const name = newName || config?.template_name const { dockerfileContent, dockerfileRelativePath } = getDockerfile( root, - dockerfile, + dockerfile ) console.log( `Found ${asLocalRelative( - dockerfileRelativePath, - )} that will be used to build the sandbox template.`, + dockerfileRelativePath + )} that will be used to build the sandbox template.` ) const body = { @@ -284,23 +285,20 @@ export const buildCommand = new commander.Command('build') if (opts.memoryMb % 2 !== 0) { console.error( `The memory in megabytes must be an even number. You provided ${asLocal( - opts.memoryMb.toFixed(0), - )}.`, + opts.memoryMb.toFixed(0) + )}.` ) process.exit(1) } } - const template = await requestBuildTemplate( - body, - templateID, - ) + const template = await requestBuildTemplate(body, templateID) templateID = template.templateID console.log( `Requested build for the sandbox template ${asFormattedSandboxTemplate( - template, - )} `, + template + )} ` ) await saveConfig( @@ -314,7 +312,7 @@ export const buildCommand = new commander.Command('build') memory_mb: memoryMB, team_id: teamID, }, - true, + true ) try { @@ -323,23 +321,24 @@ export const buildCommand = new commander.Command('build') { stdio: 'inherit', cwd: root, - }, + } ) } catch (err: any) { console.error( - 'Docker login failed. Please try to log in with `e2b auth login` and try again.', + 'Docker login failed. Please try to log in with `e2b auth login` and try again.' ) process.exit(1) } process.stdout.write('\n') console.log('Building docker image...') - const cmd = `docker build . -f ${dockerfileRelativePath} --pull --platform linux/amd64 -t docker.${connectionConfig.domain - }/e2b/custom-envs/${templateID}:${template.buildID} ${Object.entries( - dockerBuildArgs, - ) - .map(([key, value]) => `--build-arg="${key}=${value}"`) - .join(' ')}` + const cmd = `docker build . -f ${dockerfileRelativePath} --pull --platform linux/amd64 -t docker.${ + connectionConfig.domain + }/e2b/custom-envs/${templateID}:${template.buildID} ${Object.entries( + dockerBuildArgs + ) + .map(([key, value]) => `--build-arg="${key}=${value}"`) + .join(' ')}` child_process.execSync(cmd, { stdio: 'inherit', cwd: root, @@ -351,13 +350,23 @@ export const buildCommand = new commander.Command('build') console.log('Docker image built.\n') console.log('Pushing docker image...') - child_process.execSync( - `docker push docker.${connectionConfig.domain}/e2b/custom-envs/${templateID}:${template.buildID}`, - { - stdio: 'inherit', - cwd: root, - }, - ) + try { + child_process.execSync( + `docker push docker.${connectionConfig.domain}/e2b/custom-envs/${templateID}:${template.buildID}`, + { + stdio: 'inherit', + cwd: root, + } + ) + } catch (err: any) { + await buildWithProxy( + userConfig, + connectionConfig, + accessToken, + template, + root + ) + } console.log('Docker image pushed.\n') console.log('Triggering build...') @@ -365,29 +374,25 @@ export const buildCommand = new commander.Command('build') console.log( `Triggered build for the sandbox template ${asFormattedSandboxTemplate( - template, - )} `, + template + )} ` ) console.log('Waiting for build to finish...') - await waitForBuildFinish( - templateID, - template.buildID, - name, - ) + await waitForBuildFinish(templateID, template.buildID, name) process.exit(0) } catch (err: any) { console.error(err) process.exit(1) } - }, + } ) async function waitForBuildFinish( templateID: string, buildID: string, - name?: string, + name?: string ) { let logsOffset = 0 @@ -409,7 +414,7 @@ async function waitForBuildFinish( switch (template.status) { case 'building': template.logs.forEach((line) => - process.stdout.write(asBuildLogs(stripAnsi.default(line))), + process.stdout.write(asBuildLogs(stripAnsi.default(line))) ) break case 'ready': { @@ -419,15 +424,19 @@ async function waitForBuildFinish( sandbox = Sandbox("${aliases?.length ? aliases[0] : template.templateID}") # Create async sandbox -sandbox = await AsyncSandbox.create("${aliases?.length ? aliases[0] : template.templateID}")`) +sandbox = await AsyncSandbox.create("${ + aliases?.length ? aliases[0] : template.templateID + }")`) const typescriptExample = asTypescript(`import { Sandbox } from 'e2b' // Create sandbox -const sandbox = await Sandbox.create('${aliases?.length ? aliases[0] : template.templateID}')`) +const sandbox = await Sandbox.create('${ + aliases?.length ? aliases[0] : template.templateID + }')`) const examplesMessage = `You can now use the template to create custom sandboxes.\nLearn more on ${asPrimary( - 'https://e2b.dev/docs', + 'https://e2b.dev/docs' )}` const exampleHeader = boxen.default(examplesMessage, { @@ -451,28 +460,28 @@ const sandbox = await Sandbox.create('${aliases?.length ? aliases[0] : template. const exampleUsage = `${withDelimiter( pythonExample, - 'Python SDK', + 'Python SDK' )}\n${withDelimiter(typescriptExample, 'JS SDK', true)}` console.log( `\n✅ Building sandbox template ${asFormattedSandboxTemplate({ aliases, ...template, - })} finished.\n${exampleHeader}\n${exampleUsage}\n`, + })} finished.\n${exampleHeader}\n${exampleUsage}\n` ) break } case 'error': template.logs.forEach((line) => - process.stdout.write(asBuildLogs(stripAnsi.default(line))), + process.stdout.write(asBuildLogs(stripAnsi.default(line))) ) throw new Error( `\n❌ Building sandbox template ${asFormattedSandboxTemplate({ aliases, ...template, })} failed.\nCheck the logs above for more details or contact us ${asPrimary( - '(https://e2b.dev/docs/getting-help)', - )} to get help.\n`, + '(https://e2b.dev/docs/getting-help)' + )} to get help.\n` ) } } while (template.status === 'building') @@ -496,8 +505,8 @@ function getDockerfile(root: string, file?: string) { if (dockerfileContent === undefined) { throw new Error( `No ${asLocalRelative( - dockerfileRelativePath, - )} found in the root directory.`, + dockerfileRelativePath + )} found in the root directory.` ) } @@ -538,16 +547,16 @@ function getDockerfile(root: string, file?: string) { throw new Error( `No ${asLocalRelative(defaultDockerfileRelativePath)} or ${asLocalRelative( - fallbackDockerfileRelativeName, + fallbackDockerfileRelativeName )} found in the root directory (${root}). You can specify a custom Dockerfile with ${asBold( - '--dockerfile ', - )} option.`, + '--dockerfile ' + )} option.` ) } async function requestBuildTemplate( args: e2b.paths['/templates']['post']['requestBody']['content']['application/json'], - templateID?: string, + templateID?: string ): Promise< Omit< e2b.paths['/templates']['post']['responses']['202']['content']['application/json'], diff --git a/packages/cli/src/commands/template/buildWithProxy.ts b/packages/cli/src/commands/template/buildWithProxy.ts new file mode 100644 index 000000000..362f12afa --- /dev/null +++ b/packages/cli/src/commands/template/buildWithProxy.ts @@ -0,0 +1,210 @@ +import child_process from 'child_process' +import * as e2b from 'e2b' +import * as http from 'http' +import * as url from 'url' +import * as https from 'node:https' +import { confirm } from '../../utils/confirm' +import { USER_CONFIG_PATH, UserConfig } from 'src/user' +import * as fs from 'fs' + +const PORT = 49984 + +export async function buildWithProxy( + userConfig: UserConfig | null, + connectionConfig: e2b.ConnectionConfig, + accessToken: string, + template: { templateID: string; buildID: string }, + root: string +) { + if (!userConfig?.dockerProxySet) { + console.log( + "There was an issue during Docker authentication. Please follow the workaround steps from https://e2b.dev/docs/cli/build-auth-error and then continue." + ) + const yes = await confirm('Have you completed the steps from the https://e2b.dev/docs/cli/build-auth-error workaround guide?') + + if (!yes) { + console.log('Please follow the workaround steps from https://e2b.dev/docs/cli/build-auth-error and then try again.') + process.exit(1) + } + } + + let proxyStarted: ((value: unknown) => void) | undefined = undefined + + const proxyReady = new Promise((resolve) => { + proxyStarted = resolve + }) + + const accessTokenBase64Encoded = Buffer.from( + `_e2b_access_token:${accessToken}` + ).toString('base64') + + const proxyServer = await proxy( + connectionConfig, + template, + accessTokenBase64Encoded, + proxyStarted! + ) + + await proxyReady + const success = await docker(connectionConfig, template, root) + + if (!success) { + console.error('Docker push failed') + process.exit(1) + } + + if (userConfig && !userConfig.dockerProxySet) { + userConfig.dockerProxySet = true + fs.writeFileSync(USER_CONFIG_PATH, JSON.stringify(userConfig, null, 2)) + } + + proxyServer.close() +} + +async function docker( + connectionConfig: e2b.ConnectionConfig, + template: { templateID: string; buildID: string }, + root: string +) { + const localDomain = + process.platform === 'linux' ? 'localhost' : 'host.docker.internal' + let success = false + + child_process.execSync( + `docker tag docker.${connectionConfig.domain}/e2b/custom-envs/${template.templateID}:${template.buildID} ${localDomain}:${PORT}/e2b/custom-envs/${template.templateID}:${template.buildID}`, + { + stdio: 'inherit', + cwd: root, + } + ) + + let onExit: ((code: number | null) => void) | undefined = undefined + const dockerBuilt = new Promise((resolve) => { + onExit = resolve + }) + + const child = child_process.spawn( + 'docker', + [ + 'push', + `${localDomain}:${PORT}/e2b/custom-envs/${template.templateID}:${template.buildID}`, + ], + { + detached: true, + stdio: 'inherit', + cwd: root, + } + ) + child.on('exit', (code) => { + if (code !== 0) { + console.error('Docker push failed') + process.exit(1) + } + success = true + onExit!(code) + }) + + child.on('error', (err) => { + console.error('Error', err) + process.exit(1) + }) + + await dockerBuilt + return success +} + +async function proxy( + connectionConfig: e2b.ConnectionConfig, + template: { templateID: string; buildID: string }, + credsBase64: string, + proxyStarted: (value: unknown) => void +) { + const res = await fetch( + `https://docker.${connectionConfig.domain}/v2/token?account=_e2b_access_token&scope=repository%3Ae2b%2Fcustom-envs%2F${template.templateID}%3Apush%2Cpull`, + { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${credsBase64}`, + }, + } + ) + + const { token } = await res.json() + + const proxyServer = http.createServer( + (clientReq: http.IncomingMessage, clientRes: http.ServerResponse) => { + // Parse the target URL + const targetUrl = new url.URL( + clientReq.url || '/', + `https://docker.${connectionConfig.domain}` + ) + + // Construct options for the proxy request + const options = { + protocol: 'https:', + hostname: targetUrl.hostname, + method: clientReq.method, + path: targetUrl.pathname + targetUrl.search, + headers: { + ...clientReq.headers, + host: targetUrl.hostname, + }, + } as http.RequestOptions + + if (!options.headers!.Authorization) { + if (targetUrl.pathname.startsWith('/v2/token')) { + options.headers!.Authorization = `Basic ${credsBase64}` + } else if ( + targetUrl.pathname == '/v2/' || + targetUrl.pathname == '/v2' + ) { + options.headers!.Authorization = `Bearer ${credsBase64}` + } else if ( + // Exclude the artifacts-uploads namespace + !targetUrl.pathname.startsWith('/artifacts-uploads/namespaces') + ) { + options.headers!.Authorization = `Bearer ${token}` + } + } + + // Create the proxy getHeaders + const proxyReq: http.ClientRequest = https.request( + options, + (proxyRes: http.IncomingMessage) => { + // Copy status code and headers + clientRes.writeHead(proxyRes.statusCode || 500, proxyRes.headers) + // Pipe the response data + proxyRes.pipe(clientRes, { + end: true, + }) + } + ) + + // Handle proxy request errors + proxyReq.on('error', (err: Error) => { + console.error('Proxy Request Error:', err) + clientRes.statusCode = 500 + clientRes.end(`Proxy Error: ${err.message}`) + }) + + // Pipe the client request data to proxy request + clientReq.pipe(proxyReq, { + end: true, + }) + } + ) + + // Handle server errors + proxyServer.on('error', (err: Error) => { + console.error('Server Error:', err) + }) + + // Start the server + proxyServer.listen(PORT, () => { + proxyStarted(null) + console.log(`Proxy server running on port ${PORT}`) + }) + + return proxyServer +} diff --git a/packages/cli/src/user.ts b/packages/cli/src/user.ts index d90580972..b2dbbfcd5 100644 --- a/packages/cli/src/user.ts +++ b/packages/cli/src/user.ts @@ -10,6 +10,7 @@ export interface UserConfig { teamName: string teamId: string teamApiKey: string + dockerProxySet?: boolean } export const USER_CONFIG_PATH = path.join(os.homedir(), '.e2b', 'config.json') // TODO: Keep in Keychain