Skip to content

Commit

Permalink
Merge pull request #449 from alopukhov/better_socket_activation
Browse files Browse the repository at this point in the history
Add execution branch for socket activation to correct LISTEN_PID
  • Loading branch information
AkihiroSuda authored Jul 16, 2024
2 parents 4fb2e2c + ca26493 commit 3cb66b7
Show file tree
Hide file tree
Showing 8 changed files with 214 additions and 57 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ jobs:
run: docker run --rm --net=host --privileged rootlesskit:test-integration ./integration-port.sh
- name: "Integration test: IPv6 routing"
run: docker run --rm --privileged --sysctl net.ipv6.conf.all.disable_ipv6=0 rootlesskit:test-integration ./integration-ipv6.sh
- name: "Integration test: systemd socket activation"
run: docker run --rm --net=none --privileged rootlesskit:test-integration ./integration-systemd-socket.sh
- name: "Integration test: Network (network driver=slirp4netns)"
run: |
docker run --rm --privileged rootlesskit:test-integration ./integration-net.sh slirp4netns
Expand Down
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ FROM ubuntu:${UBUNTU_VERSION} AS test-integration
# sudo: only for lxc-user-nic benchmark and rootful veth benchmark (for comparison)
# libcap2-bin and curl: used by the RUN instructions in this Dockerfile.
# bind9-dnsutils: for `nslookup` command used by integration-net.sh
RUN apt-get update && apt-get install -y iproute2 liblxc-common lxc-utils iperf3 busybox sudo libcap2-bin curl bind9-dnsutils
# systemd and uuid-runtime: for systemd-socket-activate used by integration-systemd-socket.sh
RUN apt-get update && apt-get install -y iproute2 liblxc-common lxc-utils iperf3 busybox sudo libcap2-bin curl bind9-dnsutils systemd uuid-runtime
COPY --from=idmap /usr/bin/newuidmap /usr/bin/newuidmap
COPY --from=idmap /usr/bin/newgidmap /usr/bin/newgidmap
RUN /sbin/setcap cap_setuid+eip /usr/bin/newuidmap && \
Expand Down
62 changes: 50 additions & 12 deletions cmd/rootlesskit/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,14 @@ import (
"os/exec"
"path/filepath"
"strings"
"strconv"
"syscall"

"github.com/Masterminds/semver/v3"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"

"github.com/rootless-containers/rootlesskit/v2/pkg/systemd/activation"
"github.com/rootless-containers/rootlesskit/v2/pkg/child"
"github.com/rootless-containers/rootlesskit/v2/pkg/common"
"github.com/rootless-containers/rootlesskit/v2/pkg/copyup/tmpfssymlink"
Expand All @@ -29,17 +31,24 @@ import (
"github.com/rootless-containers/rootlesskit/v2/pkg/version"
)


const (
pipeFDEnvKey = "_ROOTLESSKIT_PIPEFD_UNDOCUMENTED"
childUseActivationEnvKey = "_ROOTLESSKIT_SYSTEMD_ACTIVATION_CHILD_USE_UNDOCUMENTED"
runActivationHelperEnvKey = "_ROOTLESSKIT_SYSTEMD_ACTIVATION_RUN_HELPER_UNDOCUMENTED"
stateDirEnvKey = "ROOTLESSKIT_STATE_DIR" // documented
parentEUIDEnvKey = "ROOTLESSKIT_PARENT_EUID" // documented
parentEGIDEnvKey = "ROOTLESSKIT_PARENT_EGID" // documented
)

func main() {
const (
pipeFDEnvKey = "_ROOTLESSKIT_PIPEFD_UNDOCUMENTED"
stateDirEnvKey = "ROOTLESSKIT_STATE_DIR" // documented
parentEUIDEnvKey = "ROOTLESSKIT_PARENT_EUID" // documented
parentEGIDEnvKey = "ROOTLESSKIT_PARENT_EGID" // documented
)
iAmActivationHelper := checkActivationHelper()
iAmChild := os.Getenv(pipeFDEnvKey) != ""
id := "parent"
if iAmChild {
id = "child " // padded to len("parent")
} else if iAmActivationHelper {
id = "activation_helper"
}
debug := false
app := cli.NewApp()
Expand Down Expand Up @@ -252,15 +261,21 @@ OPTIONS:
if clicontext.NArg() < 1 {
return errors.New("no command specified")
}
if iAmActivationHelper {
activationOpt, err := createActivationOpts(clicontext)
if err != nil {
return err
}
return activation.ActivationHelper(activationOpt)
}
if iAmChild {
childOpt, err := createChildOpt(clicontext, pipeFDEnvKey, stateDirEnvKey, clicontext.Args().Slice())
childOpt, err := createChildOpt(clicontext)
if err != nil {
return err
}
return child.Child(childOpt)
}
parentOpt, err := createParentOpt(clicontext, pipeFDEnvKey, stateDirEnvKey,
parentEUIDEnvKey, parentEGIDEnvKey)
parentOpt, err := createParentOpt(clicontext)
if err != nil {
return err
}
Expand Down Expand Up @@ -305,11 +320,12 @@ func parseCIDR(s string) (*net.IPNet, error) {
return ipnet, nil
}

func createParentOpt(clicontext *cli.Context, pipeFDEnvKey, stateDirEnvKey, parentEUIDEnvKey, parentEGIDEnvKey string) (parent.Opt, error) {
func createParentOpt(clicontext *cli.Context) (parent.Opt, error) {
var err error
opt := parent.Opt{
PipeFDEnvKey: pipeFDEnvKey,
StateDirEnvKey: stateDirEnvKey,
ChildUseActivationEnvKey: childUseActivationEnvKey,
CreatePIDNS: clicontext.Bool("pidns"),
CreateCgroupNS: clicontext.Bool("cgroupns"),
CreateUTSNS: clicontext.Bool("utsns"),
Expand Down Expand Up @@ -575,13 +591,15 @@ func (w *logrusDebugWriter) Write(p []byte) (int, error) {
return len(p), nil
}

func createChildOpt(clicontext *cli.Context, pipeFDEnvKey, stateDirEnvKey string, targetCmd []string) (child.Opt, error) {
func createChildOpt(clicontext *cli.Context) (child.Opt, error) {
pidns := clicontext.Bool("pidns")
detachNetNS := clicontext.Bool("detach-netns")
opt := child.Opt{
PipeFDEnvKey: pipeFDEnvKey,
RunActivationHelperEnvKey: runActivationHelperEnvKey,
ChildUseActivationEnvKey: childUseActivationEnvKey,
StateDirEnvKey: stateDirEnvKey,
TargetCmd: targetCmd,
TargetCmd: clicontext.Args().Slice(),
MountProcfs: pidns,
DetachNetNS: detachNetNS,
Propagation: clicontext.String("propagation"),
Expand Down Expand Up @@ -664,3 +682,23 @@ func unameM() string {
}
return machine
}

func checkActivationHelper() bool {
envValue, envSet := os.LookupEnv(runActivationHelperEnvKey)
if !envSet {
return false
}
activationHelperValue, err := strconv.ParseBool(envValue)
if err != nil {
panic(fmt.Sprintf("Env variable [%s] is set to [%s] and cannot be parsed", runActivationHelperEnvKey, envValue))
}
return activationHelperValue
}

func createActivationOpts(clicontext *cli.Context) (activation.Opt, error) {
opt := activation.Opt {
RunActivationHelperEnvKey: runActivationHelperEnvKey,
TargetCmd: clicontext.Args().Slice(),
}
return opt, nil
}
33 changes: 33 additions & 0 deletions hack/integration-systemd-socket-check-env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash

set -eu -o pipefail

OK_FILE=$1
ERR_FILE=$2
EXPECTED_LISTEN_FDS=$3

fail() {
echo "$@" > "$ERR_FILE"
exit 1
}

if ! [[ "${LISTEN_FDS:-}" =~ [1-9] ]]; then
fail "LISTEN_FDS (${LISTEN_FDS:-}) is not set or not positive a number."
fi

if [[ "${LISTEN_FDS:-}" != "${EXPECTED_LISTEN_FDS}" ]]; then
fail "LISTEN_FDS (${LISTEN_FDS}) is not equal to expected ${EXPECTED_LISTEN_FDS}."
fi

if [[ "${LISTEN_PID}" != "$$" ]]; then
fail "LISTEN_PID (${LISTEN_PID}) is not equal to \$\$ ($$)."
fi

for ((i=0,fdnum=3; i<LISTEN_FDS; fdnum++, i++)); do
fdpath="/proc/$$/fd/${fdnum}"
if [[ ! -e "$fdpath" ]]; then
fail "FD #${fdnum} does not exists"
fi
done

touch "${OK_FILE}"
72 changes: 55 additions & 17 deletions hack/integration-systemd-socket.sh
Original file line number Diff line number Diff line change
@@ -1,17 +1,55 @@
#!/bin/sh
set -e
if [ -z "$EXECED" ]
then
systemd-socket-activate -E EXECED=1 -l /tmp/activate.sock socat ACCEPT-FD:3 EXEC:"rootlesskit $0",nofork 2>/dev/null &
OUTPUT="$(curl --unix-socket /tmp/activate.sock http://localhost/hello 2>/dev/null)"
[ "$(printf 'Hello\n' )" = "$OUTPUT" ] || exit 1
else
[ "$LISTEN_FDS" = "1" ] || exit 1
read -r REQUEST
if [ "$(printf 'GET /hello HTTP/1.1\r\n')" = "$REQUEST" ]
then
printf 'HTTP/1.1 200 OK\r\nContent-Length: 6\r\n\r\nHello\n'
else
printf 'HTTP/1.1 400 Bad Request\r\nContent-Length: 5\r\n\r\nBad!\n'
fi
fi
#!/bin/bash

srcdir=$(realpath $(dirname $0))
source "${srcdir}/common.inc.sh"

test_with_uuidd_daemon() {
uuidd_tmpdir=$(mktemp -d)
uuidd_sock="${uuidd_tmpdir}/uuidd.sock"
systemd-socket-activate -l "${uuidd_sock}" "$ROOTLESSKIT" uuidd --no-pid --no-fork --socket-activation &
pid=$!
sleep 2
uuidd -d -r -n 1 -s "${uuidd_sock}" || return 1
uuidd -d -t -n 1 -s "${uuidd_sock}" || return 1
uuidd -d -k -s "${uuidd_sock}" || return 1
rm -r "${uuidd_tmpdir}" || return 1
wait $pid || return 1
}

test_env_variables() {
tmpdir=$(mktemp -d)
sock1="${tmpdir}/sock1.sock"
sock2="${tmpdir}/sock2.sock"
sock3="${tmpdir}/sock3.sock"
## Test 1 socket
timeout 30 systemd-socket-activate -l "${sock1}" "$ROOTLESSKIT" "${srcdir}/integration-systemd-socket-check-env.sh" "${tmpdir}/ok1" "${tmpdir}/fail1" 1 &
pid=$!
sleep 2
curl --unix-socket "${sock1}" "http//example.com" >/dev/null 2>&1 || true # just trigger
wait $pid
if [[ ! -e "${tmpdir}/ok1" ]]; then return 1; fi
## Test 2 sockets
timeout 30 systemd-socket-activate -l "${sock1}" -l "${sock2}" "$ROOTLESSKIT" "${srcdir}/integration-systemd-socket-check-env.sh" "${tmpdir}/ok2" "${tmpdir}/fail2" 2 &
pid=$!
sleep 2
curl --unix-socket "${sock1}" "http//example.com" >/dev/null 2>&1 || true
wait $pid
if [[ ! -e "${tmpdir}/ok2" ]]; then return 1; fi
## Test 3 sockets
timeout 30 systemd-socket-activate -l "${sock1}" -l "${sock2}" -l "${sock3}" "$ROOTLESSKIT" "${srcdir}/integration-systemd-socket-check-env.sh" "${tmpdir}/ok3" "${tmpdir}/fail3" 3 &
pid=$!
sleep 2
curl --unix-socket "${sock1}" "http//example.com" >/dev/null 2>&1 || true
wait $pid
if [[ ! -e "${tmpdir}/ok3" ]]; then return 1; fi

rm -r "${tmpdir}"
}

INFO "===== Systemd socket activation: uuidd daemon ====="
test_with_uuidd_daemon

INFO "===== Systemd socket activation: LISTEN_* variables check ====="
test_env_variables

INFO "===== PASSING ====="
31 changes: 23 additions & 8 deletions pkg/child/child.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,29 @@ func setupFiles(cmd *exec.Cmd) {
}


func createCmd(targetCmd []string) (*exec.Cmd, error) {
var args []string
if len(targetCmd) > 1 {
args = targetCmd[1:]
}
cmd := exec.Command(targetCmd[0], args...)
func createCmd(opt Opt) (*exec.Cmd, error) {
fixListenPidEnv, err := strconv.ParseBool(os.Getenv(opt.ChildUseActivationEnvKey))
if err != nil {
fixListenPidEnv = false
}
os.Unsetenv(opt.ChildUseActivationEnvKey)
targetCmd := opt.TargetCmd
var cmd *exec.Cmd
cmdEnv := os.Environ()
if fixListenPidEnv {
cmd = exec.Command("/proc/self/exe", os.Args[1:]...)
cmdEnv = append(cmdEnv, opt.RunActivationHelperEnvKey + "=true")
} else {
var args []string
if len(targetCmd) > 1 {
args = targetCmd[1:]
}
cmd = exec.Command(targetCmd[0], args...)
}
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Env = os.Environ()
cmd.Env = cmdEnv
cmd.SysProcAttr = &syscall.SysProcAttr{
Pdeathsig: syscall.SIGKILL,
}
Expand Down Expand Up @@ -252,6 +265,8 @@ func setupNet(stateDir string, msg *messages.ParentInitNetworkDriverCompleted, e

type Opt struct {
PipeFDEnvKey string // needs to be set
RunActivationHelperEnvKey string // needs to be set
ChildUseActivationEnvKey string // needs to be set
StateDirEnvKey string // needs to be set
TargetCmd []string // needs to be set
NetworkDriver network.ChildDriver // nil for HostNetwork
Expand Down Expand Up @@ -458,7 +473,7 @@ func Child(opt Opt) error {
}()
}

cmd, err := createCmd(opt.TargetCmd)
cmd, err := createCmd(opt)
if err != nil {
return err
}
Expand Down
40 changes: 21 additions & 19 deletions pkg/parent/parent.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

type Opt struct {
PipeFDEnvKey string // needs to be set
ChildUseActivationEnvKey string // needs to be set
StateDir string // directory needs to be precreated
StateDirEnvKey string // optional env key to propagate StateDir value
NetworkDriver network.ParentDriver // nil for HostNetwork
Expand Down Expand Up @@ -125,25 +126,26 @@ func LockStateDir(stateDir string) (*flock.Flock, error) {
return lock, nil
}

func setupFilesAndEnv(cmd *exec.Cmd, readPipe *os.File, writePipe *os.File, envKey string) {
func setupFilesAndEnv(readPipe *os.File, writePipe *os.File, opt Opt) ([]*os.File, []string) {
// 0 1 and 2 are used for stdin. stdout, and stderr
const firstExtraFD = 3
systemdActivationFDs := 0
// check for systemd socket activation sockets
if v := os.Getenv("LISTEN_FDS"); v != "" {
if num, err := strconv.Atoi(v); err == nil {
systemdActivationFDs = num
}
}
cmd.ExtraFiles = make([]*os.File, systemdActivationFDs + 2)
for fd := 0; fd < systemdActivationFDs; fd++ {
cmd.ExtraFiles[fd] = os.NewFile(uintptr(firstExtraFD + fd), "")
}
readIndex := systemdActivationFDs
writeIndex := readIndex + 1
cmd.ExtraFiles[readIndex] = readPipe
cmd.ExtraFiles[writeIndex] = writePipe
cmd.Env = append(os.Environ(), envKey+"="+strconv.Itoa(firstExtraFD+readIndex)+","+strconv.Itoa(firstExtraFD+writeIndex))
const listenFdsStart = 3
listenPid, listenPidErr := strconv.Atoi(os.Getenv("LISTEN_PID"))
listenFds, listenFdsErr := strconv.Atoi(os.Getenv("LISTEN_FDS"))
useSystemdSocketFDs := listenPidErr == nil && listenFdsErr == nil && listenFds > 0
if !useSystemdSocketFDs {
listenFds = 0
}
extraFiles := make([]*os.File, listenFds + 2)
for i, fd := 0, listenFdsStart; i < listenFds; i, fd = i + 1, fd + 1 {
name := "LISTEN_FD_" + strconv.Itoa(fd)
extraFiles[i] = os.NewFile(uintptr(fd), name)
}
extraFiles[listenFds] = readPipe
extraFiles[listenFds + 1] = writePipe
cmdEnv := os.Environ()
cmdEnv = append(cmdEnv, opt.PipeFDEnvKey + "=" + strconv.Itoa(listenFdsStart + listenFds) + "," + strconv.Itoa(listenFdsStart + listenFds + 1))
cmdEnv = append(cmdEnv, opt.ChildUseActivationEnvKey + "=" + strconv.FormatBool(listenPid == os.Getpid()))
return extraFiles, cmdEnv
}

func Parent(opt Opt) error {
Expand Down Expand Up @@ -199,7 +201,7 @@ func Parent(opt Opt) error {
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
setupFilesAndEnv(cmd, pipeR, pipe2W, opt.PipeFDEnvKey)
cmd.ExtraFiles, cmd.Env = setupFilesAndEnv(pipeR, pipe2W, opt)
if opt.StateDirEnvKey != "" {
cmd.Env = append(cmd.Env, opt.StateDirEnvKey+"="+opt.StateDir)
}
Expand Down
28 changes: 28 additions & 0 deletions pkg/systemd/activation/activation.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package activation

import (
"os"
"os/exec"
"syscall"
"strconv"
)

type Opt struct {
RunActivationHelperEnvKey string // needs to be set
TargetCmd []string // needs to be set
}

func ActivationHelper(opt Opt) error {
pid := os.Getpid()
os.Unsetenv(opt.RunActivationHelperEnvKey)
os.Setenv("LISTEN_PID", strconv.Itoa(pid))
argsv := opt.TargetCmd
execPath, err := exec.LookPath(argsv[0])
if err != nil {
return err
}
if err = syscall.Exec(execPath, argsv, os.Environ()); err != nil {
return err
}
panic("should not reach here")
}

0 comments on commit 3cb66b7

Please sign in to comment.