Skip to content

Commit

Permalink
Improved deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
darxkies committed Nov 19, 2020
1 parent d7751a3 commit 05660b2
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 35 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ require (
golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9
golang.org/x/net v0.0.0-20200707034311-ab3426394381
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68
google.golang.org/grpc v1.27.0
gopkg.in/ini.v1 v1.58.0
gopkg.in/yaml.v2 v2.2.8
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -825,6 +825,8 @@ golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf h1:kt3wY1Lu5MJAnKTfoMR52Cu4g
golang.org/x/sys v0.0.0-20201107080550-4d91cf3a1aaf/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7 h1:s330+6z/Ko3J0o6rvOcwXe5nzs7UT9tLKHoOXYn6uE0=
golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
44 changes: 12 additions & 32 deletions pkg/deployment/node_deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@ package deployment

import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
"path"
Expand All @@ -21,17 +18,18 @@ import (
)

type NodeDeployment struct {
identityFile string
name string
node *config.Node
config *config.InternalConfig
sshLimiter *utils.Limiter
parallel bool
checksumsFilename string
identityFile string
name string
node *config.Node
config *config.InternalConfig
sshLimiter *utils.Limiter
parallel bool
targetChecksumsFilename string
localChecksums *utils.Checksums
}

func NewNodeDeployment(identityFile string, name string, node *config.Node, config *config.InternalConfig, parallel bool) *NodeDeployment {
return &NodeDeployment{identityFile: identityFile, name: name, node: node, config: config, sshLimiter: utils.NewLimiter(utils.ConcurrentSshConnectionsLimit), parallel: parallel, checksumsFilename: path.Join(config.GetFullTargetAssetDirectory(utils.DirectoryDynamicData), "checksums")}
return &NodeDeployment{identityFile: identityFile, name: name, node: node, config: config, sshLimiter: utils.NewLimiter(utils.ConcurrentSshConnectionsLimit), parallel: parallel, targetChecksumsFilename: path.Join(config.GetFullTargetAssetDirectory(utils.DirectoryDynamicData), "checksums"), localChecksums: utils.NewChecksums(path.Join(config.GetFullLocalAssetDirectory(utils.DirectoryDynamicData), "checksums"), config.BaseDirectory)}
}

func (deployment *NodeDeployment) Steps(skipRestart bool) (result int) {
Expand Down Expand Up @@ -63,23 +61,7 @@ func (deployment *NodeDeployment) Steps(skipRestart bool) (result int) {
}

func (deployment *NodeDeployment) md5sum(filename string) (result string, error error) {
file, error := os.Open(filename)

if error != nil {
return
}

defer file.Close()

hash := md5.New()

if _, error = io.Copy(hash, file); error != nil {
return
}

result = hex.EncodeToString(hash.Sum(nil)[:16])

return
return deployment.localChecksums.GetChecksum(filename)
}

func (deployment *NodeDeployment) createDirectories() error {
Expand Down Expand Up @@ -154,7 +136,7 @@ func (deployment *NodeDeployment) getRemoteFileChecksums() map[string]string {
checksumCommand += fmt.Sprintf(" '%s'", toFile)
}

checksumCommand += fmt.Sprintf("; do if [ -f $i ]; then if [ $i -ot '%s' ]; then grep -e \" $i$\" '%s'; else md5sum $i; fi; fi; done", deployment.checksumsFilename, deployment.checksumsFilename)
checksumCommand += fmt.Sprintf("; do if [ -f $i ]; then if [ $i -ot '%s' ]; then grep -e \" $i$\" '%s'; else md5sum $i; fi; fi; done", deployment.targetChecksumsFilename, deployment.targetChecksumsFilename)

output, _ := deployment.Execute("get-checksums", checksumCommand)

Expand Down Expand Up @@ -298,9 +280,7 @@ func (deployment *NodeDeployment) UploadFiles(forceUpload bool, skipRestart bool
utils.IncreaseProgressStep()

if len(filesList) > 0 {
command := fmt.Sprintf("for i in %s; do if [ -f '%s' ]; then grep -ve \" $i$\" '%s' > /tmp/checksums; mv /tmp/checksums %s; fi; md5sum $i >> '%s'; done", filesList, deployment.checksumsFilename, deployment.checksumsFilename, deployment.checksumsFilename, deployment.checksumsFilename)

log.Println(command)
command := fmt.Sprintf("for i in %s; do if [ -f '%s' ]; then grep -ve \" $i$\" '%s' > /tmp/checksums; mv /tmp/checksums %s; fi; md5sum $i >> '%s'; done", filesList, deployment.targetChecksumsFilename, deployment.targetChecksumsFilename, deployment.targetChecksumsFilename, deployment.targetChecksumsFilename)

_, _error = deployment.Execute("update-checkups", command)

Expand Down
103 changes: 103 additions & 0 deletions pkg/utils/checksums.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package utils

import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
"strings"

"github.com/pkg/errors"
)

type Checksums struct {
filename string
baseDirectory string
checksums map[string]string
loaded bool
}

func NewChecksums(filename, baseDirectory string) *Checksums {
return &Checksums{filename: filename, baseDirectory: baseDirectory, checksums: map[string]string{}}
}

func (checksums *Checksums) GetChecksum(targetFilename string) (result string, error error) {
relativeFilename := targetFilename[len(checksums.baseDirectory):]

_ = checksums.load()

checksumsCache, _errorCache := os.Stat(checksums.filename)
checksumsTarget, _errorTarget := os.Stat(targetFilename)

if _errorCache == nil && _errorTarget == nil && checksumsTarget.ModTime().Before(checksumsCache.ModTime()) {
if checksum, ok := checksums.checksums[relativeFilename]; ok {
return checksum, nil
}
}

file, error := os.Open(targetFilename)

if error != nil {
return
}

defer file.Close()

hash := md5.New()

if _, error = io.Copy(hash, file); error != nil {
return
}

result = hex.EncodeToString(hash.Sum(nil)[:16])

checksums.checksums[relativeFilename] = result

_ = checksums.save()

return
}

func (checksums *Checksums) save() error {
buffer := ""

for filename, value := range checksums.checksums {
buffer += fmt.Sprintf("%s %s\n", value, filename)
}

if _error := ioutil.WriteFile(checksums.filename, []byte(buffer), 0644); _error != nil {
return errors.Wrapf(_error, "Could not write to %s", checksums.filename)
}

return nil
}

func (checksums *Checksums) load() error {
if checksums.loaded {
return nil
}

checksums.loaded = true

content, _error := ioutil.ReadFile(checksums.filename)
if _error != nil {
return _error
}

lines := strings.Split(string(content), "\n")

for _, line := range lines {
if len(line) < 35 {
continue
}

filename := line[33:]
checksum := line[:32]

checksums.checksums[filename] = checksum
}

return nil
}
4 changes: 2 additions & 2 deletions setup/multipass/Makefile
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
.EXPORT_ALL_VARIABLES:

NAME=single-node
IP=$(shell multipass info $(NAME) | grep IPv4 | sed -e "s/IPv4://" | xargs)
IP=$(shell multipass info $(NAME) 2> /dev/null | grep IPv4 | sed -e "s/IPv4://" | xargs)
SSH_KEY=$(shell cat ~/.ssh/id_rsa.pub)
CLOUD_INIT="disable_root: 0\nssh_authorized_keys:\\n - $(SSH_KEY)"
CLOUD_INIT="disable_root: 0\\nssh_authorized_keys:\\n - $(SSH_KEY)"

run: destroy create deploy

Expand Down

0 comments on commit 05660b2

Please sign in to comment.