Merge pull request #88 from Pack3tL0ss/dev

Lots of Installer & Image Creator enhancements
Pack3tL0ss · Sep 11, 2020 · fd986cc · fd986cc
2 parents b0b8a2c + 392f426
commit fd986cc
Show file tree

Hide file tree

Showing 16 changed files with 393 additions and 325 deletions.
diff --git a/.static.yaml b/.static.yaml
@@ -2,8 +2,8 @@
 # This file should not be modified.  Static variables/paths used throughout ConsolePi
 # Versioning = YYYY.Major.Patch/Minor
 ---
-CONSOLEPI_VER: 2020-4.2
-INSTALLER_VER: 52
+CONSOLEPI_VER: 2020-4.3
+INSTALLER_VER: 53
 CFG_FILE_VER: 9
 CONFIG_FILE_YAML: /etc/ConsolePi/ConsolePi.yaml
 CONFIG_FILE: /etc/ConsolePi/ConsolePi.conf # For backward compat, use yaml config going forward

diff --git a/ConsolePi.yaml.example b/ConsolePi.yaml.example
@@ -12,12 +12,12 @@ CONFIG:
   vpn_check_ip: 10.0.150.1 # used to check VPN (internal) connectivity should be ip only reachable via VPN
   net_check_ip: 8.8.4.4 # used to check Internet connectivity
   local_domain: example.com # used to bypass VPN. evals domain sent via dhcp option if matches this var will not establish vpn
-  wlan_ip: 10.112.0.1 # IP of ConsolePi when in hotspot mode
+  wlan_ip: 10.110.0.1 # IP of ConsolePi when in hotspot mode
   wlan_ssid: ConsolePi # SSID used in hotspot mode
   wlan_psk: ConsolePiR0cks!! # psk used for hotspot SSID
   wlan_country: US # regulatory domain for hotspot SSID
   wired_dhcp: false # Run dhcp on eth interface (after trying as client)
-  wired_ip: 10.12.0.1 # Fallback IP for eth interface
+  wired_ip: 10.30.110.1 # Fallback IP for eth interface
   btmode: serial # Bluetooth Mode: 'serial' or 'pan'
   cloud: false # enable ConsolePi cloud sync for Clustering (mdns enabled either way)
   cloud_svc: gdrive # must be gdrive (all that is supported now)

diff --git a/README.md b/README.md
@@ -81,6 +81,15 @@ Prior Changes can be found in the - [ChangeLog](changelog.md)
   - Additionally AutoHotSpot was added as a configurable option, but the prompt didn't display.  All of these worked via cmd-line option/silent install.
   > If you did a fresh install w/ any version from v2020-2.4 - v2020-4.2 you are likely impacted.  Just use `sudo passwd consolepi` to set the password as desired.
 
+### Sept 2020 (v2020-4.3) Installer Version 53 Sept 2020 Lots of Installer Tweaks
+- This effort was primarily around the Installer and the Image Creator.
+- Installer: Tested, re-tested, made enhancements/improvements, added more imports
+- Place home/pi home/your-user /root etc. in consolepi-stage dir and run image-creator...
+  - Image Creator will import home/pi into home/pi on the image, entire directory structure.  Same for /root.
+  - Once the installer runs on the image it will also import /home/pi (redundant, but useful if you don't use the image-creator)
+  - Installer also prompts to see if you want to create new users, once created if in the consolepi-stage dir it's structure will be imported
+> So you can import .ssh keys / known_hosts and any other files/dirs you want in the users home.
+
 
 # Features
 ## **Feature Summary Image**
@@ -570,6 +579,8 @@ Examples:
 - The ConsolePi installer will start on first login, as long as the RaspberryPi has internet access.  This can be disabled with `--auto_install=false`.
   > If you set `--auto_install=false`, `--cmd_line=...` is ignored.  You would specify arguments for the installer manually.
 - If the `consolepi-image-creator.sh` script is ran from a ConsolePi, the script will detect that it's a ConsolePi and offer to pre-staage it's existing settings.  If a file has alredy been pre-staged (via consolepi-stage dir) it will skip it.  It will give you the chance to edit ConsolePi.yaml if pre-staged, so you can deploy multiple ConsolePis and edit the specifics for each as you stage them.
+- Entire home directory imports:  If you place /root and/or /home/pi inside the consolepi-stage directory.  Those contents/subdirs will be imported to the respective users directory on the image.
+  - You can even pre-stage a users home directory for a user that doesn't exist.  When the installer runs, you are given the option to create new users.  Once created if a folder is found in consolepi-stage for that user (i.e. `home/pi/consolepi-stage/home/larry`), the contents will be coppied from the `consolepi-stage` dir to `/home/larry`.
 
 The install script (not this image-creator, the installer that actually installs ConsolePi) will look for and if found import a number of items from the consolepi-stage directory.  Gdrive credentials, ovpn settings, ssh keys refer to *TODO link to section highlighting imports*
 

diff --git a/installer/common.sh b/installer/common.sh
@@ -7,7 +7,7 @@
 # wired_dhcp=false  # temp until a config option
 cur_dir=$(pwd)
 iam=${SUDO_USER:-$(who -m | awk '{ print $1 }')}
-tty_cols=$(stty -a | grep -o "columns [0-9]*" | awk '{print $2}')
+tty_cols=$(stty -a 2>/dev/null | grep -o "columns [0-9]*" | awk '{print $2}')
 consolepi_dir="/etc/ConsolePi/"
 src_dir="${consolepi_dir}src/"
 bak_dir="${consolepi_dir}bak/"
@@ -111,29 +111,35 @@ menu_print() {
     line_len=${line_len:=121}
     while (( "$#" )); do
         case "$1" in
+            -c)
+                style="$2"
+                shift 2
+                ;;
             -L|-len)
                 line_len=$2
                 shift 2
                 ;;
             -head)
+                style=${style:-'*'}
                 str=" $2 "
                 len=${#str}
-                # ((line_len+=1)) #actual line_len ends up aw line_len +1 not sure why
+                [[ "$str" =~ "\e[" ]] && ((len-=11))
+                [[ "$str" =~ ';1m' ]] && ((len-=2))
                 left=$(( ((line_len-len))/2 ))
                 [[ $((left+len+left)) -eq $line_len ]] && right=$left || right=$((left+1))
-                printf -v pad_left "%*s" $left && pad_left=${pad_left// /*}
-                printf -v pad_right "%*s" $right && pad_right=${pad_right// /*}
-                printf "%s%s%s\n" "$pad_left" "$str" "$pad_right"
+                printf -v pad_left "%*s" $left && pad_left=${pad_left// /$style}
+                printf -v pad_right "%*s" $right && pad_right=${pad_right// /$style}
+                printf "%s%b%s\n" "$pad_left" "$str" "$pad_right"
                 shift 2
                 ;;
             -foot)
-                str="**$2"
+                str="${style}${style}$2"
                 len=${#str}
                 right=$(( ((line_len-len)) ))
-                printf -v pad_right "%*s" $right && pad_right=${pad_right// /*}
+                printf -v pad_right "%*s" $right && pad_right=${pad_right// /$style}
                 printf "%s%s\n" "$str" "$pad_right"
                 shift 2
-                unset line_len
+                unset line_len; unset style
                 ;;
             -nl|-li|*)
                 if [[ "$1" == "-nl" ]]; then
@@ -149,7 +155,7 @@ menu_print() {
                 [[ "$str" =~ ';1m' ]] && ((len-=2))
                 pad_len=$(( ((line_len-len-5)) ))
                 printf -v pad "%*s" $pad_len # && pad=${pad// /-}
-                printf '* %b %s *\n' "$str" "$pad"
+                printf '%s %b %s %s\n' "$style" "$str" "$pad" "$style"
                 shift
                 ;;
         esac
@@ -169,7 +175,7 @@ logit() {
     #
     # NOTE: Sending a status of "ERROR" results in the script exiting
     #       default status is INFO if none provided.
-    [[ $(basename "$0") == 'dhcpcd.exit-hook' ]] && stop_on_error=false || stop_on_error=true
+    [[ $(basename "$0" 2>/dev/null) == 'dhcpcd.exit-hook' ]] && stop_on_error=false || stop_on_error=true
     local args=()
     while (( "$#" )); do
         case "$1" in
@@ -222,7 +228,11 @@ logit() {
 
     # if status was ERROR which means FATAL then log and exit script
     if $fatal ; then
-        echo -e "$(date +'%b %d %T') [$$][${status}][${process}] Last Error is fatal, script exiting Please review log ${log_file}" && exit 1
+        echo -e "$(date +'%b %d %T') [$$][${status}][${process}] Last Error is fatal, script exiting Please review log ${log_file}"
+        echo -e "\n${_red}---- Error Detail ----${_norm}"
+        grep -A 999 "${log_start}" $log_file | grep -v "^WARNING: Retrying " | grep -v "apt does not have a stable CLI interface" | grep "ERROR" -B 10 | grep -v "INFO"
+        echo '--'
+        exit 1
     fi
 }
 
@@ -580,12 +590,12 @@ spaces() {
 }
 
 process_cmds() {
-    reset_vars=('cmd' 'pmsg' 'fmsg' 'cmd_pfx' 'fail_lvl' 'silent' 'out' 'stop' 'err' 'showstart' 'pname' 'pexclude' 'pkg' 'do_apt_install')
+    reset_vars=('cmd' 'pmsg' 'fmsg' 'cmd_pfx' 'fail_lvl' '_silent' 'out' 'stop' 'err' 'showstart' 'pname' 'pexclude' 'pkg' 'do_apt_install')
     local do_autoremove=false  # TODO check if the return is necessary may be relic from early testing
     $_DEBUG_ && echo "DEBUG: ${@}"  ## -- DEBUG LINE --
     while (( "$#" )); do
         if $_DEBUG_; then
-            echo -e "DEBUG:\n\tcmd=${cmd}\n\tsilent=$silent\n\tpmsg=${pmsg}\n\tfmsg=${fmsg}\n\tfail_lvl=$fail_lvl"
+            echo -e "DEBUG:\n\tcmd=${cmd}\n\t_silent=$_silent\n\tpmsg=${pmsg}\n\tfmsg=${fmsg}\n\tfail_lvl=$fail_lvl"
             echo -e "DEBUG TOP ~ Currently evaluating: '$1'"
         fi
         case "$1" in
@@ -598,7 +608,7 @@ process_cmds() {
                 shift
                 ;;
             -s) # only show msg if cmd fails
-                local silent=true
+                local _silent=true
                 shift
                 ;;
             -u) # Run Command as logged in User
@@ -654,8 +664,8 @@ process_cmds() {
                 [ -z pmsg ] && local pmsg="Success - Install $pname (apt)"
                 [ -z fmsg ] && local fmsg="Error - Install $pname (apt)"
                 local stop=true
-                [[ ! -z $pexclude ]] && local cmd="sudo apt-get -y install $pkg ${pexclude}-" ||
-                    local cmd="sudo apt-get -y install $pkg"
+                [[ ! -z $pexclude ]] && local cmd="sudo apt -y install $pkg ${pexclude}-" ||
+                    local cmd="sudo apt -y install $pkg"
                 ;;
             -apt-purge) # purge pkg followed by autoremove
                 case "$3" in
@@ -670,7 +680,7 @@ process_cmds() {
                 esac
                 [ -z pmsg ] && local pmsg="Success - Remove $pname (apt)"
                 [ -z fmsg ] && local fmsg="Error - Remove $pname (apt)"
-                local cmd="sudo apt-get -y purge $2"
+                local cmd="sudo apt -y purge $2"
                 local do_autoremove=true
                 shift $_shift
                 ;;
@@ -703,27 +713,29 @@ process_cmds() {
         # if cmd is set process cmd
         # use defaults if flag not set
         if [[ ! -z $cmd ]]; then
-            local pmsg=${pmsg:-"Success - $cmd"}
+            local pcmd=${cmd/sudo /} ; local pcmd=${pcmd/-y /}
+            local pmsg=${pmsg:-"Success - $pcmd"}
+            unset pcmd
+            # local pmsg=${pmsg:-"Success - ${cmd/-y /}"}
             local fmsg=${fmsg:-"Error - $cmd  See details in $log_file"}
             local fail_lvl=${fail_lvl:-"WARNING"}
-            local silent=${silent:-false}
+            local _silent=${_silent:-false}
             local stop=${stop:-false}
             local err=${err:-$log_file}
             local out=${out:-'/dev/null'}
             local showstart=${showstart:-true}
             local do_apt_install=${do_apt_install:-false}
             [[ ! -z $cmd_pfx ]] && local cmd="$cmd_pfx $cmd"
             if $_DEBUG_; then
-                echo -e "DEBUG:\n\tcmd=$cmd\n\tpname=$pname\n\tsilent=$silent\n\tpmsg=${pmsg}\n\tfmsg=${fmsg}\n\tfail_lvl=$fail_lvl\n\tout=$out\n\tstop=$stop\n\tret=$ret\n"
+                echo -e "DEBUG:\n\tcmd=$cmd\n\tpname=$pname\n\t_silent=$_silent\n\tpmsg=${pmsg}\n\tfmsg=${fmsg}\n\tfail_lvl=$fail_lvl\n\tout=$out\n\tstop=$stop\n\tret=$ret\n"
                 echo "------------------------------------------------------------------------------------------"
             fi
             # -- // PROCESS THE CMD \\ --
-            ! $silent && $showstart && logit "Starting ${pmsg/Success - /}"
-            # if eval "$cmd" >>"$out" 2>>"$err"; then # <-- Do the command
+            ! $_silent && $showstart && logit -E "Starting ${pmsg/Success - /}"
             logit -L "process_cmds executing: $cmd"
             if eval "$cmd" >>"$out" 2> >(grep -v "^$\|^WARNING: apt does not.*CLI.*$" >>"$err") ; then # <-- Do the command
                 local cmd_failed=false
-                ! $silent && logit "$pmsg"
+                ! $_silent && logit "$pmsg"
                 unset cmd
             else
                 local cmd_failed=true
@@ -736,7 +748,7 @@ process_cmds() {
                         logit -L "process_cmds executing: $cmd"
                         if eval "$cmd" >>"$out" 2> >(grep -v "^$\|^WARNING: apt does not.*CLI.*$" >>"$err"); then
                             local cmd_failed=false
-                            ! $silent && logit "$pmsg"
+                            ! $_silent && logit "$pmsg"
                         fi
                         ((x+=1))
                     done
@@ -765,5 +777,5 @@ process_cmds() {
             logit "Error - apt autoremove returned error-code" "WARNING"
     fi
 
-    return 0
+    ! $cmd_failed && return 0 || return 1
 }
diff --git a/installer/config.sh b/installer/config.sh
@@ -342,7 +342,6 @@ collect() {
     if ! $selected_prompts || [ -z "$rem_user" ]; then
         header
         [ -z "$rem_user" ] && rem_user=$iam
-        echo
         echo "If you have multiple ConsolePis they can discover each other over the network via mdns"
         echo "and if enabled can sync via Google Drive."
         echo

diff --git a/installer/consolepi-image-creator.sh b/installer/consolepi-image-creator.sh
@@ -187,6 +187,18 @@ get_input() {
     # return input (input is set globally)
 }
 
+do_user_dir_import(){
+    [[ $1 == root ]] && local user_home=root || local user_home="home/$1"
+    # -- Copy Prep pre-staged files if they exist (stage-dir/home/<username>) for newly created user.
+    if [[ -d "$STAGE_DIR/$user_home" ]]; then
+        dots "Found staged files for $1, cp to ${1}'s home on image"
+        res=$(
+            chown -R $(grep "^$1:" /mnt/usb2/etc/passwd | cut -d: -f3-4) "$STAGE_DIR/$user_home" 2>&1 &&
+            cp -r "$STAGE_DIR/$user_home/." "/mnt/usb2/$user_home/" 2>&1
+            ) &&
+                ( do_error $? && return 0 ) || ( do_error $? "$res" && return 1 )
+    fi
+}
 
 show_disk_details() {
     echo -e "------------------------------- // Device Details for $(green "$my_usb") \\\\\ -----------------------------------"
@@ -243,6 +255,9 @@ do_import_configs() {
             cp ${CUR_DIR}/$STAGE_DIR/known_hosts /mnt/usb2/root/.ssh/ ; do_error $((rc+=$?))
         fi
 
+        do_user_dir_import root
+        do_user_dir_import pi
+
         # -- adjust perms in .ssh directory if created imported --
         if [[ -d $IMG_HOME/.ssh ]]; then
             dots "Set Ownership of $IMG_HOME/.ssh"
@@ -315,7 +330,7 @@ do_import_configs() {
                 elif [[ ! $f =~ "/home/pi" ]] && [[ $f =~ $MY_HOME ]]; then
                     src="$f"
                     # dst is in the stage dir for non pi/root users.  After user creation installer will look for files in the stage dir
-                    dst="${IMG_STAGE}${MY_HOME}"
+                    dst="${IMG_STAGE}${f}"
                 else
                     src="$f"
                     dst="/mnt/usb2${f}"
@@ -436,9 +451,12 @@ main() {
     fi
     [[ $my_usb ]] && boot_list=($(sudo fdisk -l |grep -o '/dev/sd[a-z][0-9]  \*'| cut -d'/' -f3| awk '{print $1}'))
     [[ $boot_list =~ $my_usb ]] && my_usb=    # if usb device found make sure it's not marked as bootable if so reset my_usb so we can check for sd card adapter
+    # basename $(mount | grep 'on / '|awk '{print $1}')
     [[ -z $my_usb ]] && my_usb=$( sudo fdisk -l | grep 'Disk /dev/mmcblk' | awk '{print $2}' | cut -d: -f1 | cut -d'/' -f3)
 
-    echo -e "\n\n$(green "ConsolePi Image Creator") \n'exit' (which will terminate the script) is valid at all prompts\n"
+    ! $LOCAL_DEV && SCRIPT_TITLE=$(green "ConsolePi Image Creator") || SCRIPT_TITLE="${_green}ConsolePi Image Creator${_norm} ${_lred}${_blink}Local DEV${_norm}"
+    echo -e "\n\n$SCRIPT_TITLE \n'exit' (which will terminate the script) is valid at all prompts\n"
+
     [[ $my_usb ]] && echo -e "Script has discovered removable flash device @ $(green "${my_usb}") with the following details\n" ||
         echo -e "Script failed to detect removable flash device, you will need to specify the device"
 
@@ -558,8 +576,18 @@ main() {
 
     # Mount boot partition
     dots "Mounting boot partition to enable ssh"
-    [[ $my_usb =~ "mmcblk" ]] && res=$(sudo mount /dev/${my_usb}p1 /mnt/usb1 2>&1) || res=$(sudo mount /dev/${my_usb}1 /mnt/usb1  2>&1)
-    do_error $? "$res"
+    for i in {1..2}; do
+        [[ $my_usb =~ "mmcblk" ]] && res=$(sudo mount /dev/${my_usb}p1 /mnt/usb1 2>&1) || res=$(sudo mount /dev/${my_usb}1 /mnt/usb1  2>&1) ; rc=$?
+        if [[ $rc == 0 ]]; then
+            break
+        else
+            # mmcblk device would fail on laptop after image creation re-run with -nodd and was fine
+            echo "Sleep then Retry"
+            sleep 3
+            dots "Mounting boot partition to enable ssh"
+        fi
+    done
+    do_error $rc "$res"
 
     # Create empty file ssh in boot partition
     dots "Enabling ssh on image"
@@ -609,7 +637,7 @@ main() {
         fi
 
         $LOCAL_DEV && cmd_line="-dev $cmd_line"
-        grep -q "consolepi-install" $IMG_HOME/.bashrc || echo "consolepi-install ${cmd_line}" >> $IMG_HOME/.bashrc
+        grep -q "consolepi-install" $IMG_HOME/.profile || echo "consolepi-install ${cmd_line}" >> $IMG_HOME/.profile
 
         # make install command/script executable
         sudo chmod +x /mnt/usb2/usr/local/bin/consolepi-install &&