#!/bin/bash
#
# vim:tabstop=3:expandtab:shiftwidth=3
#
###############################################################################
#
#  Helper script to run command(s)/scriptlet(s) over multiple hosts
#
# o Multiple hosts 
# o Adhoc command(s) input
# o Adhoc command(s) as argument (behind the - argument)
# o Single password entry (ssh_askpass)
# o Parallel sessions, 1 log-file / host (--bg-log-dir)
# o No script transfer, the command(s) are run through ssh
# o Commands from (multiple) 'inline' files converted to adhoc commands
#   o 'inline' files 'look and feel' like little scripts (after -- argument)
#   o Output per 'inline' file tagged (--tags)
#   o Exit status per 'inline' (--exit-status)
# o Arguments for the 'inlines' can be passed after the --- 
#   These will be available as $1 $2 etc. Beware that they will be equal in
#   every inline file. For specific insource vars, use the next method:
#
# -> additional vars can be passed to 'inline' files, by exporting:
#      export <InLineFileName>_var=value
#    will be available as $var in the inline
#
# The --bg-session-throttle <value> will wait if the amount of parallel
# sessions exceeds the <value>, it will continue when the number of
# sessions is dropped 10% of the <value>
#
# -> The hidden option ---bg-session-throttle-hysteresis <value>, controls
#    the number of sessions to drop for continuation.
#
# An insource file can be setup to control the defaults. A search will be done
# in $HOME, the dir where the script resides and /etc. When a file is found,
# only that will be insourced and the search will be stopped.
#
# By adding the hidden option ---no-insource, or setting
#   export ___no_insource="yes"
# The insource will be skipped, so that a controlling script can setup the
# required controls, without the .ssh-batch to be in the way.
#
# The hidden option + arg ---info-sep <separators>
#   export ___info_sep_ARG=',_'
# Enables the facility to add info about the hostname to the hostname. Handy to
# add environment information. (for example lin1234,DEV). The first char will
# also be used to check for a second field, where the ip address can be placed
# when a system is not in DNS (for example lin1234,DEV_PHP,192.168.1.2). 
# Background log files will come back as lin1234_DEV_PHP
# The next char(s) in the ___info_sep_ARG will be used for alternative
# info separator using ',_' will lin1234_DEV_PHP works like lin1234,DEV_PHP
#
# When a hostname includes a '/' character, the script assumes that a filename
# is passed instead of a hostname.  Files with hostnames can hold comment
# characters. A # is a 'hard' comment, other punctuation marks like
# a % or a - are considered to be a 'soft' comment. The --all-hosts option
# enables the hostnames with a 'soft' comment. The hostnames in a file
# have to be at the start of the line. Filenames can also be placed in
# the file. Be careful it is recursively read!! The '/' character need to be in
# the name to distinct a filename from a hostname (like on the command line).
# Only hostnames can be 'soft' commented.
#
# Ansible hostfile (INI inventory file) format is supported.
# Adding a :group (./inventory:group) will select [group] from the inventory.
# The :all group wil provide all hosts, which is equal to providing no group.
# The :ungrouped will provide the hosts which are not in part of a group.
# The [group:children] method and host name range expansion is also supported.
# (host[01:20] / host[01:20:2]).
#
# When a hostname includes a '^' character, the script assumes that the
# hostname includes a jump-host (jumphost^targethost).
#
# The hidden option + arg ---check-hostname <script>, or setting
#   export ___check_hostname="<script>"
# enables a command/script to check the hostname.
#  $1=requested hostname
#  $2=user name
#  $3=host info (if enabled)
#  $4=ip address (if host info is enabled and the third field is added)
# The script should output the corrected hostname.
#
# All options can be controlled from the .ssh-batch insource file, by
# converting them to variables, where the - need to be replaced by _
# Switches can be switched to on by putting a 'yes' (lower case!) in.
# Arguments can be placed by adding _ARG and putting the value in.
# Options with arguments will be processed as array if multiple inputs
# are provided, for example:
#
#  --ssh-option NumberOfPasswordPrompts=1 --ssh-option StrictHostKeyChecking=no
#
# will be 
#
#  __ssh_option_ARG=("NumberOfPasswordPrompts=1" "StrictHostKeyChecking=no")
#
# When providing a user@host and a --login-name user, the priority will be
# the --login-user. The hidden option ---at-login-name-priority will switch
# the priority to the user@host.
#
# The hidden option ---debug will show debug output. This can be used in
# combination with ---dry-run (or) ---no-run, when inline translation errors
# occur.
#
# With the hidden ---ignore-missing-inlines option, the ERROR message into a
# WARNING, when missing an inline file.
#
# Author:   Hans Vervaart
# Requires: _option_processor
# Requires: ssh_askpass
#
########################### extra help ########################################
#Usage:# The hosts argument can be a single host name or multiple hostnames.
#Usage:#
#Usage:# Multiple hostnames are multiple arguments. Arguments including a /
#Usage:# are text files containing hostnames. Tip: Hostname files can be nested.
#Usage:# Ansible inventory (INI formated) files are supported.
#Usage:#
#Usage:# The group selection can be added to the filename by adding :group 
#Usage:# Example: ./inventory:group (will select [group]). The :ungrouped 
#Usage:# hostnames, [group:children], hostname range expansion host[01:20] and
#Usage:# host[01:20:2] are supported. Variables and group variables are ignored.
#Usage:#
#Usage:# A hostname can contain a port (hostname:port), which will be passed as:
#Usage:#
#Usage:#   #ssh -p port hostname 
#Usage:#
#Usage:# Jumphosts can be added with a ^ as concatenation character to the
#Usage:# hostname (jumphost^hostname). Multiple jumphosts are supported.
#Usage:#  
#Usage:# Users can be added with an @ sign (user@host). This also the case
#Usage:# for jumphosts.
#Usage:#
#Usage:# For target hosts, alternative users can be provided as a comma
#Usage:# separated list:
#Usage:#
#Usage:#   user,user2,user3@host or --login-name user,user2,user3
#Usage:#  
#Usage:# The user2 en user3 are the alternative users, when user cannot login,
#Usage:# the 'next' is tried. When a user succeeds to login, the other leftover
#Usage:# alternative users are skipped.
#--all-hosts#            Enables the hostnames with a 'soft' comment (- or %).
#--bg-disable#           Disable background runs, when --bg-log-dir is provided.
#--bg-extra-ssh-option#  Add extra ssh option(s) when running in the background
#--bg-log-dir#           Background logs directory and enable background runs.
#--bg-session-throttle#  Maximum background runs.
#--bg-timeout#           The wait for all background runs time-out.
#--env-var#              Environment variable(s) to export to the client.
#--exit-status#          Add the EXIT_STATUS of the inline to the output.
#--fqdn-logname#         Log file name and output name in fqdn.
#--hostnames#            Add the hostname to the output.
#--login-name#           Login user name(s) (whole batch, excluding jumphosts)
#--no-info#              No info messages, only ssh command and host output.
#--no-ssh-askpass#       Disable ssh_askpass usage (when ssh-keys are used).
#--no-pass-through-opts# Options between the last hostname and the - will be
#--no-pass-through-opts# passed through as additional ssh options. This option
#--no-pass-through-opts# will disable the pass-through.
#--quiet#                Only host output.
#--ssh-option#           Options to be bassed to the ssh command.
#--tags#                 Add tags; inline names surrounded by { } in the output.
###############################################################################

#
# By creating a $HOME/.ssh-batch with variables, these settings can be
# controlled and functionality can be enabled.
#
# By adding this variable, extra information about the host can be added
# and will be added to the output logfile
#___info_sep_ARG=',_'

# The script which can be used to check the hostname for 'correction' like
# adding a domain name.
#___check_hostname_ARG=check_hostname

# Defaults for options
__bg_timeout_ARG="10"
__bg_session_throttle_ARG="0"
__ssh_option_ARG=("NumberOfPasswordPrompts=1" "StrictHostKeyChecking=no")
_help_show_args="hosts [[- adhoc-cmds]|[-- inline-files]][--- args]"

# Add the RunDir if not in the PATH, to be sure we will find the required
# tools.
RunDir=$(cd $(dirname $0) 2> /dev/null;pwd)
if [ "$RunDir" != "" ] &&  \
   ! echo $PATH | tr ':' '\n' | grep "^$RunDir\$" > /dev/null 2>&1
then
   if [ "$PATH" = "" ]
   then
      PATH="$RunDir"
   else
      PATH="$RunDir:$PATH"
   fi
   export PATH
fi

# Check if _option_processor and ssh_askpass can be found
if [ ! -r $RunDir/_option_processor ]
then
   echo "! [ERROR] $RunDir/_option_processor not found, unable to run"
   exit 1
elif ! which ssh_askpass 2>&1 | grep '^/' > /dev/null 2>&1
then
   echo "! [ERROR] ssh_askpass not found, unable to run"
   exit 1
fi

# Run the _option_processor from the $RunDir
. $RunDir/_option_processor

if [ "$__quiet" = "yes" ]
then
   __no_info="yes"
fi

EnvVar()
{
   if [ "$__env_var" = "yes" ]
   then
      EnvRegExpr="$(echo "${__env_var_ARG[@]}" | tr ',[:space:]' '\n\n' | \
                    grep -v '^[[:space:]]*$ '| sed -e 's/$/=/' -e 's/^/^/' | \
                    tr '\n' '|' | sed 's/|$//')"
      if ! set | egrep "$EnvRegExpr" | base64 2> /dev/null | grep '.'
      then
         # Always send something
         echo
      fi
   fi
}

InLine2Command()
{
   if [ "$1" = "" ]
   then
      if [ "$__no_info" != "yes" ] && tty -s
      then
         echo "# [INFO] Enter commands and close the input with ^D" >&2
      fi
      Files=/dev/stdin
   else
      Files="$*"
   fi

   MissingInLines=0

   # Any env var to be picked up?
   if [ "$__env_var" = "yes" ]
   then
      echo -n 'eval $(base64 -d 2> /dev/null | sed '"'"'s|^|export |'"'"');'
   fi

   for File in $Files
   do
      if [ "$__tags" = "yes" ]
      then
         InLineName="{$(basename $File)}"
         SedTag='|sed s/^/'$InLineName'/g'
      else
         unset InLineName SedTag
      fi
      if [ -d $File ]
      then
         if [ "$__no_info" != "yes" ]
         then
            echo "# [INFO] Skipping directory $File" >&2
         fi
      elif [ ! -r $File ] && [ "$File" != '/dev/stdin' ]
      then
         echo "! $MissingWarnErr inline-file $File not found" >&2
         MissingInLines=$(($MissingInLines+1))
      else
         if [ "$__no_info" != "yes" ]
         then
            echo "# [INFO] Using inline file $File" >&2
         fi
         (
            # Insert passed Args
            if [ "$Args" != "" ]
            then
               echo "set -- $Args"
            fi
            # Insert exported vars
            # Format: export <InLineName>_myvar="content"
            set | grep "^$(basename $File)_[A-Za-z][A-Za-z]*=" | \
            sed "s/^$(basename $File)_//"
            # remove /dev/stdin (does not work with su - <user>)
            if [ "$File" = '/dev/stdin' ]
            then
               unset File
            fi
            # Remove comments.
            # Beware: the ' is not allowed in a trailing comment'
            sed -e "s/[[:space:]]#[^'\"]*$//g" \
                -e 's/^[[:space:]]*#.*//' \
                -e 's/^[[:space:]]*/ /' \
                -e 's/;;/;2;/' \
                -e 's/^ $//' \
                $File ) | \
         grep -v '^$' | tr '\n' ';' | \
         sed -e 's/;*;/;/g' \
             -e 's/;2;/;;/g' ${Sed_SshInNextScript[@]} \
             -e 's,\\[[:space:]]*;,,g' \
             -e 's,\([(]\)[[:space:]]*;,\1,g' \
             -e 's,[[:space:]]*([[:space:]]*)[[:space:];]*{[[:space:];]*,(){ ,g' \
             -e 's,\([[:space:]]function[[:space:]]\)[[:space:]]*\([^();[:space:]][^();[:space:]]*\)[;[:space:]]*{[;[:space:]]*,\1\2 { ,g' \
             -e 's,\([;[:space:]]do\)[[:space:]]*;,\1,g' \
             -e 's,\([;[:space:]]then\)[[:space:]]*;,\1,g' \
             -e 's,\([;[:space:]]else\)[[:space:]]*;,\1,g' \
             -e 's,\([[:space:]]in\)[[:space:]]*;,\1,g' \
             -e "s/^[[:space:]]*/$ExitStatusOpen(/" \
             -e "s,;*\$,)2>\&1${ExitStatusClose}${SedTag};,"
      fi
   done
   return $MissingInLines
}

FilterHostRegExpr='^[^[@./_A-Za-z0-9]'
if [ "$__all_hosts" = "yes" ]
then
   FilterHostEgrep='^$'
else
   FilterHostEgrep='^$|'"$FilterHostRegExpr"
fi

# One : must be in
RecurseStack=":"

FilterHostNames()
{
   local NewHostName File Group SaveStack AlNum

   # For the sed find AlphaNumeric + markers
   AlNum='\([a-zA-Z0-9][a-zA-Z0-9]*\)'

   while [ $# -ne 0 ]
   do
      if echo $1 | grep '/' > /dev/null 2>&1
      then
         File="$(echo $1 | awk -F: '{print $1}')"
         Group="$(echo $1 | awk -F: '{print $2}')"
         if [ "$Group" = "all" ]
         then
            unset Group
         fi
         if [ -r $File ]
         then
            # read the content and remove spaces from the start of the line,
            # keep the first word and remove the space after the word
            # including the rest of the line.
            cd $(dirname $File) 2> /dev/null
            sed -e '1i[ungrouped]' \
                -e 's/^[[:space:]]*//' \
                -e 's/^\([^[:space:]]*\).*/\1/' \
                -e 's/#.*//' $(basename $File) 2> /dev/null | \
            egrep -v "$FilterHostEgrep" | \
            while read NewHostName
            do
               if [ "${NewHostName:0:1}" = "[" ]
               then
                  # Flush : and dummy group (close group)
                  Group="$(echo "$Group" | sed 's/[:[:space:]]//g')"
                  if [ "$Group" = "" ]
                  then
                     if [ "${NewHostName}" != "${NewHostName##*:}" ]
                     then
                        # Dummy group to stop output (:vars / :children)
                        Group=' '
                     fi
                  elif [ "$NewHostName" = "[$Group]" ]
                  then
                     # Open for hosts
                     Group=":${Group}"
                  elif [ "$NewHostName" = "[$Group:children]" ]
                  then
                     # Open for hosts groups
                     Group="::${Group}"
                  fi
               elif [ "${Group}" = "" ] || [ "${Group:0:1}" = ":" ]
               then
                  if [ "${Group:1:1}" = ":" ]
                  then
                     # Check for re-entrance
                     if ! echo "${RecurseStack}" | grep -q ":${NewHostName}:"
                     then
                        # Process as host group (:children)
                        SaveStack="${RecurseStack}"
                        RecurseStack="${RecurseStack}${NewHostName}:"
                        FilterHostNames $File:$NewHostName
                        RecurseStack="${SaveStack}"
                     elif [ "$___debug" = "yes" ]
                     then
                        echo "# Re-entrance [$NewHostName:children] skipped" >&2
                        echo "# Stack=${RecurseStack}" >&2
                     fi
                  else
                     FilterHostNames $NewHostName
                  fi
               fi
            done
         fi
      else
         eval echo $(echo $1 | egrep -v "$FilterHostEgrep" | \
                     sed -e "s/$FilterHostRegExpr\(.*\)/\1/" \
                         -e "s/[[]$AlNum:$AlNum[]]/{\1..\2}/g" \
                         -e "s/[[]$AlNum:$AlNum:$AlNum[]]/{\1..\2..\3}/g" | \
                     tr '\n' ' ')
      fi
      shift
   done
}

FilterUniqHostNames()
{
   local UniqHostNames CheckHostName

   UniqHostNames=":"
   for CheckHostName in $(FilterHostNames $*)
   do
      if ! echo "$UniqHostNames" | grep -q ":${CheckHostName}:"
      then
         echo "$CheckHostName"
         UniqHostNames="${UniqHostNames}${CheckHostName}:"
      fi
   done | tr '\n' ' '
}

CheckHostName()
{
   if [ "$___check_hostname_ARG" != "" ] &&
      which $___check_hostname_ARG 2> /dev/null | grep '^/' > /dev/null 2>&1
   then
      $___check_hostname_ARG "$1" "$2" "$3" "$4"
   else
      echo $1
   fi
}

WaitForBackGroundSessions() # $1=Message $2=NrOfPsToContinue $3=NrOfPsToWait 
{
   TotalPs=$(echo "$PIDs" | wc -w)
   ActivPs=$(ps u $PIDs | grep -v '^USER' | wc -l)
   if [ $TotalPs -gt 0 ] && [ $ActivPs -ge $3 ] && [ $3 -gt 0 ]
   then
      echo "$1"
      if echo $__bg_timeout_ARG | grep -q '^[1-9][0-9]*$'
      then
         Wait=$(($__bg_timeout_ARG*60))
      else
         Wait=213
      fi
      Step=1
      ActivPs=$(($2+1))
      Len=$(printf "%s" "$TotalPs" | wc -m)
      while [ $Wait -gt 0 ] && [ $ActivPs -gt $2 ]
      do
         ActivPs=$(ps u $PIDs | grep -v '^USER' | wc -l)
         printf "Waiting for %0*i/%0*i sessions to finish (%02i:%02i)\r" \
                "$Len" "$ActivPs" "$Len" "$TotalPs" \
                "$(expr $Wait / 60)" "$(expr $Wait % 60)"
         Wait=$(expr $Wait - $Step)
         if [ $ActivPs -gt $2 ]
         then
            sleep $Step
         fi
      done
      echo
      ActivePids="$(ps u $PIDs | grep -v '^USER' | awk '{print $2}' | \
                   tr '\n' ' ' | sed -e 's/^ //' -e 's/ $//')"
      if [ "$ActivePids" != "" ] && [ $(echo "$ActivePids" | wc -w) -gt $3 ]
      then
         echo "# [CHECK] Still active PIDs: $ActivePids"
      fi
   fi
}

# Get overrides in
if [ "$___no_insource" != "yes" ]
then
   for InSourceDir in $HOME $(dirname $0) /etc
   do
      if [ -r $InSourceDir/.$(basename $0) ]
      then
         . $InSourceDir/.$(basename $0) > /dev/null 2>&1 
         break
      fi
   done
fi

# Check the user's whises
if [ "$___ignore_missing_inlines" = "yes" ]
then
   MissingWarnErr="[WARNING]"
else
   MissingWarnErr="[ERROR]"
fi

unset unset ExitStatusOpen ExitStatusClose
if [ "$__exit_status" = "yes" ]
then
   ExitStatusOpen='('
   ExitStatusClose=';echo EXIT_STATUS=\$?)'
fi

# Clear any content
unset HostNames AdHocCmds InLines Args

# Pull in the hostnames till the -
while [ $# -ne 0 ] && [ "${1:0:1}" != "-" ]
do
   if [ "$HostNames" = "" ]
   then
      HostNames="$(FilterUniqHostNames $1)"
   else
      HostNames="${HostNames}$(FilterUniqHostNames $1)"
   fi
   shift
done

# Collect ssh options
unset SshOpts
for SshOpt in ${__ssh_option_ARG[@]}
do
   SshOpts[${#SshOpts[@]}]='-o'
   SshOpts[${#SshOpts[@]}]="$SshOpt"
done

# Add Opts if we go to the bg (background)
if [ "$__bg_log_dir" = "yes" ] && \
   [ "$__bg_disable" != "yes" ] && \
   [ "$__bg_extra_ssh_option_ARG" != "" ]
then
   for SshOpt in ${__bg_extra_ssh_option_ARG[@]}
   do
      SshOpts[${#SshOpts[@]}]='-o'
      SshOpts[${#SshOpts[@]}]="$SshOpt"
   done
fi

# Add passthrough ssh options
while [ $# -ne 0 ] && ! echo "$1" | egrep -q '^[-]{1,3}$'
do
   if [ "$__no_pass_through_opts" = "" ]
   then
      SshOpts[${#SshOpts[@]}]="$1"
   fi
   shift
done

# Read the - contents into AdHocCmds
if [ "$1" = "-" ]
then
   shift
   while [ $# -ne 0 ] && [ "${1:0:2}" != "--" ]
   do
      if [ "$AdHocCmds" = "" ]
      then
         AdHocCmds="$1"
      else
         AdHocCmds="$AdHocCmds $1"
      fi
      shift
   done
fi

# Read the -- contents into Inlines
if [ "$1" = "--" ]
then
   shift
   while [ $# -ne 0 ] && [ "${1:0:3}" != "---" ]
   do
      if [ "$InLines" = "" ]
      then
         InLines="$1"
      else
         InLines="$InLines $1"
      fi
      shift
   done
fi

# Read the --- contents into Args
if [ "$1" = "---" ]
then
   shift
   Args="$*"
fi

if [ "$HostNames" = "" ]
then
   echo "! [ERROR] No system(s) provided" >&2
   exit 1
fi

if [ "$__bg_log_dir_ARG" != "" ] && 
   [ ! -d $__bg_log_dir_ARG ]
then
   echo "! [ERROR] Log dir $__bg_log_dir_ARG not available" >&2
   exit 1
fi

# From the insource settings
if echo $SshIsInNextScript | grep -i '^y' > /dev/null 2>&1
then
  Sed_SshInNextScript=('-e' 's,[$`"],\\\0,g')
fi

MissedInLines=0
if [ "$AdHocCmds" != "" ]
then
   Command="$(echo $AdHocCmds | InLine2Command)"
else
   Command="$(InLine2Command $InLines)"
   MissedInLines=$?
fi

if [ "$___debug" = "yes" ]
then
   echo "# HostNames: $HostNames"
   echo "# AdHocCmds: $AdHocCmds"
   echo "# InLines  : $InLines"
   echo "# Args     : $Args"
   echo "# Command  : $Command"
   echo "# SshOpts  : ${SshOpts[@]}"
   echo "# Log dir  : $__bg_log_dir_ARG"
fi

if [ $MissedInLines -gt 0 ]
then
   if [ "$___ignore_missing_inlines" = "yes" ]
   then
      if [ "$__no_info" != "yes" ]
      then
         echo "# [INFO] Ignoring missing $MissedInLines inline files" >&2
      fi
   else
      exit 1
   fi
fi

if [ "$Command" = "" ]
then
   if [ "$__no_info" != "yes" ]
   then
      echo "# [INFO] No commands found, nothing to do" >&2
   fi
   exit 0
fi

if [ "$___dry_run" = "yes" ] || [ "$___no_run" = "yes" ]
then
   exit 0
fi

# Get the ssh_askpass variables in, or not if disabled
if [ "$__no_ssh_askpass" = "yes" ]
then
   unset SSH_ASKPASS_CACHE SSH_ASKPASS SETSID
elif which ssh_askpass 2> /dev/null | grep '^/' > /dev/null 2>&1
then
   $(eval ssh_askpass --exports)
fi

# Clean start
unset PIDs

unset SedInfo
if [ "$___info_sep_ARG" != "" ]
then
   if [ "${___info_sep_ARG:1}" != "" ]
   then
      SedInfo[${#SedInfo[@]}]='sed'
      SedInfo[${#SedInfo[@]}]='-e'
      SedInfo[${#SedInfo[@]}]="s/[${___info_sep_ARG:1}]/${___info_sep_ARG:0:1}/"
   fi
fi

# Setup the backround wait levels for the number of processes
MaxPs=$(echo $__bg_session_throttle_ARG | egrep '^[0-9]{1,}$')
if [ "$MaxPs" = "" ] || [ $MaxPs -lt 2 ]
then
   MaxPs=0
   MinPs=0
else
   MinPs=$(($MaxPs-$(echo $___bg_session_throttle_hysteresis_ARG | \
                     sed 's/^$/0/' | egrep '^[0-9]{1,}$')))
   if [ $MinPs -lt 1 ] || [ $MinPs -ge $MaxPs ]
   then
      MinPs=$(($MaxPs-($MaxPs/10)))
      if [ $MinPs -lt 1 ] || [ $MinPs -eq $MaxPs ]
      then
         MinPs=1
      fi
   fi
fi

for Hostname in $HostNames
do
   # Start the collection of host specific ssh options in an array
   unset HostOpts

   # Check for Jumphost info (^). Multiple jumphosts will be comma separated
   # Only when ^ is not in use for ---info-sep ARG
   if echo $Hostname | grep -q '\^' && ! echo "$___info_sep_ARG" | grep -q '\^'
   then
      # Multiple jumphosts are converted to a , separated list
      HostOpts=($(echo $Hostname | sed 's/\^[^^]*$//' | \
                  tr '^' ',' | sed 's/^./-J \0/'))
      if [ "$__no_info" != "yes" ]
      then
         echo "${HostOpts[@]}" | sed -e 's/^/# [INFO] Jump-host:/' \
                                    -e 's/-J//g' >&2
      fi

      # Correct the Hostname
      Hostname=$(echo $Hostname | awk -F^ '{print $NF}')
   fi

   # Remove additional information in the hostname
   UserName=$(echo $Hostname | grep '@' | sed -e 's/@[^@]*$//')
   Hostname=$(echo $Hostname | sed 's/.*@//')
   if [ "$___info_sep_ARG" = "" ]
   then
      HostName=$Hostname
      unset HostInfo
      unset HostIpAd
   else
      if ! echo $Hostname | grep "${___info_sep_ARG:0:1}" > /dev/null 2>&1
      then
         Hostname=$(echo $Hostname | ${SedInfo[@]})
      fi
      HostName=$(echo $Hostname|awk -F ${___info_sep_ARG:0:1} '{print $1'})
      HostInfo=$(echo $Hostname|awk -F ${___info_sep_ARG:0:1} '{print $2'})
      HostIpAd=$(echo $Hostname|awk -F ${___info_sep_ARG:0:1} '{print $3'})
   fi

   # Check for :port to be added as host specific ssh option array
   if echo $HostName | grep -q ':'
   then
      HostOpts[${#HostOpts[@]}]='-p'
      HostOpts[${#HostOpts[@]}]=$(echo $HostName | awk -F: '{print $2}')

      # Correct the HostName
      HostName=$(echo $HostName | awk -F: '{print $1}')
   fi

   # Run the hostname correction plugin
   Host=$(CheckHostName "$HostName" "$UserName" "$HostInfo" "$HostIpAd")

   # Add username by option
   if [ "$__login_name" = "yes" ]  && \
      ( [ "$___at_login_name_priority" != "yes" ] || [ "$UserName" = "" ] )
   then
      UserName=$(echo ${__login_name_ARG[@]} | tr ' ' ',')
   fi

   # Add @ to the UserName (if not empty). Also with , separated
   UserName=$(echo $UserName | sed 's/.$/\0@/')

   # setup UserNames if we have a list, include the @
   unset UserNames
   if echo $UserName | grep -q ','
   then
      UserNames="$(echo $UserName | grep ',' | sed 's/,/@ /g')"
   fi
   # if UserNames is not empty we have a request for multiple login tries

   if [ "$Host" = "" ]
   then
      if [ "$__bg_log_dir" = "yes" ]
      then
         OutputFile="$(echo $__bg_log_dir_ARG | sed 's,/*$,,')"
         if [ "$__fqdn_logname" = "yes" ]
         then
            OutputFile="${OutputFile}/${HostName}"
         else
            OutputFile="${OutputFile}/$(echo ${HostName} | sed 's,[.].*,,')"
         fi
         if [ "$HostInfo" != "" ]
         then
            OutputFile="${OutputFile}_${HostInfo}"
         fi
      else
         OutputFile=/dev/null
      fi
      if [ "$__quiet" = "yes" ]
      then
         # Log the failure
         echo "# Hostname ($Hostname) check failed, skipping host" \
              > $OutputFile
      else
         echo "# Hostname ($Hostname) check failed, skipping host" \
              | tee $OutputFile >&2
      fi
   else
      WaitForBackGroundSessions \
         "Session throttle $MaxPs reached, continueing at $MinPs" $MinPs $MaxPs
      # Quiet mode?
      if [ "$__quiet" != "yes" ]
      then
         if [ "$HostInfo" = "" ]
         then
            echo "# ssh" ${SshOpts[@]} ${HostOpts[@]} ${UserName}${Host} >&2
         else
            echo "# ssh" ${SshOpts[@]} ${HostOpts[@]} ${UserName}${Host} \
                 "($HostInfo)" >&2
         fi
      fi
      if [ "$__bg_log_dir" = "yes" ]
      then
         OutputFile="$(echo $__bg_log_dir_ARG | sed 's,/*$,,')"
         if [ "$__fqdn_logname" = "yes" ]
         then
            OutputFile="${OutputFile}/${HostName}"
         else
            OutputFile="${OutputFile}/$(echo ${HostName} | sed 's,[.].*,,')"
         fi
         if [ "$HostInfo" != "" ]
         then
            OutputFile="${OutputFile}_${HostInfo}"
         fi
         if [ "$__quiet" != "yes" ]
         then
            echo "# Output send to ${OutputFile}" >&2
         fi
         if [ "$__bg_disable" = "yes" ] || [ "$SETSID" = "" ]
         then
            if [ "$UserNames" = "" ]
            then
               EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                    ${UserName}${Host} "${Command}" \
                                    > ${OutputFile} 2>&1
            else
               # This will overwrite UserName
               for UserName in $UserNames
               do
                  EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                       ${UserName}${Host} \
                                       "${Command}exit 0;" && break
               done > ${OutputFile} 2>&1
            fi
         else
            if [ "$UserNames" = "" ]
            then
               ( EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                      ${UserName}${Host} "${Command}" \
                                      > ${OutputFile} 2>&1 ) &
               PIDs="$PIDs $!"
            else
               # This will overwrite UserName
               for UserName in $UserNames
               do
                  EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                       ${UserName}${Host} \
                                       "${Command}exit 0;" && break
               done > ${OutputFile} 2>&1 &
               PIDs="$PIDs $!"
            fi
         fi
      else
         if [ "$__hostnames" = "yes" ]
         then
            if [ "$__fqdn_logname" = "yes" ]
            then
               HostNm=${HostName}
            else
               HostNm=$(echo ${HostName} | sed 's,[.].*,,')
            fi
            if [ "$UserNames" = "" ]
            then
               EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                     ${UserName}${Host} "${Command}" | \
                                     sed "s/^/${HostNm}:/"
            else
               # This will overwrite UserName
               for UserName in $UserNames
               do
                  EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                       ${UserName}${Host} \
                                       "${Command}exit 0;" && break
               done | sed "s/^/${HostNm}:/"
            fi
         else
            if [ "$UserNames" = "" ]
            then
               EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                    ${UserName}${Host} "${Command}"
            else
               # This will overwrite UserName
               for UserName in $UserNames
               do
                  EnvVar | $SETSID ssh ${SshOpts[@]} ${HostOpts[@]} \
                                       ${UserName}${Host} \
                                       "${Command}exit 0;" && break
               done
            fi
         fi
      fi
   fi
done

# Final wait
WaitForBackGroundSessions "Background sessions running" 0 1

