Re: Parallelism a la make -j <n> / GNU parallel

[John Kearney]

This version might be easier to follow. The last version was more for
being able to issue commands via a fifo to a job queue server.

  function check_valid_var_name {
    case "${1:?Missing Variable Name}" in
      [!a-zA-Z_]* | *[!a-zA-Z_0-9]* ) return 3;;
    esac
  }


  CNiceLevel=$(nice)
    declare -a JobArray
    function PushAdvancedCmd {
        local le="tmp_array${#JobArray[@]}"
        JobArray+=("${le}")
        eval "${le}"'=("${@}")'
    }
    function PushSimpleCmd {
        PushAdvancedCmd  WrapJob ${CNiceLevel} "${@}"
    }
    function PushNiceCmd {
        PushAdvancedCmd  WrapJob "${@}"
    }
    function UnpackCmd {
        check_valid_var_name ${1} || return $?
        eval _RETURN=('"${'"${1}"'[@]}"')
        unset "${1}[@]"
    }
    function runJobParrell {
        local mjobCnt=${1} && shift
        jcnt=0
        function WrapJob {
            [ ${1} -le ${CNiceLevel} ] || renice -n ${1}
            local Buffer=$("${@:2}")
            echo "${Buffer}"
            kill -s USR2 $$
        }
        function JobFinised {
            jcnt=$((${jcnt}-1))
        }
        trap JobFinised USR2
        while [ $# -gt 0 ] ; do
            while [ ${jcnt} -lt ${mjobCnt} ]; do
                jcnt=$((${jcnt}+1))
                if UnpackCmd "${1}" ; then
                    "${_RETURN[@]}" &
                else
                    continue
                fi
                shift
        done
        sleep 1
        done
    }





Am 03.05.2012 23:23, schrieb John Kearney:
> Am 03.05.2012 22:30, schrieb Greg Wooledge:
>> On Thu, May 03, 2012 at 10:12:17PM +0200, John Kearney wrote:
>>>     function runJobParrell {
>>>         local mjobCnt=${1} && shift
>>>         jcnt=0
>>>         function WrapJob {
>>>             "${@}"
>>>             kill -s USR2 $$
>>>         }
>>>         function JobFinised {
>>>             jcnt=$((${jcnt}-1))
>>>         }
>>>         trap JobFinised USR2
>>>         while [ $# -gt 0 ] ; do
>>>             while [ ${jcnt} -lt ${mjobCnt} ]; do
>>>                 jcnt=$((${jcnt}+1))
>>>                 echo WrapJob "${1}" "${2}"
>>>                 WrapJob "${1}" "${2}" &
>>>                 shift 2
>>>         done
>>>         sleep 1
>>>         done
>>>     }
>>>     function testProcess {
>>>         echo "${*}"
>>>         sleep 1
>>>     }
>>>     runJobParrell 2  testProcess "jiji#" testProcess "jiji#" testProcess
>>> "jiji#"
>>>
>>>     tends to work well enough.
>>> it gets a bit more complex if you want to recover output but not too much.
>> The real issue here is that there is no generalizable way to store an
>> arbitrary command for later execution.  Your example assumes that each
>> pair of arguments constitutes one simple command, which is fine if that's
>> all you need it to do.  But the next guy asking for this will want to
>> schedule arbitrarily complex shell pipelines and complex commands with
>> here documents and brace expansions and ....
>>
>
> :)
> A more complex/flexible example. More like what I actually use.
>
>
>
>
>   CNiceLevel=$(nice)
>     declare -a JobArray
>     function PushAdvancedCmd {
>         local IFS=$'\v'
>         JobArray+=("${*}")
>     }
>     function PushSimpleCmd {
>         PushAdvancedCmd  WrapJob ${CNiceLevel} "${@}"
>     }
>     function PushNiceCmd {
>         PushAdvancedCmd  WrapJob "${@}"
>     }
>     function UnpackCmd {
>         local IFS=$'\v'
>         set -o noglob
>         _RETURN=( .${1}. )  
>         set +o noglob
>         _RETURN[0]="${_RETURN[0]#.}"
>         local -i le=${#_RETURN[@]}-1
>         _RETURN[${le}]="${_RETURN[${le}]%.}"
>     }
>     function runJobParrell {
>         local mjobCnt=${1} && shift
>         jcnt=0
>         function WrapJob {
>             [ ${1} -le ${CNiceLevel} ] || renice -n ${1}
>             local Buffer=$("${@:2}")
>             echo "${Buffer}"
>             kill -s USR2 $$
>         }
>         function JobFinised {
>             jcnt=$((${jcnt}-1))
>         }
>         trap JobFinised USR2
>         while [ $# -gt 0 ] ; do
>             while [ ${jcnt} -lt ${mjobCnt} ]; do
>                 jcnt=$((${jcnt}+1))
>                 UnpackCmd "${1}"
>                 "${_RETURN[@]}" &
>                 shift
>         done
>         sleep 1
>         done
>     }
>
>
>
>     function testProcess {
>         echo "${*}"
>         sleep 1
>     }
>     #  So standard variable args can be handled in 2 ways 1
>     #  encode them as such
>     PushSimpleCmd testProcess "jiji#" dfds dfds dsfsd
>     PushSimpleCmd testProcess "jiji#" dfds dfds
>     PushNiceCmd 20 testProcess "jiji#" dfds
>     PushSimpleCmd testProcess "jiji#"
>     PushSimpleCmd testProcess "jiji#" "*" s
>     # more complex things just wrap them in a function and call it
>     function DoComplexMagicStuff1 {
>         echo "${@}" >&2
>     }
>     # Or more normally just do a hybrid of both.
>     PushSimpleCmd DoComplexMagicStuff1 "jiji#"
>
>     #
>    
>     runJobParrell 1 "${JobArray[@]}"
>
>
>
> Note there is another level of complexity where I start a JobQueue
> Process and issues it commands using a fifo.
>
>
>