Bashscripts.org

#! /bin/bash

## Tries to run commands in parallel. Commands are read from STDIN one
## per line, or from a given file specified by -f.
## Author: E. Choroba

file='-'
proc_num=$(grep -c ^processor'\b' /proc/cpuinfo)
prefix=$HOSTNAME-$USER-$$
sleep=10

children=()
names=()

if [[ $1 =~ ^--?h(elp)?$ ]] ; then
    cat <<-HELP
        Usage: ${0##*/} [-f file] [-n max-processes] [-p tmp-prefix] -s [sleep]
          Defaults:
            STDIN for file
            $proc_num for max-processes (number of processors)
            $prefix for tmp-prefix
            $sleep for sleep interval
        HELP
    exit
fi

function child_count () {
    child_count=0
    for child in "${children[@]}" ; do
        kill -0 $child 2>/dev/null && let child_count++
    done
    echo $child_count
}

while getopts 'f:n:p:s:' arg ; do
    case $arg in
        f ) file=$OPTARG ;;
        n ) proc_num=$((OPTARG)) ;;
        p ) prefix=$OPTARG;;
        s ) sleep=$OPTARG;;
        * ) echo "Warning: unknown option $arg" >&2 ;;
    esac
done

i=0
while read line ; do
    name=$prefix.$i
    let i++
    names+=($name)

    while ((`child_count`>=proc_num)) ; do
        sleep $sleep
    done

    eval $line 2>$name.e >$name.o &
    children+=($!)

done < <(cat $file)
wait

cat "${names[@]/%/.o}"
cat "${names[@]/%/.e}" >&2
rm "${names[@]/%/.o}" "${names[@]/%/.e}"

for file in *.txt ; do echo "process $file" ; done | parallel.sh

ls *.txt | xargs -n2 | sed 's/^/process /' | parallel.sh