ubuntu / PowerAI / MINSKY Shell scripts

 

nvidia-monitor.sh

#!/bin/bash

for i in hq-gpu-001 hq-gpu-002 hq-gpu-003 hq-gpu-004

do
banner $i
echo
ssh $i '(nvidia-smi )'
echo ; echo ; echo ; echo ; echo

done

root@hq-lsf-001:/gpfs/scripts# cat docker-monitor.sh
#!/bin/bash

for i in hq-gpu-001 hq-gpu-002 hq-gpu-003 hq-gpu-004

do
echo
banner $i
ssh $i '(docker ps -a )'
echo
echo ; echo ; echo ; echo ; echo

done

root@hq-lsf-001:/gpfs/scripts# cat kill.ksh
#!/usr/bin/ksh

. /gpfs/lsf/conf/profile.lsf
export PATH=/gpfs/lsf/script:$PATH

if [ "$1" = "" ]; then
        echo "You must give me a jobid..."
        ERROR=1
    else
###############################################################
### Envirionment ###
export JOBPID=$1
export HOSTNAME="`bjobs -u all | grep $JOBPID | awk '{print \$6}'`"
export USERNAME="`bjobs -u all | grep $JOBPID | awk '{print \$2}'`"

#echo $JOBPID $HOSTNAME $USERNAME
###############################################################

printf "%s:  "  "Are you sure? KILL JOB $JOBPID [y/N]"
read response
if [[ "$response" =~ ^([yY][eE][sS]|[yY])+$ ]]
then
    echo "Kill Docker Process $JOBPID $HOSTNAME $USERNAME"
    echo "ssh $HOSTNAME '(docker kill $USERNAME$JOBPID )'" | sh
    exit 0
else
    echo "EXIT KILL Process"
    exit 0
fi

###############################################################
    fi