Uživatel:Mmares/PBSUtils.sh utility pro PBS job scripty

Soubor funkcí velmi zjednodušujících základní úkony v PBS job scriptech, který můžete nasourcovat do vašeho PBS scriptu, nebo jej rovnou použít jako základ pro nový skript.

Soubor obsahuje funkce kontrolující a zajíšťující existenci $SCRATCHDIR, funkce pro hezký berevný výpis do terminálu, pro snadnou kompresi celé $SCRATCHDIR nebo naopak extrakci archivu do $SCRATCHDIR, atd.

Funkce jsou zpravidla dobře ošetřeny na všechny možné situace a v případě selhání zapisují jasnou chybovou zprávu a je-li to bezpodmínečně nutné, skončí úlohu.

Ostatně, podívejte se sami!

Seznam funkcí:

• log_red()
• log_green()
• log_yellow()
• pprint_pbs_envvars()
• is_running_as_pbs_job()
• clear_or_create_scratchdir()
• print_workdir_and_scratchdir()
• directory_exists_or_die()
• tar_xz_scratchdir_to_stdout()
• tar_xz_scratchdir_to_file()
• extract_archive_into_scratchdir()
• init_basic_things()

PBSUtils.sh

#!/bin/bash

#PBS -N SuperZajimavyVypocet_001
#PBS -l walltime=01:52:00
#PBS -j oe
#PBS -M mmares@cesnet.cz
#PBS -m abe
#PBS -l select=1:ncpus=4:cgroups=cpuacct:mem=6gb:scratch_local=12gb

declare -ir MY_TRUE=0
declare -ir MY_FALSE=1

log_red() {
  echo -e "\033[91;1m[${PBS_JOBID:-"no_jobid"}][$(date "+%Y-%m-%d %H:%M:%S %z")]" \
          "$@" "\033[39;21m"
}

log_green() {
  echo -e "\033[92;1m[${PBS_JOBID:-"no_jobid"}][$(date "+%Y-%m-%d %H:%M:%S %z")]" \
          "$@" "\033[39;21m"
}

log_yellow() {
  echo -e "\033[93;1m[${PBS_JOBID:-"no_jobid"}][$(date "+%Y-%m-%d %H:%M:%S %z")]" \
          "$@" "\033[39;21m"
}

pprint_pbs_envvars() {
  if [[ (( $# > 0 )) && "$1" = "--no-color" ]]; then
    env | grep -E "^PBS"
  else
    env | grep -E "^PBS" | perl -pe 's|^(.*?)=|\x1B[33m\1=\x1B[39m|'
  fi
}

is_running_as_pbs_job() {
  if [[ ! -v PBS_JOBID || "$PBS_JOBID" = "no_jobid" ]]; then
    return $MY_FALSE
  else
    return $MY_TRUE
  fi
}

clear_or_create_scratchdir() {
  [[ -v PBS_JOBID ]] || export PBS_JOBID="no_jobid"
  [[ -v SCRATCHDIR ]] || {
    export SCRATCHDIR="/tmp/$PBS_JOBID";
    export SCRATCH="$SCRATCHDIR"; }
  shopt -s dotglob
  [[ -d "$SCRATCHDIR" ]] && rm -rf "$SCRATCHDIR"/* || mkdir "$SCRATCHDIR"
  shopt -u dotglob
  [[ ! -d "$SCRATCHDIR" ]] && {
    log_red "FATAL ERROR: Could not create \$SCRATCHDIR (\"$SCRATCHDIR\")... Job will abort now";
    exit 10; }
  log_green "\$SCRATCHDIR (\"$SCRATCHDIR\") is clean and ready to be used!"
  is_running_as_pbs_job && \
    trap 'clean_scratch' TERM EXIT || \
    trap 'clean_scratch' TERM EXIT 1>/dev/null 2>&1
}

print_workdir_and_scratchdir() {
  [[ -v SCRATCHDIR ]] && \
    log_green  "\$SCRATCHDIR: $SCRATCHDIR" || \
    log_yellow "WARNING: \$SCRATCHDIR is not set!"
  [[ -v PBS_O_WORKDIR ]] && \
    log_green  "\$PBS_O_WORKDIR: $PBS_O_WORKDIR" || \
    log_yellow "WARNING: \$PBS_O_WORKDIR is not set!"
}

directory_exists_or_die() {
  for DIR_NAME in "$@"; do
    if [[ ! -d "$DIR_NAME" ]]; then
      log_red "FATAL ERROR: Directory \"$DIR_NAME\" does not exist... Job will abort now"
      exit 11
    fi
  done
}

tar_xz_scratchdir_to_stdout() {
  if [[ ! -d "$SCRATCHDIR" ]]; then
    log_red "ERROR: Cannot tar \$SCRATCHDIR ($SCRATCHDIR)," \
            "because the directory does not exist!"
    return $MY_FALSE
  else
    tar cvJf - "$SCRATCHDIR"
  fi
}

tar_xz_scratchdir_to_file() {
  (( $# == 1 )) || {
    log_red "ERROR: Did not create .tar.xz archive of \$SCRATCHDIR:" \
            "Must have exactly one argument (the exact path and name to the result archive)"
    return $MY_FALSE
  }
  if tar_xz_scratchdir_to_stdout 1>"$1"; then
    log_green "\$SCRATCHDIR successfuly archived into $1 archive"
    return $MY_TRUE
  else
    log_red "ERROR: Archiving \$SCRATCHDIR into $1 archive failed!"
    return $MY_FALSE
  fi
}

extract_archive_into_scratchdir() {
  [[ -v SCRATCHDIR ]] || clear_or_create_scratchdir
  for ARG_ARCHIVE in "$@"; do
    local ARG_ARCHIVE_REALPATH="$(realpath "$ARG_ARCHIVE")"
    cd "$SCRATCHDIR"
    tar xvf "$ARG_ARCHIVE_REALPATH" && \
      log_green "Successfuly extracted archive $ARG_ARCHIVE into \$SCRATCHDIR" || \
      log_red "ERROR: Failed extracting archive $ARG_ARCHIVE into \$SCRATCHDIR"
    cd -
  done
}

init_basic_things() {
  clear_or_create_scratchdir
  print_workdir_and_scratchdir
  is_running_as_pbs_job && \
    log_green "We are running as a PBS job" || \
    log_green "We are running OUTSIDE of a PBS-scheduled platform (i. e. on a work computer?)"
}