Axione-IPE-Viewer/data-ingest/fetch_latest_ipe.sh

#!/usr/bin/env bash
### --------------------------------------
###  IPE files synchro & ingest
###
###  The purpose of this script is to be used in a CRON to automatically fetch up-to-date IPE files and ingest them into sqlite.
###
###  Johan - 20240508 - Init scripts
### -------------------------------------

set -Eeuo pipefail

target=${1:-}
keep_backup=${2:-}

############### CONFIG ###############
g_base_dir=/srv/ftth_ipe_data/

# AXIONE
g_axione_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_axione
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
axione_ipe_date=$(date +"%Y%m%d" -d "2 days ago")
g_axione_files_pattern="A-IPE*$axione_ipe_date*.csv V-IPEZTD*CTYF*$axione_ipe_date*.csv"
g_axione_ftp_user=
g_axione_ftp_pass=

# ARCEP
g_arcep_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_arcep


############### CONSTANTS ###############

APP_SVC="ftth-ipe-map.service"

declare -A PROD_DBS
declare -A TMP_DBS

# AXIONE
AXIONE_CSV_DIR="$g_base_dir/axione/tmp"
PROD_DBS[axione]="$g_base_dir/axione/ipe.sqlite"
TMP_DBS[axione]="$g_base_dir/axione/tmp.sqlite"
######################################

help() {
  echo "$0 target (--keep-backup)"
  echo "Where:"
  echo "  - target: axione|arcep"
  echo "  - (--keep-backup): if set, will keep backup db as db.old"
}

[[ ! $target || "$target" == "-h" || "$target" == "--help" ]] && help && exit 0


if [[ "$target" != "axione" && "$target" != "arcep" ]]; then
  echo "Unknown target: $target"
  help
  exit 1
fi

[[ "$keep_backup" == "--keep-backup" ]] && keep_backup=true || keep_backup=false


############################################## AXIONE IMPLEMENTATION
axione_fetch_latest() {
  mkdir -p $AXIONE_CSV_DIR

  files=$(shopt -s nullglob dotglob; echo $AXIONE_CSV_DIR/*)
  if (( ${#files} )); then
    echo "Found CSV files not processed, not downloading again. to reset, delete content in $AXIONE_CSV_DIR:"
    echo "rm -f $AXIONE_CSV_DIR/*"
    return
  fi

  # FTP Sync 
  # Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
  lftp -u "$g_axione_ftp_user","$g_axione_ftp_pass" ftp.axione.fr -e "mget $g_axione_files_pattern -O $AXIONE_CSV_DIR; exit" ||:

}

axione_ingest() {
  if [[ -f $AXIONE_TMP_DB ]]; then
    echo "temp DB file already exists, not processing again. To reset delete it:"
    echo "rm -f $TMP_DBS[axione]"
  fi

  $g_axione_ingest $AXIONE_CSV_DIR $TMP_DBS[axione]
}

axione_clean() {
  echo "Clean Axione files"
  rm -rf $AXIONE_CSV_DIR/*csv*
}

############################################## ARCEP IMPLEMENTATION
arcep_fetch_latest() {
  echo "arcep_fetch_latdsfgsfest"

}

# arcep_ingest() {

# }

############################################## DEPLOY METHODS

rollback() {
  prod_db=$PROD_DBS[$target]
  echo "Putting back $prod_db.backup"
  sudo systemctl stop $APP_SVC
  mv -f "$prod_db.backup" "$prod_db"
  sudo systemctl start $APP_SVC || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $APP_SVC ne redémarre plus !"
}

deploy_swap() {
  prod_db=$PROD_DBS[$target]
  tmp_db=$TMP_DBS[$target]
  echo "Backup DB $prod_db to $prod_db.backup"
  if [[ -f $prod_db.backup ]]; then
    echo "Backup exists, restore it !"
    rollback
    return
  fi
  echo "Backup prod DB $prod_db"
  cp "$prod_db" "$prod_db.backup"

  echo "Swap $tmp_db to $prod_db"
  mv "$tmp_db" "$prod_db"

  echo "Restart service and check OK"
  if sudo systemctl restart $APP_SVC; then
    echo "App $APP_SVC restarted OK !"
    if $keep_backup; then
      echo "Save $prod_db.backup TO $prod_db.old"
      mv "$prod_db.backup" "$prod_db.old"
    else
      echo "Delete $prod_db.backup"
      rm -f "$prod_db.backup"
    fi
    ${target}_clean
  else
    # Error, put back old DB
    echo "Error, Rollback !"
    rollback
  fi
}

################################### MAIN

# First fetch IPE raw files
echo "########## FETCH LATEST IPEs FOR $target"
${target}_fetch_latest

# Then ingest into a database
echo "########## INGEST LATEST IPEs FOR $target"
${target}_ingest

# Finally Deploy new database to prod
echo "########## DEPLOY NEW DB FOR $target"
deploy_swap