Script to fetch IPEs for axione & arcep

data-ingest/fetch_axione_ipe.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+### --------------------------------------
+###  Daily IPE files FTP synchro & ingest
+###  Sacha - 20220414
+###  Johan - 20220519 - fix appel ingest & ajout City fast
+### -------------------------------------
+
+set -Eeuo pipefail
+
+# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
+date=$(date +"%Y%m%d" -d "2 days ago")
+
+# Script Configuration
+localDir="/srv/ftth_ipe_data/tmp_ipe"
+ingest="/srv/www/ftth-ipe-map/data-ingest/ingest"
+db_target_path="/srv/ftth_ipe_data/axione_data/ipe.sqlite"
+db_temp="/srv/ftth_ipe_data/axione_data/ipe_temp.sqlite"
+webapp_svc="ftth-ipe-map.service"
+axione_files_pattern="A-IPE*$date*.csv V-IPEZTD*CTYF*$date*.csv"
+axione_ftp_user=
+axione_ftp_pass=
+
+
+# First clean csv files in temp dir and temp sqlite if exists
+rm -f $localDir/*csv*
+rm -f $db_temp
+
+# FTP Sync 
+# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
+lftp -u "$axione_ftp_user","$axione_ftp_pass" ftp.axione.fr -e "mget $axione_files_pattern -O $localDir; exit" ||:
+
+# Check if no files mail
+echo "Check if found files"
+files=$(shopt -s nullglob dotglob; echo $localDir/*)
+if (( ! ${#files} ))
+then
+mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Pas de fichiers IPE synchronisés"
+fi
+
+echo "Ingest files into temp DB $db_temp"
+
+# Make sure db is empty
+rm -f $db_temp
+
+# Ingest into temporary DB
+$ingest $localDir $db_temp
+
+echo "Backup ori DB $db_target_path"
+# Make copy of original DB before swap
+cp $db_target_path $db_target_path.backup
+
+echo "Swap DBs new and original"
+# Swap DBs
+mv -f $db_temp $db_target_path
+
+#chown ftth-ipe-map:ftth-ipe-map $db_target_path 
+
+echo "Restart service $webapp_svc"
+# Restart service
+if sudo systemctl restart $webapp_svc; then
+  # Ok, delete copy of old DB
+  echo "OK, remove backup DB"
+  rm -f $db_target_path.backup
+else
+  # Error, put back old DB
+  echo "Error, Put back old DB & restart"
+  mv -f $db_target_path.backup $db_target_path
+  #chown ftth-ipe-map:ftth-ipe-map $db_target_path
+  sudo systemctl restart $webapp_svc || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $webapp_svc ne redémarre pas !"
+fi
+
+echo "Clean any downloaded csv files"
+# Delete CSV files
+rm -rf $localDir/*csv*

data-ingest/fetch_latest_ipe.sh

@@ -0,0 +1,158 @@
+#!/usr/bin/env bash
+### --------------------------------------
+###  IPE files synchro & ingest
+###
+###  The purpose of this script is to be used in a CRON to automatically fetch up-to-date IPE files and ingest them into sqlite.
+###
+###  Johan - 20240508 - Init scripts
+### -------------------------------------
+
+set -Eeuo pipefail
+
+target=${1:-}
+keep_backup=${2:-}
+
+############### CONFIG ###############
+g_base_dir=/srv/ftth_ipe_data/
+
+# AXIONE
+g_axione_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_axione
+# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
+axione_ipe_date=$(date +"%Y%m%d" -d "2 days ago")
+g_axione_files_pattern="A-IPE*$axione_ipe_date*.csv V-IPEZTD*CTYF*$axione_ipe_date*.csv"
+g_axione_ftp_user=
+g_axione_ftp_pass=
+
+# ARCEP
+g_arcep_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_arcep
+
+
+############### CONSTANTS ###############
+
+APP_SVC="ftth-ipe-map.service"
+
+declare -A PROD_DBS
+declare -A TMP_DBS
+
+# AXIONE
+AXIONE_CSV_DIR="$g_base_dir/axione/tmp"
+PROD_DBS[axione]="$g_base_dir/axione/ipe.sqlite"
+TMP_DBS[axione]="$g_base_dir/axione/tmp.sqlite"
+######################################
+
+help() {
+  echo "$0 target (--keep-backup)"
+  echo "Where:"
+  echo "  - target: axione|arcep"
+  echo "  - (--keep-backup): if set, will keep backup db as db.old"
+}
+
+[[ ! $target || "$target" == "-h" || "$target" == "--help" ]] && help && exit 0
+
+
+if [[ "$target" != "axione" && "$target" != "arcep" ]]; then
+  echo "Unknown target: $target"
+  help
+  exit 1
+fi
+
+[[ "$keep_backup" == "--keep-backup" ]] && keep_backup=true || keep_backup=false
+
+
+############################################## AXIONE IMPLEMENTATION
+axione_fetch_latest() {
+  mkdir -p $AXIONE_CSV_DIR
+
+  files=$(shopt -s nullglob dotglob; echo $AXIONE_CSV_DIR/*)
+  if (( ${#files} )); then
+    echo "Found CSV files not processed, not downloading again. to reset, delete content in $AXIONE_CSV_DIR:"
+    echo "rm -f $AXIONE_CSV_DIR/*"
+    return
+  fi
+
+  # FTP Sync 
+  # Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
+  lftp -u "$g_axione_ftp_user","$g_axione_ftp_pass" ftp.axione.fr -e "mget $g_axione_files_pattern -O $AXIONE_CSV_DIR; exit" ||:
+
+}
+
+axione_ingest() {
+  if [[ -f $AXIONE_TMP_DB ]]; then
+    echo "temp DB file already exists, not processing again. To reset delete it:"
+    echo "rm -f $TMP_DBS[axione]"
+  fi
+
+  $g_axione_ingest $AXIONE_CSV_DIR $TMP_DBS[axione]
+}
+
+axione_clean() {
+  echo "Clean Axione files"
+  rm -rf $AXIONE_CSV_DIR/*csv*
+}
+
+############################################## ARCEP IMPLEMENTATION
+arcep_fetch_latest() {
+  echo "arcep_fetch_latdsfgsfest"
+
+}
+
+# arcep_ingest() {
+
+# }
+
+############################################## DEPLOY METHODS
+
+rollback() {
+  prod_db=$PROD_DBS[$target]
+  echo "Putting back $prod_db.backup"
+  sudo systemctl stop $APP_SVC
+  mv -f "$prod_db.backup" "$prod_db"
+  sudo systemctl start $APP_SVC || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $APP_SVC ne redémarre plus !"
+}
+
+deploy_swap() {
+  prod_db=$PROD_DBS[$target]
+  tmp_db=$TMP_DBS[$target]
+  echo "Backup DB $prod_db to $prod_db.backup"
+  if [[ -f $prod_db.backup ]]; then
+    echo "Backup exists, restore it !"
+    rollback
+    return
+  fi
+  echo "Backup prod DB $prod_db"
+  cp "$prod_db" "$prod_db.backup"
+
+  echo "Swap $tmp_db to $prod_db"
+  mv "$tmp_db" "$prod_db"
+
+  echo "Restart service and check OK"
+  if sudo systemctl restart $APP_SVC; then
+    echo "App $APP_SVC restarted OK !"
+    if $keep_backup; then
+      echo "Save $prod_db.backup TO $prod_db.old"
+      mv "$prod_db.backup" "$prod_db.old"
+    else
+      echo "Delete $prod_db.backup"
+      rm -f "$prod_db.backup"
+    fi
+    ${target}_clean
+  else
+    # Error, put back old DB
+    echo "Error, Rollback !"
+    rollback
+  fi
+}
+
+################################### MAIN
+
+# First fetch IPE raw files
+echo "########## FETCH LATEST IPEs FOR $target"
+${target}_fetch_latest
+
+# Then ingest into a database
+echo "########## INGEST LATEST IPEs FOR $target"
+${target}_ingest
+
+# Finally Deploy new database to prod
+echo "########## DEPLOY NEW DB FOR $target"
+deploy_swap