diff --git a/data-ingest/fetch_axione_ipe.sh b/data-ingest/fetch_axione_ipe.sh new file mode 100755 index 0000000..7a19f00 --- /dev/null +++ b/data-ingest/fetch_axione_ipe.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +### -------------------------------------- +### Daily IPE files FTP synchro & ingest +### Sacha - 20220414 +### Johan - 20220519 - fix appel ingest & ajout City fast +### ------------------------------------- + +set -Eeuo pipefail + +# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille... +date=$(date +"%Y%m%d" -d "2 days ago") + +# Script Configuration +localDir="/srv/ftth_ipe_data/tmp_ipe" +ingest="/srv/www/ftth-ipe-map/data-ingest/ingest" +db_target_path="/srv/ftth_ipe_data/axione_data/ipe.sqlite" +db_temp="/srv/ftth_ipe_data/axione_data/ipe_temp.sqlite" +webapp_svc="ftth-ipe-map.service" +axione_files_pattern="A-IPE*$date*.csv V-IPEZTD*CTYF*$date*.csv" +axione_ftp_user= +axione_ftp_pass= + + +# First clean csv files in temp dir and temp sqlite if exists +rm -f $localDir/*csv* +rm -f $db_temp + +# FTP Sync +# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon +lftp -u "$axione_ftp_user","$axione_ftp_pass" ftp.axione.fr -e "mget $axione_files_pattern -O $localDir; exit" ||: + +# Check if no files mail +echo "Check if found files" +files=$(shopt -s nullglob dotglob; echo $localDir/*) +if (( ! ${#files} )) +then +mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Pas de fichiers IPE synchronisés" +fi + +echo "Ingest files into temp DB $db_temp" + +# Make sure db is empty +rm -f $db_temp + +# Ingest into temporary DB +$ingest $localDir $db_temp + +echo "Backup ori DB $db_target_path" +# Make copy of original DB before swap +cp $db_target_path $db_target_path.backup + +echo "Swap DBs new and original" +# Swap DBs +mv -f $db_temp $db_target_path + +#chown ftth-ipe-map:ftth-ipe-map $db_target_path + +echo "Restart service $webapp_svc" +# Restart service +if sudo systemctl restart $webapp_svc; then + # Ok, delete copy of old DB + echo "OK, remove backup DB" + rm -f $db_target_path.backup +else + # Error, put back old DB + echo "Error, Put back old DB & restart" + mv -f $db_target_path.backup $db_target_path + #chown ftth-ipe-map:ftth-ipe-map $db_target_path + sudo systemctl restart $webapp_svc || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $webapp_svc ne redémarre pas !" +fi + +echo "Clean any downloaded csv files" +# Delete CSV files +rm -rf $localDir/*csv* diff --git a/data-ingest/fetch_latest_ipe.sh b/data-ingest/fetch_latest_ipe.sh new file mode 100755 index 0000000..6d34449 --- /dev/null +++ b/data-ingest/fetch_latest_ipe.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +### -------------------------------------- +### IPE files synchro & ingest +### +### The purpose of this script is to be used in a CRON to automatically fetch up-to-date IPE files and ingest them into sqlite. +### +### Johan - 20240508 - Init scripts +### ------------------------------------- + +set -Eeuo pipefail + +target=${1:-} +keep_backup=${2:-} + +############### CONFIG ############### +g_base_dir=/srv/ftth_ipe_data/ + +# AXIONE +g_axione_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_axione +# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille... +axione_ipe_date=$(date +"%Y%m%d" -d "2 days ago") +g_axione_files_pattern="A-IPE*$axione_ipe_date*.csv V-IPEZTD*CTYF*$axione_ipe_date*.csv" +g_axione_ftp_user= +g_axione_ftp_pass= + +# ARCEP +g_arcep_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_arcep + + +############### CONSTANTS ############### + +APP_SVC="ftth-ipe-map.service" + +declare -A PROD_DBS +declare -A TMP_DBS + +# AXIONE +AXIONE_CSV_DIR="$g_base_dir/axione/tmp" +PROD_DBS[axione]="$g_base_dir/axione/ipe.sqlite" +TMP_DBS[axione]="$g_base_dir/axione/tmp.sqlite" +###################################### + +help() { + echo "$0 target (--keep-backup)" + echo "Where:" + echo " - target: axione|arcep" + echo " - (--keep-backup): if set, will keep backup db as db.old" +} + +[[ ! $target || "$target" == "-h" || "$target" == "--help" ]] && help && exit 0 + + +if [[ "$target" != "axione" && "$target" != "arcep" ]]; then + echo "Unknown target: $target" + help + exit 1 +fi + +[[ "$keep_backup" == "--keep-backup" ]] && keep_backup=true || keep_backup=false + + +############################################## AXIONE IMPLEMENTATION +axione_fetch_latest() { + mkdir -p $AXIONE_CSV_DIR + + files=$(shopt -s nullglob dotglob; echo $AXIONE_CSV_DIR/*) + if (( ${#files} )); then + echo "Found CSV files not processed, not downloading again. to reset, delete content in $AXIONE_CSV_DIR:" + echo "rm -f $AXIONE_CSV_DIR/*" + return + fi + + # FTP Sync + # Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon + lftp -u "$g_axione_ftp_user","$g_axione_ftp_pass" ftp.axione.fr -e "mget $g_axione_files_pattern -O $AXIONE_CSV_DIR; exit" ||: + +} + +axione_ingest() { + if [[ -f $AXIONE_TMP_DB ]]; then + echo "temp DB file already exists, not processing again. To reset delete it:" + echo "rm -f $TMP_DBS[axione]" + fi + + $g_axione_ingest $AXIONE_CSV_DIR $TMP_DBS[axione] +} + +axione_clean() { + echo "Clean Axione files" + rm -rf $AXIONE_CSV_DIR/*csv* +} + +############################################## ARCEP IMPLEMENTATION +arcep_fetch_latest() { + echo "arcep_fetch_latdsfgsfest" + +} + +# arcep_ingest() { + +# } + +############################################## DEPLOY METHODS + +rollback() { + prod_db=$PROD_DBS[$target] + echo "Putting back $prod_db.backup" + sudo systemctl stop $APP_SVC + mv -f "$prod_db.backup" "$prod_db" + sudo systemctl start $APP_SVC || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $APP_SVC ne redémarre plus !" +} + +deploy_swap() { + prod_db=$PROD_DBS[$target] + tmp_db=$TMP_DBS[$target] + echo "Backup DB $prod_db to $prod_db.backup" + if [[ -f $prod_db.backup ]]; then + echo "Backup exists, restore it !" + rollback + return + fi + echo "Backup prod DB $prod_db" + cp "$prod_db" "$prod_db.backup" + + echo "Swap $tmp_db to $prod_db" + mv "$tmp_db" "$prod_db" + + echo "Restart service and check OK" + if sudo systemctl restart $APP_SVC; then + echo "App $APP_SVC restarted OK !" + if $keep_backup; then + echo "Save $prod_db.backup TO $prod_db.old" + mv "$prod_db.backup" "$prod_db.old" + else + echo "Delete $prod_db.backup" + rm -f "$prod_db.backup" + fi + ${target}_clean + else + # Error, put back old DB + echo "Error, Rollback !" + rollback + fi +} + +################################### MAIN + +# First fetch IPE raw files +echo "########## FETCH LATEST IPEs FOR $target" +${target}_fetch_latest + +# Then ingest into a database +echo "########## INGEST LATEST IPEs FOR $target" +${target}_ingest + +# Finally Deploy new database to prod +echo "########## DEPLOY NEW DB FOR $target" +deploy_swap diff --git a/data-ingest/ingest b/data-ingest/ingest_axione similarity index 100% rename from data-ingest/ingest rename to data-ingest/ingest_axione