Compare commits

..

1 commit

Author SHA1 Message Date
Johan Le Baut
8088d76e91 Script to fetch IPEs for axione & arcep 2024-05-09 01:58:10 +02:00
3 changed files with 232 additions and 0 deletions

74
data-ingest/fetch_axione_ipe.sh Executable file
View file

@ -0,0 +1,74 @@
#!/usr/bin/env bash
### --------------------------------------
### Daily IPE files FTP synchro & ingest
### Sacha - 20220414
### Johan - 20220519 - fix appel ingest & ajout City fast
### -------------------------------------
set -Eeuo pipefail
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
date=$(date +"%Y%m%d" -d "2 days ago")
# Script Configuration
localDir="/srv/ftth_ipe_data/tmp_ipe"
ingest="/srv/www/ftth-ipe-map/data-ingest/ingest"
db_target_path="/srv/ftth_ipe_data/axione_data/ipe.sqlite"
db_temp="/srv/ftth_ipe_data/axione_data/ipe_temp.sqlite"
webapp_svc="ftth-ipe-map.service"
axione_files_pattern="A-IPE*$date*.csv V-IPEZTD*CTYF*$date*.csv"
axione_ftp_user=
axione_ftp_pass=
# First clean csv files in temp dir and temp sqlite if exists
rm -f $localDir/*csv*
rm -f $db_temp
# FTP Sync
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
lftp -u "$axione_ftp_user","$axione_ftp_pass" ftp.axione.fr -e "mget $axione_files_pattern -O $localDir; exit" ||:
# Check if no files mail
echo "Check if found files"
files=$(shopt -s nullglob dotglob; echo $localDir/*)
if (( ! ${#files} ))
then
mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Pas de fichiers IPE synchronisés"
fi
echo "Ingest files into temp DB $db_temp"
# Make sure db is empty
rm -f $db_temp
# Ingest into temporary DB
$ingest $localDir $db_temp
echo "Backup ori DB $db_target_path"
# Make copy of original DB before swap
cp $db_target_path $db_target_path.backup
echo "Swap DBs new and original"
# Swap DBs
mv -f $db_temp $db_target_path
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
echo "Restart service $webapp_svc"
# Restart service
if sudo systemctl restart $webapp_svc; then
# Ok, delete copy of old DB
echo "OK, remove backup DB"
rm -f $db_target_path.backup
else
# Error, put back old DB
echo "Error, Put back old DB & restart"
mv -f $db_target_path.backup $db_target_path
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
sudo systemctl restart $webapp_svc || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $webapp_svc ne redémarre pas !"
fi
echo "Clean any downloaded csv files"
# Delete CSV files
rm -rf $localDir/*csv*

158
data-ingest/fetch_latest_ipe.sh Executable file
View file

@ -0,0 +1,158 @@
#!/usr/bin/env bash
### --------------------------------------
### IPE files synchro & ingest
###
### The purpose of this script is to be used in a CRON to automatically fetch up-to-date IPE files and ingest them into sqlite.
###
### Johan - 20240508 - Init scripts
### -------------------------------------
set -Eeuo pipefail
target=${1:-}
keep_backup=${2:-}
############### CONFIG ###############
g_base_dir=/srv/ftth_ipe_data/
# AXIONE
g_axione_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_axione
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
axione_ipe_date=$(date +"%Y%m%d" -d "2 days ago")
g_axione_files_pattern="A-IPE*$axione_ipe_date*.csv V-IPEZTD*CTYF*$axione_ipe_date*.csv"
g_axione_ftp_user=
g_axione_ftp_pass=
# ARCEP
g_arcep_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_arcep
############### CONSTANTS ###############
APP_SVC="ftth-ipe-map.service"
declare -A PROD_DBS
declare -A TMP_DBS
# AXIONE
AXIONE_CSV_DIR="$g_base_dir/axione/tmp"
PROD_DBS[axione]="$g_base_dir/axione/ipe.sqlite"
TMP_DBS[axione]="$g_base_dir/axione/tmp.sqlite"
######################################
help() {
echo "$0 target (--keep-backup)"
echo "Where:"
echo " - target: axione|arcep"
echo " - (--keep-backup): if set, will keep backup db as db.old"
}
[[ ! $target || "$target" == "-h" || "$target" == "--help" ]] && help && exit 0
if [[ "$target" != "axione" && "$target" != "arcep" ]]; then
echo "Unknown target: $target"
help
exit 1
fi
[[ "$keep_backup" == "--keep-backup" ]] && keep_backup=true || keep_backup=false
############################################## AXIONE IMPLEMENTATION
axione_fetch_latest() {
mkdir -p $AXIONE_CSV_DIR
files=$(shopt -s nullglob dotglob; echo $AXIONE_CSV_DIR/*)
if (( ${#files} )); then
echo "Found CSV files not processed, not downloading again. to reset, delete content in $AXIONE_CSV_DIR:"
echo "rm -f $AXIONE_CSV_DIR/*"
return
fi
# FTP Sync
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
lftp -u "$g_axione_ftp_user","$g_axione_ftp_pass" ftp.axione.fr -e "mget $g_axione_files_pattern -O $AXIONE_CSV_DIR; exit" ||:
}
axione_ingest() {
if [[ -f $AXIONE_TMP_DB ]]; then
echo "temp DB file already exists, not processing again. To reset delete it:"
echo "rm -f $TMP_DBS[axione]"
fi
$g_axione_ingest $AXIONE_CSV_DIR $TMP_DBS[axione]
}
axione_clean() {
echo "Clean Axione files"
rm -rf $AXIONE_CSV_DIR/*csv*
}
############################################## ARCEP IMPLEMENTATION
arcep_fetch_latest() {
echo "arcep_fetch_latdsfgsfest"
}
# arcep_ingest() {
# }
############################################## DEPLOY METHODS
rollback() {
prod_db=$PROD_DBS[$target]
echo "Putting back $prod_db.backup"
sudo systemctl stop $APP_SVC
mv -f "$prod_db.backup" "$prod_db"
sudo systemctl start $APP_SVC || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $APP_SVC ne redémarre plus !"
}
deploy_swap() {
prod_db=$PROD_DBS[$target]
tmp_db=$TMP_DBS[$target]
echo "Backup DB $prod_db to $prod_db.backup"
if [[ -f $prod_db.backup ]]; then
echo "Backup exists, restore it !"
rollback
return
fi
echo "Backup prod DB $prod_db"
cp "$prod_db" "$prod_db.backup"
echo "Swap $tmp_db to $prod_db"
mv "$tmp_db" "$prod_db"
echo "Restart service and check OK"
if sudo systemctl restart $APP_SVC; then
echo "App $APP_SVC restarted OK !"
if $keep_backup; then
echo "Save $prod_db.backup TO $prod_db.old"
mv "$prod_db.backup" "$prod_db.old"
else
echo "Delete $prod_db.backup"
rm -f "$prod_db.backup"
fi
${target}_clean
else
# Error, put back old DB
echo "Error, Rollback !"
rollback
fi
}
################################### MAIN
# First fetch IPE raw files
echo "########## FETCH LATEST IPEs FOR $target"
${target}_fetch_latest
# Then ingest into a database
echo "########## INGEST LATEST IPEs FOR $target"
${target}_ingest
# Finally Deploy new database to prod
echo "########## DEPLOY NEW DB FOR $target"
deploy_swap