Script to fetch IPEs for axione & arcep
This commit is contained in:
Johan Le Baut
parent
318167467a
commit
8088d76e91
3 changed files with 232 additions and 0 deletions
74
data-ingest/fetch_axione_ipe.sh
Executable file
74
data-ingest/fetch_axione_ipe.sh
Executable file
|
|
@ -0,0 +1,74 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
### --------------------------------------
|
||||||
|
### Daily IPE files FTP synchro & ingest
|
||||||
|
### Sacha - 20220414
|
||||||
|
### Johan - 20220519 - fix appel ingest & ajout City fast
|
||||||
|
### -------------------------------------
|
||||||
|
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
|
||||||
|
date=$(date +"%Y%m%d" -d "2 days ago")
|
||||||
|
|
||||||
|
# Script Configuration
|
||||||
|
localDir="/srv/ftth_ipe_data/tmp_ipe"
|
||||||
|
ingest="/srv/www/ftth-ipe-map/data-ingest/ingest"
|
||||||
|
db_target_path="/srv/ftth_ipe_data/axione_data/ipe.sqlite"
|
||||||
|
db_temp="/srv/ftth_ipe_data/axione_data/ipe_temp.sqlite"
|
||||||
|
webapp_svc="ftth-ipe-map.service"
|
||||||
|
axione_files_pattern="A-IPE*$date*.csv V-IPEZTD*CTYF*$date*.csv"
|
||||||
|
axione_ftp_user=
|
||||||
|
axione_ftp_pass=
|
||||||
|
|
||||||
|
|
||||||
|
# First clean csv files in temp dir and temp sqlite if exists
|
||||||
|
rm -f $localDir/*csv*
|
||||||
|
rm -f $db_temp
|
||||||
|
|
||||||
|
# FTP Sync
|
||||||
|
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
|
||||||
|
lftp -u "$axione_ftp_user","$axione_ftp_pass" ftp.axione.fr -e "mget $axione_files_pattern -O $localDir; exit" ||:
|
||||||
|
|
||||||
|
# Check if no files mail
|
||||||
|
echo "Check if found files"
|
||||||
|
files=$(shopt -s nullglob dotglob; echo $localDir/*)
|
||||||
|
if (( ! ${#files} ))
|
||||||
|
then
|
||||||
|
mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Pas de fichiers IPE synchronisés"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Ingest files into temp DB $db_temp"
|
||||||
|
|
||||||
|
# Make sure db is empty
|
||||||
|
rm -f $db_temp
|
||||||
|
|
||||||
|
# Ingest into temporary DB
|
||||||
|
$ingest $localDir $db_temp
|
||||||
|
|
||||||
|
echo "Backup ori DB $db_target_path"
|
||||||
|
# Make copy of original DB before swap
|
||||||
|
cp $db_target_path $db_target_path.backup
|
||||||
|
|
||||||
|
echo "Swap DBs new and original"
|
||||||
|
# Swap DBs
|
||||||
|
mv -f $db_temp $db_target_path
|
||||||
|
|
||||||
|
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
|
||||||
|
|
||||||
|
echo "Restart service $webapp_svc"
|
||||||
|
# Restart service
|
||||||
|
if sudo systemctl restart $webapp_svc; then
|
||||||
|
# Ok, delete copy of old DB
|
||||||
|
echo "OK, remove backup DB"
|
||||||
|
rm -f $db_target_path.backup
|
||||||
|
else
|
||||||
|
# Error, put back old DB
|
||||||
|
echo "Error, Put back old DB & restart"
|
||||||
|
mv -f $db_target_path.backup $db_target_path
|
||||||
|
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
|
||||||
|
sudo systemctl restart $webapp_svc || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $webapp_svc ne redémarre pas !"
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Clean any downloaded csv files"
|
||||||
|
# Delete CSV files
|
||||||
|
rm -rf $localDir/*csv*
|
||||||
158
data-ingest/fetch_latest_ipe.sh
Executable file
158
data-ingest/fetch_latest_ipe.sh
Executable file
|
|
@ -0,0 +1,158 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
### --------------------------------------
|
||||||
|
### IPE files synchro & ingest
|
||||||
|
###
|
||||||
|
### The purpose of this script is to be used in a CRON to automatically fetch up-to-date IPE files and ingest them into sqlite.
|
||||||
|
###
|
||||||
|
### Johan - 20240508 - Init scripts
|
||||||
|
### -------------------------------------
|
||||||
|
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
target=${1:-}
|
||||||
|
keep_backup=${2:-}
|
||||||
|
|
||||||
|
############### CONFIG ###############
|
||||||
|
g_base_dir=/srv/ftth_ipe_data/
|
||||||
|
|
||||||
|
# AXIONE
|
||||||
|
g_axione_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_axione
|
||||||
|
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
|
||||||
|
axione_ipe_date=$(date +"%Y%m%d" -d "2 days ago")
|
||||||
|
g_axione_files_pattern="A-IPE*$axione_ipe_date*.csv V-IPEZTD*CTYF*$axione_ipe_date*.csv"
|
||||||
|
g_axione_ftp_user=
|
||||||
|
g_axione_ftp_pass=
|
||||||
|
|
||||||
|
# ARCEP
|
||||||
|
g_arcep_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_arcep
|
||||||
|
|
||||||
|
|
||||||
|
############### CONSTANTS ###############
|
||||||
|
|
||||||
|
APP_SVC="ftth-ipe-map.service"
|
||||||
|
|
||||||
|
declare -A PROD_DBS
|
||||||
|
declare -A TMP_DBS
|
||||||
|
|
||||||
|
# AXIONE
|
||||||
|
AXIONE_CSV_DIR="$g_base_dir/axione/tmp"
|
||||||
|
PROD_DBS[axione]="$g_base_dir/axione/ipe.sqlite"
|
||||||
|
TMP_DBS[axione]="$g_base_dir/axione/tmp.sqlite"
|
||||||
|
######################################
|
||||||
|
|
||||||
|
help() {
|
||||||
|
echo "$0 target (--keep-backup)"
|
||||||
|
echo "Where:"
|
||||||
|
echo " - target: axione|arcep"
|
||||||
|
echo " - (--keep-backup): if set, will keep backup db as db.old"
|
||||||
|
}
|
||||||
|
|
||||||
|
[[ ! $target || "$target" == "-h" || "$target" == "--help" ]] && help && exit 0
|
||||||
|
|
||||||
|
|
||||||
|
if [[ "$target" != "axione" && "$target" != "arcep" ]]; then
|
||||||
|
echo "Unknown target: $target"
|
||||||
|
help
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ "$keep_backup" == "--keep-backup" ]] && keep_backup=true || keep_backup=false
|
||||||
|
|
||||||
|
|
||||||
|
############################################## AXIONE IMPLEMENTATION
|
||||||
|
axione_fetch_latest() {
|
||||||
|
mkdir -p $AXIONE_CSV_DIR
|
||||||
|
|
||||||
|
files=$(shopt -s nullglob dotglob; echo $AXIONE_CSV_DIR/*)
|
||||||
|
if (( ${#files} )); then
|
||||||
|
echo "Found CSV files not processed, not downloading again. to reset, delete content in $AXIONE_CSV_DIR:"
|
||||||
|
echo "rm -f $AXIONE_CSV_DIR/*"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
# FTP Sync
|
||||||
|
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
|
||||||
|
lftp -u "$g_axione_ftp_user","$g_axione_ftp_pass" ftp.axione.fr -e "mget $g_axione_files_pattern -O $AXIONE_CSV_DIR; exit" ||:
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
axione_ingest() {
|
||||||
|
if [[ -f $AXIONE_TMP_DB ]]; then
|
||||||
|
echo "temp DB file already exists, not processing again. To reset delete it:"
|
||||||
|
echo "rm -f $TMP_DBS[axione]"
|
||||||
|
fi
|
||||||
|
|
||||||
|
$g_axione_ingest $AXIONE_CSV_DIR $TMP_DBS[axione]
|
||||||
|
}
|
||||||
|
|
||||||
|
axione_clean() {
|
||||||
|
echo "Clean Axione files"
|
||||||
|
rm -rf $AXIONE_CSV_DIR/*csv*
|
||||||
|
}
|
||||||
|
|
||||||
|
############################################## ARCEP IMPLEMENTATION
|
||||||
|
arcep_fetch_latest() {
|
||||||
|
echo "arcep_fetch_latdsfgsfest"
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
# arcep_ingest() {
|
||||||
|
|
||||||
|
# }
|
||||||
|
|
||||||
|
############################################## DEPLOY METHODS
|
||||||
|
|
||||||
|
rollback() {
|
||||||
|
prod_db=$PROD_DBS[$target]
|
||||||
|
echo "Putting back $prod_db.backup"
|
||||||
|
sudo systemctl stop $APP_SVC
|
||||||
|
mv -f "$prod_db.backup" "$prod_db"
|
||||||
|
sudo systemctl start $APP_SVC || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $APP_SVC ne redémarre plus !"
|
||||||
|
}
|
||||||
|
|
||||||
|
deploy_swap() {
|
||||||
|
prod_db=$PROD_DBS[$target]
|
||||||
|
tmp_db=$TMP_DBS[$target]
|
||||||
|
echo "Backup DB $prod_db to $prod_db.backup"
|
||||||
|
if [[ -f $prod_db.backup ]]; then
|
||||||
|
echo "Backup exists, restore it !"
|
||||||
|
rollback
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
echo "Backup prod DB $prod_db"
|
||||||
|
cp "$prod_db" "$prod_db.backup"
|
||||||
|
|
||||||
|
echo "Swap $tmp_db to $prod_db"
|
||||||
|
mv "$tmp_db" "$prod_db"
|
||||||
|
|
||||||
|
echo "Restart service and check OK"
|
||||||
|
if sudo systemctl restart $APP_SVC; then
|
||||||
|
echo "App $APP_SVC restarted OK !"
|
||||||
|
if $keep_backup; then
|
||||||
|
echo "Save $prod_db.backup TO $prod_db.old"
|
||||||
|
mv "$prod_db.backup" "$prod_db.old"
|
||||||
|
else
|
||||||
|
echo "Delete $prod_db.backup"
|
||||||
|
rm -f "$prod_db.backup"
|
||||||
|
fi
|
||||||
|
${target}_clean
|
||||||
|
else
|
||||||
|
# Error, put back old DB
|
||||||
|
echo "Error, Rollback !"
|
||||||
|
rollback
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
################################### MAIN
|
||||||
|
|
||||||
|
# First fetch IPE raw files
|
||||||
|
echo "########## FETCH LATEST IPEs FOR $target"
|
||||||
|
${target}_fetch_latest
|
||||||
|
|
||||||
|
# Then ingest into a database
|
||||||
|
echo "########## INGEST LATEST IPEs FOR $target"
|
||||||
|
${target}_ingest
|
||||||
|
|
||||||
|
# Finally Deploy new database to prod
|
||||||
|
echo "########## DEPLOY NEW DB FOR $target"
|
||||||
|
deploy_swap
|
||||||
Loading…
Reference in a new issue