Script to fetch IPEs for axione & arcep
This commit is contained in:
Johan Le Baut
parent
318167467a
commit
8088d76e91
3 changed files with 232 additions and 0 deletions
74
data-ingest/fetch_axione_ipe.sh
Executable file
74
data-ingest/fetch_axione_ipe.sh
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env bash
|
||||
### --------------------------------------
|
||||
### Daily IPE files FTP synchro & ingest
|
||||
### Sacha - 20220414
|
||||
### Johan - 20220519 - fix appel ingest & ajout City fast
|
||||
### -------------------------------------
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
|
||||
date=$(date +"%Y%m%d" -d "2 days ago")
|
||||
|
||||
# Script Configuration
|
||||
localDir="/srv/ftth_ipe_data/tmp_ipe"
|
||||
ingest="/srv/www/ftth-ipe-map/data-ingest/ingest"
|
||||
db_target_path="/srv/ftth_ipe_data/axione_data/ipe.sqlite"
|
||||
db_temp="/srv/ftth_ipe_data/axione_data/ipe_temp.sqlite"
|
||||
webapp_svc="ftth-ipe-map.service"
|
||||
axione_files_pattern="A-IPE*$date*.csv V-IPEZTD*CTYF*$date*.csv"
|
||||
axione_ftp_user=
|
||||
axione_ftp_pass=
|
||||
|
||||
|
||||
# First clean csv files in temp dir and temp sqlite if exists
|
||||
rm -f $localDir/*csv*
|
||||
rm -f $db_temp
|
||||
|
||||
# FTP Sync
|
||||
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
|
||||
lftp -u "$axione_ftp_user","$axione_ftp_pass" ftp.axione.fr -e "mget $axione_files_pattern -O $localDir; exit" ||:
|
||||
|
||||
# Check if no files mail
|
||||
echo "Check if found files"
|
||||
files=$(shopt -s nullglob dotglob; echo $localDir/*)
|
||||
if (( ! ${#files} ))
|
||||
then
|
||||
mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Pas de fichiers IPE synchronisés"
|
||||
fi
|
||||
|
||||
echo "Ingest files into temp DB $db_temp"
|
||||
|
||||
# Make sure db is empty
|
||||
rm -f $db_temp
|
||||
|
||||
# Ingest into temporary DB
|
||||
$ingest $localDir $db_temp
|
||||
|
||||
echo "Backup ori DB $db_target_path"
|
||||
# Make copy of original DB before swap
|
||||
cp $db_target_path $db_target_path.backup
|
||||
|
||||
echo "Swap DBs new and original"
|
||||
# Swap DBs
|
||||
mv -f $db_temp $db_target_path
|
||||
|
||||
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
|
||||
|
||||
echo "Restart service $webapp_svc"
|
||||
# Restart service
|
||||
if sudo systemctl restart $webapp_svc; then
|
||||
# Ok, delete copy of old DB
|
||||
echo "OK, remove backup DB"
|
||||
rm -f $db_target_path.backup
|
||||
else
|
||||
# Error, put back old DB
|
||||
echo "Error, Put back old DB & restart"
|
||||
mv -f $db_target_path.backup $db_target_path
|
||||
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
|
||||
sudo systemctl restart $webapp_svc || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $webapp_svc ne redémarre pas !"
|
||||
fi
|
||||
|
||||
echo "Clean any downloaded csv files"
|
||||
# Delete CSV files
|
||||
rm -rf $localDir/*csv*
|
||||
158
data-ingest/fetch_latest_ipe.sh
Executable file
158
data-ingest/fetch_latest_ipe.sh
Executable file
|
|
@ -0,0 +1,158 @@
|
|||
#!/usr/bin/env bash
|
||||
### --------------------------------------
|
||||
### IPE files synchro & ingest
|
||||
###
|
||||
### The purpose of this script is to be used in a CRON to automatically fetch up-to-date IPE files and ingest them into sqlite.
|
||||
###
|
||||
### Johan - 20240508 - Init scripts
|
||||
### -------------------------------------
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
target=${1:-}
|
||||
keep_backup=${2:-}
|
||||
|
||||
############### CONFIG ###############
|
||||
g_base_dir=/srv/ftth_ipe_data/
|
||||
|
||||
# AXIONE
|
||||
g_axione_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_axione
|
||||
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
|
||||
axione_ipe_date=$(date +"%Y%m%d" -d "2 days ago")
|
||||
g_axione_files_pattern="A-IPE*$axione_ipe_date*.csv V-IPEZTD*CTYF*$axione_ipe_date*.csv"
|
||||
g_axione_ftp_user=
|
||||
g_axione_ftp_pass=
|
||||
|
||||
# ARCEP
|
||||
g_arcep_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_arcep
|
||||
|
||||
|
||||
############### CONSTANTS ###############
|
||||
|
||||
APP_SVC="ftth-ipe-map.service"
|
||||
|
||||
declare -A PROD_DBS
|
||||
declare -A TMP_DBS
|
||||
|
||||
# AXIONE
|
||||
AXIONE_CSV_DIR="$g_base_dir/axione/tmp"
|
||||
PROD_DBS[axione]="$g_base_dir/axione/ipe.sqlite"
|
||||
TMP_DBS[axione]="$g_base_dir/axione/tmp.sqlite"
|
||||
######################################
|
||||
|
||||
help() {
|
||||
echo "$0 target (--keep-backup)"
|
||||
echo "Where:"
|
||||
echo " - target: axione|arcep"
|
||||
echo " - (--keep-backup): if set, will keep backup db as db.old"
|
||||
}
|
||||
|
||||
[[ ! $target || "$target" == "-h" || "$target" == "--help" ]] && help && exit 0
|
||||
|
||||
|
||||
if [[ "$target" != "axione" && "$target" != "arcep" ]]; then
|
||||
echo "Unknown target: $target"
|
||||
help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[[ "$keep_backup" == "--keep-backup" ]] && keep_backup=true || keep_backup=false
|
||||
|
||||
|
||||
############################################## AXIONE IMPLEMENTATION
|
||||
axione_fetch_latest() {
|
||||
mkdir -p $AXIONE_CSV_DIR
|
||||
|
||||
files=$(shopt -s nullglob dotglob; echo $AXIONE_CSV_DIR/*)
|
||||
if (( ${#files} )); then
|
||||
echo "Found CSV files not processed, not downloading again. to reset, delete content in $AXIONE_CSV_DIR:"
|
||||
echo "rm -f $AXIONE_CSV_DIR/*"
|
||||
return
|
||||
fi
|
||||
|
||||
# FTP Sync
|
||||
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
|
||||
lftp -u "$g_axione_ftp_user","$g_axione_ftp_pass" ftp.axione.fr -e "mget $g_axione_files_pattern -O $AXIONE_CSV_DIR; exit" ||:
|
||||
|
||||
}
|
||||
|
||||
axione_ingest() {
|
||||
if [[ -f $AXIONE_TMP_DB ]]; then
|
||||
echo "temp DB file already exists, not processing again. To reset delete it:"
|
||||
echo "rm -f $TMP_DBS[axione]"
|
||||
fi
|
||||
|
||||
$g_axione_ingest $AXIONE_CSV_DIR $TMP_DBS[axione]
|
||||
}
|
||||
|
||||
axione_clean() {
|
||||
echo "Clean Axione files"
|
||||
rm -rf $AXIONE_CSV_DIR/*csv*
|
||||
}
|
||||
|
||||
############################################## ARCEP IMPLEMENTATION
|
||||
arcep_fetch_latest() {
|
||||
echo "arcep_fetch_latdsfgsfest"
|
||||
|
||||
}
|
||||
|
||||
# arcep_ingest() {
|
||||
|
||||
# }
|
||||
|
||||
############################################## DEPLOY METHODS
|
||||
|
||||
rollback() {
|
||||
prod_db=$PROD_DBS[$target]
|
||||
echo "Putting back $prod_db.backup"
|
||||
sudo systemctl stop $APP_SVC
|
||||
mv -f "$prod_db.backup" "$prod_db"
|
||||
sudo systemctl start $APP_SVC || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $APP_SVC ne redémarre plus !"
|
||||
}
|
||||
|
||||
deploy_swap() {
|
||||
prod_db=$PROD_DBS[$target]
|
||||
tmp_db=$TMP_DBS[$target]
|
||||
echo "Backup DB $prod_db to $prod_db.backup"
|
||||
if [[ -f $prod_db.backup ]]; then
|
||||
echo "Backup exists, restore it !"
|
||||
rollback
|
||||
return
|
||||
fi
|
||||
echo "Backup prod DB $prod_db"
|
||||
cp "$prod_db" "$prod_db.backup"
|
||||
|
||||
echo "Swap $tmp_db to $prod_db"
|
||||
mv "$tmp_db" "$prod_db"
|
||||
|
||||
echo "Restart service and check OK"
|
||||
if sudo systemctl restart $APP_SVC; then
|
||||
echo "App $APP_SVC restarted OK !"
|
||||
if $keep_backup; then
|
||||
echo "Save $prod_db.backup TO $prod_db.old"
|
||||
mv "$prod_db.backup" "$prod_db.old"
|
||||
else
|
||||
echo "Delete $prod_db.backup"
|
||||
rm -f "$prod_db.backup"
|
||||
fi
|
||||
${target}_clean
|
||||
else
|
||||
# Error, put back old DB
|
||||
echo "Error, Rollback !"
|
||||
rollback
|
||||
fi
|
||||
}
|
||||
|
||||
################################### MAIN
|
||||
|
||||
# First fetch IPE raw files
|
||||
echo "########## FETCH LATEST IPEs FOR $target"
|
||||
${target}_fetch_latest
|
||||
|
||||
# Then ingest into a database
|
||||
echo "########## INGEST LATEST IPEs FOR $target"
|
||||
${target}_ingest
|
||||
|
||||
# Finally Deploy new database to prod
|
||||
echo "########## DEPLOY NEW DB FOR $target"
|
||||
deploy_swap
|
||||
Loading…
Reference in a new issue