Compare commits
No commits in common. "johan/ipes-update" and "master" have entirely different histories.
johan/ipes
...
master
3 changed files with 0 additions and 232 deletions
|
@ -1,74 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
### --------------------------------------
|
|
||||||
### Daily IPE files FTP synchro & ingest
|
|
||||||
### Sacha - 20220414
|
|
||||||
### Johan - 20220519 - fix appel ingest & ajout City fast
|
|
||||||
### -------------------------------------
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
|
|
||||||
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
|
|
||||||
date=$(date +"%Y%m%d" -d "2 days ago")
|
|
||||||
|
|
||||||
# Script Configuration
|
|
||||||
localDir="/srv/ftth_ipe_data/tmp_ipe"
|
|
||||||
ingest="/srv/www/ftth-ipe-map/data-ingest/ingest"
|
|
||||||
db_target_path="/srv/ftth_ipe_data/axione_data/ipe.sqlite"
|
|
||||||
db_temp="/srv/ftth_ipe_data/axione_data/ipe_temp.sqlite"
|
|
||||||
webapp_svc="ftth-ipe-map.service"
|
|
||||||
axione_files_pattern="A-IPE*$date*.csv V-IPEZTD*CTYF*$date*.csv"
|
|
||||||
axione_ftp_user=
|
|
||||||
axione_ftp_pass=
|
|
||||||
|
|
||||||
|
|
||||||
# First clean csv files in temp dir and temp sqlite if exists
|
|
||||||
rm -f $localDir/*csv*
|
|
||||||
rm -f $db_temp
|
|
||||||
|
|
||||||
# FTP Sync
|
|
||||||
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
|
|
||||||
lftp -u "$axione_ftp_user","$axione_ftp_pass" ftp.axione.fr -e "mget $axione_files_pattern -O $localDir; exit" ||:
|
|
||||||
|
|
||||||
# Check if no files mail
|
|
||||||
echo "Check if found files"
|
|
||||||
files=$(shopt -s nullglob dotglob; echo $localDir/*)
|
|
||||||
if (( ! ${#files} ))
|
|
||||||
then
|
|
||||||
mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Pas de fichiers IPE synchronisés"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Ingest files into temp DB $db_temp"
|
|
||||||
|
|
||||||
# Make sure db is empty
|
|
||||||
rm -f $db_temp
|
|
||||||
|
|
||||||
# Ingest into temporary DB
|
|
||||||
$ingest $localDir $db_temp
|
|
||||||
|
|
||||||
echo "Backup ori DB $db_target_path"
|
|
||||||
# Make copy of original DB before swap
|
|
||||||
cp $db_target_path $db_target_path.backup
|
|
||||||
|
|
||||||
echo "Swap DBs new and original"
|
|
||||||
# Swap DBs
|
|
||||||
mv -f $db_temp $db_target_path
|
|
||||||
|
|
||||||
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
|
|
||||||
|
|
||||||
echo "Restart service $webapp_svc"
|
|
||||||
# Restart service
|
|
||||||
if sudo systemctl restart $webapp_svc; then
|
|
||||||
# Ok, delete copy of old DB
|
|
||||||
echo "OK, remove backup DB"
|
|
||||||
rm -f $db_target_path.backup
|
|
||||||
else
|
|
||||||
# Error, put back old DB
|
|
||||||
echo "Error, Put back old DB & restart"
|
|
||||||
mv -f $db_target_path.backup $db_target_path
|
|
||||||
#chown ftth-ipe-map:ftth-ipe-map $db_target_path
|
|
||||||
sudo systemctl restart $webapp_svc || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $webapp_svc ne redémarre pas !"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Clean any downloaded csv files"
|
|
||||||
# Delete CSV files
|
|
||||||
rm -rf $localDir/*csv*
|
|
|
@ -1,158 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
### --------------------------------------
|
|
||||||
### IPE files synchro & ingest
|
|
||||||
###
|
|
||||||
### The purpose of this script is to be used in a CRON to automatically fetch up-to-date IPE files and ingest them into sqlite.
|
|
||||||
###
|
|
||||||
### Johan - 20240508 - Init scripts
|
|
||||||
### -------------------------------------
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
|
|
||||||
target=${1:-}
|
|
||||||
keep_backup=${2:-}
|
|
||||||
|
|
||||||
############### CONFIG ###############
|
|
||||||
g_base_dir=/srv/ftth_ipe_data/
|
|
||||||
|
|
||||||
# AXIONE
|
|
||||||
g_axione_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_axione
|
|
||||||
# Note: les fichiers IPE arrivent parfois très tard, on a vu du 4h du matin le lendemain... Pour être un peu plus sûr d'avoir les données, il vaut mieux prendre l'avant-veille...
|
|
||||||
axione_ipe_date=$(date +"%Y%m%d" -d "2 days ago")
|
|
||||||
g_axione_files_pattern="A-IPE*$axione_ipe_date*.csv V-IPEZTD*CTYF*$axione_ipe_date*.csv"
|
|
||||||
g_axione_ftp_user=
|
|
||||||
g_axione_ftp_pass=
|
|
||||||
|
|
||||||
# ARCEP
|
|
||||||
g_arcep_ingest=/srv/www/ftth-ipe-map/data-ingest/ingest_arcep
|
|
||||||
|
|
||||||
|
|
||||||
############### CONSTANTS ###############
|
|
||||||
|
|
||||||
APP_SVC="ftth-ipe-map.service"
|
|
||||||
|
|
||||||
declare -A PROD_DBS
|
|
||||||
declare -A TMP_DBS
|
|
||||||
|
|
||||||
# AXIONE
|
|
||||||
AXIONE_CSV_DIR="$g_base_dir/axione/tmp"
|
|
||||||
PROD_DBS[axione]="$g_base_dir/axione/ipe.sqlite"
|
|
||||||
TMP_DBS[axione]="$g_base_dir/axione/tmp.sqlite"
|
|
||||||
######################################
|
|
||||||
|
|
||||||
help() {
|
|
||||||
echo "$0 target (--keep-backup)"
|
|
||||||
echo "Where:"
|
|
||||||
echo " - target: axione|arcep"
|
|
||||||
echo " - (--keep-backup): if set, will keep backup db as db.old"
|
|
||||||
}
|
|
||||||
|
|
||||||
[[ ! $target || "$target" == "-h" || "$target" == "--help" ]] && help && exit 0
|
|
||||||
|
|
||||||
|
|
||||||
if [[ "$target" != "axione" && "$target" != "arcep" ]]; then
|
|
||||||
echo "Unknown target: $target"
|
|
||||||
help
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
[[ "$keep_backup" == "--keep-backup" ]] && keep_backup=true || keep_backup=false
|
|
||||||
|
|
||||||
|
|
||||||
############################################## AXIONE IMPLEMENTATION
|
|
||||||
axione_fetch_latest() {
|
|
||||||
mkdir -p $AXIONE_CSV_DIR
|
|
||||||
|
|
||||||
files=$(shopt -s nullglob dotglob; echo $AXIONE_CSV_DIR/*)
|
|
||||||
if (( ${#files} )); then
|
|
||||||
echo "Found CSV files not processed, not downloading again. to reset, delete content in $AXIONE_CSV_DIR:"
|
|
||||||
echo "rm -f $AXIONE_CSV_DIR/*"
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
|
|
||||||
# FTP Sync
|
|
||||||
# Le fichier CTYF n'a pas d'entête mais s'il est ingéré après les autres ça passe, pas ouf mais bon
|
|
||||||
lftp -u "$g_axione_ftp_user","$g_axione_ftp_pass" ftp.axione.fr -e "mget $g_axione_files_pattern -O $AXIONE_CSV_DIR; exit" ||:
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
axione_ingest() {
|
|
||||||
if [[ -f $AXIONE_TMP_DB ]]; then
|
|
||||||
echo "temp DB file already exists, not processing again. To reset delete it:"
|
|
||||||
echo "rm -f $TMP_DBS[axione]"
|
|
||||||
fi
|
|
||||||
|
|
||||||
$g_axione_ingest $AXIONE_CSV_DIR $TMP_DBS[axione]
|
|
||||||
}
|
|
||||||
|
|
||||||
axione_clean() {
|
|
||||||
echo "Clean Axione files"
|
|
||||||
rm -rf $AXIONE_CSV_DIR/*csv*
|
|
||||||
}
|
|
||||||
|
|
||||||
############################################## ARCEP IMPLEMENTATION
|
|
||||||
arcep_fetch_latest() {
|
|
||||||
echo "arcep_fetch_latdsfgsfest"
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
# arcep_ingest() {
|
|
||||||
|
|
||||||
# }
|
|
||||||
|
|
||||||
############################################## DEPLOY METHODS
|
|
||||||
|
|
||||||
rollback() {
|
|
||||||
prod_db=$PROD_DBS[$target]
|
|
||||||
echo "Putting back $prod_db.backup"
|
|
||||||
sudo systemctl stop $APP_SVC
|
|
||||||
mv -f "$prod_db.backup" "$prod_db"
|
|
||||||
sudo systemctl start $APP_SVC || mail -s "gaia: FTP IPE Sync error" admin@aquilenet.fr <<< "Le service $APP_SVC ne redémarre plus !"
|
|
||||||
}
|
|
||||||
|
|
||||||
deploy_swap() {
|
|
||||||
prod_db=$PROD_DBS[$target]
|
|
||||||
tmp_db=$TMP_DBS[$target]
|
|
||||||
echo "Backup DB $prod_db to $prod_db.backup"
|
|
||||||
if [[ -f $prod_db.backup ]]; then
|
|
||||||
echo "Backup exists, restore it !"
|
|
||||||
rollback
|
|
||||||
return
|
|
||||||
fi
|
|
||||||
echo "Backup prod DB $prod_db"
|
|
||||||
cp "$prod_db" "$prod_db.backup"
|
|
||||||
|
|
||||||
echo "Swap $tmp_db to $prod_db"
|
|
||||||
mv "$tmp_db" "$prod_db"
|
|
||||||
|
|
||||||
echo "Restart service and check OK"
|
|
||||||
if sudo systemctl restart $APP_SVC; then
|
|
||||||
echo "App $APP_SVC restarted OK !"
|
|
||||||
if $keep_backup; then
|
|
||||||
echo "Save $prod_db.backup TO $prod_db.old"
|
|
||||||
mv "$prod_db.backup" "$prod_db.old"
|
|
||||||
else
|
|
||||||
echo "Delete $prod_db.backup"
|
|
||||||
rm -f "$prod_db.backup"
|
|
||||||
fi
|
|
||||||
${target}_clean
|
|
||||||
else
|
|
||||||
# Error, put back old DB
|
|
||||||
echo "Error, Rollback !"
|
|
||||||
rollback
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
################################### MAIN
|
|
||||||
|
|
||||||
# First fetch IPE raw files
|
|
||||||
echo "########## FETCH LATEST IPEs FOR $target"
|
|
||||||
${target}_fetch_latest
|
|
||||||
|
|
||||||
# Then ingest into a database
|
|
||||||
echo "########## INGEST LATEST IPEs FOR $target"
|
|
||||||
${target}_ingest
|
|
||||||
|
|
||||||
# Finally Deploy new database to prod
|
|
||||||
echo "########## DEPLOY NEW DB FOR $target"
|
|
||||||
deploy_swap
|
|
Loading…
Reference in a new issue