Browse Source

shellcheck

johan/consolidate-ipe-data
Johan Le Baut 6 months ago
parent
commit
70ed8a22ae
  1. 37
      data-ingest/fetch_latest_arcep.sh
  2. 2
      data-ingest/ingest_arcep

37
data-ingest/fetch_latest_arcep.sh

@ -33,7 +33,8 @@ source_versions() { @@ -33,7 +33,8 @@ source_versions() {
ver_file=${dir_out}/${VERSIONS_FILENAME}
LAST_ARCEP_ZIP=""
BEFORE_ARCEP_ZIP=""
[[ -f ${ver_file} ]] && source ${ver_file}
# shellcheck source=/dev/null
[[ -f ${ver_file} ]] && source "${ver_file}"
g_last_arcep_zip=${LAST_ARCEP_ZIP}
g_before_arcep_zip=${BEFORE_ARCEP_ZIP}
}
@ -44,13 +45,13 @@ dl_latest_arcep() { @@ -44,13 +45,13 @@ dl_latest_arcep() {
rc=0
echo "Create out dir ${dir_out} if not exist"
mkdir -p ${dir_out}
mkdir -p "${dir_out}"
ver_file=${dir_out}/${VERSIONS_FILENAME}
touch ${ver_file}
touch "${ver_file}"
latest_file_url="$(curl -s ${GOUV_API_URL} | jq -r '.resources[] |objects | .url' | grep -i immeubles | head -1)"
file_date=$(echo $latest_file_url | cut -f6 -d '/')
file_name=$(echo $latest_file_url | cut -f7 -d '/')
file_date=$(echo "$latest_file_url" | cut -f6 -d '/')
file_name=$(echo "$latest_file_url" | cut -f7 -d '/')
latest_f=${file_date}__${file_name}
echo "Found ${latest_f} Check if already exist"
@ -58,14 +59,14 @@ dl_latest_arcep() { @@ -58,14 +59,14 @@ dl_latest_arcep() {
echo "File ${latest_f} is already the latest ! Do not do anything"
else
echo "File ${latest_f} not there, download it"
wget -O ${dir_out}/${latest_f} ${latest_file_url} || rc=1
wget -O "${dir_out}"/"${latest_f}" "${latest_file_url}" || rc=1
g_penultimate_arcep_zip=${g_before_arcep_zip}
g_before_arcep_zip=${g_last_arcep_zip}
g_last_arcep_zip=${latest_f}
g_arcep_to_unzip=${latest_f}
echo "OK, update versions file"
echo "LAST_ARCEP_ZIP=${g_last_arcep_zip}" > ${ver_file}
echo "BEFORE_ARCEP_ZIP=${g_before_arcep_zip}" >> ${ver_file}
echo "LAST_ARCEP_ZIP=${g_last_arcep_zip}" > "${ver_file}"
echo "BEFORE_ARCEP_ZIP=${g_before_arcep_zip}" >> "${ver_file}"
fi
return ${rc}
@ -75,10 +76,10 @@ dl_latest_arcep() { @@ -75,10 +76,10 @@ dl_latest_arcep() {
unzip_arcep() {
dir_out=$1
zip_file=$2
zip_dir=$(echo ${zip_file} | rev | cut -f2- -d '.' | rev)
zip_dir=$(echo "${zip_file}" | rev | cut -f2- -d '.' | rev)
mkdir -p "${dir_out}/$zip_dir"
echo "Unzip file ${dir_out}/${zip_file}"
unzip ${dir_out}/${zip_file} -d ${dir_out}/$zip_dir || return 1
unzip "${dir_out}"/"${zip_file}" -d "${dir_out}"/"$zip_dir" || return 1
return 0
}
@ -90,10 +91,10 @@ main () { @@ -90,10 +91,10 @@ main () {
# Read inputs
[[ $# -eq 0 ]] && usage && return 1
while [ -n $1 ] ; do
while [[ -n "$1" ]] ; do
case $1 in
-d|--dir-out)
dir_out=$(realpath $2)
dir_out=$(realpath "$2")
shift
;;
-r|--remove-penultimate)
@ -120,13 +121,13 @@ main () { @@ -120,13 +121,13 @@ main () {
rc=0
# Read existing dl versions
source_versions ${dir_out} || rc=1
source_versions "${dir_out}" || rc=1
# Download latest zip file if needed
[[ $rc -eq 0 ]] && dl_latest_arcep ${dir_out} || rc=1
[[ $rc -eq 0 ]] && dl_latest_arcep "${dir_out}" || rc=1
# If download succeeded and there is a file to unzip
if [[ $rc -eq 0 && -n $g_arcep_to_unzip ]]; then
# unzip file
unzip_arcep ${dir_out} ${g_last_arcep_zip} || rc=1
unzip_arcep "${dir_out}" "${g_last_arcep_zip}" || rc=1
# Unzip succeeded and need to remove penultimate arcep data (if exists)
if [[ $rc -eq 0 \
@ -135,11 +136,11 @@ main () { @@ -135,11 +136,11 @@ main () {
&& -f ${dir_out}/$g_penultimate_arcep_zip ]]; then
echo "Delete penultimate zip ${dir_out}/$g_penultimate_arcep_zip"
rm -f ${dir_out}/$g_penultimate_arcep_zip
zip_dir=$(echo ${g_penultimate_arcep_zip} | rev | cut -f2- -d '.' | rev)
rm -f "${dir_out}"/"$g_penultimate_arcep_zip"
zip_dir=$(echo "${g_penultimate_arcep_zip}" | rev | cut -f2- -d '.' | rev)
if [[ -d ${dir_out}/${zip_dir} ]]; then
echo "remove dir ${dir_out}/${zip_dir}"
rm -rf ${dir_out}/${zip_dir}
rm -rf "${dir_out:?}"/"${zip_dir}"
fi
elif [[ $rc -ne 0 ]]; then
echo "Failed to unzip ${g_last_arcep_zip} !"

2
data-ingest/ingest_arcep

@ -5,7 +5,7 @@ ARCEP_WWW="https://www.data.gouv.fr/fr/datasets/le-marche-du-haut-et-tres-haut-d @@ -5,7 +5,7 @@ ARCEP_WWW="https://www.data.gouv.fr/fr/datasets/le-marche-du-haut-et-tres-haut-d
if [ "$#" -ne 2 ]; then
echo "Usage: ingest path-to-arcep-ipe-csv-file path-to-generated-db"
echo "ARCEP file can be downloaded here: ${ARCEP_WWW}"
echo "ARCEP file can be downloaded here: ${ARCEP_WWW} or with script fetch_latest_arcep.sh"
echo ""
exit 1
fi

Loading…
Cancel
Save