diff --git a/data-ingest/fetch_latest_arcep.sh b/data-ingest/fetch_latest_arcep.sh index b963add..266efa6 100755 --- a/data-ingest/fetch_latest_arcep.sh +++ b/data-ingest/fetch_latest_arcep.sh @@ -33,7 +33,8 @@ source_versions() { ver_file=${dir_out}/${VERSIONS_FILENAME} LAST_ARCEP_ZIP="" BEFORE_ARCEP_ZIP="" - [[ -f ${ver_file} ]] && source ${ver_file} + # shellcheck source=/dev/null + [[ -f ${ver_file} ]] && source "${ver_file}" g_last_arcep_zip=${LAST_ARCEP_ZIP} g_before_arcep_zip=${BEFORE_ARCEP_ZIP} } @@ -44,13 +45,13 @@ dl_latest_arcep() { rc=0 echo "Create out dir ${dir_out} if not exist" - mkdir -p ${dir_out} + mkdir -p "${dir_out}" ver_file=${dir_out}/${VERSIONS_FILENAME} - touch ${ver_file} + touch "${ver_file}" latest_file_url="$(curl -s ${GOUV_API_URL} | jq -r '.resources[] |objects | .url' | grep -i immeubles | head -1)" - file_date=$(echo $latest_file_url | cut -f6 -d '/') - file_name=$(echo $latest_file_url | cut -f7 -d '/') + file_date=$(echo "$latest_file_url" | cut -f6 -d '/') + file_name=$(echo "$latest_file_url" | cut -f7 -d '/') latest_f=${file_date}__${file_name} echo "Found ${latest_f} Check if already exist" @@ -58,14 +59,14 @@ dl_latest_arcep() { echo "File ${latest_f} is already the latest ! Do not do anything" else echo "File ${latest_f} not there, download it" - wget -O ${dir_out}/${latest_f} ${latest_file_url} || rc=1 + wget -O "${dir_out}"/"${latest_f}" "${latest_file_url}" || rc=1 g_penultimate_arcep_zip=${g_before_arcep_zip} g_before_arcep_zip=${g_last_arcep_zip} g_last_arcep_zip=${latest_f} g_arcep_to_unzip=${latest_f} echo "OK, update versions file" - echo "LAST_ARCEP_ZIP=${g_last_arcep_zip}" > ${ver_file} - echo "BEFORE_ARCEP_ZIP=${g_before_arcep_zip}" >> ${ver_file} + echo "LAST_ARCEP_ZIP=${g_last_arcep_zip}" > "${ver_file}" + echo "BEFORE_ARCEP_ZIP=${g_before_arcep_zip}" >> "${ver_file}" fi return ${rc} @@ -75,10 +76,10 @@ dl_latest_arcep() { unzip_arcep() { dir_out=$1 zip_file=$2 - zip_dir=$(echo ${zip_file} | rev | cut -f2- -d '.' | rev) + zip_dir=$(echo "${zip_file}" | rev | cut -f2- -d '.' | rev) mkdir -p "${dir_out}/$zip_dir" echo "Unzip file ${dir_out}/${zip_file}" - unzip ${dir_out}/${zip_file} -d ${dir_out}/$zip_dir || return 1 + unzip "${dir_out}"/"${zip_file}" -d "${dir_out}"/"$zip_dir" || return 1 return 0 } @@ -90,10 +91,10 @@ main () { # Read inputs [[ $# -eq 0 ]] && usage && return 1 - while [ -n $1 ] ; do + while [[ -n "$1" ]] ; do case $1 in -d|--dir-out) - dir_out=$(realpath $2) + dir_out=$(realpath "$2") shift ;; -r|--remove-penultimate) @@ -120,13 +121,13 @@ main () { rc=0 # Read existing dl versions - source_versions ${dir_out} || rc=1 + source_versions "${dir_out}" || rc=1 # Download latest zip file if needed - [[ $rc -eq 0 ]] && dl_latest_arcep ${dir_out} || rc=1 + [[ $rc -eq 0 ]] && dl_latest_arcep "${dir_out}" || rc=1 # If download succeeded and there is a file to unzip if [[ $rc -eq 0 && -n $g_arcep_to_unzip ]]; then # unzip file - unzip_arcep ${dir_out} ${g_last_arcep_zip} || rc=1 + unzip_arcep "${dir_out}" "${g_last_arcep_zip}" || rc=1 # Unzip succeeded and need to remove penultimate arcep data (if exists) if [[ $rc -eq 0 \ @@ -135,11 +136,11 @@ main () { && -f ${dir_out}/$g_penultimate_arcep_zip ]]; then echo "Delete penultimate zip ${dir_out}/$g_penultimate_arcep_zip" - rm -f ${dir_out}/$g_penultimate_arcep_zip - zip_dir=$(echo ${g_penultimate_arcep_zip} | rev | cut -f2- -d '.' | rev) + rm -f "${dir_out}"/"$g_penultimate_arcep_zip" + zip_dir=$(echo "${g_penultimate_arcep_zip}" | rev | cut -f2- -d '.' | rev) if [[ -d ${dir_out}/${zip_dir} ]]; then echo "remove dir ${dir_out}/${zip_dir}" - rm -rf ${dir_out}/${zip_dir} + rm -rf "${dir_out:?}"/"${zip_dir}" fi elif [[ $rc -ne 0 ]]; then echo "Failed to unzip ${g_last_arcep_zip} !" diff --git a/data-ingest/ingest_arcep b/data-ingest/ingest_arcep index 174706f..d0bf46c 100755 --- a/data-ingest/ingest_arcep +++ b/data-ingest/ingest_arcep @@ -5,7 +5,7 @@ ARCEP_WWW="https://www.data.gouv.fr/fr/datasets/le-marche-du-haut-et-tres-haut-d if [ "$#" -ne 2 ]; then echo "Usage: ingest path-to-arcep-ipe-csv-file path-to-generated-db" - echo "ARCEP file can be downloaded here: ${ARCEP_WWW}" + echo "ARCEP file can be downloaded here: ${ARCEP_WWW} or with script fetch_latest_arcep.sh" echo "" exit 1 fi