Add script to fetch latest arcep data

This commit is contained in:
Johan Le Baut 2022-04-11 09:49:51 +02:00
parent b9df2de12a
commit 4a6f063c1f
5 changed files with 243 additions and 9 deletions

155
data-ingest/fetch_latest_arcep.sh Executable file
View file

@ -0,0 +1,155 @@
#!/usr/bin/env bash
set -eau -o pipefail
# API where to list arcep files
GOUV_API_URL=https://www.data.gouv.fr/api/1/datasets/le-marche-du-haut-et-tres-haut-debit-fixe-deploiements/
# File to store last versions downloaded
VERSIONS_FILENAME=.arcep_versions
## Content of version file:
# LAST_ARCEP_ZIP=<file date>__<file name>.zip
# BEFORE_ARCEP_ZIP=<file date>__<file name>.zip
# Global vars
g_last_arcep_zip=""
g_before_arcep_zip=""
g_penultimate_arcep_zip=""
g_arcep_to_unzip=""
# Script usage
usage() {
source
echo
echo "Usage : $0 -d|--dir-out <dir to put downloads in> (-r|--remove-penultimate)"
echo
echo " With:"
echo " -d|--dir-out: folder where to store zip files"
echo " (-r|--remove-penultimate): if set, remove 2nd before last version after dl latest file"
echo
}
# Get already dl data info
source_versions() {
dir_out=$1
ver_file=${dir_out}/${VERSIONS_FILENAME}
LAST_ARCEP_ZIP=""
BEFORE_ARCEP_ZIP=""
[[ -f ${ver_file} ]] && source ${ver_file}
g_last_arcep_zip=${LAST_ARCEP_ZIP}
g_before_arcep_zip=${BEFORE_ARCEP_ZIP}
}
# Dl arcep latest data if needed
dl_latest_arcep() {
dir_out=$1
rc=0
echo "Create out dir ${dir_out} if not exist"
mkdir -p ${dir_out}
ver_file=${dir_out}/${VERSIONS_FILENAME}
touch ${ver_file}
latest_file_url="$(curl -s ${GOUV_API_URL} | jq -r '.resources[] |objects | .url' | grep -i immeubles | head -1)"
file_date=$(echo $latest_file_url | cut -f6 -d '/')
file_name=$(echo $latest_file_url | cut -f7 -d '/')
latest_f=${file_date}__${file_name}
echo "Found ${latest_f} Check if already exist"
if [[ -n ${g_last_arcep_zip} && "${latest_f}" = "${g_last_arcep_zip}" ]]; then
echo "File ${latest_f} is already the latest ! Do not do anything"
else
echo "File ${latest_f} not there, download it"
wget -O ${dir_out}/${latest_f} ${latest_file_url} || rc=1
g_penultimate_arcep_zip=${g_before_arcep_zip}
g_before_arcep_zip=${g_last_arcep_zip}
g_last_arcep_zip=${latest_f}
g_arcep_to_unzip=${latest_f}
echo "OK, update versions file"
echo "LAST_ARCEP_ZIP=${g_last_arcep_zip}" > ${ver_file}
echo "BEFORE_ARCEP_ZIP=${g_before_arcep_zip}" >> ${ver_file}
fi
return ${rc}
}
# Unzip a dl arcep file
unzip_arcep() {
dir_out=$1
zip_file=$2
zip_dir=$(echo ${zip_file} | rev | cut -f2- -d '.' | rev)
mkdir -p "${dir_out}/$zip_dir"
echo "Unzip file ${dir_out}/${zip_file}"
unzip ${dir_out}/${zip_file} -d ${dir_out}/$zip_dir || return 1
return 0
}
# main
main () {
# Init input vars
remove_penultimate=false
dir_out=""
# Read inputs
[[ $# -eq 0 ]] && usage && return 1
while [ -n $1 ] ; do
case $1 in
-d|--dir-out)
dir_out=$(realpath $2)
shift
;;
-r|--remove-penultimate)
remove_penultimate=true
;;
-h|--help)
usage && exit 0
;;
*)
echo "Unknown command: $1"
usage && exit 1
;;
esac
[[ $# -le 1 ]] && break
shift
done
# check inputs
if [[ -z ${dir_out} ]]; then
echo "Error: You need to specify an output dir -d|--dir-out <dir path>"
usage
return 1
fi
rc=0
# Read existing dl versions
source_versions ${dir_out} || rc=1
# Download latest zip file if needed
[[ $rc -eq 0 ]] && dl_latest_arcep ${dir_out} || rc=1
# If download succeeded and there is a file to unzip
if [[ $rc -eq 0 && -n $g_arcep_to_unzip ]]; then
# unzip file
unzip_arcep ${dir_out} ${g_last_arcep_zip} || rc=1
# Unzip succeeded and need to remove penultimate arcep data (if exists)
if [[ $rc -eq 0 \
&& $remove_penultimate \
&& -n $g_penultimate_arcep_zip \
&& -f ${dir_out}/$g_penultimate_arcep_zip ]]; then
echo "Delete penultimate zip ${dir_out}/$g_penultimate_arcep_zip"
rm -f ${dir_out}/$g_penultimate_arcep_zip
zip_dir=$(echo ${g_penultimate_arcep_zip} | rev | cut -f2- -d '.' | rev)
if [[ -d ${dir_out}/${zip_dir} ]]; then
echo "remove dir ${dir_out}/${zip_dir}"
rm -rf ${dir_out}/${zip_dir}
fi
elif [[ $rc -ne 0 ]]; then
echo "Failed to unzip ${g_last_arcep_zip} !"
fi
fi
return $rc
}
### Call main
main "$@" || exit 1
exit 0

57
data-ingest/ingest_arcep Executable file
View file

@ -0,0 +1,57 @@
#!/usr/bin/env bash
set -eau -o pipefail
ARCEP_WWW="https://www.data.gouv.fr/fr/datasets/le-marche-du-haut-et-tres-haut-debit-fixe-deploiements/"
if [ "$#" -ne 2 ]; then
echo "Usage: ingest path-to-arcep-ipe-csv-file path-to-generated-db"
echo "ARCEP file can be downloaded here: ${ARCEP_WWW}"
echo ""
exit 1
fi
fullIpePath=$(realpath "${1}")
fullDbPath=$(realpath "${2}")
tmpSql=$(mktemp)
clean_tmp () {
rm "${tmpSql}"
}
trap clean_tmp EXIT
echo "[+] Ingesting IPE data from ${fullIpePath}"
echo ""
cat > "${tmpSql}" <<EOF
.separator ","
.import ${fullIpePath} arcep
EOF
sqlite3 "${fullDbPath}" < "${tmpSql}"
echo "[+] Ingesting spatial data."
cat > "${tmpSql}" <<EOF
SELECT load_extension("mod_spatialite");
SELECT InitSpatialMetaData();
-- Despite the SELECT, we're actually creating a new ImmeubleGeoPoint
-- column here. The spatialite API is a bit weird...
SELECT AddGeometryColumn('arcep','ImmeubleGeoPoint',4326,'POINT');
-- The geodata is expressed in RGF93 (SRID 2154). We need to project
-- it to WSG84 to display it on a OSM map.
UPDATE arcep SET ImmeubleGeoPoint =
MakePoint(
CAST(x as DOUBLE),
CAST(y as DOUBLE),
4326);
EOF
sqlite3 "${fullDbPath}" < "${tmpSql}"
echo "[+] Creating Rtree index (spatial index). We're almost done."
cat > "${tmpSql}" <<EOF
SELECT load_extension("mod_spatialite");
SELECT CreateSpatialIndex('arcep','ImmeubleGeoPoint');
EOF
sqlite3 "${fullDbPath}" < "${tmpSql}"
echo "[+] SQLite database generated at ${fullDbPath}"

View file

@ -38,20 +38,38 @@ class Axione:
) )
req_area = cur.fetchone()[0] req_area = cur.fetchone()[0]
if req_area <= 0.08: if req_area <= 0.08:
# cur.execute(
# """
# SELECT
# X(ImmeubleGeoPoint),
# Y(ImmeubleGeoPoint),
# IdentifiantImmeuble,
# EtatImmeuble,
# NumeroVoieImmeuble,
# TypeVoieImmeuble,
# NomVoieImmeuble
# FROM ipe
# WHERE ROWID IN (
# SELECT ROWID FROM SpatialIndex
# WHERE f_table_name = 'ipe' AND
# search_frame = BuildMBR(:swx, :swy, :nex, :ney, 4326))
# """,
# areaCoordinates,
# )
cur.execute( cur.execute(
""" """
SELECT SELECT
X(ImmeubleGeoPoint), X(ImmeubleGeoPoint),
Y(ImmeubleGeoPoint), Y(ImmeubleGeoPoint),
IdentifiantImmeuble, imb_id,
EtatImmeuble, imb_etat,
NumeroVoieImmeuble, num_voie,
TypeVoieImmeuble, type_voie,
NomVoieImmeuble nom_voie
FROM ipe FROM arcep
WHERE ROWID IN ( WHERE ROWID IN (
SELECT ROWID FROM SpatialIndex SELECT ROWID FROM SpatialIndex
WHERE f_table_name = 'ipe' AND WHERE f_table_name = 'arcep' AND
search_frame = BuildMBR(:swx, :swy, :nex, :ney, 4326)) search_frame = BuildMBR(:swx, :swy, :nex, :ney, 4326))
""", """,
areaCoordinates, areaCoordinates,
@ -60,9 +78,10 @@ class Axione:
existing_buildings = dict() existing_buildings = dict()
buildings = existing_buildings buildings = existing_buildings
for b in cur.fetchall(): for b in cur.fetchall():
print(b)
etatImm = b[3] etatImm = b[3]
idImm = b[2] idImm = b[2]
isEligible = etatImm == AXIONE_ETAT_DEPLOYE isEligible = etatImm == AXIONE_ETAT_DEPLOYE or etatImm == "deploye"
aquilenetEligStatus = FAIEligibilityStatus( aquilenetEligStatus = FAIEligibilityStatus(
isEligible=isEligible, isEligible=isEligible,
ftthStatus=etatImm, ftthStatus=etatImm,

View file

@ -48,7 +48,7 @@ def getEligData():
except ValueError as err: except ValueError as err:
print("Could not get Axione data for this area:", err) print("Could not get Axione data for this area:", err)
buildings = liazo.getAreaBuildings(processed_args["centerlat"], processed_args["centerlng"], buildings) # buildings = liazo.getAreaBuildings(processed_args["centerlat"], processed_args["centerlng"], buildings)
return {"buildings": buildings} return {"buildings": buildings}
else: else:

View file

@ -70,6 +70,9 @@ function updateEligData(map, eligData) {
let buildings = eligData.buildings; let buildings = eligData.buildings;
console.log(buildings) console.log(buildings)
markers = Object.values(buildings).map(building => { markers = Object.values(buildings).map(building => {
console.log(building.numVoieImm)
console.log("lat", building.y)
console.log("lng", building.x)
const latlng = new L.latLng(building.y, building.x); const latlng = new L.latLng(building.y, building.x);
const addrImm = `${building.numVoieImm} ${building.typeVoieImm} ${building.nomVoieImm}` const addrImm = `${building.numVoieImm} ${building.typeVoieImm} ${building.nomVoieImm}`
const marker = new L.marker(latlng) const marker = new L.marker(latlng)