Add script to fetch latest arcep data
This commit is contained in:
Johan Le Baut
parent
b9df2de12a
commit
4a6f063c1f
|
|
@ -0,0 +1,155 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eau -o pipefail
|
||||
|
||||
# API where to list arcep files
|
||||
GOUV_API_URL=https://www.data.gouv.fr/api/1/datasets/le-marche-du-haut-et-tres-haut-debit-fixe-deploiements/
|
||||
# File to store last versions downloaded
|
||||
VERSIONS_FILENAME=.arcep_versions
|
||||
## Content of version file:
|
||||
# LAST_ARCEP_ZIP=<file date>__<file name>.zip
|
||||
# BEFORE_ARCEP_ZIP=<file date>__<file name>.zip
|
||||
|
||||
# Global vars
|
||||
g_last_arcep_zip=""
|
||||
g_before_arcep_zip=""
|
||||
g_penultimate_arcep_zip=""
|
||||
g_arcep_to_unzip=""
|
||||
|
||||
# Script usage
|
||||
usage() {
|
||||
source
|
||||
echo
|
||||
echo "Usage : $0 -d|--dir-out <dir to put downloads in> (-r|--remove-penultimate)"
|
||||
echo
|
||||
echo " With:"
|
||||
echo " -d|--dir-out: folder where to store zip files"
|
||||
echo " (-r|--remove-penultimate): if set, remove 2nd before last version after dl latest file"
|
||||
echo
|
||||
}
|
||||
|
||||
# Get already dl data info
|
||||
source_versions() {
|
||||
dir_out=$1
|
||||
ver_file=${dir_out}/${VERSIONS_FILENAME}
|
||||
LAST_ARCEP_ZIP=""
|
||||
BEFORE_ARCEP_ZIP=""
|
||||
[[ -f ${ver_file} ]] && source ${ver_file}
|
||||
g_last_arcep_zip=${LAST_ARCEP_ZIP}
|
||||
g_before_arcep_zip=${BEFORE_ARCEP_ZIP}
|
||||
}
|
||||
|
||||
# Dl arcep latest data if needed
|
||||
dl_latest_arcep() {
|
||||
dir_out=$1
|
||||
rc=0
|
||||
|
||||
echo "Create out dir ${dir_out} if not exist"
|
||||
mkdir -p ${dir_out}
|
||||
ver_file=${dir_out}/${VERSIONS_FILENAME}
|
||||
touch ${ver_file}
|
||||
|
||||
latest_file_url="$(curl -s ${GOUV_API_URL} | jq -r '.resources[] |objects | .url' | grep -i immeubles | head -1)"
|
||||
file_date=$(echo $latest_file_url | cut -f6 -d '/')
|
||||
file_name=$(echo $latest_file_url | cut -f7 -d '/')
|
||||
latest_f=${file_date}__${file_name}
|
||||
|
||||
echo "Found ${latest_f} Check if already exist"
|
||||
if [[ -n ${g_last_arcep_zip} && "${latest_f}" = "${g_last_arcep_zip}" ]]; then
|
||||
echo "File ${latest_f} is already the latest ! Do not do anything"
|
||||
else
|
||||
echo "File ${latest_f} not there, download it"
|
||||
wget -O ${dir_out}/${latest_f} ${latest_file_url} || rc=1
|
||||
g_penultimate_arcep_zip=${g_before_arcep_zip}
|
||||
g_before_arcep_zip=${g_last_arcep_zip}
|
||||
g_last_arcep_zip=${latest_f}
|
||||
g_arcep_to_unzip=${latest_f}
|
||||
echo "OK, update versions file"
|
||||
echo "LAST_ARCEP_ZIP=${g_last_arcep_zip}" > ${ver_file}
|
||||
echo "BEFORE_ARCEP_ZIP=${g_before_arcep_zip}" >> ${ver_file}
|
||||
fi
|
||||
|
||||
return ${rc}
|
||||
}
|
||||
|
||||
# Unzip a dl arcep file
|
||||
unzip_arcep() {
|
||||
dir_out=$1
|
||||
zip_file=$2
|
||||
zip_dir=$(echo ${zip_file} | rev | cut -f2- -d '.' | rev)
|
||||
mkdir -p "${dir_out}/$zip_dir"
|
||||
echo "Unzip file ${dir_out}/${zip_file}"
|
||||
unzip ${dir_out}/${zip_file} -d ${dir_out}/$zip_dir || return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
# main
|
||||
main () {
|
||||
# Init input vars
|
||||
remove_penultimate=false
|
||||
dir_out=""
|
||||
|
||||
# Read inputs
|
||||
[[ $# -eq 0 ]] && usage && return 1
|
||||
while [ -n $1 ] ; do
|
||||
case $1 in
|
||||
-d|--dir-out)
|
||||
dir_out=$(realpath $2)
|
||||
shift
|
||||
;;
|
||||
-r|--remove-penultimate)
|
||||
remove_penultimate=true
|
||||
;;
|
||||
-h|--help)
|
||||
usage && exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown command: $1"
|
||||
usage && exit 1
|
||||
;;
|
||||
esac
|
||||
[[ $# -le 1 ]] && break
|
||||
shift
|
||||
done
|
||||
|
||||
# check inputs
|
||||
if [[ -z ${dir_out} ]]; then
|
||||
echo "Error: You need to specify an output dir -d|--dir-out <dir path>"
|
||||
usage
|
||||
return 1
|
||||
fi
|
||||
|
||||
rc=0
|
||||
# Read existing dl versions
|
||||
source_versions ${dir_out} || rc=1
|
||||
# Download latest zip file if needed
|
||||
[[ $rc -eq 0 ]] && dl_latest_arcep ${dir_out} || rc=1
|
||||
# If download succeeded and there is a file to unzip
|
||||
if [[ $rc -eq 0 && -n $g_arcep_to_unzip ]]; then
|
||||
# unzip file
|
||||
unzip_arcep ${dir_out} ${g_last_arcep_zip} || rc=1
|
||||
|
||||
# Unzip succeeded and need to remove penultimate arcep data (if exists)
|
||||
if [[ $rc -eq 0 \
|
||||
&& $remove_penultimate \
|
||||
&& -n $g_penultimate_arcep_zip \
|
||||
&& -f ${dir_out}/$g_penultimate_arcep_zip ]]; then
|
||||
|
||||
echo "Delete penultimate zip ${dir_out}/$g_penultimate_arcep_zip"
|
||||
rm -f ${dir_out}/$g_penultimate_arcep_zip
|
||||
zip_dir=$(echo ${g_penultimate_arcep_zip} | rev | cut -f2- -d '.' | rev)
|
||||
if [[ -d ${dir_out}/${zip_dir} ]]; then
|
||||
echo "remove dir ${dir_out}/${zip_dir}"
|
||||
rm -rf ${dir_out}/${zip_dir}
|
||||
fi
|
||||
elif [[ $rc -ne 0 ]]; then
|
||||
echo "Failed to unzip ${g_last_arcep_zip} !"
|
||||
fi
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
|
||||
### Call main
|
||||
main "$@" || exit 1
|
||||
|
||||
exit 0
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eau -o pipefail
|
||||
|
||||
ARCEP_WWW="https://www.data.gouv.fr/fr/datasets/le-marche-du-haut-et-tres-haut-debit-fixe-deploiements/"
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "Usage: ingest path-to-arcep-ipe-csv-file path-to-generated-db"
|
||||
echo "ARCEP file can be downloaded here: ${ARCEP_WWW}"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
fullIpePath=$(realpath "${1}")
|
||||
fullDbPath=$(realpath "${2}")
|
||||
|
||||
tmpSql=$(mktemp)
|
||||
clean_tmp () {
|
||||
rm "${tmpSql}"
|
||||
}
|
||||
trap clean_tmp EXIT
|
||||
|
||||
echo "[+] Ingesting IPE data from ${fullIpePath}"
|
||||
echo ""
|
||||
cat > "${tmpSql}" <<EOF
|
||||
.separator ","
|
||||
.import ${fullIpePath} arcep
|
||||
EOF
|
||||
|
||||
sqlite3 "${fullDbPath}" < "${tmpSql}"
|
||||
|
||||
echo "[+] Ingesting spatial data."
|
||||
|
||||
cat > "${tmpSql}" <<EOF
|
||||
SELECT load_extension("mod_spatialite");
|
||||
SELECT InitSpatialMetaData();
|
||||
|
||||
-- Despite the SELECT, we're actually creating a new ImmeubleGeoPoint
|
||||
-- column here. The spatialite API is a bit weird...
|
||||
SELECT AddGeometryColumn('arcep','ImmeubleGeoPoint',4326,'POINT');
|
||||
|
||||
-- The geodata is expressed in RGF93 (SRID 2154). We need to project
|
||||
-- it to WSG84 to display it on a OSM map.
|
||||
UPDATE arcep SET ImmeubleGeoPoint =
|
||||
MakePoint(
|
||||
CAST(x as DOUBLE),
|
||||
CAST(y as DOUBLE),
|
||||
4326);
|
||||
EOF
|
||||
sqlite3 "${fullDbPath}" < "${tmpSql}"
|
||||
|
||||
echo "[+] Creating Rtree index (spatial index). We're almost done."
|
||||
cat > "${tmpSql}" <<EOF
|
||||
SELECT load_extension("mod_spatialite");
|
||||
SELECT CreateSpatialIndex('arcep','ImmeubleGeoPoint');
|
||||
EOF
|
||||
sqlite3 "${fullDbPath}" < "${tmpSql}"
|
||||
|
||||
echo "[+] SQLite database generated at ${fullDbPath}"
|
||||
|
|
@ -38,20 +38,38 @@ class Axione:
|
|||
)
|
||||
req_area = cur.fetchone()[0]
|
||||
if req_area <= 0.08:
|
||||
# cur.execute(
|
||||
# """
|
||||
# SELECT
|
||||
# X(ImmeubleGeoPoint),
|
||||
# Y(ImmeubleGeoPoint),
|
||||
# IdentifiantImmeuble,
|
||||
# EtatImmeuble,
|
||||
# NumeroVoieImmeuble,
|
||||
# TypeVoieImmeuble,
|
||||
# NomVoieImmeuble
|
||||
# FROM ipe
|
||||
# WHERE ROWID IN (
|
||||
# SELECT ROWID FROM SpatialIndex
|
||||
# WHERE f_table_name = 'ipe' AND
|
||||
# search_frame = BuildMBR(:swx, :swy, :nex, :ney, 4326))
|
||||
# """,
|
||||
# areaCoordinates,
|
||||
# )
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
X(ImmeubleGeoPoint),
|
||||
Y(ImmeubleGeoPoint),
|
||||
IdentifiantImmeuble,
|
||||
EtatImmeuble,
|
||||
NumeroVoieImmeuble,
|
||||
TypeVoieImmeuble,
|
||||
NomVoieImmeuble
|
||||
FROM ipe
|
||||
imb_id,
|
||||
imb_etat,
|
||||
num_voie,
|
||||
type_voie,
|
||||
nom_voie
|
||||
FROM arcep
|
||||
WHERE ROWID IN (
|
||||
SELECT ROWID FROM SpatialIndex
|
||||
WHERE f_table_name = 'ipe' AND
|
||||
WHERE f_table_name = 'arcep' AND
|
||||
search_frame = BuildMBR(:swx, :swy, :nex, :ney, 4326))
|
||||
""",
|
||||
areaCoordinates,
|
||||
|
|
@ -60,9 +78,10 @@ class Axione:
|
|||
existing_buildings = dict()
|
||||
buildings = existing_buildings
|
||||
for b in cur.fetchall():
|
||||
print(b)
|
||||
etatImm = b[3]
|
||||
idImm = b[2]
|
||||
isEligible = etatImm == AXIONE_ETAT_DEPLOYE
|
||||
isEligible = etatImm == AXIONE_ETAT_DEPLOYE or etatImm == "deploye"
|
||||
aquilenetEligStatus = FAIEligibilityStatus(
|
||||
isEligible=isEligible,
|
||||
ftthStatus=etatImm,
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def getEligData():
|
|||
except ValueError as err:
|
||||
print("Could not get Axione data for this area:", err)
|
||||
|
||||
buildings = liazo.getAreaBuildings(processed_args["centerlat"], processed_args["centerlng"], buildings)
|
||||
# buildings = liazo.getAreaBuildings(processed_args["centerlat"], processed_args["centerlng"], buildings)
|
||||
|
||||
return {"buildings": buildings}
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -70,6 +70,9 @@ function updateEligData(map, eligData) {
|
|||
let buildings = eligData.buildings;
|
||||
console.log(buildings)
|
||||
markers = Object.values(buildings).map(building => {
|
||||
console.log(building.numVoieImm)
|
||||
console.log("lat", building.y)
|
||||
console.log("lng", building.x)
|
||||
const latlng = new L.latLng(building.y, building.x);
|
||||
const addrImm = `${building.numVoieImm} ${building.typeVoieImm} ${building.nomVoieImm}`
|
||||
const marker = new L.marker(latlng)
|
||||
|
|
|
|||
Loading…
Reference in New Issue