Add script to fetch latest arcep data

data-ingest/fetch_latest_arcep.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+set -eau -o pipefail
+
+# API where to list arcep files
+GOUV_API_URL=https://www.data.gouv.fr/api/1/datasets/le-marche-du-haut-et-tres-haut-debit-fixe-deploiements/
+# File to store last versions downloaded
+VERSIONS_FILENAME=.arcep_versions
+## Content of version file: 
+# LAST_ARCEP_ZIP=<file date>__<file name>.zip
+# BEFORE_ARCEP_ZIP=<file date>__<file name>.zip
+
+# Global vars
+g_last_arcep_zip=""
+g_before_arcep_zip=""
+g_penultimate_arcep_zip=""
+g_arcep_to_unzip=""
+
+# Script usage
+usage() {
+  source 
+  echo
+  echo "Usage : $0 -d|--dir-out <dir to put downloads in> (-r|--remove-penultimate)"
+  echo
+  echo "  With:"
+  echo "    -d|--dir-out: folder where to store zip files"
+  echo "    (-r|--remove-penultimate): if set, remove 2nd before last version after dl latest file"
+  echo
+}
+
+# Get already dl data info
+source_versions() {
+  dir_out=$1
+  ver_file=${dir_out}/${VERSIONS_FILENAME}
+  LAST_ARCEP_ZIP=""
+  BEFORE_ARCEP_ZIP=""
+  [[ -f ${ver_file} ]] && source ${ver_file}
+  g_last_arcep_zip=${LAST_ARCEP_ZIP}
+  g_before_arcep_zip=${BEFORE_ARCEP_ZIP}
+}
+
+# Dl arcep latest data if needed
+dl_latest_arcep() {
+  dir_out=$1
+  rc=0
+
+  echo "Create out dir ${dir_out} if not exist"
+  mkdir -p ${dir_out}
+  ver_file=${dir_out}/${VERSIONS_FILENAME}
+  touch ${ver_file}
+
+  latest_file_url="$(curl -s ${GOUV_API_URL} | jq -r '.resources[] |objects | .url'  | grep -i immeubles | head -1)"
+  file_date=$(echo $latest_file_url | cut -f6 -d '/')
+  file_name=$(echo $latest_file_url | cut -f7 -d '/')
+  latest_f=${file_date}__${file_name}
+
+  echo "Found ${latest_f} Check if already exist"
+  if [[ -n ${g_last_arcep_zip} && "${latest_f}" = "${g_last_arcep_zip}" ]]; then
+    echo "File ${latest_f} is already the latest ! Do not do anything"
+  else
+    echo "File ${latest_f} not there, download it"
+    wget -O ${dir_out}/${latest_f} ${latest_file_url} || rc=1
+    g_penultimate_arcep_zip=${g_before_arcep_zip}
+    g_before_arcep_zip=${g_last_arcep_zip}
+    g_last_arcep_zip=${latest_f}
+    g_arcep_to_unzip=${latest_f}
+    echo "OK, update versions file"
+    echo "LAST_ARCEP_ZIP=${g_last_arcep_zip}" > ${ver_file}
+    echo "BEFORE_ARCEP_ZIP=${g_before_arcep_zip}" >> ${ver_file}
+  fi
+
+  return ${rc}
+}
+
+# Unzip a dl arcep file
+unzip_arcep() {
+  dir_out=$1
+  zip_file=$2
+  zip_dir=$(echo ${zip_file} | rev | cut -f2- -d '.' | rev)
+  mkdir -p "${dir_out}/$zip_dir"
+  echo "Unzip file ${dir_out}/${zip_file}"
+  unzip ${dir_out}/${zip_file} -d ${dir_out}/$zip_dir || return 1
+  return 0
+}
+
+# main
+main () {
+  # Init input vars
+  remove_penultimate=false
+  dir_out=""
+
+  # Read inputs
+  [[ $# -eq 0 ]] && usage && return 1
+  while [ -n $1 ] ; do
+    case $1 in
+      -d|--dir-out)
+        dir_out=$(realpath $2)
+        shift
+        ;;
+      -r|--remove-penultimate)
+        remove_penultimate=true
+        ;;
+      -h|--help)
+        usage && exit 0
+        ;;
+      *)
+        echo "Unknown command: $1"
+        usage && exit 1
+        ;;
+    esac
+    [[ $# -le 1 ]] && break
+    shift
+  done
+
+  # check inputs
+  if [[ -z ${dir_out} ]]; then
+    echo "Error: You need to specify an output dir -d|--dir-out <dir path>"
+    usage
+    return 1
+  fi
+
+  rc=0
+  # Read existing dl versions
+  source_versions ${dir_out} || rc=1
+  # Download latest zip file if needed
+  [[ $rc -eq 0 ]] && dl_latest_arcep ${dir_out} || rc=1
+  # If download succeeded and there is a file to unzip
+  if [[ $rc -eq 0 && -n $g_arcep_to_unzip ]]; then
+    # unzip file
+    unzip_arcep ${dir_out} ${g_last_arcep_zip} || rc=1
+    
+    # Unzip succeeded and need to remove penultimate arcep data (if exists)
+    if [[ $rc -eq 0 \
+        && $remove_penultimate \
+        && -n $g_penultimate_arcep_zip  \
+        && -f ${dir_out}/$g_penultimate_arcep_zip ]]; then
+
+      echo "Delete penultimate zip ${dir_out}/$g_penultimate_arcep_zip"
+      rm -f ${dir_out}/$g_penultimate_arcep_zip
+      zip_dir=$(echo ${g_penultimate_arcep_zip} | rev | cut -f2- -d '.' | rev)
+      if [[ -d ${dir_out}/${zip_dir} ]]; then
+        echo "remove dir ${dir_out}/${zip_dir}"
+        rm -rf ${dir_out}/${zip_dir}
+      fi
+    elif [[ $rc -ne 0 ]]; then
+      echo "Failed to unzip ${g_last_arcep_zip} !"
+    fi
+  fi
+  return $rc
+}
+
+
+### Call main
+main "$@" || exit 1
+
+exit 0

data-ingest/ingest_arcep

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -eau -o pipefail
+
+ARCEP_WWW="https://www.data.gouv.fr/fr/datasets/le-marche-du-haut-et-tres-haut-debit-fixe-deploiements/"
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: ingest path-to-arcep-ipe-csv-file path-to-generated-db"
+    echo "ARCEP file can be downloaded here: ${ARCEP_WWW}"
+    echo ""
+    exit 1
+fi
+fullIpePath=$(realpath "${1}")
+fullDbPath=$(realpath "${2}")
+
+tmpSql=$(mktemp)
+clean_tmp () {
+    rm "${tmpSql}"
+}
+trap clean_tmp EXIT
+
+echo "[+] Ingesting IPE data from ${fullIpePath}"
+echo ""
+cat > "${tmpSql}" <<EOF
+.separator ","
+.import ${fullIpePath} arcep
+EOF
+
+sqlite3 "${fullDbPath}" < "${tmpSql}"
+
+echo "[+] Ingesting spatial data."
+
+cat > "${tmpSql}" <<EOF
+SELECT load_extension("mod_spatialite");
+SELECT InitSpatialMetaData();
+
+-- Despite the SELECT, we're actually creating a new ImmeubleGeoPoint
+-- column here. The spatialite API is a bit weird...
+SELECT AddGeometryColumn('arcep','ImmeubleGeoPoint',4326,'POINT');
+
+-- The geodata is expressed in RGF93 (SRID 2154). We need to project
+-- it to WSG84 to display it on a OSM map.
+UPDATE arcep SET ImmeubleGeoPoint =
+    MakePoint(
+        CAST(x as DOUBLE),
+        CAST(y as DOUBLE),
+        4326);
+EOF
+sqlite3 "${fullDbPath}" < "${tmpSql}"
+
+echo "[+] Creating Rtree index (spatial index). We're almost done."
+cat > "${tmpSql}" <<EOF
+SELECT load_extension("mod_spatialite");
+SELECT CreateSpatialIndex('arcep','ImmeubleGeoPoint');
+EOF
+sqlite3 "${fullDbPath}" < "${tmpSql}"
+
+echo "[+] SQLite database generated at ${fullDbPath}"

webapp/ipe_fetcher/axione.py

@@ -38,20 +38,38 @@ class Axione:
         )
         req_area = cur.fetchone()[0]
         if req_area <= 0.08:
+            # cur.execute(
+            #     """
+            #     SELECT
+            #         X(ImmeubleGeoPoint),
+            #         Y(ImmeubleGeoPoint),
+            #         IdentifiantImmeuble,
+            #         EtatImmeuble,
+            #         NumeroVoieImmeuble,
+            #         TypeVoieImmeuble,
+            #         NomVoieImmeuble
+            #     FROM ipe
+            #     WHERE ROWID IN (
+            #         SELECT ROWID FROM SpatialIndex
+            #         WHERE f_table_name = 'ipe' AND
+            #               search_frame = BuildMBR(:swx, :swy, :nex, :ney, 4326))
+            # """,
+            #     areaCoordinates,
+            # )
             cur.execute(
                 """
                 SELECT
                     X(ImmeubleGeoPoint),
                     Y(ImmeubleGeoPoint),
-                    IdentifiantImmeuble,
-                    EtatImmeuble,
-                    NumeroVoieImmeuble,
-                    TypeVoieImmeuble,
-                    NomVoieImmeuble
-                FROM ipe
+                    imb_id,
+                    imb_etat,
+                    num_voie,
+                    type_voie,
+                    nom_voie
+                FROM arcep
                 WHERE ROWID IN (
                     SELECT ROWID FROM SpatialIndex
-                    WHERE f_table_name = 'ipe' AND
+                    WHERE f_table_name = 'arcep' AND
                           search_frame = BuildMBR(:swx, :swy, :nex, :ney, 4326))
             """,
                 areaCoordinates,
@@ -60,9 +78,10 @@ class Axione:
                 existing_buildings = dict()
             buildings = existing_buildings
             for b in cur.fetchall():
+                print(b)
                 etatImm = b[3]
                 idImm = b[2]
-                isEligible = etatImm == AXIONE_ETAT_DEPLOYE
+                isEligible = etatImm == AXIONE_ETAT_DEPLOYE or etatImm == "deploye"
                 aquilenetEligStatus = FAIEligibilityStatus(
                     isEligible=isEligible,
                     ftthStatus=etatImm,

webapp/main.py

@@ -48,7 +48,7 @@ def getEligData():
         except ValueError as err:
             print("Could not get Axione data for this area:", err)
 
-        buildings = liazo.getAreaBuildings(processed_args["centerlat"], processed_args["centerlng"], buildings)
+        # buildings = liazo.getAreaBuildings(processed_args["centerlat"], processed_args["centerlng"], buildings)
 
         return {"buildings": buildings}
     else:

webapp/templates/app.js

@@ -70,6 +70,9 @@ function updateEligData(map, eligData) {
   let buildings = eligData.buildings;
   console.log(buildings)
   markers = Object.values(buildings).map(building => {
+    console.log(building.numVoieImm)
+    console.log("lat", building.y)
+    console.log("lng", building.x)
     const latlng = new L.latLng(building.y, building.x);
     const addrImm = `${building.numVoieImm} ${building.typeVoieImm} ${building.nomVoieImm}`
     const marker = new L.marker(latlng)