Ajout d'un script d'ingestion des données laposte-insee

Je n'ai pas encore proprement releasé fast-fantoir. On pointe pour
l'instant sur une branche WIP. Le format de la DB générée ne devrait
pas cependant changer.

Ça vaudrait probablement le coup de wrapper tout le pipeline (ie.
génération des données FANTOIR *et* insee) dans un seul et même script
provisionné par Nix a terme.

.gitignore

@@ -1,3 +1,3 @@
 __pycache__
 /elig-test.ini
-.vscode
+*~

README.md

@@ -10,6 +10,24 @@ Nous utilisons [poetry](https://python-poetry.org/) pour gérer les dépendances
 
 Une fois poetry installé, vous pouvez obtenir les dépendances Python à l'aide de `poetry install`. Vous pouvez ensuite obtenir un interprêteur Python pré-configuré pour utiliser ces dépendances à l'aide de `poetry shell`.
 
+## Générer/Mettre à jour la base d'adresse.
+
+1. Récupérer les données FANTOIR les plus récentes sur cette page: https://www.data.gouv.fr/fr/datasets/fichier-fantoir-des-voies-et-lieux-dits/.
+1. Transformez ce fichier texte en une base de données SQLite a l'aide de ce logiciel: https://git.alternativebit.fr/NinjaTrappeur/fast-fantoir/src/branch/fastestest
+1. Récupérez la base de données générée puis injectez les données insee à l'aide du script `./data/prepare-fantoir-db`
+
+```
+ » ./data/prepare-fantoir-db fantoir.sqlite
+Downloading the latest laposte insee data
+
+  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
+                                 Dload  Upload   Total   Spent    Left  Speed
+100 2625k    0 2625k    0     0  89006      0 --:--:--  0:00:30 --:--:--   99k
+Importing data to DB
+```
+
+La base de données est alors prête, vous pouvez la mettre où vous voulez tant qu'elle est correctement pointée par `db_addresses_sqlite_path` dans le fichier de configuration `axione-elig-test.ini`.
+
 ## Jouer les Tests
 
 Nous avons quelques tests pour le parseur. Vous pouvez les jouer à l'aide de:
@@ -45,9 +63,3 @@ curl -v http://127.0.0.1:5000/addresses/communes?s=29530
 # Chercher les voies d'une commune via son code insee
 curl -v http://127.0.0.1:5000/addresses/fantoirvoies/29036
 ```
-
-### Jeux de données
-
-#### Insee
-
-Importez ce CSV https://www.data.gouv.fr/fr/datasets/base-officielle-des-codes-postaux/ dans la base de données

address_finder/api.py

@@ -1,110 +1,79 @@
 import sqlite3
 import sys
 import json
-from .model import Commune, FantoirVoie
-import re
+from .model import Commune,FantoirVoie
 
 # DB with addresses info
-DB_ADDRESSES_PATH_ENV = "DB_ADDRESSES_PATH"
-DB_ADDRESSES_DEFAULT_PATH = "/etc/fantoir.sqlite"
+DB_ADDRESSES_PATH_ENV="DB_ADDRESSES_PATH"
+DB_ADDRESSES_DEFAULT_PATH="/etc/fantoir.sqlite"
 
 # Table for insee codes
-DB_TABLE_INSEE_NAME = "insee"
-DB_COL_COMMUNE_INSEE = "Code_commune_INSEE"
-DB_COL_COMMUNE_NAME = "Nom_commune"
-DB_COL_COMMUNE_POSTE = "Code_postal"
+DB_TABLE_INSEE_NAME="insee"
+DB_COL_COMMUNE_INSEE="Code_commune_INSEE"
+DB_COL_COMMUNE_NAME="Nom_commune"
+DB_COL_COMMUNE_POSTE="Code_postal"
+
+# Table for Fantoir voies (code Rivoli)
+DB_TABLE_FANTOIR_NAME="keyv"
+DB_COL_FANTOIR_INSEE="key"
+DB_FANTOIR_INSEE_KEY_SUFFIX="keyv:"
 
 # Utility to find an address
 class AddressFinder:
 
-    def __init__(self, db_insee_communes_sqlite_path: str, db_fantoir_voies_sqlite_path: str):
-        self.dbCommunesPath = db_insee_communes_sqlite_path
-        self.dbFantoirPath = db_fantoir_voies_sqlite_path
-        print("DB insee communes Path : " + self.dbCommunesPath)
-        print("DB Fantoir voies Path : " + self.dbFantoirPath)
+    def __init__(self, db_addresses_sqlite_path: str):
+        self.dbPath = db_addresses_sqlite_path
+        print("DB addresses Path : " + self.dbPath)
 
-    def getCommunesFromNameOrZip(self, communeNameOrZip: str, limit: int = None) -> list[Commune]:
-        con = sqlite3.connect(self.dbCommunesPath)
+    def getCommunesFromNameOrZip(self, communeNameOrZip: str) -> list[Commune]:
+        con = sqlite3.connect(self.dbPath)
         con.row_factory = sqlite3.Row
         cur = con.cursor()
-
-        # Check if a search limit is specified, make sure it is an integer
-        select_limit = ""
-        if limit is not None:
-            try:
-                select_limit = f"LIMIT {int(limit)}"
-            except ValueError:
-                print("Error, limit arg not a valid int: ", limit)
-
+        communes: list[Commune] = []
         try:
-            # If no search parameter, select all
-            if communeNameOrZip is None:
-                cur.execute(
-                    f"SELECT * from \"{DB_TABLE_INSEE_NAME}\" {select_limit}")
-            else:
-                communeSearch = communeNameOrZip
-                zipSearch = communeNameOrZip
-                searchOpertor = "OR"
-
-                # Allow search zip and commune at the same time, in the format "29530 PLO"
-                regexCommuneAndZip = r"[0-9]{5} .+"
-                if re.match(regexCommuneAndZip, communeNameOrZip):
-                    splitSearch = communeNameOrZip.split(' ')
-                    zipSearch = splitSearch[0]
-                    communeSearch = ' '.join(splitSearch[1:])
-                    searchOpertor = "AND"
-                cur.execute(
-                    f"SELECT * from \"{DB_TABLE_INSEE_NAME}\" WHERE {DB_COL_COMMUNE_NAME} LIKE \"%{communeSearch}%\" COLLATE nocase {searchOpertor} {DB_COL_COMMUNE_POSTE} LIKE \"{zipSearch}%\" {select_limit}")
+          if communeNameOrZip is None:
+            cur.execute(f"SELECT * from \"{DB_TABLE_INSEE_NAME}\"")
+          else:
+            cur.execute(f"SELECT * from \"{DB_TABLE_INSEE_NAME}\" WHERE {DB_COL_COMMUNE_NAME}=\"{communeNameOrZip}\" COLLATE nocase OR {DB_COL_COMMUNE_POSTE}=\"{communeNameOrZip}\"")
         except sqlite3.OperationalError as err:
-            print("Error querying DB : {0}".format(err), file=sys.stderr)
-            return []
-
-        communesMap = dict()
-        for row in cur.fetchall():
-            row_obj = dict(row)
-            commune = Commune(
-                codeInsee=row_obj[DB_COL_COMMUNE_INSEE],
-                nom=row_obj[DB_COL_COMMUNE_NAME],
-                codeZip=row_obj[DB_COL_COMMUNE_POSTE])
-            # This way we avoid duplicates in DB
-            communesMap[commune["codeInsee"]] = commune
-
+          print("Error querying DB : {0}".format(err), file=sys.stderr)
+          return []
+        rows = [dict(row) for row in cur.fetchall()]
         con.close()
-        return list(communesMap.values())
-
-    def getCommuneFantoirVoies(self, communeInseeCode: str, voieSearch: str = None, limit: int = None) -> list[FantoirVoie]:
+        for row in rows:
+          commune=Commune(
+            codeInsee=row[DB_COL_COMMUNE_INSEE],
+            nom=row[DB_COL_COMMUNE_NAME],
+            codeZip=row[DB_COL_COMMUNE_POSTE])
+          communes.append(commune)
+        return communes
 
+    def getCommuneFantoirVoies(self, communeInseeCode: str) -> list[FantoirVoie]:
+        
         # Extract data from DB
-        con = sqlite3.connect(self.dbFantoirPath)
+        con = sqlite3.connect(self.dbPath)
         con.row_factory = sqlite3.Row
         cur = con.cursor()
-        if voieSearch is None:
-          voieSearch=''
-
-        # Check if a search limit is specified, make sure it is an integer
-        select_limit = ""
-        if limit is not None:
-            try:
-                select_limit = f"LIMIT {int(limit)}"
-            except ValueError:
-                print("Error, limit arg not a valid int: ", limit)
-
         try:
-            cur.execute(
-                f"SELECT trim(libelle), rivoli_with_key from fantoir WHERE full_insee=\"{communeInseeCode}\" AND libelle like \"%{voieSearch}%\" {select_limit}")
+          cur.execute(f"SELECT value from \"{DB_TABLE_FANTOIR_NAME}\" WHERE {DB_COL_FANTOIR_INSEE}=\"{DB_FANTOIR_INSEE_KEY_SUFFIX}{communeInseeCode}\"")
         except sqlite3.OperationalError as err:
-            print("Error querying DB : {0}".format(err), file=sys.stderr)
-            return []
-        data_raw = cur.fetchall()
+          print("Error querying DB : {0}".format(err), file=sys.stderr)
+          return []
+        data_raw = cur.fetchone()
         con.close()
 
+        ## Get JSON payload
+        
         fantoir_dict = []
         # Check if data where found
         if data_raw is not None:
-            fantoir_dict = dict(data_raw)
+          data = dict(data_raw)
+          # Extract the data behind "value" which is a JSON structure
+          data_dict=json.loads(data.get("value"))
+          # In extracted JSON data, the interesting payload is behind "value" key
+          fantoir_dict = data_dict.get("value")
         else:
-            print("Did not found any data matching Insee code " +
-                  str(communeInseeCode))
-
+          print("Did not found any data matching Insee code " + str(communeInseeCode))
+        
         # Return the json dump
         return fantoir_dict

address_finder/model.py

@@ -6,5 +6,12 @@ class Commune(TypedDict):
   codeZip: str
 
 class FantoirVoie(TypedDict):
-  libelle: str
-  rivoli_with_key: int
+  id: str
+  dateAjout: int
+  libelle: list[str]
+  typeVoie:str
+  codeCommune: str
+  codeFantoir: str
+  cleRivoli: str
+  nomCommune: str
+  predecesseur: bool

axione_api/api.py

@@ -23,37 +23,14 @@ def ptoRequest(ptoRef):
         </soapenv:Envelope>
     """
 
-def fantoirRequest(insee, rivoli, numVoie):
-    ts = datetime.now(timezone.utc).isoformat()
-    return f"""
-        <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ent="http://structureadresseftth.axione.fr/model/entreprise" xmlns:com="http://structureadresseftth.axione.fr/model/commun">
-        <soapenv:Header/>
-        <soapenv:Body>
-            <ent:obtentionStructureAdresseDemandeSoap>
-                <ent:entete versionWS="3.0" horodatageRequete="{ts}">
-                    <com:operateurCommercial nom="AQUILENET" identifiant=""/>
-                </ent:entete>
-                <ent:referenceAdresse>
-                        <com:referenceRivoli codeInsee="{insee}" codeRivoli="{rivoli}" numeroVoie="{numVoie}"/>
-                </ent:referenceAdresse>
-            </ent:obtentionStructureAdresseDemandeSoap>
-        </soapenv:Body>
-        </soapenv:Envelope>
-    """
-
-def query_axione_fantoir(cfg, insee, rivoli, numVoie):
-    body = fantoirRequest(insee, rivoli, numVoie)
-    return query_axione(cfg, body)
-
 
 def query_axione_pto(cfg, ptoRef):
     body = ptoRequest(ptoRef)
-    return query_axione(cfg, body)
-
-def query_axione(cfg, body):
     # Note: the password should be the base64 of username:password.
     # Don't ask why.
-    passwd = cfg.password
+    passwd = base64.b64encode(f"{cfg.username}:{cfg.password}".encode("utf8")).decode(
+        "utf8"
+    )
     headers = {
         "User-Agent": "aquilenet-elig-test/0.1",
         "Accept": "*/*",
@@ -61,7 +38,7 @@ def query_axione(cfg, body):
         "Connection": "Keep-Alive",
         "Authorization": passwd,
     }
-    respData = None
+    resp = None
     if not cfg.debug:
         try:
             conn = http.client.HTTPSConnection(
@@ -90,10 +67,11 @@ def query_axione(cfg, body):
         print(headers)
         print("BODY: ")
         print(body)
+        print("===================")
         with open("./fixtures/dummy-data-1.xml", "r") as f:
             dummyData = f.read()
         return dummyData
-    return respData
+    return resp
 
 
 class LigneResult(TypedDict):
@@ -120,45 +98,17 @@ class BatimentResult(TypedDict):
     referenceBatiment: str
     etages: list[EtageResult]
 
-class AxioneErreur(TypedDict):
-    codeErreur: str
-    libelleErreur: str
 
-class AxioneResult(TypedDict):
-    codeRetour: int
-    axioneErreur: AxioneErreur
-    batiments: list[BatimentResult]
-
-
-def parse_response(resp_str) -> AxioneResult:
+def parse_response(resp_str) -> list[BatimentResult]:
     root = ET.fromstring(resp_str)
-    codeRetourXml = root.find(
-        ".//{http://structureadresseftth.axione.fr/model/commun}codeRetour"
-    )
-    codeRetour = int(codeRetourXml.text.strip())
-    axioneErreur = None
-    parsedBatiments = []
-    if codeRetour == 0:
-      parsedBatiments = [
-          parse_batiment(b)
-          for b in root.findall(
-              ".//{http://structureadresseftth.axione.fr/model/commun}batiment"
-          )
-      ]
-    else:
-      axioneErreur = AxioneErreur(
-          codeErreur = root.find(
-              ".//{http://structureadresseftth.axione.fr/model/commun}codeErreur"
-          ).text.strip(),
-          libelleErreur = root.find(
-              ".//{http://structureadresseftth.axione.fr/model/commun}libelleErreur"
-          ).text.strip()
-      )
-    return AxioneResult(
-      codeRetour=codeRetour,
-      axioneErreur=axioneErreur,
-      batiments=parsedBatiments
-    )
+    parsedBatiments = [
+        parse_batiment(b)
+        for b in root.findall(
+            ".//{http://structureadresseftth.axione.fr/model/commun}batiment"
+        )
+    ]
+    return parsedBatiments
+
 
 def parse_batiment(batiment) -> BatimentResult:
     etatBatiment = batiment.get("etatBatiment", None)

config/config.py

@@ -2,12 +2,11 @@ import configparser
 
 
 class Config:
-    def __init__(self, username, password, source_addr, db_insee_communes_sqlite_path, db_fantoir_voies_sqlite_path):
+    def __init__(self, username, password, source_addr, db_addresses_sqlite_path):
         self.username = username
         self.password = password
         self.source_addr = source_addr
-        self.db_insee_communes_sqlite_path = db_insee_communes_sqlite_path
-        self.db_fantoir_voies_sqlite_path = db_fantoir_voies_sqlite_path
+        self.db_addresses_sqlite_path = db_addresses_sqlite_path
         self.debug = False
 
 
@@ -18,6 +17,5 @@ def parse_config(cfgPath):
     username = cfg.get("API", "username")
     passwd = cfg.get("API", "password")
     source_addr = cfg.get("API", "source_addr")
-    db_insee_communes_sqlite_path = cfg.get("ADDRESSES","db_insee_communes_sqlite_path")
-    db_fantoir_voies_sqlite_path = cfg.get("ADDRESSES","db_fantoir_voies_sqlite_path")
-    return Config(username, passwd, source_addr,db_insee_communes_sqlite_path, db_fantoir_voies_sqlite_path)
+    db_addresses_sqlite_path = cfg.get("ADDRESSES","db_addresses_sqlite_path")
+    return Config(username, passwd, source_addr,db_addresses_sqlite_path)

data/prepare-fantoir-db

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [ "$#" -ne 1 ]; then
+    echo "Usage: prepare-fantoir-db path-to-fantoir-db"
+    echo ""
+    echo "ERROR: Missing fantoir db"
+    exit 1
+fi
+fantoirDb="$1"
+
+# Setup tmp working dir
+tmpDir=$(mktemp -d)
+posteData="${tmpDir}"/poste.csv
+tmpSql="${tmpDir}"/import-insee-codes.sql
+clean_tmp () {
+    rm -r "${tmpDir}"
+}
+trap clean_tmp EXIT
+
+echo "Downloading the latest laposte insee data"
+echo ""
+curl "https://datanova.laposte.fr/explore/dataset/laposte_hexasmal/download/?format=csv&timezone=Europe/Berlin&lang=fr&use_labels_for_header=true&csv_separator=%3B" > "${posteData}"
+
+cat >"${tmpSql}" <<EOF
+.separator ";"
+.import ${posteData} poste_insee
+EOF
+
+echo "Importing data to DB"
+echo ""
+sqlite3 "${fantoirDb}" < "${tmpSql}"

elig-test.ini.sample

@@ -5,5 +5,4 @@
   # to send the requests from.
   source_addr = xxx.xxx.xxx.xxx
 [ADDRESSES]
-  db_insee_communes_sqlite_path = path/to/db.sqlite
-  db_fantoir_voies_sqlite_path = path/to/db.sqlite
+  db_addresses_sqlite_path = path/to/db.sqlite

fixtures/dummy-data-2.xml

@@ -1,46 +0,0 @@
-<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
-  <SOAP-ENV:Header />
-  <SOAP-ENV:Body>
-    <ns3:obtentionStructureAdresseReponseSoap xmlns:ns2="http://structureadresseftth.axione.fr/model/commun" xmlns:ns3="http://structureadresseftth.axione.fr/model/entreprise">
-      <ns3:entete horodatageReponse="2022-02-13T00:57:50.773+01:00" horodatageRequete="2022-02-12T23:57:48.974060Z" identifiantReponse="1644710270773" versionWS="3.0">
-        <ns2:operateurCommercial identifiant="" nom="AQUILENET" />
-      </ns3:entete>
-      <ns3:codeRetour>
-        <ns2:codeRetour>0</ns2:codeRetour>
-      </ns3:codeRetour>
-      <ns3:codeOI>BEFO</ns3:codeOI>
-      <ns3:etatImmeuble>true</ns3:etatImmeuble>
-      <ns3:structureDetaillee>
-        <ns2:adresse referenceHexacle="361992226N">
-          <ns2:referenceRivoli codeInsee="36199" codeRivoli="0031" numeroVoie="48" />
-        </ns2:adresse>
-        <ns2:batiment conditionsSyndic="false" dateDebutAcceptationCmdAcces="2021-09-14T00:00:00.000+02:00" dateDebutFournitureCRCmdAcces="2021-09-14T00:00:00.000+02:00" etatBatiment="COMMERCIALISABLE" identifiantImmeuble="BAT36-7C32FE6AD11F4A6797E4F29E" nombreLogementsImmeuble="1" referenceBatiment="NA">
-          <ns2:referenceGeographique coordonneeImmeubleX="2.027143270510003" coordonneeImmeubleY="47.01243491007431" typeProjection="WGS84" />
-          <ns2:escalier reference="NA">
-            <ns2:etage nombreLignesActives="0" nombreLignesExistantes="0" nombreLocauxFTTH="1" refPriseCommandeObligatoire="false" reference="RDC">
-              <ns2:listePbo>
-                <ns2:pbo referencePBO="PBO36_ISS1_139" typePbo="AERIEN" />
-              </ns2:listePbo>
-              <ns2:listeLignesFTTH>
-                <ns2:ligneFTTH>
-                  <ns2:prise etiquetteAPoser="Non" referencePBO="PBO36_ISS1_139">
-                    <ns2:referencePTO>BF-ISS1-0110</ns2:referencePTO>
-                    <ns2:referencePrisePromoteur />
-                    <ns2:statutLigneFTTH actif="false" commercialisable="true" existant="false" raccordable="true" rompu="false" />
-                  </ns2:prise>
-                </ns2:ligneFTTH>
-                <ns2:ligneFTTH>
-                  <ns2:local localisationLocalOC="" localisationLocalOI="" />
-                </ns2:ligneFTTH>
-              </ns2:listeLignesFTTH>
-              <ns2:pm referencePM="ADR_36199_ISS1" referencePMT="PMT_36199_ISS1">
-                <ns2:typeEmplacementPM>PME</ns2:typeEmplacementPM>
-                <ns2:responsableBrassage>OC</ns2:responsableBrassage>
-              </ns2:pm>
-            </ns2:etage>
-          </ns2:escalier>
-        </ns2:batiment>
-      </ns3:structureDetaillee>
-    </ns3:obtentionStructureAdresseReponseSoap>
-  </SOAP-ENV:Body>
-</SOAP-ENV:Envelope>

fixtures/dummy-data-3.xml

@@ -1,36 +0,0 @@
-<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/">
-  <SOAP-ENV:Header />
-  <SOAP-ENV:Body>
-    <ns3:obtentionStructureAdresseReponseSoap xmlns:ns2="http://structureadresseftth.axione.fr/model/commun" xmlns:ns3="http://structureadresseftth.axione.fr/model/entreprise">
-      <ns3:entete horodatageReponse="2022-02-13T00:55:44.339+01:00" horodatageRequete="2022-02-12T23:55:42.382844Z" identifiantReponse="1644710144339" versionWS="3.0">
-        <ns2:operateurCommercial identifiant="" nom="AQUILENET" />
-      </ns3:entete>
-      <ns3:codeRetour>
-        <ns2:codeRetour>1</ns2:codeRetour>
-        <ns2:codeErreur>I01</ns2:codeErreur>
-        <ns2:libelleErreur>Code Rivoli introuvable, manquant ou incomplet</ns2:libelleErreur>
-      </ns3:codeRetour>
-      <ns3:codeOI />
-      <ns3:structureDetaillee>
-        <ns2:adresse />
-        <ns2:batiment referenceBatiment="">
-          <ns2:referenceGeographique coordonneeImmeubleX="" coordonneeImmeubleY="" typeProjection="" />
-          <ns2:escalier reference="">
-            <ns2:etage refPriseCommandeObligatoire="false" reference="">
-              <ns2:listePbo>
-                <ns2:pbo referencePBO="" typePbo="" typeRaccoPbPTO="" />
-              </ns2:listePbo>
-              <ns2:listeLignesFTTH>
-                <ns2:ligneFTTH />
-              </ns2:listeLignesFTTH>
-              <ns2:pm referencePM="" referencePMT="">
-                <ns2:typeEmplacementPM>PME</ns2:typeEmplacementPM>
-                <ns2:responsableBrassage>OI</ns2:responsableBrassage>
-              </ns2:pm>
-            </ns2:etage>
-          </ns2:escalier>
-        </ns2:batiment>
-      </ns3:structureDetaillee>
-    </ns3:obtentionStructureAdresseReponseSoap>
-  </SOAP-ENV:Body>
-</SOAP-ENV:Envelope>

run-dev-server

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-DEBUG=true TEMPLATES_AUTO_RELOAD=true FLASK_ENV=development CONFIG=./elig-test.ini FLASK_APP=webapp poetry run flask run --reload
+DEBUG=true CONFIG=./elig-test.ini FLASK_APP=webapp poetry run flask run --reload

scripts/import-laposte-insee.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-set -eau -o pipefail
-
-if [ "$#" -ne 2 ]; then
-    echo "Usage: import-laposte-insee.sh path-to-laposte-insee-CSV fantoir-sqlite-db-path"
-    echo ""
-    echo "ERROR: Missing laposte CSV."
-    echo "You can download it at https://www.data.gouv.fr/fr/datasets/base-officielle-des-codes-postaux/"
-    exit 1
-fi
-
-tmpDir=$(mktemp -d)
-clean_tmp () {
-    rm -r "${tmpDir}"
-}
-trap clean_tmp EXIT
-tmpSql="${tmpDir}"/import-laposte-hexasmal.sql
-
-echo "Importing laposte/insee hexasmal data into the fantoir db."
-
-cat >"${tmpSql}" <<EOF
-.separator ";"
-.import $1 insee
-EOF
-
-sqlite3 "${2}" < "${tmpSql}"
-
-
-echo "Data imported"

templates/landing_form.html

@@ -1,235 +1,25 @@
 <!doctype html>
 
 <html lang="fr">
-
 <head>
   <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-  <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
-    integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
-  <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
-    integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"
-    integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1"
-    crossorigin="anonymous"></script>
-  <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"
-    integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM"
-    crossorigin="anonymous"></script>
-  <script src="https://cdn.jsdelivr.net/npm/@tarekraafat/autocomplete.js@10.2.6/dist/autoComplete.min.js"></script>
-<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@tarekraafat/autocomplete.js@10.2.6/dist/css/autoComplete.02.min.css">
-
+  <meta name="viewport" content="width=device-width, initial-scale=1">
   <title>Aquilenet: Éligibilité FTTH</title>
   <style>
-    {% include 'style.css'%}
+    {% include 'style.css' %}
   </style>
 </head>
 
 <body>
-  <h1 class="text-center">
-    <a href="https://www.aquilenet.fr/" id="aquilenet-title">AQUILENET</a>
-  </h1>
-  <div id="mainContainer" class="container">
-
-    <div class="row d-flex justify-content-between align-items-center" data-parent="#mainContainer">
-
-      <div class="col-12 text-intro">
-        <div id="methodPto" class="collapse show testMethods">
-
-          <h2 class="text-center form-title">Test d'éligibilité par PTO</h2>
-            <form id="formPtoTest" method="post" action="/test/pto">
-              <div class="form-group" id="ptoForm">
-                <label class="form-label" for="pto-number">Numéro PTO</label>
-                <input autofocus required type="text" class="form-control" name="pto" id="pto-number" aria-describedby="ptoHelp"
-                  placeholder="OOOO-XXXX-XXXX" oninvalid="this.setCustomValidity('Veuillez renseigner le PTO')"
-                  oninput="setCustomValidity('')">
-
-                <small id="ptoHelp" class="form-text btn btn-link" data-toggle="collapse" data-target="#ptoInfo"
-                  aria-expanded="true" aria-controls="ptoInfo">Où trouver mon numéro de PTO ?</small>
-                <div id="ptoInfo" class="collapse" aria-labelledby="ptoForm" data-parent="#ptoForm">
-                  <div class="card-body">
-                    PTO (Point de terminaison optique) est un numéro unique que vous pouvez trouver sur le boîtier de
-                    raccordement de la fibre.
-                    C'est un petit boîtier blanc installé dans la maison
-                    <br>
-                    <img src="{{url_for('static', filename='find_pto.jpg')}}" class="img-fluid" alt="Responsive image">
-                  </div>
-                </div>
-              </div>
-              <button id="buttonPtoTest" type="submit" class="btn btn-sable">Tester le PTO</button>
-            </form>
-
-            <br>
-            <button id="buttonNoPto" type="button" data-toggle="collapse" data-target=".testMethods"
-              class="btn btn-link page-nav-btn" aria-expanded="false" aria-controls="methodPto methodAddress">Je n'ai pas/ne trouve
-              pas le PTO, tester autrement</button>
-        </div>
-
-        <div id="methodAddress" class="collapse testMethods">
-          <h2 class="text-center form-title">Test d'éligibilité par adresse</h2>
-            <form id="formAddressTest" method="post" action="/test/address">
-                <div class="form-group row" id="communeForm">
-                    <label class="form-label col-sm-2 my-1" for="commune-autocomplete">Commune</label>
-                    <input id="commune-autocomplete" class="col-sm-9" type="search" dir="ltr" spellcheck=false autocorrect="off" autocomplete="off" class="form-control"  autocapitalize="off"/>
-                </div>
-              <div class="form-row collapse" id="voieForm">
-                <div class="col-sm-3 my-1">
-                  <label class="form-label" for="numeroVoieInput">Numéro de voie</label>
-                  <input required type="text" name="numeroVoie" class="form-control" id="numeroVoieInput"
-                    aria-describedby="numeroVoieHelp" placeholder="Numéro de voie"
-                    oninvalid="this.setCustomValidity('Veuillez renseigner le numéro de voie')"
-                    oninput="setCustomValidity('')">
-                </div>
-                <div class="col-sm-7 my-1">
-                  <label class="form-label" for="fantoir-autocomplete">Nom de voie</label>
-                  <input id="fantoir-autocomplete" class="form-control" type="search" dir="ltr" spellcheck=false autocorrect="off" autocomplete="off" autocapitalize="off"/>
-                </div>
-              </div>
-              <br>
-              <button id="btnTestAdresse" type="submit" class="btn btn-sable collapse">Tester l'adresse</button>
-            </form>
-            <br>
-            <button id="buttonReturnPto" type="button" data-toggle="collapse" data-target=".testMethods"
-              class="btn btn-link page-nav-btn" aria-expanded="false" aria-controls="methodPto methodAddress">Revenir au test par
-              PTO</button>
-        </div>
-      </div>
-    </div>
+  <h1 id="aquilenet-title">Aquilenet</h1>
+  <div id="container">
+    <h1 id="main-title">Test d'Éligibilité FTTH Aquilenet</h1>
+    <form method="post" action="/result">
+      <label>Numéro de PTO :
+        <input name="pto"/>
+      </label>
+      <button>Tester</button>
+    </form>
   </div>
-
-  <script>
-    $(document).ready(function () {
-
-      // AbortController allows to cancel promises
-      const controllerCommunes = new AbortController();
-      const { signalCommunes } = controllerCommunes;
-      const controllerVoies = new AbortController();
-      const { signalVoies } = controllerVoies;
-
-      // Global variables
-      var communes = []
-      var codeInsee = 0
-      var codeRivoli = ""
-      var voies = []
-      var voie = ""
-
-      function sanitizeInputStr(inputStr) {
-
-        inputStr = inputStr.replace(/[ÀÁÂÃÄÅàáâãäå]/g, "a");
-        inputStr = inputStr.replace(/[Ææ]/g, "ae");
-        inputStr = inputStr.replace(/[Çç]/g, "c");
-        inputStr = inputStr.replace(/[ÈÉÊËèéêë]/g, "e");
-        inputStr = inputStr.replace(/[ÌÍÎÏìíîï]/g, "i");
-        inputStr = inputStr.replace(/[Ññ]/g, "n");
-        inputStr = inputStr.replace(/[ÒÓÔÕÖòóôõö]/g, "o");
-        inputStr = inputStr.replace(/[ÙÚÛÜùúûü]/g, "u");
-        inputStr = inputStr.replace(/[Ýý]/g, "y");
-        inputStr = inputStr.replace(/[ß]/g, "ss");
-
-        return inputStr;
-      }
-
-      function setButtonSpinner(buttonSelector, buttonText) {
-        // disable button
-        $(buttonSelector).prop("disabled", true);
-        // add spinner to button
-        $(buttonSelector).html(
-          `<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span>${buttonText}`
-        ); 
-      }
-
-      $('#methodAddress').on('show.bs.collapse', function () {
-        $('#communeInput').trigger('input')
-        $('#communeInput').trigger('keyup')
-      });
-      $('#methodAddress').on('shown.bs.collapse', function () {
-        $('#communeInput').focus();
-      });
-      $('#voieForm').on('show.bs.collapse', function () {
-        $('#voieInput').trigger('input')
-        $('#voieInput').trigger('keyup')
-      });
-      $('#voieForm').on('shown.bs.collapse', function () {
-        $('#numeroVoieInput').focus();
-      });
-
-      const autoCompleteCommune = new autoComplete({
-          selector: "#commune-autocomplete",
-          placeHolder: "Code postal/nom de commune...",
-          data: {
-              src: async (query) => {
-                  const api = "addresses/communes?limit=100";
-                  const reqUrl = query === '' ? api : api + "&s=" + query
-                  const source = await fetch(reqUrl, { signalCommunes });
-                  const data = await source.json();
-                  return data;
-              },
-              keys: [ "codeZip", "nom" ],
-          },
-          resultList: {
-              element: (list, data) => {
-              }
-          },
-          resultItem: {
-              highlight: true
-          },
-          debounce: 300,
-          events: {
-              input: {
-                  selection: (event) => {
-                      const selection = event.detail.selection.value;
-                      autoCompleteCommune.input.value = selection.codeZip + " - " + selection.nom ;
-                      codeInsee = selection.codeInsee;
-                      $("#voieForm").collapse("show");
-                  }
-              }
-          }
-      });
-
-      const autoCompleteFantoir = new autoComplete({
-          selector: "#fantoir-autocomplete",
-          placeHolder: "Nom de la voie...",
-          data: {
-              src: async (query) => {
-                  const api = "addresses/fantoirvoies/" + codeInsee + "?limit=100";
-                  const reqUrl = query === '' ? api : api + "&s=" + query
-                  const source = await fetch(reqUrl, { signalVoies });
-                  const data = await source.json();
-                  return Object.entries(data).map(e => {return {"name": e[0], "value": e[1]}; });
-              },
-              keys: [ "name" ],
-          },
-          resultList: {
-              element: (list, data) => {
-              }
-          },
-          resultItem: {
-              highlight: true
-          },
-          debounce: 300,
-          events: {
-              input: {
-                  selection: (event) => {
-                      const selection = event.detail.selection.value;
-                      autoCompleteFantoir.input.value = selection.name;
-                      codeRivoli = selection.value;
-                      $('#btnTestAdresse').collapse('show');
-                  }
-              }
-          }
-      });
-
-      $('#formAddressTest').submit(function(eventObj) {
-        setButtonSpinner("#btnTestAdresse", "Test...");
-        $('#formAddressTest').append(`<input type="hidden" class="form-control" name="codeInsee" value="${codeInsee}" />`);
-        $('#formAddressTest').append(`<input type="hidden" class="form-control" name="codeRivoli" value="${codeRivoli}" />`);
-        return true;
-      });
-      $('#formPtoTest').submit(function(eventObj) {
-        setButtonSpinner("#buttonPtoTest", "Test...");
-        return true;
-      });
-    });
-  </script>
 </body>
-
 </html>

templates/result.html

@@ -3,19 +3,7 @@
 <html lang="fr">
 <head>
   <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-  <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
-    integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
-  <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"
-    integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"
-    integrity="sha384-UO2eT0CpHqdSJQ6hJty5KVphtPhzWj9WO1clHTMGa3JDZwrnQq4sF86dIHNDz0W1"
-    crossorigin="anonymous"></script>
-  <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"
-    integrity="sha384-JjSmVgyd0p3pXB1rRibZUAYoIIy6OrQ6VrjIEaFf/nJGzIxFDsf4x0xIM+B07jRM"
-    crossorigin="anonymous"></script>
-  <script src="https://cdn.jsdelivr.net/npm/@tarekraafat/autocomplete.js@10.2.6/dist/autoComplete.min.js"></script>
-
+  <meta name="viewport" content="width=device-width, initial-scale=1">
   <title>Aquilenet: Éligibilité FTTH</title>
   <style>
     {% include 'style.css' %}
@@ -23,20 +11,11 @@
 </head>
 
 <body>
-  <h1 class="text-center">
-    <a href="https://www.aquilenet.fr/" id="aquilenet-title">AQUILENET</a>
-  </h1>
-
-  
-  <div id="containerResults" class="container">
-    <a id="buttonReturnTest" type="button" class="btn btn-link page-nav-btn" href="/">
-      Retour test
-    </a>
-  
-  {% if result['codeRetour'] == 0 %}
+  <h1 id="aquilenet-title">Aquilenet</h1>
+  <div id="container">
     <h1 id="main-title">Test d'Éligibilité FTTH Aquilenet: Résultats</h1>
     <p>Résultat pour le PTO: {{ pto }}</p>
-    {% for batiment in result['batiments'] %}
+    {% for batiment in result %}
     <table>
       <thead>
         <tr>
@@ -78,12 +57,6 @@
       {% endfor %}
     </table>
     {% endfor %}
-  
-  {% else %}
-    <h1 id="main-title">Erreur</h1>
-    <p>Code d'erreur: {{ result['axioneErreur']['codeErreur'] }}</p>
-    <p>Description de l'erreur: {{ result['axioneErreur']['libelleErreur'] }}</p>
-  {% endif %}
   </div>
 </body>
 </html>

templates/style.css

@@ -1,71 +1,16 @@
-html, body {
-  height : 100%;
-  /* font-size: 20px; */
-}
 body {
     background-color:  #1787c2;
-    /* display: flex;
-    flex-direction: column; */
+    display: flex;
+    flex-direction: column;
     font-family: 'Titillium Web', sans-serif;
 }
 #aquilenet-title {
-    color: whitesmoke;
+    color: white;
     align-self: center;
-    font-size: 2em;
-    margin-top: .3em;
-    margin-bottom: .5em;
-    text-decoration: none;
-}
-.form-title {
-    color: whitesmoke;
-    align-self: center;
-    font-size: 2em;
+    font-size: 4em;
     margin-top: .3em;
     margin-bottom: .5em;
 }
-.form-label, .card-body, #containerResults {
-  color: whitesmoke;
-}
-#ptoHelp {
-  color: rgb(192, 192, 192);
-  text-decoration: underline;
-}
-
-.sable .btn-ciel, .ciel .btn-sable{
-  padding: 8px;
-  border-radius: 10px;
-  box-shadow: 3px 3px 3px rgba(0,0,0,.3);
-  font-size: 1.1em
-}
-.sable .btn-ciel:hover, .ciel .btn-sable:hover{
-  box-shadow: none;
-}
-
-.page-nav-btn {
-  color: whitesmoke;
-  text-decoration: underline;
-}
-
-.btn-sable:hover {
-  background-position: right center; /* change the direction of the change here */
-}
-
-.btn-sable {
-  /* box-shadow: 0 0 20px #eee; */
-  /* border-radius: 10px; */
-  color: darkslategray;
-  transition: 0.5s;
-  background-size: 200% auto;
-  text-shadow: dimgray 0 1px 1px;
-  background-image: linear-gradient(to bottom, #ffd38c 0%, #ffedd0 51%,  #e0e0e0 100%);
-}
-
-/* Autocomplete */
-
-
-/* #fda085 51%, */
-
-/*
 #container {
     width: 80%;
     background-color: #ffd38c;
@@ -84,7 +29,7 @@ form {
     align-self: center;
     margin-bottom: 2em;
 }
-*/
+
 table,td {
     border: 2px solid #333;
     border-collapse: collapse;

webapp.py

@@ -3,7 +3,7 @@ from flask import Flask, render_template, request, escape
 import json
 
 from config.config import parse_config
-from axione_api.api import query_axione_pto, parse_response, query_axione_fantoir
+from axione_api.api import query_axione_pto, parse_response
 from address_finder.api import AddressFinder
 
 def load_config():
@@ -22,7 +22,7 @@ def load_config():
 
 cfg = load_config()
 
-addressFinder = AddressFinder(cfg.db_insee_communes_sqlite_path, cfg.db_fantoir_voies_sqlite_path)
+addressFinder = AddressFinder(cfg.db_addresses_sqlite_path)
 
 app = Flask(__name__)
 
@@ -30,11 +30,17 @@ app = Flask(__name__)
 def get_form():
     return render_template("landing_form.html")
 
+@app.route("/result", methods=['POST'])
+def show_result():
+    pto = escape(request.form['pto'])
+    result = parse_response(query_axione_pto(cfg, pto))
+    return render_template("result.html", pto=pto, result=result)
+
 @app.route("/addresses/communes", methods=['GET'])
 def get_communes():
     to_search=request.args.get('s')
-    limit=request.args.get('limit')
-    communes=addressFinder.getCommunesFromNameOrZip(to_search,limit)
+    print(to_search)
+    communes=addressFinder.getCommunesFromNameOrZip(to_search)
     response = app.response_class(
         response=json.dumps(communes),
         mimetype='application/json'
@@ -43,32 +49,9 @@ def get_communes():
 
 @app.route("/addresses/fantoirvoies/<codeInsee>", methods=['GET'])
 def get_fantoir_voies(codeInsee):
-    to_search=request.args.get('s')
-    limit=request.args.get('limit')
-    fantoirVoies=addressFinder.getCommuneFantoirVoies(codeInsee,to_search,limit)
+    fantoirVoies=addressFinder.getCommuneFantoirVoies(codeInsee)
     response = app.response_class(
         response=json.dumps(fantoirVoies),
         mimetype='application/json'
     )
-    return response
-
-@app.route("/test/address", methods=['POST'])
-def test_address():
-    codeInsee = escape(request.form['codeInsee'])
-    numeroVoie = escape(request.form['numeroVoie'])
-    # Trimming rivoli's key
-    codeRivoli = escape(request.form['codeRivoli'])[:-1]
-    result = parse_response(query_axione_fantoir(cfg, codeInsee, codeRivoli, numeroVoie))
-    if cfg.debug:
-        print("===================")
-        print("Dummy response: ")
-        print(result)
-        print("===================")
-    return render_template("result.html", pto="", result=result)
-
-
-@app.route("/test/pto", methods=['POST'])
-def test_pto():
-    pto = escape(request.form['pto'])
-    result = parse_response(query_axione_pto(cfg, pto))
-    return render_template("result.html", pto=pto, result=result)
+    return response