Adapt backend to new fantoir data schema

We updated the fantoir DB schema. See
https://git.alternativebit.fr/NinjaTrappeur/fast-fantoir/src/branch/master
for more informations.
This commit is contained in:
Félix Baylac-Jacqué 2022-02-12 11:50:57 +01:00
parent 46aeaf7da4
commit 3acd1dd349
4 changed files with 59 additions and 50 deletions

View file

@ -45,3 +45,9 @@ curl -v http://127.0.0.1:5000/addresses/communes?s=29530
# Chercher les voies d'une commune via son code insee # Chercher les voies d'une commune via son code insee
curl -v http://127.0.0.1:5000/addresses/fantoirvoies/29036 curl -v http://127.0.0.1:5000/addresses/fantoirvoies/29036
``` ```
### Jeux de données
#### Insee
Importez ce CSV https://www.data.gouv.fr/fr/datasets/base-officielle-des-codes-postaux/ dans la base de données

View file

@ -85,15 +85,13 @@ class AddressFinder:
cur = con.cursor() cur = con.cursor()
try: try:
cur.execute( cur.execute(
f"SELECT value from \"{DB_TABLE_FANTOIR_NAME}\" WHERE {DB_COL_FANTOIR_INSEE}=\"{DB_FANTOIR_INSEE_KEY_SUFFIX}{communeInseeCode}\"") f"SELECT trim(libelle), rivoli_with_key from fantoir WHERE full_insee=\"{communeInseeCode}\" AND libelle like \"%{voieSearch}%\"")
except sqlite3.OperationalError as err: except sqlite3.OperationalError as err:
print("Error querying DB : {0}".format(err), file=sys.stderr) print("Error querying DB : {0}".format(err), file=sys.stderr)
return [] return []
data_raw = cur.fetchone() data_raw = cur.fetchall()
con.close() con.close()
# Get JSON payload
# Check if a search limit is specified, make sure it is an integer # Check if a search limit is specified, make sure it is an integer
if limit is not None: if limit is not None:
try: try:
@ -104,33 +102,7 @@ class AddressFinder:
fantoir_dict = [] fantoir_dict = []
# Check if data where found # Check if data where found
if data_raw is not None: if data_raw is not None:
data = dict(data_raw) fantoir_dict = dict(data_raw)
# Extract the data behind "value" which is a JSON structure
data_dict = json.loads(data.get("value"))
# In extracted JSON data, the interesting payload is behind "value" key
fantoir_dict = data_dict.get("value")
# Apply search filter if any
if voieSearch is not None:
# Can match multiple words, for example "avenue noe" matches "avenue de noes"
regexSearch = r".*"
for expr in voieSearch.split(' '):
regexSearch += r"(?=" + expr + r").*"
regexSearch += r".*"
fantoir_voies_filtered = []
nb_match = 0
for voie in fantoir_dict:
for libelle in voie['libelle']:
if re.search(regexSearch, libelle, re.IGNORECASE):
fantoir_voies_filtered.append(voie)
nb_match += 1
break
if limit is not None and nb_match >= limit:
break
fantoir_dict = fantoir_voies_filtered
else:
# Apply search limit if any
fantoir_dict = fantoir_dict[:limit]
else: else:
print("Did not found any data matching Insee code " + print("Did not found any data matching Insee code " +
str(communeInseeCode)) str(communeInseeCode))

29
scripts/import-laposte-insee.sh Executable file
View file

@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -eau -o pipefail
if [ "$#" -ne 2 ]; then
echo "Usage: import-laposte-insee.sh path-to-laposte-insee-CSV fantoir-sqlite-db-path"
echo ""
echo "ERROR: Missing laposte CSV."
echo "You can download it at https://www.data.gouv.fr/fr/datasets/base-officielle-des-codes-postaux/"
exit 1
fi
tmpDir=$(mktemp -d)
clean_tmp () {
rm -r "${tmpDir}"
}
trap clean_tmp EXIT
tmpSql="${tmpDir}"/import-laposte-hexasmal.sql
echo "Importing laposte/insee hexasmal data into the fantoir db."
cat >"${tmpSql}" <<EOF
.separator ";"
.import $1 insee
EOF
sqlite3 "${2}" < "${tmpSql}"
echo "Data imported"

View file

@ -127,11 +127,13 @@
success: function (data, status, xhr) { success: function (data, status, xhr) {
if (JSON.stringify(data) !== JSON.stringify(voies)){ if (JSON.stringify(data) !== JSON.stringify(voies)){
$("#voies").empty(); $("#voies").empty();
voies=data voies=Object.entries(data);
voies.forEach(voie => { let voie;
$("#voies").append("<option id=" + voie.id + " value='" + for (i in voies) {
voie.libelle[0] + "'></option>"); voie = voies[i]
}); $("#voies").append("<option id=" + voie[1] + " value='" +
voie[0] + "'></option>");
}
if (voies.length == 1) { if (voies.length == 1) {
$('#btnTestAdresse').collapse('show'); $('#btnTestAdresse').collapse('show');
voie=voies[0].libelle[0] voie=voies[0].libelle[0]