Aquilenet/Axione-IPE-Viewer
7
1
Fork 0
Code Issues 5 Pull requests Projects Releases Wiki Activity

update axione ingest script

Browse source
This commit is contained in:
Johan Le Baut 2022-12-08 12:28:33 +01:00
parent 270812f6d6
commit 90bcc50087
1 changed files with 29 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
data-ingest/ingest
View file

@ -1,6 +1,8 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -eau -o pipefail set -eau -o pipefail
NEEDED_COLUMNS=("IdentifiantImmeuble" "EtatImmeuble" "CoordonneeImmeubleX" "CoordonneeImmeubleY" "NumeroVoieImmeuble" "TypeVoieImmeuble" "NomVoieImmeuble" "CodePostalImmeuble" "CommuneImmeuble" "DateDebutAcceptationCmdAcces" "DateMiseEnServiceCommercialeImmeuble")
if [ "$#" -ne 2 ]; then if [ "$#" -ne 2 ]; then
echo "Usage: ingest path-to-directory-containing-IPE-CSVs path-to-generated-db" echo "Usage: ingest path-to-directory-containing-IPE-CSVs path-to-generated-db"
echo "" echo ""
@ -35,13 +37,36 @@ for ipeFile in ${ipeFiles}; do
fi fi
if ! $firstFile; then if ! $firstFile; then
import_opt="-skip 1" import_opt="-skip 1"
else
header=$(head -n1 $ipeFile)
OLD_IFS=$IFS
export IFS=";"
idx=1
idx_to_keep=()
for column in $header; do
export IFS=$OLD_IFS
if [[ " ${NEEDED_COLUMNS[*]} " =~ " ${column} " ]]; then
idx_to_keep+=("$idx")
fi
idx=$((idx+1))
export IFS=";"
done
export IFS=$OLD_IFS
cut_idx_to_keep=$(IFS=',';echo "${idx_to_keep[*]}";IFS=$' \t\n')
echo " Column indexes that will be kept in csv files: $cut_idx_to_keep (matching columns ${NEEDED_COLUMNS[*]})"
fi fi
firstFile=false firstFile=false
fi fi
cat >> "${tmpSql}" <<EOF
.import ${import_opt} ${ipeFile} ipe
EOF
useIpeFile=${ipeFile}
if [[ "$cut_idx_to_keep" != "" ]]; then
cut -d';' -f$cut_idx_to_keep $ipeFile > ${ipeFile}.cut
useIpeFile=${ipeFile}.cut
fi
cat >> "${tmpSql}" <<EOF
.import ${import_opt} ${useIpeFile} ipe
EOF
rm -f ${ipeFile}.cut
done done
echo "" echo ""
@ -57,7 +82,7 @@ echo "[+] Create separate table with id immeuble index and its state."
cat > "${tmpSql}" <<EOF cat > "${tmpSql}" <<EOF
CREATE TABLE refimm (IdentifiantImmeuble text NOT NULL, EtatImmeuble text NOT NULL); CREATE TABLE refimm (IdentifiantImmeuble text NOT NULL, EtatImmeuble text NOT NULL);
CREATE UNIQUE INDEX idx_IdentifiantImmeuble on refimm (IdentifiantImmeuble); CREATE UNIQUE INDEX idx_IdentifiantImmeuble on refimm (IdentifiantImmeuble);
INSERT INTO refimm SELECT IdentifiantImmeuble,EtatImmeuble FROM ipe; INSERT OR REPLACE INTO refimm SELECT IdentifiantImmeuble,EtatImmeuble FROM ipe;
EOF EOF
sqlite3 "${fullDbPath}" < "${tmpSql}" sqlite3 "${fullDbPath}" < "${tmpSql}"