Axione-IPE-Viewer/data-ingest/ingest

94 lines
2.4 KiB
Bash
Executable File

#!/usr/bin/env bash
set -eau -o pipefail
if [ "$#" -ne 2 ]; then
echo "Usage: ingest path-to-directory-containing-IPE-CSVs path-to-generated-db"
echo ""
exit 1
fi
fullIpeDirPath=$(realpath "${1}")
fullDbPath=$(realpath "${2}")
tmpSql=$(mktemp)
clean_tmp () {
rm "${tmpSql}"
}
trap clean_tmp EXIT
echo "[+] Ingesting IPE data."
echo " The following files will be ingested."
echo ""
ipeFiles=$(find "${fullIpeDirPath}" -name '*.csv')
cat > "${tmpSql}" <<EOF
.separator ";"
EOF
firstFile=true
for ipeFile in ${ipeFiles}; do
echo " ${ipeFile}"
head -n1 $ipeFile | grep -q IdentifiantImmeuble && header=true || header=false
import_opt=""
if $firstFile || $header; then
if ! $header; then
echo "ERROR: first file ${ipeFile} does not have a good header"
exit 1
fi
if ! $firstFile; then
import_opt="-skip 1"
fi
firstFile=false
fi
cat >> "${tmpSql}" <<EOF
.import ${import_opt} ${ipeFile} ipe
EOF
done
echo ""
rc=0
sqlite3 "${fullDbPath}" < "${tmpSql}" 2>&1 | grep -v "filling the rest with NULL" || rc="${PIPESTATUS[0]}"
if [[ $rc -ne 0 ]]; then
echo "Error executing sqlite import"
exit "$rc"
fi
echo "[+] Create separate table with id immeuble index and its state."
cat > "${tmpSql}" <<EOF
CREATE TABLE refimm (IdentifiantImmeuble text NOT NULL, EtatImmeuble text NOT NULL);
CREATE UNIQUE INDEX idx_IdentifiantImmeuble on refimm (IdentifiantImmeuble);
INSERT INTO refimm SELECT IdentifiantImmeuble,EtatImmeuble FROM ipe;
EOF
sqlite3 "${fullDbPath}" < "${tmpSql}"
echo "[+] Ingesting spatial data."
cat > "${tmpSql}" <<EOF
SELECT load_extension("mod_spatialite");
SELECT InitSpatialMetaData();
-- Despite the SELECT, we're actually creating a new ImmeubleGeoPoint
-- column here. The spatialite API is a bit weird...
SELECT AddGeometryColumn('ipe','ImmeubleGeoPoint',4326,'POINT');
-- The geodata is expressed in RGF93 (SRID 2154). We need to project
-- it to WSG84 to display it on a OSM map.
UPDATE ipe SET ImmeubleGeoPoint =
Transform(
MakePoint(
CAST(CoordonneeImmeubleX as DOUBLE),
CAST(CoordonneeImmeubleY as DOUBLE),
2154)
,4326);
EOF
sqlite3 "${fullDbPath}" < "${tmpSql}"
echo "[+] Creating Rtree index (spatial index). We're almost done."
cat > "${tmpSql}" <<EOF
SELECT load_extension("mod_spatialite");
SELECT CreateSpatialIndex('ipe','ImmeubleGeoPoint');
EOF
sqlite3 "${fullDbPath}" < "${tmpSql}"
echo "[+] SQLite database generated at ${fullDbPath}"