117 lines
3.5 KiB
Bash
Executable file
117 lines
3.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -eau -o pipefail
|
|
|
|
NEEDED_COLUMNS=("IdentifiantImmeuble" "EtatImmeuble" "CoordonneeImmeubleX" "CoordonneeImmeubleY" "NumeroVoieImmeuble" "TypeVoieImmeuble" "NomVoieImmeuble" "CodePostalImmeuble" "CommuneImmeuble" "DateDebutAcceptationCmdAcces" "DateMiseEnServiceCommercialeImmeuble")
|
|
|
|
if [ "$#" -ne 2 ]; then
|
|
echo "Usage: ingest path-to-directory-containing-IPE-CSVs path-to-generated-db"
|
|
echo ""
|
|
exit 1
|
|
fi
|
|
fullIpeDirPath=$(realpath "${1}")
|
|
fullDbPath=$(realpath "${2}")
|
|
|
|
tmpSql=$(mktemp)
|
|
clean_tmp () {
|
|
rm "${tmpSql}"
|
|
}
|
|
trap clean_tmp EXIT
|
|
|
|
echo "[+] Ingesting IPE data."
|
|
echo " The following files will be ingested."
|
|
echo ""
|
|
ipeFiles=$(find "${fullIpeDirPath}" -name '*.csv')
|
|
cat > "${tmpSql}" <<EOF
|
|
.separator ";"
|
|
EOF
|
|
|
|
firstFile=true
|
|
for ipeFile in ${ipeFiles}; do
|
|
echo " ${ipeFile}"
|
|
head -n1 $ipeFile | grep -q IdentifiantImmeuble && header=true || header=false
|
|
import_opt=""
|
|
if $firstFile || $header; then
|
|
if ! $header; then
|
|
echo "ERROR: first file ${ipeFile} does not have a good header"
|
|
exit 1
|
|
fi
|
|
if ! $firstFile; then
|
|
import_opt="-skip 1"
|
|
else
|
|
header=$(head -n1 $ipeFile)
|
|
OLD_IFS=$IFS
|
|
export IFS=";"
|
|
idx=1
|
|
idx_to_keep=()
|
|
for column in $header; do
|
|
export IFS=$OLD_IFS
|
|
if [[ " ${NEEDED_COLUMNS[*]} " =~ " ${column} " ]]; then
|
|
idx_to_keep+=("$idx")
|
|
fi
|
|
idx=$((idx+1))
|
|
export IFS=";"
|
|
done
|
|
export IFS=$OLD_IFS
|
|
cut_idx_to_keep=$(IFS=',';echo "${idx_to_keep[*]}";IFS=$' \t\n')
|
|
echo " Column indexes that will be kept in csv files: $cut_idx_to_keep (matching columns ${NEEDED_COLUMNS[*]})"
|
|
fi
|
|
firstFile=false
|
|
fi
|
|
|
|
useIpeFile=${ipeFile}
|
|
if [[ "$cut_idx_to_keep" != "" ]]; then
|
|
cut -d';' -f$cut_idx_to_keep $ipeFile > ${ipeFile}.cut
|
|
useIpeFile=${ipeFile}.cut
|
|
fi
|
|
cat >> "${tmpSql}" <<EOF
|
|
.import ${import_opt} ${useIpeFile} ipe
|
|
EOF
|
|
done
|
|
|
|
echo ""
|
|
|
|
rc=0
|
|
sqlite3 "${fullDbPath}" < "${tmpSql}" 2>&1 | grep -v "filling the rest with NULL" || rc="${PIPESTATUS[0]}"
|
|
if [[ $rc -ne 0 ]]; then
|
|
echo "Error executing sqlite import"
|
|
exit "$rc"
|
|
fi
|
|
|
|
echo "[+] Create separate table with id immeuble index and its state."
|
|
cat > "${tmpSql}" <<EOF
|
|
CREATE TABLE refimm (IdentifiantImmeuble text NOT NULL, EtatImmeuble text NOT NULL, DateDebutAcceptationCmdAcces text NOT NULL);
|
|
CREATE UNIQUE INDEX idx_IdentifiantImmeuble on refimm (IdentifiantImmeuble);
|
|
INSERT OR REPLACE INTO refimm SELECT IdentifiantImmeuble,EtatImmeuble, DateDebutAcceptationCmdAcces FROM ipe;
|
|
EOF
|
|
sqlite3 "${fullDbPath}" < "${tmpSql}"
|
|
|
|
echo "[+] Ingesting spatial data."
|
|
|
|
cat > "${tmpSql}" <<EOF
|
|
SELECT load_extension("mod_spatialite");
|
|
SELECT InitSpatialMetaData();
|
|
|
|
-- Despite the SELECT, we're actually creating a new ImmeubleGeoPoint
|
|
-- column here. The spatialite API is a bit weird...
|
|
SELECT AddGeometryColumn('ipe','ImmeubleGeoPoint',4326,'POINT');
|
|
|
|
-- The geodata is expressed in RGF93 (SRID 2154). We need to project
|
|
-- it to WSG84 to display it on a OSM map.
|
|
UPDATE ipe SET ImmeubleGeoPoint =
|
|
Transform(
|
|
MakePoint(
|
|
CAST(CoordonneeImmeubleX as DOUBLE),
|
|
CAST(CoordonneeImmeubleY as DOUBLE),
|
|
2154)
|
|
,4326);
|
|
EOF
|
|
sqlite3 "${fullDbPath}" < "${tmpSql}"
|
|
|
|
echo "[+] Creating Rtree index (spatial index). We're almost done."
|
|
cat > "${tmpSql}" <<EOF
|
|
SELECT load_extension("mod_spatialite");
|
|
SELECT CreateSpatialIndex('ipe','ImmeubleGeoPoint');
|
|
EOF
|
|
sqlite3 "${fullDbPath}" < "${tmpSql}"
|
|
|
|
echo "[+] SQLite database generated at ${fullDbPath}"
|