From 77f9be057ed5ac65cbc3c27658e9c51f3cd94750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr>
Date: Mon, 21 Feb 2022 12:47:59 +0100
Subject: [PATCH] Add primitive IPE data ingestor.

---
 data-ingest/ingest | 59 ++++++++++++++++++++++++++++++++++++++++++++++
 notes.org          | 48 +++++++++++++++++++++++++++++++++++++
 shell.nix          |  9 +++++++
 3 files changed, 116 insertions(+)
 create mode 100755 data-ingest/ingest
 create mode 100644 notes.org
 create mode 100644 shell.nix

diff --git a/data-ingest/ingest b/data-ingest/ingest
new file mode 100755
index 0000000..e2d3082
--- /dev/null
+++ b/data-ingest/ingest
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -eau -o pipefail
+
+if [ "$#" -ne 2 ]; then
+    echo "Usage: ingest path-to-directory-containing-IPE-CSVs path-to-generated-db"
+    echo ""
+    exit 1
+fi
+fullIpeDirPath=$(realpath "${1}")
+fullDbPath=$(realpath "${2}")
+
+tmpSql=$(mktemp)
+clean_tmp () {
+    rm "${tmpSql}"
+}
+trap clean_tmp EXIT
+
+echo "[+] Ingesting IPE ata."
+echo "    The following files will be ingested."
+echo ""
+ipeFiles=$(find "${fullIpeDirPath}" -name '*.csv')
+cat > "${tmpSql}" <<EOF
+.separator ";"
+EOF
+
+for ipeFile in ${ipeFiles}; do
+echo "    ${ipeFile}"
+cat >> "${tmpSql}" <<EOF
+.import ${ipeFile} ipe
+EOF
+done
+echo ""
+
+sqlite3 "${fullDbPath}" < "${tmpSql}"
+
+echo "[+] Ingesting spatial data."
+
+cat > "${tmpSql}" <<EOF
+SELECT load_extension("mod_spatialite");
+SELECT InitSpatialMetaData();
+
+-- Despite the SELECT, we're actually creating a new ImmeubleGeoPoint
+-- column here. The spatialite API is a bit weird...
+SELECT AddGeometryColumn('ipe','ImmeubleGeoPoint',4326,'POINT');
+
+-- The geodata is expressed in RGF93 (SRID 2154). We need to project
+-- it to WSG84 to display it on a OSM map.
+UPDATE ipe SET ImmeubleGeoPoint =
+    Transform(
+        MakePoint(
+            CAST(CoordonneeImmeubleX as DOUBLE),
+            CAST(CoordonneeImmeubleY as DOUBLE),
+            2154)
+        ,4326);
+EOF
+
+sqlite3 "${fullDbPath}" < "${tmpSql}"
+
+echo "[+] SQLite database generated at ${fullDbPath}"
diff --git a/notes.org b/notes.org
new file mode 100644
index 0000000..d92428c
--- /dev/null
+++ b/notes.org
@@ -0,0 +1,48 @@
+* Notes
+
+  Vous pouvez trouver ici les notes plus ou moins en vrac que j'ai
+  prise lors de l'étude de ce projet. C'était la première fois que je
+  touchais a un GIS. Si c'est également votre cas, il y a
+  potentiellement des réponses a vos futures questions ici.
+
+
+** Load Spatialite
+  1. Have libspatialite in your LD_LIBRARY_PATH.
+  2. Run =select load_extension("mod_spatialite");= at SQLite startup.
+** How to Create a point
+  Example in this tutorial: https://gist.github.com/sixman9/805823
+
+** How to Test Points
+   Use the spatial index table.
+
+   /!\ DO NOT USE =CONTAINS= to filter out data. It's not as performant as the spatial index.
+
+   See https://www.gaia-gis.it/gaia-sins/spatialite-cookbook/html/rtree.html for what's happening under the hood.
+
+** Spatialite Getting Started
+
+   Petit test pour voir comment ça marche.
+
+  #+BEGIN_SRC
+  spatialite> CREATE TABLE points (id INTEGER PRIMARY KEY NOT NULL);
+  spatialite> SELECT AddGeometryColumn('points','dummypoints',3857,'POINT');
+  1
+  spatialite> INSERT INTO points (dummypoints) VALUES(Transform(MakePoint(384998.246399999,6366249.435399996366249.43539999,4964),3857));
+  spatialite> select * from points;
+  1|
+  spatialite> select AsText(dummypoints) from points;
+
+  #+END_SRC
+
+** SRID, quezako?
+  - Bonne référence: https://www.gaia-gis.it/gaia-sins/spatialite-cookbook/html/srid.html
+
+  En pratique, on spécifie le SRID a spatialite en utiliant le code EPSG
+
+  - Projection règlementaire en France: RGF93, EPSG:2154
+  - Mercator (OSM/Google Maps): WGS84, EPSG:4326
+
+  D'après https://geodesie.ign.fr/index.php?page=rgf93, le RGF93 est compatible avec Mercator.
+
+** DSPs to Ingest
+ADTH AISN BEFO BTHD CTYF EURE FI44 NATH NIVE NPDC SART SHSN SIEL SPTH VAUC
diff --git a/shell.nix b/shell.nix
new file mode 100644
index 0000000..afde95e
--- /dev/null
+++ b/shell.nix
@@ -0,0 +1,9 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+pkgs.mkShell {
+  buildInputs = [ pkgs.libspatialite ];
+  nativeBuildInputs = [ pkgs.sqlite pkgs.cargo pkgs.rustc ];
+  shellHook = ''
+    LD_LIBRARY_PATH=${pkgs.libspatialite}/lib:LD_LIBRARY_PATH
+  '';
+}