little morning programming

flake.lock

@@ -50,24 +50,11 @@
         "type": "github"
       }
     },
-    "quebec-osm-data": {
-      "flake": false,
-      "locked": {
-        "narHash": "sha256-Mm7tzput0a4BiUeafCtVpuzPKnT13NfU6zU7o/bJZ3k=",
-        "type": "file",
-        "url": "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf"
-      },
-      "original": {
-        "type": "file",
-        "url": "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf"
-      }
-    },
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
         "nixpkgs": "nixpkgs",
-        "quebec-cities-poly": "quebec-cities-poly",
-        "quebec-osm-data": "quebec-osm-data"
+        "quebec-cities-poly": "quebec-cities-poly"
       }
     },
     "systems": {

flake.nix

@@ -4,21 +4,22 @@
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
-    quebec-osm-data = {
-      url =
-        "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf";
-      flake = false;
-    };
     quebec-cities-poly = {
       url = "github:didiercrunch/cities";
       flake = false;
     };
   };
-  outputs = { self, nixpkgs, flake-utils, quebec-osm-data, quebec-cities-poly }:
+  outputs = { self, nixpkgs, flake-utils, quebec-cities-poly }:
     flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
 
+        # osm-data = pkgs.fetchurl {
+        #   url =
+        #     "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf";
+        #   hash = "sha256-JkDJxODHnZQLK4+5Bjdu5GkbmHSK9TPt2ep6nsykCRg=";
+        # };
+
         poly2geojson = pkgs.rustPlatform.buildRustPackage rec {
           pname = "poly2geojson";
           version = "0.1.3";
@@ -46,21 +47,98 @@
           '';
         };
 
-        data-dir = pkgs.stdenv.mkDerivation {
+        custom-osmium-tool = pkgs.stdenv.mkDerivation rec {
+          pname = "osmium-tool";
+          version = "1.16.0";
+
+          src = pkgs.fetchFromGitHub {
+            owner = "osmcode";
+            repo = "osmium-tool";
+            rev = "8ed94a5da2a24b233a77732bfaa8c75fc32ea505";
+            sha256 = "sha256-YxjInUqcVfrBvAdBA/pd2J7NEBp6FvoQ97S+aW9x7jE=";
+          };
+
+          nativeBuildInputs = with pkgs; [ cmake installShellFiles pandoc ];
+
+          buildInputs = with pkgs; [
+            boost
+            bzip2
+            expat
+            libosmium
+            lz4
+            protozero
+            zlib
+            nlohmann_json
+          ];
+
+          doCheck = false;
+
+          postInstall = ''
+            installShellCompletion --zsh ../zsh_completion/_osmium
+          '';
+        };
+
+        geofabrik-base-url =
+          "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/";
+
+        osm-quebec."2020" = pkgs.fetchurl {
+          url = "${geofabrik-base-url}quebec-200101.osm.pbf";
+          hash =
+            "sha256-458a33e10a3fc70884541a00e314471e9b7c92f352fe61ec909066478fb7bd09";
+        };
+        osm-quebec."2021" = pkgs.fetchurl {
+          url = "${geofabrik-base-url}quebec-210101.osm.pbf";
+          hash =
+            "sha256-90a8bfe610951da56f315140256a4918fc9b98ac03bb441778f8a66271c128a9";
+        };
+        osm-quebec."2022" = pkgs.fetchurl {
+          url = "${geofabrik-base-url}quebec-220101.osm.pbf";
+          hash =
+            "sha256-899526faf3220c7e9ec9fe2499e7ba3bccfe94dfefa328c0b7846dce6494e96a";
+        };
+        osm-quebec."2023" = pkgs.fetchurl {
+          url = "${geofabrik-base-url}quebec-230101.osm.pbf";
+          hash = "sha256-AReFCvBIz42RBcOYgC25ZJaBXADW1lIzSpCI+vdrWX0=";
+        };
+        osm-quebec."2024" = pkgs.fetchurl {
+          url = "${geofabrik-base-url}quebec-240101.osm.pbf";
+          hash = "sha256-lcbn4mks4NlnwU3QwQOFdLICxFGapDC0PR/h0HlIR1E=";
+        };
+        osm-quebec."2025" = pkgs.fetchurl {
+          url = "${geofabrik-base-url}quebec-250101.osm.pbf";
+          hash = "sha256-QLIflsZuv8r2+mGVARrGkXyjWZ7qGzdSODpVka8iK8c=";
+        };
+
+        data-dir = osm-data:
+          pkgs.stdenv.mkDerivation {
             name = "data-dir";
             src = ./scripts;
-          buildInputs = [ poly2geojson pkgs.jdk23 osmosis ];
+            buildInputs =
+              [ poly2geojson pkgs.jdk23 osmosis custom-osmium-tool ];
             buildPhase = ''
               mkdir -p cities/canada/quebec
               cp ${quebec-cities-poly}/canada/quebec/* cities/canada/quebec
               chmod u+w cities/canada/quebec
-            ${pkgs.babashka}/bin/bb  to-geojson.clj --osm-file ${quebec-osm-data} --paths cities/canada/quebec/*.poly
+              ${pkgs.babashka}/bin/bb  to-geojson.clj --osm-file ${osm-data} --paths cities/canada/quebec/mon*.poly
             '';
             installPhase = ''
               cp -r cities $out
             '';
           };
 
+        historical-data-dir = pkgs.stdenv.mkDerivation {
+          name = "historical-data-dir";
+          src = ./scripts;
+          buildInputs = [ ];
+          buildPhase = "";
+          installPhase = ''
+            mkdir -p $out
+            ln -s ${data-dir osm-quebec."2023"} $out/2023
+            ln -s ${data-dir osm-quebec."2024"} $out/2024
+            ln -s ${data-dir osm-quebec."2025"} $out/2025
+          '';
+        };
+
         python-virtual-env = pkgs.python312.withPackages (ps: [
           ps.matplotlib
           ps.jupyter
@@ -77,6 +155,7 @@
           default = data-dir;
           poly2geojson = poly2geojson;
           osmosis = osmosis;
+          historical-data-dir = historical-data-dir;
         };
         devShell = pkgs.mkShell {
           name = "spacial-data-experiments";
@@ -86,7 +165,7 @@
             sqlfluff
             poly2geojson
             gdal
-            osmosis
+            custom-osmium-tool
             python-virtual-env
           ];
         };

scripts/generate_duckdb.py

@@ -4,6 +4,7 @@ from pathlib import Path
 import click
 from tqdm import tqdm
 from posix import mkdir
+from datetime import datetime, date
 
 sql_tmpl = jinja2.Template('''
 LOAD spatial;
@@ -14,6 +15,7 @@ CREATE TABLE IF NOT EXISTS city_road(
     points DOUBLE[][],
     tags MAP(VARCHAR, VARCHAR),
     highway TEXT,
+    extract_date Date,
     length DOUBLE
 );
 
@@ -59,7 +61,6 @@ CREATE TEMPORARY VIEW road_points_with_distance AS (
 
 CREATE TEMPORARY VIEW data AS (
     SELECT
-        '{{ city }}' AS city_name,
         relation_id AS relation_id,
         LIST([road_points_with_distance.lat, road_points_with_distance.lon]) AS points,
         ANY_VALUE(tags) AS tags,
@@ -71,8 +72,15 @@ CREATE TEMPORARY VIEW data AS (
 
 
 
-INSERT INTO city_road(city_name, relation_id, points, tags, highway, length) (
-    SELECT city_name, relation_id, points, tags, highway, length FROM data
+INSERT INTO city_road(city_name, extract_date, relation_id, points, tags, highway, length) (
+    SELECT '{{ city }}' AS city_name,
+           '{{extract_date}}' AS extract_date,
+           relation_id,
+           points,
+           tags,
+           highway,
+           length
+    FROM data
 );
 
 ''')
@@ -86,19 +94,23 @@ def sql_safe(w: str) -> str:
     return str(w).replace("'", r"\\'")
 
 
-def append_data_to_duckdb(conn: duckdb.DuckDBPyConnection, osm_pbf_path: Path) -> None:
+def append_data_to_duckdb(conn: duckdb.DuckDBPyConnection, osm_pbf_path: Path, extract_date: date) -> None:
     city = extract_city(osm_pbf_path)
-    sql = sql_tmpl.render(city=sql_safe(city), file_name=sql_safe(osm_pbf_path))
+    sql = sql_tmpl.render(city=sql_safe(city),
+                          file_name=sql_safe(osm_pbf_path),
+                          extract_date=str(extract_date))
     conn.execute(sql)
 
-def append_data_to_duckdb_file(duckdb_file: Path, osm_pbf_path: Path) -> None:
+def append_data_to_duckdb_file(duckdb_file: Path, osm_pbf_path: Path, extract_date: date) -> None:
     with duckdb.connect(str(duckdb_file)) as conn:
-        append_data_to_duckdb(conn, osm_pbf_path)
+        append_data_to_duckdb(conn, osm_pbf_path, extract_date)
 
 @click.command()
 @click.option('--output-file', type=click.Path(exists=False, file_okay=True, dir_okay=False, writable=True, path_type=Path))
 @click.option('--input-dir', type=click.Path(exists=False, file_okay=False, dir_okay=True, readable=True, path_type=Path))
-def create_duckdb_file(output_file: Path, input_dir: Path) -> None:
+@click.option('--date', '--date-end', type=click.DateTime(formats=["%Y-%m-%d"]))
+def create_duckdb_file(output_file: Path, input_dir: Path, date: datetime) -> None:
+
     output_file.parent.mkdir(parents=True, exist_ok=True)
     files = [f for f in input_dir.glob('**/*osm.pbf') if f.is_file()]
 
@@ -106,7 +118,7 @@ def create_duckdb_file(output_file: Path, input_dir: Path) -> None:
         if "'" in str(osm_file):
             continue
         try:
-            append_data_to_duckdb_file(output_file, osm_file)
+            append_data_to_duckdb_file(output_file, osm_file, date.date())
         except Exception as e:
             click.echo(f"Could not process {osm_file}")
 

scripts/to-geojson.clj

@@ -2,35 +2,35 @@
          '[clojure.string :as strings]
          '[babashka.cli :as cli])
 
-
-(defn extract-data [file]
-    (let [data (slurp file)
-          geojson (shell {:out :string :continue true :in data} "poly2geojson")
-          city-name (subs file
-                          (inc (strings/last-index-of file "/"))
-                          (strings/last-index-of file "_"))]
-        {:originalFile file
-         :geojson (some-> geojson
-                      :out
-                      (json/parse-string true)
-                      :geometry
-                      json/generate-string)
-         :province "québec"
-         :cityName city-name}))
-
-(defn extract-osm-pbf [osm-file file]
-    (let [output-file (strings/replace file ".poly" ".osm.pbf")]
-        (println "Processing " file " file to " output-file)
-        (shell {:continue true}
-            "osmosis"
+(defn osmosis-command [osm-file poly-file output-file]
+    ["osmosis"
     "--read-pbf-fast" (str "file=" osm-file)
     "--bounding-polygon"
-            (str"file=" file)
+    (str"file=" poly-file)
     "--write-pbf"
-            (str "file=" output-file))))
+    (str "file=" output-file)])
+
+(defn osmium-command [osm-file poly-file output-file]
+    ["osmium"
+    "extract"
+    "-p"
+    poly-file
+    osm-file
+    "-o"
+    output-file])
+
+(defn extract-osm-pbf [osm-file poly-file]
+    (let [output-file (strings/replace poly-file ".poly" ".osm.pbf")
+          cmd (osmium-command osm-file poly-file output-file)]
+        (println "Processing " poly-file " file to " output-file)
+        (println cmd)
+        (apply shell (into [{:continue false}] cmd))))
 
 (def opts (cli/parse-opts *command-line-args* {:coerce {:paths []}}))
 
-(doall (pmap
+(println "ok")
+(println opts)
+
+(doall (map
     #(extract-osm-pbf (:osm-file opts) %)
     (:paths opts)))