diff --git a/flake.lock b/flake.lock index 6d0e1c6..15cfefc 100644 --- a/flake.lock +++ b/flake.lock @@ -50,24 +50,11 @@ "type": "github" } }, - "quebec-osm-data": { - "flake": false, - "locked": { - "narHash": "sha256-Mm7tzput0a4BiUeafCtVpuzPKnT13NfU6zU7o/bJZ3k=", - "type": "file", - "url": "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf" - }, - "original": { - "type": "file", - "url": "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf" - } - }, "root": { "inputs": { "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", - "quebec-cities-poly": "quebec-cities-poly", - "quebec-osm-data": "quebec-osm-data" + "quebec-cities-poly": "quebec-cities-poly" } }, "systems": { diff --git a/flake.nix b/flake.nix index b4c6e1f..32debe8 100644 --- a/flake.nix +++ b/flake.nix @@ -4,21 +4,22 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; - quebec-osm-data = { - url = - "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf"; - flake = false; - }; quebec-cities-poly = { url = "github:didiercrunch/cities"; flake = false; }; }; - outputs = { self, nixpkgs, flake-utils, quebec-osm-data, quebec-cities-poly }: + outputs = { self, nixpkgs, flake-utils, quebec-cities-poly }: flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + # osm-data = pkgs.fetchurl { + # url = + # "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/quebec-241223.osm.pbf"; + # hash = "sha256-JkDJxODHnZQLK4+5Bjdu5GkbmHSK9TPt2ep6nsykCRg="; + # }; + poly2geojson = pkgs.rustPlatform.buildRustPackage rec { pname = "poly2geojson"; version = "0.1.3"; @@ -46,18 +47,95 @@ ''; }; - data-dir = pkgs.stdenv.mkDerivation { - name = "data-dir"; - src = ./scripts; - buildInputs = [ poly2geojson pkgs.jdk23 osmosis ]; - buildPhase = '' - mkdir -p cities/canada/quebec - cp ${quebec-cities-poly}/canada/quebec/* cities/canada/quebec - chmod u+w cities/canada/quebec - ${pkgs.babashka}/bin/bb to-geojson.clj --osm-file ${quebec-osm-data} --paths cities/canada/quebec/*.poly + custom-osmium-tool = pkgs.stdenv.mkDerivation rec { + pname = "osmium-tool"; + version = "1.16.0"; + + src = pkgs.fetchFromGitHub { + owner = "osmcode"; + repo = "osmium-tool"; + rev = "8ed94a5da2a24b233a77732bfaa8c75fc32ea505"; + sha256 = "sha256-YxjInUqcVfrBvAdBA/pd2J7NEBp6FvoQ97S+aW9x7jE="; + }; + + nativeBuildInputs = with pkgs; [ cmake installShellFiles pandoc ]; + + buildInputs = with pkgs; [ + boost + bzip2 + expat + libosmium + lz4 + protozero + zlib + nlohmann_json + ]; + + doCheck = false; + + postInstall = '' + installShellCompletion --zsh ../zsh_completion/_osmium ''; + }; + + geofabrik-base-url = + "https://static.home.all-dressed-programming.com/download.geofabrik.de/north-america/canada/"; + + osm-quebec."2020" = pkgs.fetchurl { + url = "${geofabrik-base-url}quebec-200101.osm.pbf"; + hash = + "sha256-458a33e10a3fc70884541a00e314471e9b7c92f352fe61ec909066478fb7bd09"; + }; + osm-quebec."2021" = pkgs.fetchurl { + url = "${geofabrik-base-url}quebec-210101.osm.pbf"; + hash = + "sha256-90a8bfe610951da56f315140256a4918fc9b98ac03bb441778f8a66271c128a9"; + }; + osm-quebec."2022" = pkgs.fetchurl { + url = "${geofabrik-base-url}quebec-220101.osm.pbf"; + hash = + "sha256-899526faf3220c7e9ec9fe2499e7ba3bccfe94dfefa328c0b7846dce6494e96a"; + }; + osm-quebec."2023" = pkgs.fetchurl { + url = "${geofabrik-base-url}quebec-230101.osm.pbf"; + hash = "sha256-AReFCvBIz42RBcOYgC25ZJaBXADW1lIzSpCI+vdrWX0="; + }; + osm-quebec."2024" = pkgs.fetchurl { + url = "${geofabrik-base-url}quebec-240101.osm.pbf"; + hash = "sha256-lcbn4mks4NlnwU3QwQOFdLICxFGapDC0PR/h0HlIR1E="; + }; + osm-quebec."2025" = pkgs.fetchurl { + url = "${geofabrik-base-url}quebec-250101.osm.pbf"; + hash = "sha256-QLIflsZuv8r2+mGVARrGkXyjWZ7qGzdSODpVka8iK8c="; + }; + + data-dir = osm-data: + pkgs.stdenv.mkDerivation { + name = "data-dir"; + src = ./scripts; + buildInputs = + [ poly2geojson pkgs.jdk23 osmosis custom-osmium-tool ]; + buildPhase = '' + mkdir -p cities/canada/quebec + cp ${quebec-cities-poly}/canada/quebec/* cities/canada/quebec + chmod u+w cities/canada/quebec + ${pkgs.babashka}/bin/bb to-geojson.clj --osm-file ${osm-data} --paths cities/canada/quebec/mon*.poly + ''; + installPhase = '' + cp -r cities $out + ''; + }; + + historical-data-dir = pkgs.stdenv.mkDerivation { + name = "historical-data-dir"; + src = ./scripts; + buildInputs = [ ]; + buildPhase = ""; installPhase = '' - cp -r cities $out + mkdir -p $out + ln -s ${data-dir osm-quebec."2023"} $out/2023 + ln -s ${data-dir osm-quebec."2024"} $out/2024 + ln -s ${data-dir osm-quebec."2025"} $out/2025 ''; }; @@ -77,6 +155,7 @@ default = data-dir; poly2geojson = poly2geojson; osmosis = osmosis; + historical-data-dir = historical-data-dir; }; devShell = pkgs.mkShell { name = "spacial-data-experiments"; @@ -86,7 +165,7 @@ sqlfluff poly2geojson gdal - osmosis + custom-osmium-tool python-virtual-env ]; }; diff --git a/scripts/generate_duckdb.py b/scripts/generate_duckdb.py index 716e327..803ee72 100644 --- a/scripts/generate_duckdb.py +++ b/scripts/generate_duckdb.py @@ -4,6 +4,7 @@ from pathlib import Path import click from tqdm import tqdm from posix import mkdir +from datetime import datetime, date sql_tmpl = jinja2.Template(''' LOAD spatial; @@ -14,6 +15,7 @@ CREATE TABLE IF NOT EXISTS city_road( points DOUBLE[][], tags MAP(VARCHAR, VARCHAR), highway TEXT, + extract_date Date, length DOUBLE ); @@ -59,7 +61,6 @@ CREATE TEMPORARY VIEW road_points_with_distance AS ( CREATE TEMPORARY VIEW data AS ( SELECT - '{{ city }}' AS city_name, relation_id AS relation_id, LIST([road_points_with_distance.lat, road_points_with_distance.lon]) AS points, ANY_VALUE(tags) AS tags, @@ -71,8 +72,15 @@ CREATE TEMPORARY VIEW data AS ( -INSERT INTO city_road(city_name, relation_id, points, tags, highway, length) ( - SELECT city_name, relation_id, points, tags, highway, length FROM data +INSERT INTO city_road(city_name, extract_date, relation_id, points, tags, highway, length) ( + SELECT '{{ city }}' AS city_name, + '{{extract_date}}' AS extract_date, + relation_id, + points, + tags, + highway, + length + FROM data ); ''') @@ -86,19 +94,23 @@ def sql_safe(w: str) -> str: return str(w).replace("'", r"\\'") -def append_data_to_duckdb(conn: duckdb.DuckDBPyConnection, osm_pbf_path: Path) -> None: +def append_data_to_duckdb(conn: duckdb.DuckDBPyConnection, osm_pbf_path: Path, extract_date: date) -> None: city = extract_city(osm_pbf_path) - sql = sql_tmpl.render(city=sql_safe(city), file_name=sql_safe(osm_pbf_path)) + sql = sql_tmpl.render(city=sql_safe(city), + file_name=sql_safe(osm_pbf_path), + extract_date=str(extract_date)) conn.execute(sql) -def append_data_to_duckdb_file(duckdb_file: Path, osm_pbf_path: Path) -> None: +def append_data_to_duckdb_file(duckdb_file: Path, osm_pbf_path: Path, extract_date: date) -> None: with duckdb.connect(str(duckdb_file)) as conn: - append_data_to_duckdb(conn, osm_pbf_path) + append_data_to_duckdb(conn, osm_pbf_path, extract_date) @click.command() @click.option('--output-file', type=click.Path(exists=False, file_okay=True, dir_okay=False, writable=True, path_type=Path)) @click.option('--input-dir', type=click.Path(exists=False, file_okay=False, dir_okay=True, readable=True, path_type=Path)) -def create_duckdb_file(output_file: Path, input_dir: Path) -> None: +@click.option('--date', '--date-end', type=click.DateTime(formats=["%Y-%m-%d"])) +def create_duckdb_file(output_file: Path, input_dir: Path, date: datetime) -> None: + output_file.parent.mkdir(parents=True, exist_ok=True) files = [f for f in input_dir.glob('**/*osm.pbf') if f.is_file()] @@ -106,7 +118,7 @@ def create_duckdb_file(output_file: Path, input_dir: Path) -> None: if "'" in str(osm_file): continue try: - append_data_to_duckdb_file(output_file, osm_file) + append_data_to_duckdb_file(output_file, osm_file, date.date()) except Exception as e: click.echo(f"Could not process {osm_file}") diff --git a/scripts/to-geojson.clj b/scripts/to-geojson.clj index c338283..d0cce53 100644 --- a/scripts/to-geojson.clj +++ b/scripts/to-geojson.clj @@ -2,35 +2,35 @@ '[clojure.string :as strings] '[babashka.cli :as cli]) +(defn osmosis-command [osm-file poly-file output-file] + ["osmosis" + "--read-pbf-fast" (str "file=" osm-file) + "--bounding-polygon" + (str"file=" poly-file) + "--write-pbf" + (str "file=" output-file)]) -(defn extract-data [file] - (let [data (slurp file) - geojson (shell {:out :string :continue true :in data} "poly2geojson") - city-name (subs file - (inc (strings/last-index-of file "/")) - (strings/last-index-of file "_"))] - {:originalFile file - :geojson (some-> geojson - :out - (json/parse-string true) - :geometry - json/generate-string) - :province "québec" - :cityName city-name})) +(defn osmium-command [osm-file poly-file output-file] + ["osmium" + "extract" + "-p" + poly-file + osm-file + "-o" + output-file]) -(defn extract-osm-pbf [osm-file file] - (let [output-file (strings/replace file ".poly" ".osm.pbf")] - (println "Processing " file " file to " output-file) - (shell {:continue true} - "osmosis" - "--read-pbf-fast" (str "file=" osm-file) - "--bounding-polygon" - (str"file=" file) - "--write-pbf" - (str "file=" output-file)))) +(defn extract-osm-pbf [osm-file poly-file] + (let [output-file (strings/replace poly-file ".poly" ".osm.pbf") + cmd (osmium-command osm-file poly-file output-file)] + (println "Processing " poly-file " file to " output-file) + (println cmd) + (apply shell (into [{:continue false}] cmd)))) (def opts (cli/parse-opts *command-line-args* {:coerce {:paths []}})) -(doall (pmap +(println "ok") +(println opts) + +(doall (map #(extract-osm-pbf (:osm-file opts) %) (:paths opts)))