diff options
| -rw-r--r-- | .woodpecker/tests.yaml | 1 | ||||
| -rw-r--r-- | CHANGELOG.adoc | 17 | ||||
| -rw-r--r-- | README.adoc | 61 | ||||
| -rw-r--r-- | hittekaart-cli/src/main.rs | 39 | ||||
| -rw-r--r-- | hittekaart-py/hittekaart_py/hittekaart_py.pyi | 5 | ||||
| -rw-r--r-- | hittekaart-py/src/lib.rs | 32 | ||||
| -rw-r--r-- | hittekaart/src/lib.rs | 2 | ||||
| -rw-r--r-- | hittekaart/src/storage.rs | 190 |
8 files changed, 288 insertions, 59 deletions
diff --git a/.woodpecker/tests.yaml b/.woodpecker/tests.yaml index 2c9c5b2..4ca95a0 100644 --- a/.woodpecker/tests.yaml +++ b/.woodpecker/tests.yaml @@ -1,6 +1,5 @@ when: - event: push - branch: master steps: - name: test diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc new file mode 100644 index 0000000..eeb2bc0 --- /dev/null +++ b/CHANGELOG.adoc @@ -0,0 +1,17 @@ += CHANGELOG + +All notable changes to this project will be documented in this file. + +== UNRELEASED + +Added: + +- The `storage::OsmAnd` and `storage::MbTiles` formats + +Removed: + +- The `storage::Sqlite` struct (`storage::OsmAnd` can be used instead) + +== 0.1.0 (2025-11-29) + +Initial release. diff --git a/README.adoc b/README.adoc index bb779b5..0692a05 100644 --- a/README.adoc +++ b/README.adoc @@ -8,7 +8,7 @@ hittekaart - A GPX track heatmap generator. == SYNOPSIS ---- -hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] [--sqlite] FILES... +hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] [--format=...] FILES... ---- == INSTALLATION @@ -42,24 +42,54 @@ the `--output/-o` option. In order to overcome storage overhead when saving many small files (see the tip and table further below), `hittekaart` can instead output a SQLite database -with the heatmap tile data. To do so, use the `--sqlite` command line option, -and control where the SQLite file should be placed with `--output`/`-o`. +with the heatmap tile data. Two formats are available for this: -While this does not allow you to immediately serve the tiles with a HTTP -server, it does cut down on the wasted space on non-optimal file systems. - -The generated SQLite file will have one table with the following schema: +*OSMAND* compatible output can be generated by the `--format=osm-and` option. +In this case, `--output`/`-o` can be used to control the output filename. The +OsmAnd format is relatively simple, its main data lies in a single `tiles` +table: [source,sql] ----- +----- CREATE TABLE tiles ( - zoom INTEGER, x INTEGER, y INTEGER, - data BLOB, - PRIMARY KEY (zoom, x, y) + z INTEGER, + image BLOB, + time INTEGER, + PRIMARY KEY (x, y, z) ); ----- +----- + +The `time` column is always `NULL`. + +To use the map in the app, make sure to: + +* Give it a `.sqlitedb` extension, and +* place it under `Android/data/net.osmand.plus/files/tiles/`. + +For a full reference of the format, see +https://osmand.net/docs/technical/osmand-file-formats/osmand-sqlite/. + +*MBTILES* files can be generated by the `--format=mb-tiles` option. Like OsmAnd +files, MBTiles are also SQLite databases. In this case, the main table has the +following scheme: + +[source,sql] +----- +CREATE TABLE tiles ( + zoom_level INTEGER, + tile_column INTEGER, + tile_row INTEGER, + tile_data BLOB, + PRIMARY KEY (zoom_level, tile_column, tile_row) +); +----- + +Note that the `tile_row` is inverted. + +For a full reference of the format, see +https://github.com/mapbox/mbtiles-spec/blob/master/1.3/spec.md. === INPUT FILES @@ -204,10 +234,9 @@ The following options are supported: when generating single files, and `tiles.sqlite` when storing the tiles in a SQLite database. -`--sqlite`:: - Output a single SQLite file with all tiles instead of saving each tile as a - separate PNG file. In this case, `-o` can be used to set the location of - the SQLite database. The schema is described above. +`--format=FORMAT`:: + Sets the output format (folder, osm-and, mb-tiles). See the sections above + on OUTPUT FORMAT and SQLITE OUTPUT for more information. `-m MODE`, `--mode=MODE`:: Sets the overlay generation mode (heatmap, marktile, tilehunter). See diff --git a/hittekaart-cli/src/main.rs b/hittekaart-cli/src/main.rs index 11133af..e4a227c 100644 --- a/hittekaart-cli/src/main.rs +++ b/hittekaart-cli/src/main.rs @@ -7,8 +7,8 @@ use color_eyre::{ }; use hittekaart::{ gpx::{self, Compression}, - renderer::{self, heatmap, marktile, tilehunt, Renderer}, - storage::{Folder, Sqlite, Storage}, + renderer::{self, Renderer, heatmap, marktile, tilehunt}, + storage::{Folder, MbTiles, OsmAnd, Storage}, }; use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; use is_terminal::IsTerminal; @@ -25,6 +25,14 @@ enum Mode { Tilehunter, } +/// Output format. +#[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum Format { + Folder, + OsmAnd, + MbTiles, +} + #[derive(Parser, Debug, Clone)] #[command(author, version, about)] struct Args { @@ -45,14 +53,14 @@ struct Args { threads: usize, /// The output directory. Will be created if it does not exist. Defaults to "tiles" for the - /// folder-based storage, and "tiles.sqlite" for the SQLite-based storage. + /// folder-based storage, "tiles.sqlitedb" for the OsmAnd format, and "tiles.mbtiles" for the + /// MBTiles format. #[arg(long, short)] output: Option<PathBuf>, - /// Store the tiles in a SQLite database. If given, `--output` will determine the SQLite - /// filename. + /// Output format. #[arg(long)] - sqlite: bool, + format: Format, /// Generation mode. #[arg(value_enum, long, short, default_value_t = Mode::Heatmap)] @@ -115,12 +123,19 @@ fn run<R: Renderer>(renderer: R, args: Args) -> Result<()> { let tracks = tracks.into_iter().collect::<Result<Vec<_>>>()?; bar.finish(); - let mut storage: Box<dyn Storage + Send> = if args.sqlite { - let output = args.output.unwrap_or_else(|| "tiles.sqlite".into()); - Box::new(Sqlite::connect(output)?) - } else { - let output = args.output.unwrap_or_else(|| "tiles".into()); - Box::new(Folder::new(output)) + let mut storage: Box<dyn Storage + Send> = match args.format { + Format::Folder => { + let output = args.output.unwrap_or_else(|| "tiles".into()); + Box::new(Folder::new(output)) + } + Format::MbTiles => { + let output = args.output.unwrap_or_else(|| "tiles.mbtiles".into()); + Box::new(MbTiles::open(output)?) + } + Format::OsmAnd => { + let output = args.output.unwrap_or_else(|| "tiles.sqlitedb".into()); + Box::new(OsmAnd::open(output)?) + } }; storage.prepare()?; diff --git a/hittekaart-py/hittekaart_py/hittekaart_py.pyi b/hittekaart-py/hittekaart_py/hittekaart_py.pyi index 0efe011..b3d110c 100644 --- a/hittekaart-py/hittekaart_py/hittekaart_py.pyi +++ b/hittekaart-py/hittekaart_py/hittekaart_py.pyi @@ -14,7 +14,10 @@ class Storage: def Folder(path: bytes) -> "Storage": ... @staticmethod - def Sqlite(path: bytes) -> "Storage": ... + def OsmAnd(path: bytes) -> "Storage": ... + + @staticmethod + def MbTiles(path: bytes) -> "Storage": ... class HeatmapRenderer: diff --git a/hittekaart-py/src/lib.rs b/hittekaart-py/src/lib.rs index c0f3f6c..7a88bab 100644 --- a/hittekaart-py/src/lib.rs +++ b/hittekaart-py/src/lib.rs @@ -87,7 +87,8 @@ impl Track { #[derive(Debug, Clone, PartialEq, Eq)] enum StorageType { Folder(PathBuf), - Sqlite(PathBuf), + OsmAnd(PathBuf), + MbTiles(PathBuf), } /// Represents a storage target. @@ -113,22 +114,28 @@ impl Storage { Storage(StorageType::Folder(path.into())) } - /// Output to the given SQLite file. - /// - /// This will create a single table 'tiles' with the columns 'zoom', 'x', 'y' and 'data'. + /// Output to the given SQLite file in a OsmAnd compatible format. /// /// Note that you cannot "append" to an existing database, it must be a non-existing file. #[staticmethod] - #[pyo3(name = "Sqlite")] - fn sqlite(path: &[u8]) -> Self { + #[pyo3(name = "OsmAnd")] + fn osmand(path: &[u8]) -> Self { + let path = OsStr::from_bytes(path); + Storage(StorageType::OsmAnd(path.into())) + } + + #[staticmethod] + #[pyo3(name = "MbTiles")] + fn mbtiles(path: &[u8]) -> Self { let path = OsStr::from_bytes(path); - Storage(StorageType::Sqlite(path.into())) + Storage(StorageType::MbTiles(path.into())) } fn __repr__(&self) -> String { match self.0 { StorageType::Folder(ref path) => format!("<Storage.Folder path='{}'>", path.display()), - StorageType::Sqlite(ref path) => format!("<Storage.Sqlite path='{}'>", path.display()), + StorageType::OsmAnd(ref path) => format!("<Storage.OsmAnd path='{}'>", path.display()), + StorageType::MbTiles(ref path) => format!("<Storage.MbTiles path='{}'>", path.display()), } } } @@ -140,8 +147,13 @@ impl Storage { let storage = hittekaart::storage::Folder::new(path.clone()); Ok(Box::new(storage)) } - StorageType::Sqlite(ref path) => { - let storage = hittekaart::storage::Sqlite::connect(path.clone()) + StorageType::OsmAnd(ref path) => { + let storage = hittekaart::storage::OsmAnd::open(path.clone()) + .map_err(|e| err_to_py(&e))?; + Ok(Box::new(storage)) + } + StorageType::MbTiles(ref path) => { + let storage = hittekaart::storage::MbTiles::open(path.clone()) .map_err(|e| err_to_py(&e))?; Ok(Box::new(storage)) } diff --git a/hittekaart/src/lib.rs b/hittekaart/src/lib.rs index 5d71d2c..a885480 100644 --- a/hittekaart/src/lib.rs +++ b/hittekaart/src/lib.rs @@ -31,7 +31,7 @@ //! )?; //! // Before we run the second step, we set up the storage system. You can save the bytes in any //! // way you want, but there are convenience functions in this crate: -//! let mut store = storage::Sqlite::connect(":memory:")?; +//! let mut store = storage::OsmAnd::open(":memory:")?; //! store.prepare()?; //! store.prepare_zoom(ZOOM)?; //! // Now we're ready to do the actual colorizing diff --git a/hittekaart/src/storage.rs b/hittekaart/src/storage.rs index c21627f..137bf5d 100644 --- a/hittekaart/src/storage.rs +++ b/hittekaart/src/storage.rs @@ -97,49 +97,102 @@ impl Storage for Folder { } } -/// SQLite based storage. +/// OsmAnd compatible storage (SQLite backed). /// -/// This stores tiles in a SQLite database. The database will have a single table: +/// This stores tiles into a SQLite database with the following tables: /// /// ```sql /// CREATE TABLE tiles ( -/// zoom INTEGER, /// x INTEGER, /// y INTEGER, -/// data BLOB, -/// PRIMARY KEY (zoom, x, y) +/// z INTEGER, +/// image BLOB, +/// time INTEGER, +/// PRIMARY KEY (x, y, z) +/// ); +/// CREATE TABLE info ( +/// url TEXT, +/// randoms TEXT, +/// referer TEXT, +/// rule TEXT, +/// useragent TEXT, +/// minzoom INTEGER, +/// maxzoom INTEGER, +/// ellipsoid INTEGER, +/// inverted_y INTEGER, +/// timecolumn TEXT, +/// expireminutes INTEGER, +/// tilenumbering TEXT, +/// tilesize INTEGER /// ); /// ``` +/// +/// The tile data lands in the `tiles` table, with `x`, `y` and `z` being the x/y/zoom coordinates. +/// Note that the coordinates and zoom are not inverted, the same values as with the folder storage +/// will be used. The `time` column is set to `NULL`. +/// +/// The `info` table contains metadata that is required by OsmAnd to properly load the tiles. +/// +/// To use the resulting file, give it an `.sqlitedb` extension and place it under +/// `Android/data/net.osmand.plus/files/tiles/` on your phone. +/// +/// See <https://osmand.net/docs/technical/osmand-file-formats/osmand-sqlite/> for the technical +/// reference. #[derive(Debug)] -pub struct Sqlite { +pub struct OsmAnd { connection: Connection, + min_zoom: u32, + max_zoom: u32, } -impl Sqlite { - /// Create a new SQLite backed tile store. +impl OsmAnd { + /// Create a new OsmAnd tile store. /// /// The database will be saved at the given location. Note that the database must not yet /// exist. - pub fn connect<P: AsRef<Path>>(file: P) -> Result<Self> { + pub fn open<P: AsRef<Path>>(file: P) -> Result<Self> { let path = file.as_ref(); if fs::metadata(path).is_ok() { return Err(Error::OutputAlreadyExists(path.to_path_buf())); } let connection = Connection::open(path)?; - Ok(Sqlite { connection }) + Ok(OsmAnd { + connection, + min_zoom: u32::MAX, + max_zoom: u32::MIN, + }) } } -impl Storage for Sqlite { +impl Storage for OsmAnd { fn prepare(&mut self) -> Result<()> { self.connection.execute( + "CREATE TABLE info ( + url TEXT, + randoms TEXT, + referer TEXT, + rule TEXT, + useragent TEXT, + minzoom INTEGER, + maxzoom INTEGER, + ellipsoid INTEGER, + inverted_y INTEGER, + timecolumn TEXT, + expireminutes INTEGER, + tilenumbering TEXT, + tilesize INTEGER + );", + (), + )?; + self.connection.execute( "CREATE TABLE tiles ( - zoom INTEGER, - x INTEGER, - y INTEGER, - data BLOB, - PRIMARY KEY (zoom, x, y) - );", + x INTEGER, + y INTEGER, + z INTEGER, + image BLOB, + time INTEGER, + PRIMARY KEY (x, y, z) + );", (), )?; self.connection.execute("BEGIN;", ())?; @@ -152,13 +205,114 @@ impl Storage for Sqlite { fn store(&mut self, zoom: u32, x: u64, y: u64, data: &[u8]) -> Result<()> { self.connection.execute( - "INSERT INTO tiles (zoom, x, y, data) VALUES (?, ?, ?, ?)", + "INSERT INTO tiles (z, x, y, image) VALUES (?, ?, ?, ?)", params![zoom, x, y, data], )?; + self.min_zoom = self.min_zoom.min(zoom); + self.max_zoom = self.max_zoom.max(zoom); Ok(()) } fn finish(&mut self) -> Result<()> { + self.connection.execute( + "INSERT INTO info (inverted_y, tilenumbering, minzoom, maxzoom) VALUES (?, ?, ?, ?);", + params![0, "", self.min_zoom, self.max_zoom], + )?; + self.connection.execute("COMMIT;", ())?; + Ok(()) + } +} + +/// MBTiles storage (SQLite backed). +/// +/// This stores tiles into a SQLite database with the following tables: +/// +/// ```sql +/// CREATE TABLE metadata (name TEXT, value TEXT); +/// CREATE TABLE tiles ( +/// zoom_level INTEGER, +/// tile_column INTEGER, +/// tile_row INTEGER, +/// tile_data BLOB, +/// PRIMARY KEY (zoom_level, tile_column, tile_row) +/// ); +/// ``` +/// +/// The tiles end up in the `tiles` table. Note that the `y` coordinate (`tile_row`) is inverted, +/// meaning that `tile_row = 2^zoom - 1 - y`. +/// +/// The metadata table will contain two rows, one with `name = "name"` and one with `name = +/// "format"`. You can set a custom name/title of the map with the following SQL statement: +/// +/// ```sql +/// UPDATE metadata SET value = "My cool heatmap!" WHERE name = "name"; +/// ``` +#[derive(Debug)] +pub struct MbTiles { + connection: Connection, +} + +impl MbTiles { + /// Create a new MBTiles tile store. + /// + /// The database will be saved at the given location. Note that the database must not yet + /// exist. + pub fn open<P: AsRef<Path>>(file: P) -> Result<Self> { + let path = file.as_ref(); + if fs::metadata(path).is_ok() { + return Err(Error::OutputAlreadyExists(path.to_path_buf())); + } + let connection = Connection::open(path)?; + Ok(MbTiles { connection }) + } +} + +impl Storage for MbTiles { + fn prepare(&mut self) -> Result<()> { + self.connection.execute( + "PRAGMA application_id = 0x4d504258;", + (), + )?; + self.connection.execute( + "CREATE TABLE metadata (name TEXT, value TEXT);", + (), + )?; + self.connection.execute( + "CREATE TABLE tiles ( + zoom_level INTEGER, + tile_column INTEGER, + tile_row INTEGER, + tile_data BLOB, + PRIMARY KEY (zoom_level, tile_column, tile_row) + );", + (), + )?; + self.connection.execute("BEGIN;", ())?; + Ok(()) + } + + fn prepare_zoom(&mut self, _zoom: u32) -> Result<()> { + Ok(()) + } + + fn store(&mut self, zoom: u32, x: u64, y: u64, data: &[u8]) -> Result<()> { + let inverted_y = 2u64.pow(zoom) - 1 - y; + self.connection.execute( + "INSERT INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)", + params![zoom, x, inverted_y, data], + )?; + Ok(()) + } + + fn finish(&mut self) -> Result<()> { + self.connection.execute( + "INSERT INTO metadata (name, value) VALUES (?, ?);", + params!["name", "Heatmap"], + )?; + self.connection.execute( + "INSERT INTO metadata (name, value) VALUES (?, ?);", + params!["format", "png"], + )?; self.connection.execute("COMMIT;", ())?; Ok(()) } |