From ac3afadba547b4b9a4063da567acd6d2f4f74554 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 18 Jan 2023 18:46:39 +0100 Subject: add support for SQLite output --- Cargo.lock | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + README.adoc | 38 ++++++++++++++++++++++++++---- src/main.rs | 22 ++++++++++++++---- src/renderer.rs | 2 +- src/storage.rs | 56 +++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 180 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 03fb13b..c3d5c36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,6 +23,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.8", + "once_cell", + "version_check", +] + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -470,6 +481,18 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "flate2" version = "1.0.25" @@ -571,6 +594,18 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" +dependencies = [ + "hashbrown", +] [[package]] name = "heck" @@ -616,6 +651,7 @@ dependencies = [ "num-traits", "rayon", "roxmltree", + "rusqlite", ] [[package]] @@ -756,6 +792,16 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "libsqlite3-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -1073,6 +1119,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + [[package]] name = "plotters" version = "0.3.4" @@ -1284,6 +1336,20 @@ dependencies = [ "xmlparser", ] +[[package]] +name = "rusqlite" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -1567,6 +1633,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index 3c9b45d..bbbab82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ nalgebra = "0.31.4" num-traits = "0.2.15" rayon = "1.6.1" roxmltree = "0.17.0" +rusqlite = "0.28.0" [dev-dependencies] criterion = "0.4.0" diff --git a/README.adoc b/README.adoc index 8636e74..9c673d9 100644 --- a/README.adoc +++ b/README.adoc @@ -8,7 +8,7 @@ hittekaart - A GPX track heatmap generator. == SYNOPSIS ---- -hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] FILES... +hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] [--sqlite] FILES... ---- == INSTALLATION @@ -38,6 +38,29 @@ By default, the directory `tiles/` will be used as the root directory, so a tile would be saved as `tiles/{zoom}/{x}/{y}.png`. You can change this by using the `--output/-o` option. +=== SQLITE OUTPUT + +In order to overcome storage overhead when saving many small files (see the tip +and table further below), `hittekaart` can instead output a SQLite database +with the heatmap tile data. To do so, use the `--sqlite` command line option, +and control where the SQLite file should be placed with `--output`/`-o`. + +While this does not allow you to immediately serve the tiles with a HTTP +server, it does cut down on the wasted space on non-optimal file systems. + +The generated SQLite file will have one table with the following schema: + +[source,sql] +---- +CREATE TABLE tiles ( + zoom INTEGER, + x INTEGER, + y INTEGER, + data BLOB, + PRIMARY KEY (zoom, x, y) +); +---- + === INPUT FILES `hittekaart` expects GPX track files with the `.gpx` extension. It will parse @@ -140,8 +163,8 @@ system that is optimized for a large amount of small files, for example by setting a smaller block size. Many of the PNG images are smaller than 2 KiB (half a standard block); for those 50% of storage is wasted already. -Currently, `hittekaart` does not provide a built-in way to store or serve the -tiles more efficiently. +If you don't need the tiles in separate files, you can use the SQLite output +mode. For the same data as above, the SQLite database would be 73 MiB in size. ==== == OPTIONS @@ -161,7 +184,14 @@ The following options are supported: will automatically pick a default. `-o DIRECTORY`, `--output=DIRECTORY`:: - Generate the output tiles into the given directory. Defaults to `tiles/`. + Generate the output tiles into the given directory. Defaults to `tiles/` + when generating single files, and `tiles.sqlite` when storing the tiles in + a SQLite database. + +`--sqlite`:: + Output a single SQLite file with all tiles instead of saving each tile as a + separate PNG file. In this case, `-o` can be used to set the location of + the SQLite database. The schema is described above. == EXAMPLE diff --git a/src/main.rs b/src/main.rs index 7230dce..e5b835e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,7 @@ use color_eyre::{ use hittekaart::{ gpx::{self, Compression}, renderer, - storage::{Folder, Storage}, + storage::{Folder, Sqlite, Storage}, }; use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle}; use is_terminal::IsTerminal; @@ -36,9 +36,15 @@ struct Args { #[arg(long, short, default_value_t = 0)] threads: usize, - /// The output directory. Will be created if it does not exist. - #[arg(long, short, default_value = "tiles")] - output_directory: PathBuf, + /// The output directory. Will be created if it does not exist. Defaults to "tiles" for the + /// folder-based storage, and "tiles.sqlite" for the SQLite-based storage. + #[arg(long, short)] + output: Option, + + /// Store the tiles in a SQLite database. If given, `--output` will determine the SQLite + /// filename. + #[arg(long)] + sqlite: bool, } fn main() -> Result<()> { @@ -85,7 +91,13 @@ fn main() -> Result<()> { let tracks = tracks.into_iter().collect::>>()?; bar.finish(); - let mut storage = Folder::new(args.output_directory.clone()); + let mut storage: Box = if args.sqlite { + let output = args.output.unwrap_or_else(|| "tiles.sqlite".into()); + Box::new(Sqlite::connect(output)?) + } else { + let output = args.output.unwrap_or_else(|| "tiles".into()); + Box::new(Folder::new(output)) + }; storage.prepare()?; let multibar = MultiProgress::new(); diff --git a/src/renderer.rs b/src/renderer.rs index 4840e73..74a321c 100644 --- a/src/renderer.rs +++ b/src/renderer.rs @@ -139,7 +139,7 @@ fn colorize_tile(tile: &ImageBuffer, Vec>, max: u32) -> RgbaImage { /// rendering the next one. /// /// This has a way lower memory usage than [`colorize_heatcounter`]. -pub fn lazy_colorization Result<()> + Send + Sync>( +pub fn lazy_colorization Result<()> + Send>( layer: HeatCounter, mut save_callback: F, ) -> Result<()> { diff --git a/src/storage.rs b/src/storage.rs index fef8194..51a418e 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -1,4 +1,8 @@ -use color_eyre::{eyre::{bail, WrapErr}, Result}; +use color_eyre::{ + eyre::{bail, WrapErr}, + Result, +}; +use rusqlite::{params, Connection}; use std::{ fs, io::ErrorKind, @@ -65,3 +69,53 @@ impl Storage for Folder { Ok(()) } } + +#[derive(Debug)] +pub struct Sqlite { + connection: Connection, +} + +impl Sqlite { + pub fn connect>(file: P) -> Result { + let path = file.as_ref(); + if fs::metadata(path).is_ok() { + bail!("Path {path:?} already exists, refusing to open") + } + let connection = Connection::open(path)?; + Ok(Sqlite { connection }) + } +} + +impl Storage for Sqlite { + fn prepare(&mut self) -> Result<()> { + self.connection.execute( + "CREATE TABLE tiles ( + zoom INTEGER, + x INTEGER, + y INTEGER, + data BLOB, + PRIMARY KEY (zoom, x, y) + );", + (), + )?; + self.connection.execute("BEGIN;", ())?; + Ok(()) + } + + fn prepare_zoom(&mut self, _zoom: u32) -> Result<()> { + Ok(()) + } + + fn store(&mut self, zoom: u32, x: u64, y: u64, data: &[u8]) -> Result<()> { + self.connection.execute( + "INSERT INTO tiles (zoom, x, y, data) VALUES (?, ?, ?, ?)", + params![zoom, x, y, data], + )?; + Ok(()) + } + + fn finish(&mut self) -> Result<()> { + self.connection.execute("COMMIT;", ())?; + Ok(()) + } +} -- cgit v1.2.3