aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Cargo.lock72
-rw-r--r--Cargo.toml1
-rw-r--r--README.adoc38
-rw-r--r--src/main.rs22
-rw-r--r--src/renderer.rs2
-rw-r--r--src/storage.rs56
6 files changed, 180 insertions, 11 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 03fb13b..c3d5c36 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,6 +24,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
+name = "ahash"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
+dependencies = [
+ "getrandom 0.2.8",
+ "once_cell",
+ "version_check",
+]
+
+[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -471,6 +482,18 @@ dependencies = [
]
[[package]]
+name = "fallible-iterator"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
+[[package]]
name = "flate2"
version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -571,6 +594,18 @@ name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+dependencies = [
+ "ahash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"
+dependencies = [
+ "hashbrown",
+]
[[package]]
name = "heck"
@@ -616,6 +651,7 @@ dependencies = [
"num-traits",
"rayon",
"roxmltree",
+ "rusqlite",
]
[[package]]
@@ -757,6 +793,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]]
+name = "libsqlite3-sys"
+version = "0.25.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
+dependencies = [
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
name = "linux-raw-sys"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1074,6 +1120,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
[[package]]
+name = "pkg-config"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
+
+[[package]]
name = "plotters"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1285,6 +1337,20 @@ dependencies = [
]
[[package]]
+name = "rusqlite"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
+dependencies = [
+ "bitflags",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "smallvec",
+]
+
+[[package]]
name = "rustc-demangle"
version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1568,6 +1634,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
name = "version_check"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 3c9b45d..bbbab82 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,6 +26,7 @@ nalgebra = "0.31.4"
num-traits = "0.2.15"
rayon = "1.6.1"
roxmltree = "0.17.0"
+rusqlite = "0.28.0"
[dev-dependencies]
criterion = "0.4.0"
diff --git a/README.adoc b/README.adoc
index 8636e74..9c673d9 100644
--- a/README.adoc
+++ b/README.adoc
@@ -8,7 +8,7 @@ hittekaart - A GPX track heatmap generator.
== SYNOPSIS
----
-hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] FILES...
+hittekaart [--output=...] [--min-zoom=...] [--max-zoom=...] [--threads=...] [--sqlite] FILES...
----
== INSTALLATION
@@ -38,6 +38,29 @@ By default, the directory `tiles/` will be used as the root directory, so a
tile would be saved as `tiles/{zoom}/{x}/{y}.png`. You can change this by using
the `--output/-o` option.
+=== SQLITE OUTPUT
+
+In order to overcome storage overhead when saving many small files (see the tip
+and table further below), `hittekaart` can instead output a SQLite database
+with the heatmap tile data. To do so, use the `--sqlite` command line option,
+and control where the SQLite file should be placed with `--output`/`-o`.
+
+While this does not allow you to immediately serve the tiles with a HTTP
+server, it does cut down on the wasted space on non-optimal file systems.
+
+The generated SQLite file will have one table with the following schema:
+
+[source,sql]
+----
+CREATE TABLE tiles (
+ zoom INTEGER,
+ x INTEGER,
+ y INTEGER,
+ data BLOB,
+ PRIMARY KEY (zoom, x, y)
+);
+----
+
=== INPUT FILES
`hittekaart` expects GPX track files with the `.gpx` extension. It will parse
@@ -140,8 +163,8 @@ system that is optimized for a large amount of small files, for example by
setting a smaller block size. Many of the PNG images are smaller than 2 KiB
(half a standard block); for those 50% of storage is wasted already.
-Currently, `hittekaart` does not provide a built-in way to store or serve the
-tiles more efficiently.
+If you don't need the tiles in separate files, you can use the SQLite output
+mode. For the same data as above, the SQLite database would be 73 MiB in size.
====
== OPTIONS
@@ -161,7 +184,14 @@ The following options are supported:
will automatically pick a default.
`-o DIRECTORY`, `--output=DIRECTORY`::
- Generate the output tiles into the given directory. Defaults to `tiles/`.
+ Generate the output tiles into the given directory. Defaults to `tiles/`
+ when generating single files, and `tiles.sqlite` when storing the tiles in
+ a SQLite database.
+
+`--sqlite`::
+ Output a single SQLite file with all tiles instead of saving each tile as a
+ separate PNG file. In this case, `-o` can be used to set the location of
+ the SQLite database. The schema is described above.
== EXAMPLE
diff --git a/src/main.rs b/src/main.rs
index 7230dce..e5b835e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -8,7 +8,7 @@ use color_eyre::{
use hittekaart::{
gpx::{self, Compression},
renderer,
- storage::{Folder, Storage},
+ storage::{Folder, Sqlite, Storage},
};
use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
use is_terminal::IsTerminal;
@@ -36,9 +36,15 @@ struct Args {
#[arg(long, short, default_value_t = 0)]
threads: usize,
- /// The output directory. Will be created if it does not exist.
- #[arg(long, short, default_value = "tiles")]
- output_directory: PathBuf,
+ /// The output directory. Will be created if it does not exist. Defaults to "tiles" for the
+ /// folder-based storage, and "tiles.sqlite" for the SQLite-based storage.
+ #[arg(long, short)]
+ output: Option<PathBuf>,
+
+ /// Store the tiles in a SQLite database. If given, `--output` will determine the SQLite
+ /// filename.
+ #[arg(long)]
+ sqlite: bool,
}
fn main() -> Result<()> {
@@ -85,7 +91,13 @@ fn main() -> Result<()> {
let tracks = tracks.into_iter().collect::<Result<Vec<_>>>()?;
bar.finish();
- let mut storage = Folder::new(args.output_directory.clone());
+ let mut storage: Box<dyn Storage + Send> = if args.sqlite {
+ let output = args.output.unwrap_or_else(|| "tiles.sqlite".into());
+ Box::new(Sqlite::connect(output)?)
+ } else {
+ let output = args.output.unwrap_or_else(|| "tiles".into());
+ Box::new(Folder::new(output))
+ };
storage.prepare()?;
let multibar = MultiProgress::new();
diff --git a/src/renderer.rs b/src/renderer.rs
index 4840e73..74a321c 100644
--- a/src/renderer.rs
+++ b/src/renderer.rs
@@ -139,7 +139,7 @@ fn colorize_tile(tile: &ImageBuffer<Luma<u8>, Vec<u8>>, max: u32) -> RgbaImage {
/// rendering the next one.
///
/// This has a way lower memory usage than [`colorize_heatcounter`].
-pub fn lazy_colorization<F: FnMut(RenderedTile) -> Result<()> + Send + Sync>(
+pub fn lazy_colorization<F: FnMut(RenderedTile) -> Result<()> + Send>(
layer: HeatCounter,
mut save_callback: F,
) -> Result<()> {
diff --git a/src/storage.rs b/src/storage.rs
index fef8194..51a418e 100644
--- a/src/storage.rs
+++ b/src/storage.rs
@@ -1,4 +1,8 @@
-use color_eyre::{eyre::{bail, WrapErr}, Result};
+use color_eyre::{
+ eyre::{bail, WrapErr},
+ Result,
+};
+use rusqlite::{params, Connection};
use std::{
fs,
io::ErrorKind,
@@ -65,3 +69,53 @@ impl Storage for Folder {
Ok(())
}
}
+
+#[derive(Debug)]
+pub struct Sqlite {
+ connection: Connection,
+}
+
+impl Sqlite {
+ pub fn connect<P: AsRef<Path>>(file: P) -> Result<Self> {
+ let path = file.as_ref();
+ if fs::metadata(path).is_ok() {
+ bail!("Path {path:?} already exists, refusing to open")
+ }
+ let connection = Connection::open(path)?;
+ Ok(Sqlite { connection })
+ }
+}
+
+impl Storage for Sqlite {
+ fn prepare(&mut self) -> Result<()> {
+ self.connection.execute(
+ "CREATE TABLE tiles (
+ zoom INTEGER,
+ x INTEGER,
+ y INTEGER,
+ data BLOB,
+ PRIMARY KEY (zoom, x, y)
+ );",
+ (),
+ )?;
+ self.connection.execute("BEGIN;", ())?;
+ Ok(())
+ }
+
+ fn prepare_zoom(&mut self, _zoom: u32) -> Result<()> {
+ Ok(())
+ }
+
+ fn store(&mut self, zoom: u32, x: u64, y: u64, data: &[u8]) -> Result<()> {
+ self.connection.execute(
+ "INSERT INTO tiles (zoom, x, y, data) VALUES (?, ?, ?, ?)",
+ params![zoom, x, y, data],
+ )?;
+ Ok(())
+ }
+
+ fn finish(&mut self) -> Result<()> {
+ self.connection.execute("COMMIT;", ())?;
+ Ok(())
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 07:35:24 +0000