Merge branch 'external-languages'

11 files changed, 252 insertions, 35 deletions

diff --git a/.gitignore b/.gitignore
index a3cbe22..259f717 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,4 @@ test
 .venv/
 /data
 poetry.toml
+/language-packs
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 1fe2185..0000000
--- a/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-PYBABEL=".venv/bin/pybabel"
-LOCALE="en"
-
-babel-extract:
-	$(PYBABEL) extract -F babel.cfg -o fietsboek/locale/fietslog.pot --input-dirs=fietsboek
-
-babel-update:
-	$(PYBABEL) update -d fietsboek/locale -l $(LOCALE) -i fietsboek/locale/fietslog.pot
-
-babel-compile:
-	$(PYBABEL) compile -d fietsboek/locale -l $(LOCALE) -i fietsboek/locale/$(LOCALE)/LC_MESSAGES/messages.po
-
-babel-init:
-	$(PYBABEL) init -d fietsboek/locale -l $(LOCALE) -i fietsboek/locale/fietslog.pot
-
-.PHONY: babel-extract babel-update babel-compile
diff --git a/doc/administration/configuration.rst b/doc/administration/configuration.rst
index 379c683..5f0e3d4 100644
--- a/doc/administration/configuration.rst
+++ b/doc/administration/configuration.rst
@@ -72,6 +72,18 @@ Currently, Fietsboek ships with English ("en") and German ("de"). Removing a
 language from this list will make it unavailable. If you create a custom
 language locally, make sure to add it to this list here!
 
+Fietsboek also allows you to install "language packs", providing languages from
+third-party sources. Language packs are normal Python packages that must be
+installed via the package manager (e.g. by using ``pip`` in the same
+environment that you installed Fietsboek in), and then their names can be
+listed as ``fietsboek.language_packs`` in the configuration. Note that you must
+still add the locales to ``available_locales`` for them to work.
+
+.. warning::
+
+    Since language packs are just Python packages, they can contain and execute
+    arbitrary code. Do not install untrusted language packs.
+
 Database Settings
 -----------------
 
diff --git a/doc/developer.rst b/doc/developer.rst
index 6b99851..b47b0a4 100644
--- a/doc/developer.rst
+++ b/doc/developer.rst
@@ -6,6 +6,7 @@ Developer Guide
     :caption: Contents
 
     developer/localize
+    developer/language-pack
 
 This guide contains information for developers that want to modify or extend
 Fietsboek. This includes information on how to localize Fietsboek to new
diff --git a/doc/developer/language-pack.rst b/doc/developer/language-pack.rst
new file mode 100644
index 0000000..71cf9e8
--- /dev/null
+++ b/doc/developer/language-pack.rst
@@ -0,0 +1,140 @@
+Language Packs
+==============
+
+Fietsboek allows language files to be distributed as third-party-packages, such
+that you can localize Fietsboek without needing to add the files to the main
+repository. This way, language packs can be updated independently of Fietsboek,
+and packs can be provided for languages that are not "officially supported".
+
+The basic concept behind language packs is the same as in normal
+:doc:`localization <localize>`, except that the files are in a different Python
+package. If you are familiar with the ``gettext`` machinery, you can simply
+create a Python package that has a ``locale/`` folder in the same structure as
+Fietsboek's.
+
+In the following guide, we will create an example language pack for Dutch
+(``"nl"``).
+
+Preqrequisites
+-------------------
+
+In order to create a translation package, you need access to the source code of
+Fietsboek and Babel_. The easiest way to get both is to clone the git
+repository, and use a virtual environment to install babel:
+
+.. code-block:: bash
+
+    git clone https://gitlab.com/dunj3/fietsboek.git
+    FB_PATH=$PWD/fietsboek/fietsboek
+    virtualenv /tmp/babel
+    /tmp/babel/pip install Babel
+    BABEL=/tmp/babel/bin/pybabel
+
+
+Creating a Package
+------------------
+
+Language packs are normal Python packages, as such, you can use any tool you'd
+like to create a pack --- as long as you make sure to include the package data.
+In our example, we will use Poetry_, as that is what Fietsboek itself uses.
+
+To create a basic package, we will pick a path in which to create our pack:
+:file:`~/fb-i18n-nl` and create the basic structure:
+
+.. code-block:: bash
+
+    PACK_PATH=~/fb-i18n-nl
+    mkdir -p $PACK_PATH/fb_i18n_nl
+    touch $PACK_PATH/fb_i18n_nl/__init__.py
+    mkdir $PACK_PATH/fb_i18n_nl/locale
+
+And the content of :file:`$PACK_PATH/pyproject.toml`:
+
+.. code-block:: toml
+
+    [tool.poetry]
+    name = "fb-i18n-nl"
+    version = "0.1.0"
+    description = ""
+    authors = ["Jan Modaal <jan.modaal@example.com>"]
+    packages = [{include = "fb_i18n_nl"}]
+
+    [tool.poetry.dependencies]
+    python = "^3.7"
+
+    [build-system]
+    requires = ["poetry-core"]
+    build-backend = "poetry.core.masonry.api"
+
+.. note::
+
+    You can also use ``poetry new`` to create the basic package, just make sure
+    to remove the unnecessary files.
+
+Initializing the Language
+-------------------------
+
+This is the same process as for built-in languages, except that you need to
+adjust the paths:
+
+.. code-block:: bash
+
+    $BABEL init -d $PACK_PATH/fb_i18n_nl/locale -l nl -i $FB_PATH/locale/fietslog.pot
+
+You can also copy over the English HTML files, which makes it easier to
+translate them:
+
+.. code-block:: bash
+
+    cp -rv $FB_PATH/locale/en/html
+
+Translating & Compiling
+-----------------------
+
+Update the ``messages.po`` file in
+:file:`$PACK_PATH/fb_i18n_nl/locale/nl/LC_MESSAGES/messages.po`, as well as the
+HTML files in :file:`$PACK_PATH/fb_18n_nl/locale/nl/html`.
+
+Once the messages have been translated, you can compile the resulting file:
+
+.. code-block:: bash
+
+    $BABEL compile -d $PACK_PATH/fb_i18n_nl/locale -l nl -i $PACK_PATH/fb_i18n_nl/locale/nl/LC_MESSAGES/messages.po
+
+Installing the Language Pack
+----------------------------
+
+You can install the language pack like a normal Python package:
+
+.. code-block:: bash
+
+    pip install $PACK_PATH
+
+And enable it in your settings:
+
+.. code-block:: ini
+
+    fietsboek.language_packs =
+        fb_i18n_nl
+
+    available_locales = en nl
+
+Automation
+----------
+
+If you are fine with the following default values, then you can use the
+provided :file:`justfile` with just_ to automate most of the boring processes:
+
+* Poetry will be used
+* The pack will be saved in :file:`language-packs/fietsboek-i18n-LOCALE`
+* The module name will be ``fietsboek_i18n_LOCALE``
+
+.. code-block:: bash
+
+    just create-language-pack nl
+    just update-language-pack nl
+    just compile-language-pack nl
+
+.. _Poetry: https://python-poetry.org/
+.. _Babel: https://babel.pocoo.org/en/latest/index.html
+.. _just: https://github.com/casey/just
diff --git a/doc/developer/localize.rst b/doc/developer/localize.rst
index c345c3e..91a51eb 100644
--- a/doc/developer/localize.rst
+++ b/doc/developer/localize.rst
@@ -43,11 +43,11 @@ In order to do so, use the :program:`pybabel` binary:
 
     .venv/bin/pybabel extract -F babel.cfg -o fietsboek/locale/fietslog.pot --input-dirs=fietsboek
 
-The :file:`Makefile` contains a shortcut for this command:
+The :file:`justfile` (requires just_) contains a shortcut for this command:
 
 .. code:: bash
 
-    make babel-extract
+    just extract-messages
 
 Creating a New Language
 -----------------------
@@ -60,12 +60,6 @@ generate the ``.po`` file containing the messages using :program:`pybabel`:
     # Replace LOCALE with the locale name, for example "nl" or "fr"
     .venv/bin/pybabel init -d fietsboek/locale -l LOCALE -i fietsboek/locale/fietslog.pot
 
-Again, there is a shortcut in the :file:`Makefile`:
-
-.. code:: bash
-
-    make babel-init LOCALE=nl
-
 This will create the directory :file:`fietsboek/locale/{language}`.
 
 Finally, you need to copy the non-gettext messages. This is best done by
@@ -76,6 +70,12 @@ copying over the english original texts:
     # Replace nl with the locale that you just generated
     cp -r fietsboek/locale/en/html fietsboek/locale/nl/
 
+Again, there is a shortcut in the :file:`justfile` that does both steps:
+
+.. code:: bash
+
+    just init-language nl
+
 Updating a Language
 -------------------
 
@@ -94,7 +94,7 @@ Alternatively, you can also use the shortcut again:
 
 .. code:: bash
 
-    make babel-update LOCALE=nl
+    just update-language nl
 
 Translating
 -----------
@@ -125,10 +125,12 @@ Or using the shortcut:
 
 .. code:: bash
 
-    make babel-compile LOCALE=nl
+    just compile-language nl
 
 Further Reading
 ---------------
 
 * The Pyramid documentation: `Internationalization and Localization
   <https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/i18n.html>`__
+
+.. _just: https://github.com/casey/just
diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py
index e648cfa..a248dc9 100644
--- a/fietsboek/__init__.py
+++ b/fietsboek/__init__.py
@@ -79,6 +79,8 @@ def main(_global_config, **settings):
         config.include(".models")
         config.scan()
         config.add_translation_dirs("fietsboek:locale/")
+        for pack in parsed_config.language_packs:
+            config.add_translation_dirs(f"{pack}:locale/")
         config.set_session_factory(my_session_factory)
         config.set_security_policy(SecurityPolicy())
         config.set_csrf_storage_policy(CookieCSRFStoragePolicy())
diff --git a/fietsboek/config.py b/fietsboek/config.py
index da31061..41f6a64 100644
--- a/fietsboek/config.py
+++ b/fietsboek/config.py
@@ -155,6 +155,9 @@ class Config(BaseModel):
     session_key: str
     """Session key."""
 
+    language_packs: PyramidList = Field([], alias="fietsboek.language_packs")
+    """Additional language packs to load."""
+
     available_locales: PyramidList = PyramidList(["en", "de"])
     """Available locales."""
 
diff --git a/fietsboek/util.py b/fietsboek/util.py
index c0a59a5..63414ae 100644
--- a/fietsboek/util.py
+++ b/fietsboek/util.py
@@ -4,7 +4,7 @@ import re
 import os
 import unicodedata
 import secrets
-from typing import Optional
+from typing import Optional, List
 
 # Compat for Python < 3.9
 import importlib_resources
@@ -276,7 +276,9 @@ def check_password_constraints(password: str, repeat_password: Optional[str] = N
         raise ValueError(_("password_constraint.length"))
 
 
-def read_localized_resource(locale_name: str, path: str, raise_on_error: bool = False) -> str:
+def read_localized_resource(
+    locale_name: str, path: str, locale_packages: List[str], raise_on_error: bool = False
+) -> str:
     """Reads a localized resource.
 
     Localized resources are located in the ``fietsboek/locale/**`` directory.
@@ -286,6 +288,8 @@ def read_localized_resource(locale_name: str, path: str, raise_on_error: bool =
 
     :param locale_name: Name of the locale.
     :param path: Path of the resource.
+    :param locale_packages: Names of packages in which locale data is searched.
+        By default, only built-in locales are searched.
     :param raise_on_error: Raise an error instead of returning a placeholder.
     :raises FileNotFoundError: If the path could not be found and
         ``raise_on_error`` is ``True``.
@@ -297,13 +301,17 @@ def read_localized_resource(locale_name: str, path: str, raise_on_error: bool =
     if "_" in locale_name:
         locales.append(locale_name.split("_", 1)[0])
 
+    if locale_packages is None:
+        locale_packages = ["fietsboek"]
+
     for locale in locales:
-        locale_dir = importlib_resources.files("fietsboek") / "locale" / locale
-        resource_path = locale_dir / path
-        try:
-            return resource_path.read_text()
-        except (FileNotFoundError, ModuleNotFoundError, NotADirectoryError):
-            pass
+        for package in locale_packages:
+            locale_dir = importlib_resources.files(package) / "locale" / locale
+            resource_path = locale_dir / path
+            try:
+                return resource_path.read_text()
+            except (FileNotFoundError, ModuleNotFoundError, NotADirectoryError):
+                pass
     if raise_on_error:
         raise FileNotFoundError(f"Resource {path!r} not found")
     return f"{locale_name}:{path}"
diff --git a/fietsboek/views/default.py b/fietsboek/views/default.py
index 0a2d7e2..883d8d7 100644
--- a/fietsboek/views/default.py
+++ b/fietsboek/views/default.py
@@ -40,7 +40,11 @@ def home(request):
 
         # Show the default home page
         locale = request.localizer.locale_name
-        content = util.read_localized_resource(locale, "html/home.html")
+        content = util.read_localized_resource(
+            locale,
+            "html/home.html",
+            locale_packages=request.config.language_packs,
+        )
         return {
             "home_content": content,
         }
diff --git a/justfile b/justfile
new file mode 100644
index 0000000..fc29e00
--- /dev/null
+++ b/justfile
@@ -0,0 +1,60 @@
+_default:
+    @just --list
+
+# Create a new language pack
+create-language-pack locale:
+    #!/bin/bash
+    set -euo pipefail
+    [ -e "language-packs/fietsboek-i18n-{{ locale }}" ] && (echo "Already exists!" ; exit 1)
+    FB_PATH="$PWD"
+    mkdir -p "language-packs/fietsboek-i18n-{{ locale }}"
+    cd "language-packs/fietsboek-i18n-{{ locale }}"
+    {
+        echo '[tool.poetry]'
+        echo 'name = "fietsboek-i18n-{{ locale }}"'
+        echo 'version = "0.1.0"'
+        echo 'description = ""'
+        echo 'authors = ["John Doe <john@example.com>"]'
+        echo 'packages = [{include = "fietsboek_i18n_{{ locale }}"}]'
+        echo '[tool.poetry.dependencies]'
+        echo 'python = "^3.7"'
+        echo '[build-system]'
+        echo 'requires = ["poetry-core"]'
+        echo 'build-backend = "poetry.core.masonry.api"'
+    } >> pyproject.toml
+    mkdir -p "fietsboek_i18n_{{ locale }}/locale"
+    touch "fietsboek_i18n_{{ locale }}/__init__.py"
+    cp -r "$FB_PATH/fietsboek/locale/en/html" "fietsboek_i18n_{{ locale }}/locale/"
+    pybabel init -d "fietsboek_i18n_{{ locale }}/locale" -l {{ locale }} -i "$FB_PATH/fietsboek/locale/fietslog.pot"
+
+# Compile the messages of an existing language pack
+compile-language-pack locale:
+    #!/bin/bash
+    set -euo pipefail
+    cd "language-packs/fietsboek-i18n-{{ locale }}"
+    pybabel compile -d "fietsboek_i18n_{{ locale }}/locale" -l {{ locale }} -i "fietsboek_i18n_{{ locale }}/locale//{{ locale }}/LC_MESSAGES/messages.po"
+
+# Update the messages contained in the given pack
+update-language-pack locale:
+    #!/bin/bash
+    set -euo pipefail
+    FB_PATH="$PWD"
+    cd "language-packs/fietsboek-i18n-{{ locale }}"
+    pybabel update -d "fietsboek_i18n_{{ locale }}/locale" -l {{ locale }} -i "$FB_PATH/fietsboek/locale/fietslog.pot"
+
+# Initializes a new built-in language
+init-language locale:
+    pybabel init -d fietsboek/locale -l {{ locale }} -i fietsboek/locale/fietslog.pot
+    cp -r fietsboek/locale/en/html fietsboek/locale/{{ locale }}/
+
+# Update the built-in message catalogue
+update-language locale:
+    pybabel update -d fietsboek/locale -l {{ locale }} -i fietsboek/locale/fietslog.pot
+
+# Compile the given built-in language
+compile-language locale:
+    pybabel compile -d fietsboek/locale -l {{ locale }} -i fietsboek/locale/{{ locale }}/LC_MESSAGES/messages.po
+
+# Extract new messages from the source files
+extract-messages:
+    pybabel extract -F babel.cfg -o fietsboek/locale/fietslog.pot --input-dirs=fietsboek