From 810ccfe9feccff0d5ccecea57aad11491d62a6e5 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 3 Dec 2022 16:38:39 +0100 Subject: enable loading of foreign locale packages --- fietsboek/__init__.py | 2 ++ fietsboek/config.py | 3 +++ fietsboek/util.py | 21 ++++++++++++++------- fietsboek/views/default.py | 6 +++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py index e208a61..fe6bf52 100644 --- a/fietsboek/__init__.py +++ b/fietsboek/__init__.py @@ -80,6 +80,8 @@ def main(_global_config, **settings): config.include('.models') config.scan() config.add_translation_dirs('fietsboek:locale/') + for pack in parsed_config.language_packs: + config.add_translation_dirs(f'{pack}:locale/') config.set_session_factory(my_session_factory) config.set_security_policy(SecurityPolicy()) config.set_csrf_storage_policy(CookieCSRFStoragePolicy()) diff --git a/fietsboek/config.py b/fietsboek/config.py index d2efe77..b8a2cfb 100644 --- a/fietsboek/config.py +++ b/fietsboek/config.py @@ -141,6 +141,9 @@ class Config(BaseModel): session_key: str """Session key.""" + language_packs: PyramidList = Field([], alias="fietsboek.language_packs") + """Additional language packs to load.""" + available_locales: PyramidList = ["en", "de"] """Available locales.""" diff --git a/fietsboek/util.py b/fietsboek/util.py index 71f5d16..6b2190b 100644 --- a/fietsboek/util.py +++ b/fietsboek/util.py @@ -284,7 +284,7 @@ def check_password_constraints(password, repeat_password=None): raise ValueError(_("password_constraint.length")) -def read_localized_resource(locale_name, path, raise_on_error=False): +def read_localized_resource(locale_name, path, locale_packages=None, raise_on_error=False): """Reads a localized resource. Localized resources are located in the ``fietsboek/locale/**`` directory. @@ -294,6 +294,9 @@ def read_localized_resource(locale_name, path, raise_on_error=False): :param locale_name: Name of the locale. :type locale_name: str + :param locale_packages: Names of packages in which locale data is searched. + By default, only built-in locales are searched. + :type locale_packages: list[str] :param raise_on_error: Raise an error instead of returning a placeholder. :type raise_on_error: bool :raises FileNotFoundError: If the path could not be found and @@ -307,13 +310,17 @@ def read_localized_resource(locale_name, path, raise_on_error=False): if "_" in locale_name: locales.append(locale_name.split("_", 1)[0]) + if locale_packages is None: + locale_packages = ['fietsboek'] + for locale in locales: - locale_dir = importlib_resources.files('fietsboek') / 'locale' / locale - resource_path = locale_dir / path - try: - return resource_path.read_text() - except (FileNotFoundError, ModuleNotFoundError, NotADirectoryError): - pass + for package in locale_packages: + locale_dir = importlib_resources.files(package) / 'locale' / locale + resource_path = locale_dir / path + try: + return resource_path.read_text() + except (FileNotFoundError, ModuleNotFoundError, NotADirectoryError): + pass if raise_on_error: raise FileNotFoundError(f"Resource {path!r} not found") return f"{locale_name}:{path}" diff --git a/fietsboek/views/default.py b/fietsboek/views/default.py index f4aaa8f..516aef7 100644 --- a/fietsboek/views/default.py +++ b/fietsboek/views/default.py @@ -40,7 +40,11 @@ def home(request): # Show the default home page locale = request.localizer.locale_name - content = util.read_localized_resource(locale, "html/home.html") + content = util.read_localized_resource( + locale, + "html/home.html", + locale_packages=request.config.language_packs, + ) return { 'home_content': content, } -- cgit v1.2.3 From 191ebd7f707552029c6666cdb7f5bd16e5f491f3 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 3 Dec 2022 17:28:55 +0100 Subject: add first documentation about language packs --- doc/developer.rst | 1 + doc/developer/language-pack.rst | 123 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 doc/developer/language-pack.rst diff --git a/doc/developer.rst b/doc/developer.rst index 6b99851..b47b0a4 100644 --- a/doc/developer.rst +++ b/doc/developer.rst @@ -6,6 +6,7 @@ Developer Guide :caption: Contents developer/localize + developer/language-pack This guide contains information for developers that want to modify or extend Fietsboek. This includes information on how to localize Fietsboek to new diff --git a/doc/developer/language-pack.rst b/doc/developer/language-pack.rst new file mode 100644 index 0000000..a39617b --- /dev/null +++ b/doc/developer/language-pack.rst @@ -0,0 +1,123 @@ +Language Packs +============== + +Fietsboek allows language files to be distributed as third-party-packages, such +that you can localize Fietsboek without needing to add the files to the main +repository. This way, language packs can be updated independently of Fietsboek, +and packs can be provided for languages that are not "officially supported". + +The basic concept behind language packs is the same as in normal +:doc:`localization `, except that the files are in a different Python +package. If you are familiar with the ``gettext`` machinery, you can simply +create a Python package that has a ``locale/`` folder in the same structure as +Fietsboek's. + +In the following guide, we will create an example language pack for Dutch +(``"nl"``). + +Preqrequisites +------------------- + +In order to create a translation package, you need access to the source code of +Fietsboek and Babel_. The easiest way to get both is to clone the git +repository, and use a virtual environment to install babel: + +.. code-block:: bash + + git clone https://gitlab.com/dunj3/fietsboek.git + FB_PATH=$PWD/fietsboek/fietsboek + virtualenv /tmp/babel + /tmp/babel/pip install Babel + BABEL=/tmp/babel/bin/pybabel + + +Creating a Package +------------------ + +Language packs are normal Python packages, as such, you can use any tool you'd +like to create a pack --- as long as you make sure to include the package data. +In our example, we will use Poetry_, as that is what Fietsboek itself uses. + +To create a basic package, we will pick a path in which to create our pack: +:file:`~/fb-i18n-nl` and create the basic structure: + +.. code-block:: bash + + PACK_PATH=~/fb-i18n-nl + mkdir -p $PACK_PATH/fb_i18n_nl + touch $PACK_PATH/fb_i18n_nl/__init__.py + mkdir $PACK_PATH/fb_i18n_nl/locale + +And the content of :file:`$PACK_PATH/pyproject.toml`: + +.. code-block:: toml + + [tool.poetry] + name = "fb-i18n-nl" + version = "0.1.0" + description = "" + authors = ["Jan Modaal "] + packages = [{include = "fb_i18n_nl"}] + + [tool.poetry.dependencies] + python = "^3.7" + + [build-system] + requires = ["poetry-core"] + build-backend = "poetry.core.masonry.api" + +.. note:: + + You can also use ``poetry new`` to create the basic package, just make sure + to remove the unnecessary files. + +Initializing the Language +------------------------- + +This is the same process as for built-in languages, except that you need to +adjust the paths: + +.. code-block:: bash + + $BABEL init -d $PACK_PATH/fb_i18n_nl/locale -l nl -i $FB_PATH/locale/fietslog.pot + +You can also copy over the English HTML files, which makes it easier to +translate them: + +.. code-block:: bash + + cp -rv $FB_PATH/locale/en/html + +Translating & Compiling +----------------------- + +Update the ``messages.po`` file in +:file:`$PACK_PATH/fb_i18n_nl/locale/nl/LC_MESSAGES/messages.po`, as well as the +HTML files in :file:`$PACK_PATH/fb_18n_nl/locale/nl/html`. + +Once the messages have been translated, you can compile the resulting file: + +.. code-block:: bash + + $BABEL compile -d $PACK_PATH/fb_i18n_nl/locale -l nl -i $PACK_PATH/fb_i18n_nl/locale/nl/LC_MESSAGES/messages.po + +Installing the Language Pack +---------------------------- + +You can install the language pack like a normal Python package: + +.. code-block:: bash + + pip install $PACK_PATH + +And enable it in your settings: + +.. code-block:: ini + + fietsboek.language_packs = + fb_i18n_nl + + available_locales = en nl + +.. _Poetry: https://python-poetry.org/ +.. _Babel: https://babel.pocoo.org/en/latest/index.html -- cgit v1.2.3 From b9628678de72793e1bf0ad10f8c842c5bcb7dc10 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 3 Dec 2022 18:02:18 +0100 Subject: add justfile for language pack processing --- .gitignore | 1 + doc/developer/language-pack.rst | 17 ++++++++++++++++ justfile | 43 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 justfile diff --git a/.gitignore b/.gitignore index a3cbe22..259f717 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ test .venv/ /data poetry.toml +/language-packs diff --git a/doc/developer/language-pack.rst b/doc/developer/language-pack.rst index a39617b..71cf9e8 100644 --- a/doc/developer/language-pack.rst +++ b/doc/developer/language-pack.rst @@ -119,5 +119,22 @@ And enable it in your settings: available_locales = en nl +Automation +---------- + +If you are fine with the following default values, then you can use the +provided :file:`justfile` with just_ to automate most of the boring processes: + +* Poetry will be used +* The pack will be saved in :file:`language-packs/fietsboek-i18n-LOCALE` +* The module name will be ``fietsboek_i18n_LOCALE`` + +.. code-block:: bash + + just create-language-pack nl + just update-language-pack nl + just compile-language-pack nl + .. _Poetry: https://python-poetry.org/ .. _Babel: https://babel.pocoo.org/en/latest/index.html +.. _just: https://github.com/casey/just diff --git a/justfile b/justfile new file mode 100644 index 0000000..e5625fc --- /dev/null +++ b/justfile @@ -0,0 +1,43 @@ +default: + @just --list + +# Create a new language pack +create-language-pack locale: + #!/bin/bash + set -euo pipefail + [ -e "language-packs/fietsboek-i18n-{{ locale }}" ] && (echo "Already exists!" ; exit 1) + FB_PATH="$PWD" + mkdir -p "language-packs/fietsboek-i18n-{{ locale }}" + cd "language-packs/fietsboek-i18n-{{ locale }}" + { + echo '[tool.poetry]' + echo 'name = "fietsboek-i18n-{{ locale }}"' + echo 'version = "0.1.0"' + echo 'description = ""' + echo 'authors = ["John Doe "]' + echo 'packages = [{include = "fietsboek_i18n_{{ locale }}"}]' + echo '[tool.poetry.dependencies]' + echo 'python = "^3.7"' + echo '[build-system]' + echo 'requires = ["poetry-core"]' + echo 'build-backend = "poetry.core.masonry.api"' + } >> pyproject.toml + mkdir -p "fietsboek_i18n_{{ locale }}/locale" + touch "fietsboek_i18n_{{ locale }}/__init__.py" + cp -r "$FB_PATH/fietsboek/locale/en/html" "fietsboek_i18n_{{ locale }}/locale/" + pybabel init -d "fietsboek_i18n_{{ locale }}/locale" -l {{ locale }} -i "$FB_PATH/fietsboek/locale/fietslog.pot" + +# Compile the messages of an existing language pack +compile-language-pack locale: + #!/bin/bash + set -euo pipefail + cd "language-packs/fietsboek-i18n-{{ locale }}" + pybabel compile -d "fietsboek_i18n_{{ locale }}/locale" -l {{ locale }} -i "fietsboek_i18n_{{ locale }}/locale//{{ locale }}/LC_MESSAGES/messages.po" + +# Update the messages contained in the given pack +update-language-pack locale: + #!/bin/bash + set -euo pipefail + FB_PATH="$PWD" + cd "language-packs/fietsboek-i18n-{{ locale }}" + pybabel update -d "fietsboek_i18n_{{ locale }}/locale" -l {{ locale }} -i "$FB_PATH/fietsboek/locale/fietslog.pot" -- cgit v1.2.3 From acce7c38d3833b59c4eb2e99dbe7f0393b6a9cbd Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 3 Dec 2022 18:56:32 +0100 Subject: replace Makefile targets with justfile We might re-introduce Makefiles, but for different purposes (SASS or TypeScript compilation). --- Makefile | 16 ---------------- doc/developer/localize.rst | 22 ++++++++++++---------- justfile | 19 ++++++++++++++++++- 3 files changed, 30 insertions(+), 27 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 1fe2185..0000000 --- a/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -PYBABEL=".venv/bin/pybabel" -LOCALE="en" - -babel-extract: - $(PYBABEL) extract -F babel.cfg -o fietsboek/locale/fietslog.pot --input-dirs=fietsboek - -babel-update: - $(PYBABEL) update -d fietsboek/locale -l $(LOCALE) -i fietsboek/locale/fietslog.pot - -babel-compile: - $(PYBABEL) compile -d fietsboek/locale -l $(LOCALE) -i fietsboek/locale/$(LOCALE)/LC_MESSAGES/messages.po - -babel-init: - $(PYBABEL) init -d fietsboek/locale -l $(LOCALE) -i fietsboek/locale/fietslog.pot - -.PHONY: babel-extract babel-update babel-compile diff --git a/doc/developer/localize.rst b/doc/developer/localize.rst index c345c3e..91a51eb 100644 --- a/doc/developer/localize.rst +++ b/doc/developer/localize.rst @@ -43,11 +43,11 @@ In order to do so, use the :program:`pybabel` binary: .venv/bin/pybabel extract -F babel.cfg -o fietsboek/locale/fietslog.pot --input-dirs=fietsboek -The :file:`Makefile` contains a shortcut for this command: +The :file:`justfile` (requires just_) contains a shortcut for this command: .. code:: bash - make babel-extract + just extract-messages Creating a New Language ----------------------- @@ -60,12 +60,6 @@ generate the ``.po`` file containing the messages using :program:`pybabel`: # Replace LOCALE with the locale name, for example "nl" or "fr" .venv/bin/pybabel init -d fietsboek/locale -l LOCALE -i fietsboek/locale/fietslog.pot -Again, there is a shortcut in the :file:`Makefile`: - -.. code:: bash - - make babel-init LOCALE=nl - This will create the directory :file:`fietsboek/locale/{language}`. Finally, you need to copy the non-gettext messages. This is best done by @@ -76,6 +70,12 @@ copying over the english original texts: # Replace nl with the locale that you just generated cp -r fietsboek/locale/en/html fietsboek/locale/nl/ +Again, there is a shortcut in the :file:`justfile` that does both steps: + +.. code:: bash + + just init-language nl + Updating a Language ------------------- @@ -94,7 +94,7 @@ Alternatively, you can also use the shortcut again: .. code:: bash - make babel-update LOCALE=nl + just update-language nl Translating ----------- @@ -125,10 +125,12 @@ Or using the shortcut: .. code:: bash - make babel-compile LOCALE=nl + just compile-language nl Further Reading --------------- * The Pyramid documentation: `Internationalization and Localization `__ + +.. _just: https://github.com/casey/just diff --git a/justfile b/justfile index e5625fc..fc29e00 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,4 @@ -default: +_default: @just --list # Create a new language pack @@ -41,3 +41,20 @@ update-language-pack locale: FB_PATH="$PWD" cd "language-packs/fietsboek-i18n-{{ locale }}" pybabel update -d "fietsboek_i18n_{{ locale }}/locale" -l {{ locale }} -i "$FB_PATH/fietsboek/locale/fietslog.pot" + +# Initializes a new built-in language +init-language locale: + pybabel init -d fietsboek/locale -l {{ locale }} -i fietsboek/locale/fietslog.pot + cp -r fietsboek/locale/en/html fietsboek/locale/{{ locale }}/ + +# Update the built-in message catalogue +update-language locale: + pybabel update -d fietsboek/locale -l {{ locale }} -i fietsboek/locale/fietslog.pot + +# Compile the given built-in language +compile-language locale: + pybabel compile -d fietsboek/locale -l {{ locale }} -i fietsboek/locale/{{ locale }}/LC_MESSAGES/messages.po + +# Extract new messages from the source files +extract-messages: + pybabel extract -F babel.cfg -o fietsboek/locale/fietslog.pot --input-dirs=fietsboek -- cgit v1.2.3 From 07512ee824c5d453cf32b7cb0fea83973f40dd0c Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Tue, 6 Dec 2022 20:08:07 +0100 Subject: add language packs to configuration docs --- doc/administration/configuration.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/administration/configuration.rst b/doc/administration/configuration.rst index 379c683..5f0e3d4 100644 --- a/doc/administration/configuration.rst +++ b/doc/administration/configuration.rst @@ -72,6 +72,18 @@ Currently, Fietsboek ships with English ("en") and German ("de"). Removing a language from this list will make it unavailable. If you create a custom language locally, make sure to add it to this list here! +Fietsboek also allows you to install "language packs", providing languages from +third-party sources. Language packs are normal Python packages that must be +installed via the package manager (e.g. by using ``pip`` in the same +environment that you installed Fietsboek in), and then their names can be +listed as ``fietsboek.language_packs`` in the configuration. Note that you must +still add the locales to ``available_locales`` for them to work. + +.. warning:: + + Since language packs are just Python packages, they can contain and execute + arbitrary code. Do not install untrusted language packs. + Database Settings ----------------- -- cgit v1.2.3