diff options
| -rw-r--r-- | Dockerfile | 17 | ||||
| -rw-r--r-- | container/entrypoint | 62 | ||||
| -rw-r--r-- | container/gunicorn.conf.py | 1 | ||||
| -rw-r--r-- | doc/administration.rst | 1 | ||||
| -rw-r--r-- | doc/administration/docker.rst | 160 | ||||
| -rw-r--r-- | fietsboek/__init__.py | 30 | ||||
| -rw-r--r-- | fietsboek/scripts/__init__.py | 37 | ||||
| -rw-r--r-- | fietsboek/scripts/fietscron.py | 9 | ||||
| -rw-r--r-- | fietsboek/updater/__init__.py | 125 |
9 files changed, 386 insertions, 56 deletions
diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..262cfac --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.11 +RUN pip install gunicorn psycopg2-binary mysqlclient + +RUN mkdir /package +WORKDIR /package/ +COPY ["fietsboek", "fietsboek"] +COPY ["pyproject.toml", "README.md", "LICENSE.txt", "CHANGELOG.rst", "production.ini", "."] +RUN pip install . + +COPY --chmod=755 ["container/entrypoint", "/bin/entrypoint"] +COPY ["container/gunicorn.conf.py", "/fietsboek/gunicorn.conf.py"] + +VOLUME /fietsboek/data /fietsboek/database /fietsboek/pages +WORKDIR /fietsboek + +ENTRYPOINT ["/bin/entrypoint"] +CMD ["gunicorn", "--paste", "/fietsboek/fietsboek.ini"] diff --git a/container/entrypoint b/container/entrypoint new file mode 100644 index 0000000..c7f3b2d --- /dev/null +++ b/container/entrypoint @@ -0,0 +1,62 @@ +#!/bin/bash +set -euo pipefail + +error() { + >&2 echo "[ERROR] $1" +} + +CONFIG="/fietsboek/fietsboek.ini" +if [[ ! -f "$CONFIG" ]] ; then + # Copy the default config over + cp /package/production.ini "$CONFIG" + # Change default settings + : ${SQLALCHEMY_URL:=sqlite:////fietsboek/database/fietsboek.sqlite} + sed -i '/^sqlalchemy\.url = /c sqlalchemy.url = '"$SQLALCHEMY_URL" "$CONFIG" + + if [[ ! -v REDIS_URL ]] ; then + error "need REDIS_URL set" + exit 1 + fi + sed -i '/^\[app:main\]/a redis.url = '"$REDIS_URL" "$CONFIG" + + : ${DATA_DIR:=/fietsboek/data} + mkdir -p "$DATA_DIR" + sed -i '/^\[app:main\]/a fietsboek.data_dir = '"$DATA_DIR" "$CONFIG" + + : ${SESSION_KEY:=$(openssl rand -hex 20)} + sed -i '/^session_key = /c session_key = '"$SESSION_KEY" "$CONFIG" + + sed -i '/^\[app:main\]/a fietsboek.pages = /fietsboek/pages' "$CONFIG" + + : ${ENABLE_ACCOUNT_REGISTRATION:=true} + sed -i '/^enable_account_registration =/c enable_account_registration = '"$ENABLE_ACCOUNT_REGISTRATION" "$CONFIG" + + if [[ -v DEFAULT_LOCALE_NAME ]] ; then + sed -i '/^pyramid\.default_locale_name =/c pyramid.default_locale_name = '"$DEFAULT_LOCALE_NAME" "$CONFIG" + fi + + if [[ -v EMAIL_FROM ]] ; then + sed -i '/^email\.from =/c email.from = '"$EMAIL_FROM" "$CONFIG" + fi + if [[ -v EMAIL_SMTP_URL ]] ; then + sed -i '/^email\.smtp_url =/c email.smtp_url = '"$EMAIL_SMTP_URL" "$CONFIG" + fi + if [[ -v EMAIL_USERNAME ]] ; then + sed -i '/^email\.username =/c email.username = '"$EMAIL_USERNAME" "$CONFIG" + fi + if [[ -v EMAIL_PASSWORD ]] ; then + sed -i '/^email\.password =/c email.password = '"$EMAIL_PASSWORD" "$CONFIG" + fi + + if [[ -v LOGLEVEL ]] ; then + # We are only changing the fietsboek log level here, as SQLAlchemy + # produces a lot of noise. + sed -i '/\[logger_fietsboek\]/{N;s/level = .*/level = '"$LOGLEVEL"'/}' "$CONFIG" + fi +fi + +# Make sure the data schema is up to date, or rather initialize it if this is +# the first time +(cd /package && fietsupdate update -fc "$CONFIG") + +exec "$@" diff --git a/container/gunicorn.conf.py b/container/gunicorn.conf.py new file mode 100644 index 0000000..dc574f2 --- /dev/null +++ b/container/gunicorn.conf.py @@ -0,0 +1 @@ +bind = "0.0.0.0:8000" diff --git a/doc/administration.rst b/doc/administration.rst index 6cf0f52..e09258b 100644 --- a/doc/administration.rst +++ b/doc/administration.rst @@ -12,6 +12,7 @@ Administration Guide administration/cronjobs administration/custom-pages administration/maintenance-mode + administration/docker This guide contains information pertaining to the administration of a Fietsboek instance. This includes the installation, updating, configuration, and backing diff --git a/doc/administration/docker.rst b/doc/administration/docker.rst new file mode 100644 index 0000000..ed44a05 --- /dev/null +++ b/doc/administration/docker.rst @@ -0,0 +1,160 @@ +Container Setup +=============== + +Like most applications, Fietsboek can be installed in a Docker container. This +allows you to have Fietsboek isolated from the rest of the system, e.g. if you +need to run a newer Python version. + +.. note:: + + Fietsboek's ``Dockerfile`` has been tested with Podman_, but it should + work just as well with Docker_. + + The commands in this article use ``docker`` for the sake of consistency, + but it will work the same if you replace ``docker`` with ``podman`` (or use + ``podman-docker``). + +.. _Podman: https://podman.io/ +.. _Docker: https://www.docker.com/ + +Building the Image +------------------ + +You can build the image by running ``docker build`` in the main directory. Note +that the build will use the currently checked out version of Fietsboek, so make +sure you are on the right version that you want to use:: + + docker build -t fietsboek . + +Container Layout +---------------- + +The container reserves three volumes: + +* ``/fietsboek/database`` for the SQL database. +* ``/fietsboek/data`` for the track data. +* ``/fietsboek/pages`` for the custom pages. + +Per default, the configuration file is expected at +``/fietsboek/fietsboek.ini``, and the `Gunicorn configuration`_ at +``/fietsboek/gunicorn.conf.py``. + +.. _Gunicorn configuration: https://docs.gunicorn.org/en/latest/configure.html#configuration-file + +Configuration +------------- + +There are two ways to configure the Fietsboek instance in the container. You +can either mount a custom configuration file to ``/fietsboek/fietsboek.ini``, +or you can change common settings via environment variables. Note that you need +to choose one way, as mounting a custom configuration will disable the +environment variable parsing. + +If you need to change a lot of settings, using a custom configuration file is +the prefered way. It follows the same syntax and settings as described in +:doc:`configuration`. Keep the container layout in mind when setting the data +directory and database location (or make sure that they are on other volumes). + +An example command could look like this:: + + docker run -v ./production.ini:/fietsboek/fietsboek.ini fietsboek + +If you simply want to get a quick instance up and running, the method of +overwriting single configuration values via environment variables is quicker. +This way, you do not need to provide the full configuration file, but simply +the options that diverge from the default values:: + + docker run -e REDIS_URL=redis://localhost fietsboek + +The following environment variables are supported: + +=============================== =============================== +Environment variable Setting +=============================== =============================== +``REDIS_URL`` [#f1]_ ``redis.url`` +``SQLALCHEMY_URL`` ``sqlalchemy.url`` +``DATA_DIR`` ``fietsboek.data_dir`` +``SESSION_KEY`` [#f2]_ ``session_key`` +``ENABLE_ACCOUNT_REGISTRATION`` ``enable_account_registration`` +``DEFAULT_LOCALE_NAME`` ``pyramid.default_locale_name`` +``EMAIL_FROM`` ``email.from`` +``EMAIL_SMTP_URL`` ``email.smtp_url`` +``EMAIL_USERNAME`` ``email.username`` +``EMAIL_PASSWORD`` ``email.password`` +``LOGLEVEL`` ``[logger_fietsboek] level`` +=============================== =============================== + +.. [#f1] Required. +.. [#f2] Defaults to a random string. + +The ``fietsboek.pages`` setting is not exposed, as you can simply mount the +directory containing the pages to ``/fietsboek/pages``. + +Upgrading +--------- + +The image is configured to automatically run ``fietsupdate`` at the start. +Therefore, simply updating the image to the new Fietsboek version should work +and the container will automatically migrate the data: + +.. code-block:: bash + + # ... assume we checked out the new code, build the new image: + docker build -t fietsboek . + # Run the new code, taking the data from the old version: + docker run --volumes-from=OLD_CONTAINER_ID -e ... fietsboek + +.. warning:: + + As always, back up your data before doing an update! + +Maintenance & Cronjobs +---------------------- + +The image contains the maintenance tools ``fietsctl`` and ``fietscron``. You +can execute them in the container: + +.. code-block:: bash + + # Get a user list ... + docker exec -ti CONTAINER_ID fietsctl userlist + # Run the cronjobs ... + docker exec -ti CONTAINER_ID fietscron + # ... and so on + +Docker Compose +-------------- + +Since Fietsboek needs other services (such as a Redis_ instance, and optionally +a SQL server like PostgreSQL_ or MariaDB_), it makes sense to put those +services together in a ``docker-compose.yml`` file. + +A minimal example is given here: + +.. code-block:: yaml + + services: + redis: + image: redis + + fietsboek: + image: fietsboek + ports: + - "8000:8000" + environment: + - REDIS_URL=redis://redis + volumes: + - fietsboek-data:/fietsboek/data + - fietsboek-database:/fietsboek/database + + volumes: + fietsboek-data: + fietsboek-database: + + +This example shows how to use the environment-based configuration mechanism to +get a instance up quickly, without changing the defaults too much. + +.. _Redis: https://redis.com/ +.. _PostgreSQL: https://www.postgresql.org/ +.. _MariaDB: https://mariadb.org/ diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py index 41e8a04..ee7d8b7 100644 --- a/fietsboek/__init__.py +++ b/fietsboek/__init__.py @@ -14,6 +14,7 @@ Content ------- """ import importlib.metadata +import logging from pathlib import Path from typing import Callable, Optional @@ -33,9 +34,12 @@ from . import transformers from .data import DataManager from .pages import Pages from .security import SecurityPolicy +from .updater import Updater, UpdateState __VERSION__ = importlib.metadata.version("fietsboek") +LOGGER = logging.getLogger(__name__) + def locale_negotiator(request: Request) -> Optional[str]: """Negotiates the right locale to use. @@ -88,12 +92,36 @@ def maintenance_mode( return tween -def main(_global_config, **settings): +def check_update_state(config_uri: str): + """Checks the update state of the data, and logs a warning if there is a + mismatch. + + :param config_uri: Path to the configuration file. + """ + updater = Updater(config_uri) + updater.load() + state = updater.state() + + if state == UpdateState.OUTDATED: + LOGGER.warning( + "The data seems to be outdated - make sure to run the fietsupdate migrations!" + ) + elif state == UpdateState.TOO_NEW: + LOGGER.warning("The data seems to be too new, make sure to update the code accordingly!") + elif state == UpdateState.UNKNOWN: + LOGGER.warning("Could not determine version state of the data - check `fietsupdate status`") + + +def main(global_config, **settings): """This function returns a Pyramid WSGI application.""" # Avoid a circular import by not importing at the top level # pylint: disable=import-outside-toplevel,cyclic-import from .views.tileproxy import TileRequester + # In tests this isn't passed, so guard against it + if "__file__" in global_config: + check_update_state(global_config["__file__"]) + parsed_config = mod_config.parse(settings) settings["jinja2.newstyle"] = True diff --git a/fietsboek/scripts/__init__.py b/fietsboek/scripts/__init__.py index ea36d96..b8a883a 100644 --- a/fietsboek/scripts/__init__.py +++ b/fietsboek/scripts/__init__.py @@ -1,5 +1,39 @@ """Various command line scripts to interact with the fietsboek installation.""" +from typing import Any, Optional + import click +from click.core import ParameterSource + + +class ConfigOption(click.Path): + """Special :class:`~click.Path` subclass to print a more helpful error + message. + + This class recognizes if the config option is not given and prints a hint + if the file does not exist in this case. + """ + + def __init__(self): + super().__init__(exists=True, dir_okay=False) + + def convert( + self, + value: Any, + param: Optional[click.Parameter], + ctx: Optional[click.Context], + ) -> Any: + try: + return super().convert(value, param, ctx) + except click.BadParameter as exc: + if param is not None and param.name is not None and ctx is not None: + source = ctx.get_parameter_source(param.name) + if source == ParameterSource.DEFAULT: + exc.message += ( + f"\n\nHint: {value!r} is the default, " + "try specifying a configuration file explicitely." + ) + raise exc + # We keep this as a separate option that is added to each subcommand as Click # (unlike argparse) cannot handle "--help" without the required arguments of @@ -10,8 +44,9 @@ import click config_option = click.option( "-c", "--config", - type=click.Path(exists=True, dir_okay=False), + type=ConfigOption(), required=True, + default="fietsboek.ini", help="Path to the Fietsboek configuration file", ) diff --git a/fietsboek/scripts/fietscron.py b/fietsboek/scripts/fietscron.py index f0dc438..d8476a9 100644 --- a/fietsboek/scripts/fietscron.py +++ b/fietsboek/scripts/fietscron.py @@ -16,18 +16,13 @@ from .. import config as mod_config from .. import hittekaart, models from ..config import Config from ..data import DataManager +from . import config_option LOGGER = logging.getLogger(__name__) @click.command() -@click.option( - "-c", - "--config", - type=click.Path(exists=True, dir_okay=False), - required=True, - help="Path to the Fietsboek configuration file", -) +@config_option def cli(config): """Runs regular maintenance operations on the instance. diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index 9e2e7f0..5faa805 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -1,12 +1,13 @@ """Updating (data migration) logic for fietsboek.""" import datetime +import enum import importlib.resources import importlib.util import logging import random import string from pathlib import Path -from typing import List +from typing import Optional import alembic.command import alembic.config @@ -50,6 +51,26 @@ class Down(UpdateScript): """ +class UpdateState(enum.Enum): + """State of the applied updates. + + This represents a "summary" of the output that ``fietsupdate status`` + produces. + """ + + OKAY = enum.auto() + """Everything is good, the data is up to date.""" + + OUTDATED = enum.auto() + """The data is outdated, the update process should be run.""" + + TOO_NEW = enum.auto() + """The data contains revisions that are not known to Fietsboek yet.""" + + UNKNOWN = enum.auto() + """The data version could not be determined.""" + + class Updater: """A class that implements the updating logic. @@ -57,20 +78,19 @@ class Updater: them in the right order. """ - def __init__(self, config_path): + def __init__(self, config_path: str): self.config_path = config_path self.settings = pyramid.paster.get_appsettings(config_path) self.alembic_config = alembic.config.Config(config_path) - self.scripts = {} - self.forward_dependencies = {} - self.backward_dependencies = {} + self.scripts: dict[str, "UpdateScript"] = {} + self.forward_dependencies: dict[str, list[str]] = {} + self.backward_dependencies: dict[str, list[str]] = {} @property - def version_file(self): + def version_file(self) -> Path: """Returns the path to the version file. :return: The path to the data's version file. - :rytpe: pathlib.Path """ data_dir = Path(self.settings["fietsboek.data_dir"]) return data_dir / "VERSION" @@ -99,20 +119,17 @@ class Updater: down_alembic = possible_alembic script.down_alembic = down_alembic - def exists(self, revision_id): + def exists(self, revision_id: str) -> bool: """Checks if the revision with the given ID exists. :param revision_id: ID of the revision to check. - :type revision_id: str :return: True if the revision exists. - :rtype: bool """ return revision_id in self.scripts - def current_versions(self): + def current_versions(self) -> list[str]: """Reads the current version of the data. - :rtype: list[str] :return: The versions, or an empty list if no versions are found. """ try: @@ -121,7 +138,7 @@ class Updater: except FileNotFoundError: return [] - def _transitive_versions(self): + def _transitive_versions(self) -> set[str]: versions = set() queue = self.current_versions() while queue: @@ -131,21 +148,25 @@ class Updater: queue.extend(self.scripts[current].previous) return versions - def _reverse_versions(self): + def _reverse_versions(self) -> set[str]: all_versions = set(script.id for script in self.scripts.values()) return all_versions - self._transitive_versions() - def stamp(self, versions): + def stamp(self, versions: list[str]): """Stampts the given version into the version file. This does not run any updates, it simply updates the version information. :param version: The versions to stamp. - :type version: list[str] """ self.version_file.write_text("\n".join(versions), encoding="utf-8") - def _pick_updates(self, wanted, applied, dependencies): + def _pick_updates( + self, + wanted: str, + applied: set[str], + dependencies: dict[str, list[str]], + ) -> set[str]: to_apply = set() queue = [wanted] while queue: @@ -156,9 +177,9 @@ class Updater: queue.extend(dependencies[current]) return to_apply - def _make_schedule(self, wanted, dependencies): + def _make_schedule(self, wanted: set[str], dependencies: dict[str, list[str]]) -> list[str]: wanted = set(wanted) - queue: List[str] = [] + queue: list[str] = [] while wanted: next_updates = { update @@ -169,19 +190,18 @@ class Updater: wanted -= next_updates return queue - def _stamp_versions(self, old, new): + def _stamp_versions(self, old: list[str], new: list[str]): versions = self.current_versions() versions = [version for version in versions if version not in old] versions.extend(new) self.stamp(versions) - def upgrade(self, target): + def upgrade(self, target: str): """Run the tasks to upgrade to the given target. This ensures that all previous migrations are also run. :param target: The target revision. - :type target: str """ # First, we figure out which tasks we have already applied and which # still need applying. This is pretty much a BFS over the current @@ -198,13 +218,12 @@ class Updater: script.upgrade(self.settings, self.alembic_config) self._stamp_versions(script.previous, [script.id]) - def downgrade(self, target): + def downgrade(self, target: str): """Run the tasks to downgrade to the given target. This ensures that all succeeding down-migrations are also run. :param target: The target revision. - :type target: str """ # This is basically the same as upgrade() but with the reverse # dependencies instead. @@ -218,16 +237,14 @@ class Updater: script.downgrade(self.settings, self.alembic_config) self._stamp_versions([script.id], script.previous) - def new_revision(self, revision_id=None): + def new_revision(self, revision_id: Optional[str] = None) -> str: """Creates a new revision with the current versions as dependencies and the current alembic version. :param revision_id: The revision ID to use. By default, a random string will be generated. - :type revision_id: str :return: The filename of the revision file in the ``updater/`` directory. - :rtype: str """ if not revision_id: revision_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=16)) @@ -260,15 +277,14 @@ class Updater: fobj.write(revision) return filename - def heads(self): + def heads(self) -> list[str]: """Returns all "heads", that are the latest revisions. :return: The heads. - :rtype: list[str] """ return [rev_id for (rev_id, deps) in self.backward_dependencies.items() if not deps] - def has_applied(self, revision_id, backward=False): + def has_applied(self, revision_id: str, backward: bool = False) -> bool: """Checks whether the given revision is applied. By default, this checks if a given update is applied, i.e. the current @@ -281,60 +297,79 @@ class Updater: :meth:`exists` to check whether the revision actually exists. :param revision_id: The revision to check. - :type revision_id: str :param backward: Whether to switch the comparison direction. - :type backward: bool :return: ``True`` if the current version at least matches the asked revision ID. - :rtype: bool """ if not backward: return revision_id in self._transitive_versions() return revision_id in self._reverse_versions() | set(self.current_versions()) + def state(self) -> UpdateState: + """Checks the update state of the instance. + + This returns a condensed version of what ``fietsupdate status`` + outputs. + + :return: The update state of the data. + """ + state = UpdateState.OKAY + current = self.current_versions() + heads = self.heads() + if current: + for i in current: + if not self.exists(i): + state = UpdateState.TOO_NEW + else: + return UpdateState.UNKNOWN + updates = set(heads) - set(current) + if updates: + if state != UpdateState.OKAY: + # We are both too new and too old, so something is pretty wrong + return UpdateState.UNKNOWN + return UpdateState.OUTDATED + return state + class UpdateScript: """Represents an update script.""" - def __init__(self, source, name): + def __init__(self, source: str, name: str): self.name = name spec = importlib.util.spec_from_loader(f"{__name__}.{name}", None) self.module = importlib.util.module_from_spec(spec) # type: ignore assert self.module exec(source, self.module.__dict__) # pylint: disable=exec-used - self.down_alembic = None + self.down_alembic: Optional[str] = None def __repr__(self): return f"<{__name__}.{self.__class__.__name__} name={self.name!r} id={self.id!r}>" @property - def id(self): + def id(self) -> str: """Returns the ID of the update. - :rtype: str :return: The id of the update """ return self.module.update_id @property - def previous(self): + def previous(self) -> list[str]: """Returns all dependencies of the update. - :rtype: list[str] :return: The IDs of all dependencies of the update. """ return getattr(self.module, "previous", []) @property - def alembic_version(self): + def alembic_version(self) -> str: """Returns the alembic revisions of the update. - :rtype: list[str] :return: The needed alembic revisions. """ return self.module.alembic_revision - def upgrade(self, config, alembic_config): + def upgrade(self, config: dict, alembic_config: alembic.config.Config): """Runs the upgrade migrations of this update script. This first runs the pre_alembic task, then the alembic migration, and @@ -344,9 +379,7 @@ class UpdateScript: executed. :param config: The app configuration. - :type config: dict :param alembic_config: The alembic config to use. - :type alembic_config: alembic.config.Config """ LOGGER.info("[up] Running pre-alembic task for %s", self.id) self.module.Up().pre_alembic(config) @@ -355,15 +388,13 @@ class UpdateScript: LOGGER.info("[up] Running post-alembic task for %s", self.id) self.module.Up().post_alembic(config) - def downgrade(self, config, alembic_config): + def downgrade(self, config: dict, alembic_config: alembic.config.Config): """Runs the downgrade migrations of this update script. See also :meth:`upgrade`. :param config: The app configuration. - :type config: dict :param alembic_config: The alembic config to use. - :type alembic_config: alembic.config.Config """ LOGGER.info("[down] Running pre-alembic task for %s", self.id) self.module.Down().pre_alembic(config) |