From d4d5964765163afe75fc272a87a7b1ec2b582263 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 10 Sep 2022 15:55:47 +0200 Subject: first implementation of update logic --- fietsboek/updater/__init__.py | 343 +++++++++++++++++++++++++++ pylint.toml | 523 ++++++++++++++++++++++++++++++++++++++++++ tox.ini | 2 +- 3 files changed, 867 insertions(+), 1 deletion(-) create mode 100644 fietsboek/updater/__init__.py create mode 100644 pylint.toml diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py new file mode 100644 index 0000000..bc1bc96 --- /dev/null +++ b/fietsboek/updater/__init__.py @@ -0,0 +1,343 @@ +"""Updating (data migration) logic for fietsboek.""" +import logging +import datetime +import random +import string +import importlib.util +from pathlib import Path + +# Compat for Python < 3.9 +import importlib_resources +import pyramid.paster +import alembic.runtime +import alembic.config +import alembic.command +import sqlalchemy +import jinja2 + + +LOGGER = logging.getLogger(__name__) + +TEMPLATE = """\ +\"\"\"Revision upgrade script {{ update_id }} + +Date created: {{ date }} +\"\"\" +update_id = {{ "{!r}".format(update_id) }} +previous = [ +{%- for prev in previous %} + {{ "{!r}".format(prev) }}, +{% endfor -%} +] +alembic_revision = {{ "{!r}".format(alembic_revision) }} + + +class Up: + def pre_alembic(self, config): + pass + + def post_alembic(self, config): + pass + + +class Down: + def pre_alembic(self, config): + pass + + def post_alembic(self, config): + pass +""" + + +class Updater: + """A class that implements the updating logic. + + This class is responsible for holding all of the update scripts and running + them in the right order. + """ + + def __init__(self, config_path): + self.config_path = config_path + self.settings = pyramid.paster.get_appsettings(config_path) + self.alembic_config = alembic.config.Config(config_path) + self.scripts = {} + self.forward_dependencies = {} + self.backward_dependencies = {} + + @property + def version_file(self): + """Returns the path to the version file. + + :return: The path to the data's version file. + :rytpe: pathlib.Path + """ + data_dir = Path(self.settings["fietsboek.data_dir"]) + return data_dir / "VERSION" + + def load(self): + """Load all update scripts into memory.""" + scripts = _load_update_scripts() + for script in scripts: + self.scripts[script.id] = script + self.forward_dependencies = { + script.id: script.previous for script in self.scripts.values() + } + # Ensure that each script has an entry + self.backward_dependencies = { + script.id: [] for script in self.scripts.values() + } + for script in self.scripts.values(): + for prev_id in script.previous: + self.backward_dependencies[prev_id].append(script.id) + + def current_versions(self): + """Reads the current version of the data. + + :rtype: list[str] + :return: The versions, or an empty list if no versions are found. + """ + try: + versions = self.version_file.read_text().split("\n") + return [version.strip() for version in versions if version.strip()] + except FileNotFoundError: + return [] + + def _transitive_versions(self): + versions = set() + queue = self.current_versions() + while queue: + current = queue.pop() + versions.add(current) + if current in self.scripts: + queue.extend(self.scripts[current].previous) + return versions + + def _reverse_versions(self): + all_versions = set(script.id for script in self.scripts.values()) + return (all_versions - self._transitive_versions()) | set(self.current_versions()) + + def stamp(self, versions): + """Stampts the given version into the version file. + + This does not run any updates, it simply updates the version information. + + :param version: The versions to stamp. + :type version: list[str] + """ + self.version_file.write_text("\n".join(versions)) + + def _pick_updates(self, wanted, applied, dependencies): + to_apply = set() + queue = [wanted] + while queue: + current = queue.pop(0) + if current in applied or current in to_apply: + continue + to_apply.add(current) + queue.extend(dependencies[current]) + return to_apply + + def _make_schedule(self, wanted, dependencies): + wanted = set(wanted) + queue = [] + while wanted: + next_updates = { + update + for update in wanted + if all(previous not in wanted for previous in dependencies[update]) + } + queue.extend(next_updates) + wanted -= next_updates + return queue + + def _stamp_versions(self, old, new): + versions = self.current_versions() + versions = [version for version in versions if version not in old] + versions.extend(new) + self.stamp(versions) + + def upgrade(self, target): + """Run the tasks to upgrade to the given target. + + This ensures that all previous migrations are also run. + + :param target: The target revision. + :type target: str + """ + # First, we figure out which tasks we have already applied and which + # still need applying. This is pretty much a BFS over the current + # version and its dependencies. + applied_versions = self._transitive_versions() + to_apply = self._pick_updates(target, applied_versions, self.forward_dependencies) + # Second, we need to ensure that the tasks are applied in the right + # order (topological sort) + application_queue = self._make_schedule(to_apply, self.forward_dependencies) + # Finally, we can run the updates + LOGGER.debug("Planned update: %s", application_queue) + for update in application_queue: + script = self.scripts[update] + script.upgrade(self.settings, self.alembic_config) + self._stamp_versions(script.previous, [script.id]) + + def downgrade(self, target): + """Run the tasks to downgrade to the given target. + + This ensures that all succeeding down-migrations are also run. + + :param target: The target revision. + :type target: str + """ + # This is basically the same as upgrade() but with the reverse + # dependencies instead. + applied_versions = self._reverse_versions() + to_apply = self._pick_updates(target, applied_versions, self.backward_dependencies) + application_queue = self._make_schedule(to_apply, self.backward_dependencies) + LOGGER.debug("Planned downgrade: %s", application_queue) + for downgrade in application_queue: + script = self.scripts[downgrade] + script.downgrade(self.settings, self.alembic_config) + self._stamp_versions(self.backward_dependencies[script.id], [script.id]) + + def new_revision(self, revision_id=None): + """Creates a new revision with the current versions as dependencies and + the current alembic version. + + :param revision_id: The revision ID to use. By default, a random string + will be generated. + :type revision_id: str + :return: The filename of the revision file in the ``updater/`` + directory. + :rtype: str + """ + if not revision_id: + revision_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=16)) + + current_versions = self.current_versions() + + engine = sqlalchemy.create_engine(self.settings["sqlalchemy.url"]) + with engine.connect() as conn: + context = alembic.runtime.migration.MigrationContext.configure(conn) + current_alembic = context.get_current_heads() + LOGGER.debug("Found alembic versions: %s", current_alembic) + assert len(current_alembic) == 1 + current_alembic = current_alembic[0] + + loader = jinja2.DictLoader({"revision.py": TEMPLATE}) + env = jinja2.Environment(loader=loader, autoescape=False) + template = env.get_template("revision.py") + revision = template.render( + update_id=revision_id, + previous=current_versions, + alembic_revision=current_alembic, + date=datetime.datetime.now(), + ) + + filename = f"upd_{revision_id}.py" + filepath = Path(__file__).parent / filename + LOGGER.info("Writing new revision (%s) to %r", revision_id, filepath) + with open(filepath, "x", encoding="utf-8") as fobj: + fobj.write(revision) + return filename + + def heads(self): + """Returns all "heads", that are the latest revisions. + + :return: The heads. + :rtype: list[str] + """ + return [rev_id for (rev_id, deps) in self.backward_dependencies.items() if not deps] + + +class UpdateScript: + """Represents an update script.""" + + def __init__(self, source, name): + self.name = name + spec = importlib.util.spec_from_loader(f"{__name__}.{name}", None) + self.module = importlib.util.module_from_spec(spec) + exec(source, self.module.__dict__) # pylint: disable=exec-used + + def __repr__(self): + return f"<{__name__}.{self.__class__.__name__} name={self.name!r} id={self.id!r}>" + + @property + def id(self): + """Returns the ID of the update. + + :rtype: str + :return: The id of the update + """ + return self.module.update_id + + @property + def previous(self): + """Returns all dependencies of the update. + + :rtype: list[str] + :return: The IDs of all dependencies of the update. + """ + return getattr(self.module, "previous", []) + + @property + def alembic_version(self): + """Returns the alembic revisions of the update. + + :rtype: list[str] + :return: The needed alembic revisions. + """ + return self.module.alembic_revision + + def upgrade(self, config, alembic_config): + """Runs the upgrade migrations of this update script. + + This first runs the pre_alembic task, then the alembic migration, and + finally the post_alembic task. + + Note that this does not ensure that all previous scripts have also been + executed. + + :param config: The app configuration. + :type config: dict + :param alembic_config: The alembic config to use. + :type alembic_config: alembic.config.Config + """ + LOGGER.info("[up] Running pre-alembic task for %s", self.id) + self.module.Up().pre_alembic(config) + LOGGER.info("[up] Running alembic upgrade for %s to %s", self.id, self.alembic_version) + alembic.command.upgrade(alembic_config, self.alembic_version) + LOGGER.info("[up] Running post-alembic task for %s", self.id) + self.module.Up().post_alembic(config) + + def downgrade(self, config, alembic_config): + """Runs the downgrade migrations of this update script. + + See also :meth:`upgrade`. + + :param config: The app configuration. + :type config: dict + :param alembic_config: The alembic config to use. + :type alembic_config: alembic.config.Config + """ + LOGGER.info("[down] Running pre-alembic task for %s", self.id) + self.module.Down().pre_alembic(config) + LOGGER.info("[down] Running alembic downgrade for %s to %s", self.id, self.alembic_version) + alembic.command.downgrade(alembic_config, self.alembic_version) + LOGGER.info("[down] Running post-alembic task for %s", self.id) + self.module.Down().post_alembic(config) + + +def _filename_to_modname(name): + if name.endswith(".py"): + name = name[:-3] + name = name.replace(".", "_") + return name + + +def _load_update_scripts(): + """Loads all available import scripts.""" + files = importlib_resources.files(__name__) + return [ + UpdateScript(file.read_text(), _filename_to_modname(file.name)) + for file in files.iterdir() + if file.name.startswith("upd_") + ] diff --git a/pylint.toml b/pylint.toml new file mode 100644 index 0000000..7495cf7 --- /dev/null +++ b/pylint.toml @@ -0,0 +1,523 @@ +[tool.pylint.main] +# Analyse import fallback blocks. This can be used to support both Python 2 and 3 +# compatible code, which means that the block might have code that exists only in +# one or another interpreter, leading to false positives when analysed. +# analyse-fallback-blocks = + +# Always return a 0 (non-error) status code, even if lint errors are found. This +# is primarily useful in continuous integration scripts. +# exit-zero = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +# extension-pkg-allow-list = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +# extension-pkg-whitelist = + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +# fail-on = + +# Specify a score threshold to be exceeded before program exits with error. +fail-under = 10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +# from-stdin = + +# Files or directories to be skipped. They should be base names, not paths. +ignore = ["CVS"] + +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths = ["fietsboek/updater/upd_.*.py"] + +# Files or directories matching the regex patterns are skipped. The regex matches +# against base names, not paths. The default value ignores Emacs file locks +ignore-patterns = ["^\\.#"] + +# List of module names for which member attributes should not be checked (useful +# for modules/projects where namespaces are manipulated during runtime and thus +# existing member attributes cannot be deduced by static analysis). It supports +# qualified module names, as well as Unix pattern matching. +# ignored-modules = + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +# init-hook = + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs = 1 + +# Control the amount of potential inferred values when inferring a single object. +# This can help the performance when dealing with large functions or complex, +# nested conditions. +limit-inference-results = 100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +# load-plugins = + +# Pickle collected data for later comparisons. +persistent = true + +# Minimum Python version to use for version dependent checks. Will default to the +# version used to run pylint. +py-version = "3.10" + +# Discover python modules and packages in the file system subtree. +# recursive = + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode = true + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +# unsafe-load-any-extension = + +[tool.pylint.basic] +# Naming style matching correct argument names. +argument-naming-style = "snake_case" + +# Regular expression matching correct argument names. Overrides argument-naming- +# style. If left empty, argument names will be checked with the set naming style. +# argument-rgx = + +# Naming style matching correct attribute names. +attr-naming-style = "snake_case" + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +# attr-rgx = + +# Bad variable names which should always be refused, separated by a comma. +bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"] + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +# bad-names-rgxs = + +# Naming style matching correct class attribute names. +class-attribute-naming-style = "any" + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +# class-attribute-rgx = + +# Naming style matching correct class constant names. +class-const-naming-style = "UPPER_CASE" + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +# class-const-rgx = + +# Naming style matching correct class names. +class-naming-style = "PascalCase" + +# Regular expression matching correct class names. Overrides class-naming-style. +# If left empty, class names will be checked with the set naming style. +# class-rgx = + +# Naming style matching correct constant names. +const-naming-style = "UPPER_CASE" + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming style. +# const-rgx = + +# Minimum line length for functions/classes that require docstrings, shorter ones +# are exempt. +docstring-min-length = -1 + +# Naming style matching correct function names. +function-naming-style = "snake_case" + +# Regular expression matching correct function names. Overrides function-naming- +# style. If left empty, function names will be checked with the set naming style. +# function-rgx = + +# Good variable names which should always be accepted, separated by a comma. +good-names = ["i", "j", "k", "ex", "Run", "_", "id"] + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +# good-names-rgxs = + +# Include a hint for the correct naming format with invalid-name. +# include-naming-hint = + +# Naming style matching correct inline iteration names. +inlinevar-naming-style = "any" + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +# inlinevar-rgx = + +# Naming style matching correct method names. +method-naming-style = "snake_case" + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +# method-rgx = + +# Naming style matching correct module names. +module-naming-style = "snake_case" + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +# module-rgx = + +# Colon-delimited sets of names that determine each other's naming style when the +# name regexes allow several styles. +# name-group = + +# Regular expression which should only match function or class names that do not +# require a docstring. +no-docstring-rgx = "^_" + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. These +# decorators are taken in consideration only for invalid-name. +property-classes = ["abc.abstractproperty"] + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +# typevar-rgx = + +# Naming style matching correct variable names. +variable-naming-style = "snake_case" + +# Regular expression matching correct variable names. Overrides variable-naming- +# style. If left empty, variable names will be checked with the set naming style. +# variable-rgx = + +[tool.pylint.classes] +# Warn about protected attribute access inside special methods +# check-protected-access-in-special-methods = + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods = ["__init__", "__new__", "setUp", "__post_init__"] + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make"] + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg = ["cls"] + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg = ["cls"] + +[tool.pylint.design] +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +# exclude-too-few-public-methods = + +# List of qualified class names to ignore when counting class parents (see R0901) +# ignored-parents = + +# Maximum number of arguments for function / method. +max-args = 5 + +# Maximum number of attributes for a class (see R0902). +max-attributes = 7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr = 5 + +# Maximum number of branch for function / method body. +max-branches = 12 + +# Maximum number of locals for function / method body. +max-locals = 15 + +# Maximum number of parents for a class (see R0901). +max-parents = 7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods = 20 + +# Maximum number of return / yield for function / method body. +max-returns = 6 + +# Maximum number of statements in function / method body. +max-statements = 50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods = 2 + +[tool.pylint.exceptions] +# Exceptions that will emit a warning when caught. +overgeneral-exceptions = ["BaseException", "Exception"] + +[tool.pylint.format] +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +# expected-line-ending-format = + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines = "^\\s*(# )??$" + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren = 4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string = " " + +# Maximum number of characters on a single line. +max-line-length = 100 + +# Maximum number of lines in a module. +max-module-lines = 1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +# single-line-class-stmt = + +# Allow the body of an if to be on the same line as the test if there is no else. +# single-line-if-stmt = + +[tool.pylint.imports] +# List of modules that can be imported at any level, not just the top level one. +# allow-any-import-level = + +# Allow wildcard imports from modules that define __all__. +# allow-wildcard-with-all = + +# Deprecated modules which should not be used, separated by a comma. +# deprecated-modules = + +# Output a graph (.gv or any supported image format) of external dependencies to +# the given file (report RP0402 must not be disabled). +# ext-import-graph = + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be disabled). +# import-graph = + +# Output a graph (.gv or any supported image format) of internal dependencies to +# the given file (report RP0402 must not be disabled). +# int-import-graph = + +# Force import order to recognize a module as part of the standard compatibility +# libraries. +# known-standard-library = + +# Force import order to recognize a module as part of a third party library. +known-third-party = ["enchant"] + +# Couples of modules and preferred modules, separated by a comma. +# preferred-modules = + +[tool.pylint.logging] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style = "old" + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules = ["logging"] + +[tool.pylint."messages control"] +# Only show warnings with the listed confidence levels. Leave empty to show all. +# Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] + +# Disable the message, report, category or checker with the given id(s). You can +# either give multiple identifiers separated by comma (,) or put this option +# multiple times (only on the command line, not in the configuration file where +# it should appear only once). You can also use "--disable=all" to disable +# everything first and then re-enable specific checks. For example, if you want +# to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable = ["raw-checker-failed", "bad-inline-option", "locally-disabled", "file-ignored", "suppressed-message", "useless-suppression", "deprecated-pragma", "use-symbolic-message-instead"] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where it +# should appear only once). See also the "--disable" option for examples. +enable = ["c-extension-no-member"] + +[tool.pylint.miscellaneous] +# List of note tags to take in consideration, separated by a comma. +notes = ["FIXME", "XXX", "TODO"] + +# Regular expression of note tags to take in consideration. +# notes-rgx = + +[tool.pylint.refactoring] +# Maximum number of nested blocks for function / method body +max-nested-blocks = 5 + +# Complete name of functions that never returns. When checking for inconsistent- +# return-statements if a never returning function is called then it will be +# considered as an explicit return statement and no message will be printed. +never-returning-functions = ["sys.exit", "argparse.parse_error"] + +[tool.pylint.reports] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each category, +# as well as 'statement' which is the total number of statements analyzed. This +# score is used by the global evaluation report (RP0004). +evaluation = "max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10))" + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +# msg-template = + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +# output-format = + +# Tells whether to display a full report or only the messages. +# reports = + +# Activate the evaluation score. +score = true + +[tool.pylint.similarities] +# Comments are removed from the similarity computation +ignore-comments = true + +# Docstrings are removed from the similarity computation +ignore-docstrings = true + +# Imports are removed from the similarity computation +ignore-imports = true + +# Signatures are removed from the similarity computation +ignore-signatures = true + +# Minimum lines number of a similarity. +min-similarity-lines = 4 + +[tool.pylint.spelling] +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions = 4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +# spelling-dict = + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" + +# List of comma separated words that should not be checked. +# spelling-ignore-words = + +# A path to a file that contains the private dictionary; one word per line. +# spelling-private-dict-file = + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +# spelling-store-unknown-words = + +[tool.pylint.string] +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +# check-quote-consistency = + +# This flag controls whether the implicit-str-concat should generate a warning on +# implicit string concatenation in sequences defined over several lines. +# check-str-concat-over-line-jumps = + +[tool.pylint.typecheck] +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators = ["contextlib.contextmanager"] + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +# generated-members = + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +# Tells whether to warn about missing members when the owner of the attribute is +# inferred to be None. +ignore-none = true + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference can +# return multiple potential results while evaluating a Python object, but some +# branches might not be evaluated, which results in partial inference. In that +# case, it might be useful to still emit no-member and other checks for the rest +# of the inferred objects. +ignore-on-opaque-inference = true + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes = ["optparse.Values", "thread._local", "_thread._local", "argparse.Namespace"] + +# Show a hint with possible names when a member name was not found. The aspect of +# finding the hint is based on edit distance. +missing-member-hint = true + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance = 1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices = 1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx = ".*[Mm]ixin" + +# List of decorators that change the signature of a decorated function. +# signature-mutators = + +[tool.pylint.variables] +# List of additional names supposed to be defined in builtins. Remember that you +# should avoid defining new builtins when possible. +# additional-builtins = + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables = true + +# List of names allowed to shadow builtins +# allowed-redefined-builtins = + +# List of strings which can identify a callback function by name. A callback name +# must start or end with one of those strings. +callbacks = ["cb_", "_cb"] + +# A regular expression matching the name of dummy variables (i.e. expected to not +# be used). +dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" + +# Argument names that match this expression will be ignored. Default to name with +# leading underscore. +ignored-argument-names = "_.*|^ignored_|^unused_" + +# Tells whether we should check for unused import in __init__ files. +# init-import = + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] + + diff --git a/tox.ini b/tox.ini index da41521..91cda20 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ commands = deps = pylint usedevelop = true commands = - pylint fietsboek + pylint --rcfile=pylint.toml fietsboek [testenv:pylint-tests] deps = pylint -- cgit v1.2.3 From 914843fa1eb8d557d6c0d7b50030f7f7645d200a Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 24 Sep 2022 14:59:59 +0200 Subject: add a CLI frontend for the updater logic --- fietsboek/updater/__init__.py | 10 +++ fietsboek/updater/__main__.py | 137 ++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 3 files changed, 148 insertions(+) create mode 100644 fietsboek/updater/__main__.py diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index bc1bc96..6c154e7 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -90,6 +90,16 @@ class Updater: for prev_id in script.previous: self.backward_dependencies[prev_id].append(script.id) + def exists(self, revision_id): + """Checks if the revision with the given ID exists. + + :param revision_id: ID of the revision to check. + :type revision_id: str + :return: True if the revision exists. + :rtype: bool + """ + return revision_id in self.scripts + def current_versions(self): """Reads the current version of the data. diff --git a/fietsboek/updater/__main__.py b/fietsboek/updater/__main__.py new file mode 100644 index 0000000..38297d7 --- /dev/null +++ b/fietsboek/updater/__main__.py @@ -0,0 +1,137 @@ +"""Updater for Fietsboek. + +While this script does not download and install the latest Fietsboek version +itself (as this step depends on where you got Fietsboek from), it takes care of +managing migrations between Fietsboek versions. In particular, the updater +takes care of running the database migrations, migrating the data directory and +migrating the configuration. +""" +import click + +from pyramid.paster import setup_logging + +from . import Updater + + +def user_confirm(): + click.secho("Warning:", fg="yellow") + click.echo( + "Updating *may* cause data loss. Make sure to have a backup of the " + "database and the data directory!" + ) + click.echo("For more information, please consult the documentation.") + click.confirm("Proceed?", abort=True) + + +@click.group(help=__doc__) +@click.option( + "-c", "--config", + type=click.Path(exists=True, dir_okay=False), + required=True, + help="Path to the Fietsboek configuration file", +) +@click.pass_context +def cli(ctx, config): + ctx.ensure_object(dict) + setup_logging(config) + ctx.obj["INIFILE"] = config + + +@cli.command("update") +@click.option( + "-f", "--force", + is_flag=True, + help="Skip the safety question and just run the update", +) +@click.argument("VERSION", required=False) +@click.pass_context +def update(ctx, version, force): + """Run the update process. + + Make sure to consult the documentation and ensure that you have a backup + before starting the update, to prevent any data loss! + + VERSION specifies the version you want to update to. Leave empty to choose + the latest version. + """ + updater = Updater(ctx.obj["INIFILE"]) + updater.load() + if version and not updater.exists(version): + click.secho("Revision not found", fg="red") + return + version_str = ", ".join(updater.current_versions()) + click.echo(f"Current version: {version_str}") + if not version: + heads = updater.heads() + if len(heads) != 1: + click.secho("Ambiguous heads, please specify the version to update to", fg="red") + return + version = heads[0] + click.echo(f"Selected version: {version}") + if not force: + user_confirm() + updater.upgrade(version) + version_str = ", ".join(updater.current_versions()) + click.secho(f"Update succeeded, version: {version_str}", fg="green") + + +@cli.command("downgrade") +@click.option( + "-f", "--force", + is_flag=True, + help="Skip the safety question and just run the downgrade", +) +@click.argument("VERSION") +@click.pass_context +def downgrade(ctx, version, force): + """Run the downgrade process. + + Make sure to consult the documentation and ensure that you have a backup + before starting the downgrade, to prevent any data loss! + + VERSION specifies the version you want to downgrade to. + """ + updater = Updater(ctx.obj["INIFILE"]) + updater.load() + if version and not updater.exists(version): + click.secho("Revision not found", fg="red") + return + version_str = ", ".join(updater.current_versions()) + click.echo(f"Current version: {version_str}") + click.echo(f"Downgrade to version {version}") + if not force: + user_confirm() + updater.downgrade(version) + version_str = ", ".join(updater.current_versions()) + click.secho(f"Downgrade succeeded, version: {version_str}", fg="green") + + +@cli.command("revision") +@click.argument("REVISION_ID", required=False) +@click.pass_context +def revision(ctx, revision_id): + """Create a new revision. + + This automatically populates the revision dependencies and alembic + versions, based on the current state. + + This command is useful for developers who work on Fietsboek. + """ + updater = Updater(ctx.obj["INIFILE"]) + updater.load() + current = updater.current_versions() + heads = updater.heads() + if not any(version in heads for version in current): + click.secho("Warning:", fg="yellow") + click.echo("Your current revision is not a head. This will create a branch!") + click.echo( + "If this is not what you intended, make sure to update to the latest " + "version first before creating a new revision." + ) + click.confirm("Proceed?", abort=True) + filename = updater.new_revision(revision_id) + click.echo(f"Revision saved to {filename}") + + +if __name__ == "__main__": + cli() diff --git a/setup.py b/setup.py index b53a875..ad759ef 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ requires = [ 'gpxpy', 'markdown', 'bleach', + 'Click', ] tests_require = [ -- cgit v1.2.3 From 7e60a117d854bd569a7aafbd09dff4bf4e425115 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 24 Sep 2022 15:09:58 +0200 Subject: fix downgrade not picking the right revisions If we apply a migration that takes us from "v1" to "v2", then a downgrade needs to apply the reverse of this migration - not the one that goes from "v1" to "v0", and not "no migration". The previous code managed to not do this, as it would see "v1-v2" as applied and therefore skip it. --- fietsboek/updater/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index 6c154e7..8ca4991 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -124,7 +124,7 @@ class Updater: def _reverse_versions(self): all_versions = set(script.id for script in self.scripts.values()) - return (all_versions - self._transitive_versions()) | set(self.current_versions()) + return (all_versions - self._transitive_versions()) def stamp(self, versions): """Stampts the given version into the version file. @@ -201,6 +201,7 @@ class Updater: # dependencies instead. applied_versions = self._reverse_versions() to_apply = self._pick_updates(target, applied_versions, self.backward_dependencies) + to_apply -= {target} application_queue = self._make_schedule(to_apply, self.backward_dependencies) LOGGER.debug("Planned downgrade: %s", application_queue) for downgrade in application_queue: -- cgit v1.2.3 From f2ed0d5d7b3fbd45d68692185781d0d7e7742159 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 24 Sep 2022 15:22:28 +0200 Subject: fix stamp logic for downgrades --- fietsboek/updater/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index 8ca4991..038fcd5 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -207,7 +207,7 @@ class Updater: for downgrade in application_queue: script = self.scripts[downgrade] script.downgrade(self.settings, self.alembic_config) - self._stamp_versions(self.backward_dependencies[script.id], [script.id]) + self._stamp_versions([script.id], script.previous) def new_revision(self, revision_id=None): """Creates a new revision with the current versions as dependencies and -- cgit v1.2.3 From ccbf216aaf6e10d60e3138cbbed93359acfe71a2 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 28 Sep 2022 18:55:04 +0200 Subject: reorganize updater CLI This moves the updater scripts into a subfolder, which keeps them separated better from the rest of the package. In addition, we now have the "fietsupdate" command instead of using "python -m fietsboek.updater". --- fietsboek/updater/__init__.py | 4 +- fietsboek/updater/__main__.py | 137 ------------------------------------------ fietsboek/updater/cli.py | 137 ++++++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 4 files changed, 140 insertions(+), 139 deletions(-) delete mode 100644 fietsboek/updater/__main__.py create mode 100644 fietsboek/updater/cli.py diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index 038fcd5..724defa 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -244,7 +244,7 @@ class Updater: ) filename = f"upd_{revision_id}.py" - filepath = Path(__file__).parent / filename + filepath = Path(__file__).parent / "scripts" / filename LOGGER.info("Writing new revision (%s) to %r", revision_id, filepath) with open(filepath, "x", encoding="utf-8") as fobj: fobj.write(revision) @@ -346,7 +346,7 @@ def _filename_to_modname(name): def _load_update_scripts(): """Loads all available import scripts.""" - files = importlib_resources.files(__name__) + files = importlib_resources.files(__name__) / "scripts" return [ UpdateScript(file.read_text(), _filename_to_modname(file.name)) for file in files.iterdir() diff --git a/fietsboek/updater/__main__.py b/fietsboek/updater/__main__.py deleted file mode 100644 index 38297d7..0000000 --- a/fietsboek/updater/__main__.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Updater for Fietsboek. - -While this script does not download and install the latest Fietsboek version -itself (as this step depends on where you got Fietsboek from), it takes care of -managing migrations between Fietsboek versions. In particular, the updater -takes care of running the database migrations, migrating the data directory and -migrating the configuration. -""" -import click - -from pyramid.paster import setup_logging - -from . import Updater - - -def user_confirm(): - click.secho("Warning:", fg="yellow") - click.echo( - "Updating *may* cause data loss. Make sure to have a backup of the " - "database and the data directory!" - ) - click.echo("For more information, please consult the documentation.") - click.confirm("Proceed?", abort=True) - - -@click.group(help=__doc__) -@click.option( - "-c", "--config", - type=click.Path(exists=True, dir_okay=False), - required=True, - help="Path to the Fietsboek configuration file", -) -@click.pass_context -def cli(ctx, config): - ctx.ensure_object(dict) - setup_logging(config) - ctx.obj["INIFILE"] = config - - -@cli.command("update") -@click.option( - "-f", "--force", - is_flag=True, - help="Skip the safety question and just run the update", -) -@click.argument("VERSION", required=False) -@click.pass_context -def update(ctx, version, force): - """Run the update process. - - Make sure to consult the documentation and ensure that you have a backup - before starting the update, to prevent any data loss! - - VERSION specifies the version you want to update to. Leave empty to choose - the latest version. - """ - updater = Updater(ctx.obj["INIFILE"]) - updater.load() - if version and not updater.exists(version): - click.secho("Revision not found", fg="red") - return - version_str = ", ".join(updater.current_versions()) - click.echo(f"Current version: {version_str}") - if not version: - heads = updater.heads() - if len(heads) != 1: - click.secho("Ambiguous heads, please specify the version to update to", fg="red") - return - version = heads[0] - click.echo(f"Selected version: {version}") - if not force: - user_confirm() - updater.upgrade(version) - version_str = ", ".join(updater.current_versions()) - click.secho(f"Update succeeded, version: {version_str}", fg="green") - - -@cli.command("downgrade") -@click.option( - "-f", "--force", - is_flag=True, - help="Skip the safety question and just run the downgrade", -) -@click.argument("VERSION") -@click.pass_context -def downgrade(ctx, version, force): - """Run the downgrade process. - - Make sure to consult the documentation and ensure that you have a backup - before starting the downgrade, to prevent any data loss! - - VERSION specifies the version you want to downgrade to. - """ - updater = Updater(ctx.obj["INIFILE"]) - updater.load() - if version and not updater.exists(version): - click.secho("Revision not found", fg="red") - return - version_str = ", ".join(updater.current_versions()) - click.echo(f"Current version: {version_str}") - click.echo(f"Downgrade to version {version}") - if not force: - user_confirm() - updater.downgrade(version) - version_str = ", ".join(updater.current_versions()) - click.secho(f"Downgrade succeeded, version: {version_str}", fg="green") - - -@cli.command("revision") -@click.argument("REVISION_ID", required=False) -@click.pass_context -def revision(ctx, revision_id): - """Create a new revision. - - This automatically populates the revision dependencies and alembic - versions, based on the current state. - - This command is useful for developers who work on Fietsboek. - """ - updater = Updater(ctx.obj["INIFILE"]) - updater.load() - current = updater.current_versions() - heads = updater.heads() - if not any(version in heads for version in current): - click.secho("Warning:", fg="yellow") - click.echo("Your current revision is not a head. This will create a branch!") - click.echo( - "If this is not what you intended, make sure to update to the latest " - "version first before creating a new revision." - ) - click.confirm("Proceed?", abort=True) - filename = updater.new_revision(revision_id) - click.echo(f"Revision saved to {filename}") - - -if __name__ == "__main__": - cli() diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py new file mode 100644 index 0000000..38297d7 --- /dev/null +++ b/fietsboek/updater/cli.py @@ -0,0 +1,137 @@ +"""Updater for Fietsboek. + +While this script does not download and install the latest Fietsboek version +itself (as this step depends on where you got Fietsboek from), it takes care of +managing migrations between Fietsboek versions. In particular, the updater +takes care of running the database migrations, migrating the data directory and +migrating the configuration. +""" +import click + +from pyramid.paster import setup_logging + +from . import Updater + + +def user_confirm(): + click.secho("Warning:", fg="yellow") + click.echo( + "Updating *may* cause data loss. Make sure to have a backup of the " + "database and the data directory!" + ) + click.echo("For more information, please consult the documentation.") + click.confirm("Proceed?", abort=True) + + +@click.group(help=__doc__) +@click.option( + "-c", "--config", + type=click.Path(exists=True, dir_okay=False), + required=True, + help="Path to the Fietsboek configuration file", +) +@click.pass_context +def cli(ctx, config): + ctx.ensure_object(dict) + setup_logging(config) + ctx.obj["INIFILE"] = config + + +@cli.command("update") +@click.option( + "-f", "--force", + is_flag=True, + help="Skip the safety question and just run the update", +) +@click.argument("VERSION", required=False) +@click.pass_context +def update(ctx, version, force): + """Run the update process. + + Make sure to consult the documentation and ensure that you have a backup + before starting the update, to prevent any data loss! + + VERSION specifies the version you want to update to. Leave empty to choose + the latest version. + """ + updater = Updater(ctx.obj["INIFILE"]) + updater.load() + if version and not updater.exists(version): + click.secho("Revision not found", fg="red") + return + version_str = ", ".join(updater.current_versions()) + click.echo(f"Current version: {version_str}") + if not version: + heads = updater.heads() + if len(heads) != 1: + click.secho("Ambiguous heads, please specify the version to update to", fg="red") + return + version = heads[0] + click.echo(f"Selected version: {version}") + if not force: + user_confirm() + updater.upgrade(version) + version_str = ", ".join(updater.current_versions()) + click.secho(f"Update succeeded, version: {version_str}", fg="green") + + +@cli.command("downgrade") +@click.option( + "-f", "--force", + is_flag=True, + help="Skip the safety question and just run the downgrade", +) +@click.argument("VERSION") +@click.pass_context +def downgrade(ctx, version, force): + """Run the downgrade process. + + Make sure to consult the documentation and ensure that you have a backup + before starting the downgrade, to prevent any data loss! + + VERSION specifies the version you want to downgrade to. + """ + updater = Updater(ctx.obj["INIFILE"]) + updater.load() + if version and not updater.exists(version): + click.secho("Revision not found", fg="red") + return + version_str = ", ".join(updater.current_versions()) + click.echo(f"Current version: {version_str}") + click.echo(f"Downgrade to version {version}") + if not force: + user_confirm() + updater.downgrade(version) + version_str = ", ".join(updater.current_versions()) + click.secho(f"Downgrade succeeded, version: {version_str}", fg="green") + + +@cli.command("revision") +@click.argument("REVISION_ID", required=False) +@click.pass_context +def revision(ctx, revision_id): + """Create a new revision. + + This automatically populates the revision dependencies and alembic + versions, based on the current state. + + This command is useful for developers who work on Fietsboek. + """ + updater = Updater(ctx.obj["INIFILE"]) + updater.load() + current = updater.current_versions() + heads = updater.heads() + if not any(version in heads for version in current): + click.secho("Warning:", fg="yellow") + click.echo("Your current revision is not a head. This will create a branch!") + click.echo( + "If this is not what you intended, make sure to update to the latest " + "version first before creating a new revision." + ) + click.confirm("Proceed?", abort=True) + filename = updater.new_revision(revision_id) + click.echo(f"Revision saved to {filename}") + + +if __name__ == "__main__": + cli() diff --git a/setup.py b/setup.py index ad759ef..5fd76e0 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,7 @@ setup( ], 'console_scripts': [ 'fietsctl=fietsboek.scripts.fietsctl:main', + 'fietsupdate=fietsboek.updater.cli:cli', ], }, ) -- cgit v1.2.3 From 282366e11f9cdda54dbedeb673bb894d53ec9147 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Thu, 29 Sep 2022 20:17:17 +0200 Subject: fietsupdater: better help behaviour This does two things: 1. fietsupdate update --help works (before it errored because the required argument is not given) 2. fietsupdate help works (like git help ...) --- fietsboek/updater/cli.py | 76 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py index 38297d7..6a59100 100644 --- a/fietsboek/updater/cli.py +++ b/fietsboek/updater/cli.py @@ -13,6 +13,20 @@ from pyramid.paster import setup_logging from . import Updater +# We keep this as a separate option that is added to each subcommand as Click +# (unlike argparse) cannot handle "--help" without the required arguments of +# the parent (so "fietsupdate update --help" would error out) +# See also +# https://github.com/pallets/click/issues/295 +# https://github.com/pallets/click/issues/814 +config_option = click.option( + "-c", "--config", + type=click.Path(exists=True, dir_okay=False), + required=True, + help="Path to the Fietsboek configuration file", +) + + def user_confirm(): click.secho("Warning:", fg="yellow") click.echo( @@ -23,29 +37,24 @@ def user_confirm(): click.confirm("Proceed?", abort=True) -@click.group(help=__doc__) -@click.option( - "-c", "--config", - type=click.Path(exists=True, dir_okay=False), - required=True, - help="Path to the Fietsboek configuration file", +@click.group( + help=__doc__, + context_settings={'help_option_names': ['-h', '--help']}, ) -@click.pass_context -def cli(ctx, config): - ctx.ensure_object(dict) - setup_logging(config) - ctx.obj["INIFILE"] = config +def cli(): + """CLI main entry point.""" + pass @cli.command("update") +@config_option @click.option( "-f", "--force", is_flag=True, help="Skip the safety question and just run the update", ) -@click.argument("VERSION", required=False) -@click.pass_context -def update(ctx, version, force): +@click.argument("version", required=False) +def update(config, version, force): """Run the update process. Make sure to consult the documentation and ensure that you have a backup @@ -54,7 +63,7 @@ def update(ctx, version, force): VERSION specifies the version you want to update to. Leave empty to choose the latest version. """ - updater = Updater(ctx.obj["INIFILE"]) + updater = Updater(config) updater.load() if version and not updater.exists(version): click.secho("Revision not found", fg="red") @@ -76,14 +85,14 @@ def update(ctx, version, force): @cli.command("downgrade") +@config_option @click.option( "-f", "--force", is_flag=True, help="Skip the safety question and just run the downgrade", ) -@click.argument("VERSION") -@click.pass_context -def downgrade(ctx, version, force): +@click.argument("version") +def downgrade(config, version, force): """Run the downgrade process. Make sure to consult the documentation and ensure that you have a backup @@ -91,7 +100,7 @@ def downgrade(ctx, version, force): VERSION specifies the version you want to downgrade to. """ - updater = Updater(ctx.obj["INIFILE"]) + updater = Updater(config) updater.load() if version and not updater.exists(version): click.secho("Revision not found", fg="red") @@ -106,10 +115,10 @@ def downgrade(ctx, version, force): click.secho(f"Downgrade succeeded, version: {version_str}", fg="green") -@cli.command("revision") -@click.argument("REVISION_ID", required=False) -@click.pass_context -def revision(ctx, revision_id): +@cli.command("revision", hidden=True) +@config_option +@click.argument("revision_id", required=False) +def revision(config, revision_id): """Create a new revision. This automatically populates the revision dependencies and alembic @@ -133,5 +142,26 @@ def revision(ctx, revision_id): click.echo(f"Revision saved to {filename}") +@cli.command("help", hidden=True) +@click.pass_context +@click.argument("subcommand", required=False) +def help(ctx, subcommand): + """Shows the help for a given subcommand. + + This is equivalent to using the --help option. + """ + if not subcommand: + click.echo(cli.get_help(ctx.parent)) + return + cmd = cli.get_command(ctx, subcommand) + if cmd is None: + click.echo(f"Error: Command {subcommand} not found") + else: + # Create a new context so that the usage shows "fietsboek subcommand" + # instead of "fietsboek help" + with click.Context(cmd, ctx.parent, subcommand) as subcontext: + click.echo(cmd.get_help(subcontext)) + + if __name__ == "__main__": cli() -- cgit v1.2.3 From a90d279ca27293efcb5dac244f4fdf3498d75529 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Thu, 29 Sep 2022 20:36:59 +0200 Subject: fietsupdater: use Context.fail to abort This makes sure that the exit code is set properly --- fietsboek/updater/cli.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py index 6a59100..45655d4 100644 --- a/fietsboek/updater/cli.py +++ b/fietsboek/updater/cli.py @@ -54,7 +54,8 @@ def cli(): help="Skip the safety question and just run the update", ) @click.argument("version", required=False) -def update(config, version, force): +@click.pass_context +def update(ctx, config, version, force): """Run the update process. Make sure to consult the documentation and ensure that you have a backup @@ -66,16 +67,17 @@ def update(config, version, force): updater = Updater(config) updater.load() if version and not updater.exists(version): - click.secho("Revision not found", fg="red") - return + ctx.fail(f"Version {version!r} not found") + version_str = ", ".join(updater.current_versions()) click.echo(f"Current version: {version_str}") if not version: heads = updater.heads() if len(heads) != 1: - click.secho("Ambiguous heads, please specify the version to update to", fg="red") - return + ctx.fail("Ambiguous heads, please specify the version to update to") + version = heads[0] + click.echo(f"Selected version: {version}") if not force: user_confirm() @@ -92,7 +94,8 @@ def update(config, version, force): help="Skip the safety question and just run the downgrade", ) @click.argument("version") -def downgrade(config, version, force): +@click.pass_context +def downgrade(ctx, config, version, force): """Run the downgrade process. Make sure to consult the documentation and ensure that you have a backup @@ -103,8 +106,8 @@ def downgrade(config, version, force): updater = Updater(config) updater.load() if version and not updater.exists(version): - click.secho("Revision not found", fg="red") - return + ctx.fail(f"Version {version!r} not found") + version_str = ", ".join(updater.current_versions()) click.echo(f"Current version: {version_str}") click.echo(f"Downgrade to version {version}") -- cgit v1.2.3 From 4a391609e8432782f3911464d303b2a62c7b6c2c Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Thu, 29 Sep 2022 20:53:27 +0200 Subject: fietsupdtr: noop if the update is already applied --- fietsboek/updater/__init__.py | 24 ++++++++++++++++++++++++ fietsboek/updater/cli.py | 22 ++++++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index 724defa..1c26715 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -258,6 +258,30 @@ class Updater: """ return [rev_id for (rev_id, deps) in self.backward_dependencies.items() if not deps] + def has_applied(self, revision_id, backward=False): + """Checks whether the given revision is applied. + + By default, this checks if a given update is applied, i.e. the current + version is greater-or-equal to the given revision ID. If ``backward`` + is ``True``, we instead check if the current version is lower-or-equal + to the given revision ID. + + Note that this function does not raise an error if the given revision + ID cannot be found and instead simply returns ``False``. Use + :meth:`exists` to check whether the revision actually exists. + + :param revision_id: The revision to check. + :type revision_id: str + :param backward: Whether to switch the comparison direction. + :type backward: bool + :return: ``True`` if the current version at least matches the asked + revision ID. + :rtype: bool + """ + if not backward: + return revision_id in self._transitive_versions() + return revision_id in self._reverse_versions() | set(self.current_versions()) + class UpdateScript: """Represents an update script.""" diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py index 45655d4..c71ec17 100644 --- a/fietsboek/updater/cli.py +++ b/fietsboek/updater/cli.py @@ -27,10 +27,17 @@ config_option = click.option( ) -def user_confirm(): +def user_confirm(verb): + """Ask the user for confirmation before proceeding. + + Aborts the program if no confirmation is given. + + :param verb: The verb to use in the text. + :type verb: str + """ click.secho("Warning:", fg="yellow") click.echo( - "Updating *may* cause data loss. Make sure to have a backup of the " + f"{verb} *may* cause data loss. Make sure to have a backup of the " "database and the data directory!" ) click.echo("For more information, please consult the documentation.") @@ -79,8 +86,12 @@ def update(ctx, config, version, force): version = heads[0] click.echo(f"Selected version: {version}") + if updater.has_applied(version): + click.secho("Nothing to do", fg="green") + ctx.exit() + if not force: - user_confirm() + user_confirm("Updating") updater.upgrade(version) version_str = ", ".join(updater.current_versions()) click.secho(f"Update succeeded, version: {version_str}", fg="green") @@ -111,8 +122,11 @@ def downgrade(ctx, config, version, force): version_str = ", ".join(updater.current_versions()) click.echo(f"Current version: {version_str}") click.echo(f"Downgrade to version {version}") + if updater.has_applied(version, backward=True): + click.secho("Nothing to do", fg="green") + ctx.exit() if not force: - user_confirm() + user_confirm("Downgrading") updater.downgrade(version) version_str = ", ".join(updater.current_versions()) click.secho(f"Downgrade succeeded, version: {version_str}", fg="green") -- cgit v1.2.3 From 3402387360a5712c96d60c0ee67c03b268e4c55d Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Mon, 10 Oct 2022 20:52:43 +0200 Subject: fietsupdater: fix revision subcommand --- fietsboek/updater/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py index c71ec17..2a7fc14 100644 --- a/fietsboek/updater/cli.py +++ b/fietsboek/updater/cli.py @@ -143,7 +143,7 @@ def revision(config, revision_id): This command is useful for developers who work on Fietsboek. """ - updater = Updater(ctx.obj["INIFILE"]) + updater = Updater(config) updater.load() current = updater.current_versions() heads = updater.heads() -- cgit v1.2.3 From e1361e556840a1a710385cd1a09bfa1a9f91742a Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Mon, 10 Oct 2022 20:53:04 +0200 Subject: add a common base class to update scripts This makes it easier to add some useful hooks, such as "self.tell" to output information about the update to the user, and it ensures the pre_alembic/post_alembic methods exist. --- fietsboek/updater/__init__.py | 6 ++++-- fietsboek/updater/script.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 fietsboek/updater/script.py diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index 1c26715..e81a515 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -23,6 +23,8 @@ TEMPLATE = """\ Date created: {{ date }} \"\"\" +from fietsboek.updater.script import UpdateScript + update_id = {{ "{!r}".format(update_id) }} previous = [ {%- for prev in previous %} @@ -32,7 +34,7 @@ previous = [ alembic_revision = {{ "{!r}".format(alembic_revision) }} -class Up: +class Up(UpdateScript): def pre_alembic(self, config): pass @@ -40,7 +42,7 @@ class Up: pass -class Down: +class Down(UpdateScript): def pre_alembic(self, config): pass diff --git a/fietsboek/updater/script.py b/fietsboek/updater/script.py new file mode 100644 index 0000000..15e1d59 --- /dev/null +++ b/fietsboek/updater/script.py @@ -0,0 +1,32 @@ +# Placed in a separate file to avoid cyclic dependencies +class UpdateScript: + def tell(self, text): + """Output a message to the user. + + This function should be used in update scripts instead of :func:`print` + to ensure the right stream is selected. + + :param text: The text to show to the user. + :type text: str + """ + print(text) + + def pre_alembic(self, config): + """Script that is run before the alembic migration is run. + + This method is to be overridden by subclasses. + + :param config: The app configuration. + :type config: dict + """ + pass + + def post_alembic(self, config): + """Script that is run after the alembic migrations have been run. + + This method is to be overridden by subclasses. + + :param config: The app configuration. + :type config: dict + """ + pass -- cgit v1.2.3 From 46fe6f404cfa090f7afab00de7037f1231388250 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Mon, 10 Oct 2022 21:03:15 +0200 Subject: fietsupdater: add a "status" subcommand --- fietsboek/updater/cli.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py index 2a7fc14..f00dfde 100644 --- a/fietsboek/updater/cli.py +++ b/fietsboek/updater/cli.py @@ -159,6 +159,29 @@ def revision(config, revision_id): click.echo(f"Revision saved to {filename}") +@cli.command("status") +@config_option +def status(config): + """Display information about the current version and available updates.""" + updater = Updater(config) + updater.load() + current = updater.current_versions() + heads = updater.heads() + click.secho("Current versions:", fg="yellow") + if current: + for i in current: + click.echo(i) + else: + click.secho("No current version", fg="red") + click.secho("Available updates:", fg="yellow") + updates = set(heads) - set(current) + if updates: + for i in updates: + click.echo(i) + else: + click.secho("All updates applied!", fg="green") + + @cli.command("help", hidden=True) @click.pass_context @click.argument("subcommand", required=False) -- cgit v1.2.3 From 3e900b56cf7f642d0851dfe50914c80f449d18df Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Mon, 10 Oct 2022 21:09:18 +0200 Subject: lint fixes --- fietsboek/updater/__init__.py | 2 +- fietsboek/updater/cli.py | 5 +---- fietsboek/updater/script.py | 11 +++++++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fietsboek/updater/__init__.py b/fietsboek/updater/__init__.py index e81a515..7878f8e 100644 --- a/fietsboek/updater/__init__.py +++ b/fietsboek/updater/__init__.py @@ -126,7 +126,7 @@ class Updater: def _reverse_versions(self): all_versions = set(script.id for script in self.scripts.values()) - return (all_versions - self._transitive_versions()) + return all_versions - self._transitive_versions() def stamp(self, versions): """Stampts the given version into the version file. diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py index f00dfde..9a64a24 100644 --- a/fietsboek/updater/cli.py +++ b/fietsboek/updater/cli.py @@ -8,8 +8,6 @@ migrating the configuration. """ import click -from pyramid.paster import setup_logging - from . import Updater @@ -50,7 +48,6 @@ def user_confirm(verb): ) def cli(): """CLI main entry point.""" - pass @cli.command("update") @@ -185,7 +182,7 @@ def status(config): @cli.command("help", hidden=True) @click.pass_context @click.argument("subcommand", required=False) -def help(ctx, subcommand): +def help_(ctx, subcommand): """Shows the help for a given subcommand. This is equivalent to using the --help option. diff --git a/fietsboek/updater/script.py b/fietsboek/updater/script.py index 15e1d59..305c949 100644 --- a/fietsboek/updater/script.py +++ b/fietsboek/updater/script.py @@ -1,5 +1,14 @@ +"""Base class definition for update scripts.""" # Placed in a separate file to avoid cyclic dependencies + + class UpdateScript: + """Base class for update scripts. + + This class provides stub methods for the update script hooks as well as + methods for user interaction. + """ + def tell(self, text): """Output a message to the user. @@ -19,7 +28,6 @@ class UpdateScript: :param config: The app configuration. :type config: dict """ - pass def post_alembic(self, config): """Script that is run after the alembic migrations have been run. @@ -29,4 +37,3 @@ class UpdateScript: :param config: The app configuration. :type config: dict """ - pass -- cgit v1.2.3 From aa7ffe837fe5e0739c32a633d2d02544ca5d72ba Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Mon, 10 Oct 2022 21:18:57 +0200 Subject: make fietsupdater status warn about unknown update --- fietsboek/updater/cli.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fietsboek/updater/cli.py b/fietsboek/updater/cli.py index 9a64a24..a6ea6c1 100644 --- a/fietsboek/updater/cli.py +++ b/fietsboek/updater/cli.py @@ -166,8 +166,17 @@ def status(config): heads = updater.heads() click.secho("Current versions:", fg="yellow") if current: + has_unknown = False for i in current: - click.echo(i) + if updater.exists(i): + click.echo(i) + else: + click.echo(f"{i} [unknown]") + has_unknown = True + if has_unknown: + click.echo("[*] Your version contains revisions that are unknown to me") + click.echo("[*] This can happen if you apply an update and then downgrade the code") + click.echo("[*] Make sure to keep your code and data in sync!") else: click.secho("No current version", fg="red") click.secho("Available updates:", fg="yellow") -- cgit v1.2.3 From 16216e721e0ecd981f9f30443b89d01033cdf63e Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Thu, 20 Oct 2022 21:49:47 +0200 Subject: adjust docs for new updater script --- doc/administration/upgrading.rst | 75 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/doc/administration/upgrading.rst b/doc/administration/upgrading.rst index 4bdc656..9d19f90 100644 --- a/doc/administration/upgrading.rst +++ b/doc/administration/upgrading.rst @@ -53,15 +53,84 @@ Upgrading the Database is your last chance. Some updates might change the database schema. Those updates come with database -migrations that can adapt an existing database to the new schema. In order to -do so, use ``alembic``: +migrations that can adapt an existing database to the new schema. In addition +to database migrations, some updates also modify the on-disk data that +Fietsboek stores. + +Fietsboek comes with a handy tool that allows you to run the right database and +file migrations: ``fietsupdate``. + +You can use the following command to upgrade to the latest installed version: .. code:: bash - .venv/bin/alembic -c production.ini upgrade head + .venv/bin/fietsupdate update -c production.ini + +Or you can view the status of your installation: + +.. code:: bash + + .venv/bin/fietsupdate status -c production.ini + +.. note:: + + The ``fietsupdate`` tool is only there to run database migrations and other + update tasks. It does *not* retrieve or install the correct code, please + see the above sections to do that. Restarting Fietsboek -------------------- You can now run Fietsboek again. This step depends on the method that you use to deploy Fietsboek. + +Downgrading +=========== + +In some cases, it might be required that you uninstall an update to Fietsboek, +for example if it introduced a bug. Downgrading is possible and generally +consists of two steps: + +1) Restore the old data format (backwards migration) +2) Restore the old code + +.. warning:: + + As with updates, make sure you have backups of your important data ready! + +Restoring Data: Backups +----------------------- + +The preferred way to use the old data is by restoring a backup, see +:doc:`backup`. This method ensures that your data is original and prevents the +upgrade/downgrade process from inducing errors. + +Restoring Data: Downgrading +--------------------------- + +If you do not have old backups of your data, you can use ``fietsupdate`` to +convert your data from the new format to the old one. Please note that the +upgrade process can not always be undone: If an update deletes data, even the +downgrade cannot fix it. In such cases, the only way to restore the data is +through a backup. + +To use ``fietsupdate`` to downgrade your data, run the following command, +replacing ``VERSION`` with the version you want to downgrade to: + +.. code:: bash + + .venv/bin/fietsupdate downgrade -c production.ini VERSION + +Restoring Code +-------------- + +Now that the data has been restored to the old state, you need to restore the +Fietsboek code to the old version. This works similarly to the update, with the +difference that you should download and install the version of Fietsboek that +you downgraded your data to. + +If you use ``git``, you can check out old versions of the code by using ``git +checkout``. + +After obtaining the old code, don't forget to install it into your virtual +environment! -- cgit v1.2.3 From 67282cd30605172730bb7ba16f977c30e70c1a62 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 12 Nov 2022 18:34:01 +0100 Subject: add initial update script This doesn't really do anything yet, but it serves as a starting point and sets the alembic version. --- fietsboek/updater/scripts/upd_initial.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 fietsboek/updater/scripts/upd_initial.py diff --git a/fietsboek/updater/scripts/upd_initial.py b/fietsboek/updater/scripts/upd_initial.py new file mode 100644 index 0000000..cc5e5d7 --- /dev/null +++ b/fietsboek/updater/scripts/upd_initial.py @@ -0,0 +1,26 @@ +"""Revision upgrade script initial. + +Date created: 2022-11-12 18:20:07.214366 +""" +from fietsboek.updater.script import UpdateScript + +update_id = 'initial' +previous = [ +] +alembic_revision = 'd085998b49ca' + + +class Up(UpdateScript): + def pre_alembic(self, config): + pass + + def post_alembic(self, config): + pass + + +class Down(UpdateScript): + def pre_alembic(self, config): + pass + + def post_alembic(self, config): + pass -- cgit v1.2.3