diff --git a/contrib/plugins/command.py b/contrib/plugins/command.py new file mode 100644 index 00000000..82a0ea10 --- /dev/null +++ b/contrib/plugins/command.py @@ -0,0 +1,25 @@ +# Sample plugin that adds a new top level command: 'bob hello' +# +# Run e.g. 'bob hello root' to print the path of all matching packages. + +import argparse + +def doHello(packages, argv, bobRoot): + parser = argparse.ArgumentParser(prog="bob hello", + description="Print the path of all matching packages.") + parser.add_argument('package', nargs='?', default="", + help="Package to query (defaults to all root packages)") + args = parser.parse_args(argv) + + for (stack, node) in packages.queryTreePath(args.package): + print("/".join(stack) if stack else "/") + +manifest = { + 'apiVersion' : "1.2.1.dev1", + 'commands' : { + 'hello' : { + 'func' : doHello, + 'help' : "Print all package paths", + }, + }, +} diff --git a/doc/manual/extending.rst b/doc/manual/extending.rst index 10ce9074..40d0c598 100644 --- a/doc/manual/extending.rst +++ b/doc/manual/extending.rst @@ -60,6 +60,18 @@ internal and might change without notice. .. autoclass:: bob.input.Tool() :members: +.. autoclass:: bob.pathspec.PackageSet() + :members: getRootPackage, queryPackagePath, getAliases + +.. autoclass:: bob.errors.BobError + :members: + +.. autoclass:: bob.errors.ParseError + :members: + +.. autoclass:: bob.errors.BuildError + :members: + Hooks ----- @@ -300,6 +312,59 @@ generator are the package objects returned by } +.. _extending-commands: + +Commands +-------- + +A plugin may register additional top level commands, e.g. to run ``bob +mycmd ...`` like any built-in command such as ``bob ls``. Bob takes care of +parsing the arguments that are common to (almost) all commands -- ``-D``, +``-c`` and the sandbox mode switches (``--sandbox``, ``--slim-sandbox``, +``--dev-sandbox``, ``--strict-sandbox``, ``--no-sandbox``) -- and of parsing +the recipes and generating the package graph. The plugin must not (and +cannot) parse recipes or generate packages itself. + +A command function is called with 3 arguments: + +* ``packages``: the :class:`bob.pathspec.PackageSet` holding the generated + package graph. Use e.g. :func:`bob.pathspec.PackageSet.queryPackagePath` or + :func:`bob.pathspec.PackageSet.getRootPackage` to inspect it. +* ``argv``: the list of arguments that were not consumed by Bob's standard + argument handling. The plugin is free to parse these with its own + ``argparse.ArgumentParser`` (and should handle ``-h``/``--help`` itself, as + Bob does not intercept it). +* ``bobRoot``: the fully qualified path name to the Bob executable, as passed + to generators. + +The function may return an integer that is used as the process exit code. Any +other return value (e.g. ``None``) is treated as success. + +A simple command may look like this:: + + def doHello(packages, argv, bobRoot): + import argparse + parser = argparse.ArgumentParser(prog="bob hello") + parser.add_argument('package', nargs='?', default="") + args = parser.parse_args(argv) + for (stack, node) in packages.queryTreePath(args.package): + print("/".join(stack) if stack else "/") + + manifest = { + 'apiVersion' : "1.2.1.dev1", + 'commands' : { + 'hello' : { + 'func' : doHello, + 'help' : "Print all package paths", + }, + }, + } + +Command names of Bob's built-in commands (e.g. ``build``, ``dev``, ``ls``, +...) are reserved and always take precedence over a plugin provided command +of the same name. It is not possible to re-define an already existing command +name defined by another plugin. + .. _extending-settings: Plugin settings diff --git a/pym/bob/cmds/build/query.py b/pym/bob/cmds/build/query.py index 70e959b6..ca0a8f39 100644 --- a/pym/bob/cmds/build/query.py +++ b/pym/bob/cmds/build/query.py @@ -6,7 +6,7 @@ from ...builder import LocalBuilder from ...errors import ParseError from ...input import RecipeSet -from ..helpers import processDefines +from ..helpers import processDefines, addStandardArgs from string import Formatter import argparse import os @@ -35,26 +35,12 @@ def doQueryPath(argv, bobRoot): parser.add_argument('packages', metavar='PACKAGE', type=str, nargs='+', help="(Sub-)package to query") parser.add_argument('-f', help='Output format string', default='{name}\t{dist}', metavar='FORMAT') - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") parser.add_argument('-q', dest="quiet", action="store_true", help="Be quiet in case of errors") parser.add_argument('--fail', action="store_true", help="Return a non-zero error code in case of errors") - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing") + addStandardArgs(parser) parser.set_defaults(sandbox=None) group = parser.add_mutually_exclusive_group() diff --git a/pym/bob/cmds/graph.py b/pym/bob/cmds/graph.py index b73a91a3..ceb74e9c 100644 --- a/pym/bob/cmds/graph.py +++ b/pym/bob/cmds/graph.py @@ -7,7 +7,7 @@ from ..input import RecipeSet from ..tty import colorize from ..utils import runInEventLoop -from .helpers import processDefines +from .helpers import processDefines, addStandardArgs import argparse import asyncio import json @@ -507,21 +507,7 @@ def doGraph(argv, bobRoot): parser = argparse.ArgumentParser(prog="bob graph", description='Generate dependency graph') parser.add_argument('packages', metavar='PACKAGE', type=str, nargs='+', help="Graph entry (sub-)package") - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing") + addStandardArgs(parser) parser.add_argument('--destination', metavar="DEST", help="Destination of graph files.") parser.add_argument('-e', '--exclude', default=[], action='append', dest="excludes", diff --git a/pym/bob/cmds/help.py b/pym/bob/cmds/help.py index baa86ee4..522a875f 100644 --- a/pym/bob/cmds/help.py +++ b/pym/bob/cmds/help.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later from ..errors import BobError +from ..input import RecipeSet import argparse import os.path import subprocess @@ -14,11 +15,37 @@ def doHelp(availableCommands, argv, bobRoot): description="Display help information about command.") # Help without a command parameter gets handled by the main argument parser # in pym/bob/scripts.py. - parser.add_argument('command', help="Command to get help for") + parser.add_argument('command', nargs='?', help="Command to get help for") + parser.add_argument('-a', '--all', action='store_true', + help="print all available commands") args = parser.parse_args(argv) - if args.command in availableCommands: + recipes = RecipeSet() + recipes.parseConfigs() + + if args.command is None: + lines = ["The following high level commands are available:", ""] + lines.extend(sorted([ " {:16s}{}".format(k, v[2]) + for (k,v) in availableCommands.items() if v[0] == 'hl' ])) + lines.extend(["", "The following scripting commands are available:", ""]) + lines.extend(sorted([ " {:16s}{}".format(k, v[2]) + for (k,v) in availableCommands.items() if v[0] == 'll' ])) + if args.all: + lines.extend(["", "The following plugin defined commands are available:", ""]) + + for name, spec in sorted(recipes.getCommands().items()): + lines.append(" {:16s}{}".format(name, spec.get("help", ""))) + + print("\n".join(lines)) + return 0 + + if args.command in recipes.getCommands().keys(): + h = recipes.getCommands()[args.command].get("help", "") + print(f"'{args.command}' is provided by a plugin: {h}") + return 0 + + if args.command in availableCommands.keys(): manPage = "bob-" + args.command manSection = "1" elif args.command == "bob": diff --git a/pym/bob/cmds/helpers.py b/pym/bob/cmds/helpers.py index e27d550e..84971ff7 100644 --- a/pym/bob/cmds/helpers.py +++ b/pym/bob/cmds/helpers.py @@ -8,6 +8,24 @@ def processDefines(defs): defines[key] = value return defines +def addStandardArgs(parser): + """Add the -D/-c/sandbox arguments common to most Bob sub-commands""" + parser.add_argument('-D', default=[], action='append', dest="defines", + help="Override default environment variable") + parser.add_argument('-c', dest="configFile", default=[], action='append', + help="Use config File") + group = parser.add_mutually_exclusive_group() + group.add_argument('--sandbox', action='store_true', default=False, + help="Enable sandboxing") + group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', + help="Enable slim sandboxing") + group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', + help="Enable development sandboxing") + group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', + help="Enable strict sandboxing") + group.add_argument('--no-sandbox', action='store_false', dest='sandbox', + help="Disable sandboxing") + def dumpYaml(doc, indent): if indent is None: diff --git a/pym/bob/cmds/misc.py b/pym/bob/cmds/misc.py index caa9e5f3..43d3f812 100644 --- a/pym/bob/cmds/misc.py +++ b/pym/bob/cmds/misc.py @@ -7,7 +7,7 @@ from ..input import RecipeSet from ..errors import ParseError, BuildError -from .helpers import processDefines +from .helpers import processDefines, addStandardArgs import argparse import codecs import sys @@ -83,21 +83,7 @@ def doLS(argv, bobRoot): help="Prints the full path prefix for each package") group.add_argument('-d', '--direct', default=False, action='store_true', help="List packages themselves, not their contents") - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing") + addStandardArgs(parser) args = parser.parse_args(argv) defines = processDefines(args.defines) @@ -135,23 +121,9 @@ def doQueryMeta(argv, bobRoot): formatter_class=argparse.RawDescriptionHelpFormatter, description="""Query meta information of packages.""") parser.add_argument('packages', nargs='+', help="(Sub-)packages to query") - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") parser.add_argument('-r', '--recursive', default=False, action='store_true', help="Recursively display dependencies") - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing") + addStandardArgs(parser) args = parser.parse_args(argv) defines = processDefines(args.defines) @@ -196,26 +168,12 @@ def doQuerySCM(argv, bobRoot): """) parser.add_argument('packages', nargs='+', help="(Sub-)packages to query") - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") parser.add_argument('-f', default=[], action='append', dest="formats", help="Output format for scm (syntax: scm=format). Can be specified multiple times.") parser.add_argument('--default', default="", help='Default for missing attributes (default: "")') parser.add_argument('-r', '--recursive', default=False, action='store_true', help="Recursively display dependencies") - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing") + addStandardArgs(parser) formats = { 'git' : "git {package} {dir} {url} {branch}", @@ -269,21 +227,7 @@ def doQueryRecipe(argv, bobRoot): parser = argparse.ArgumentParser(prog="bob query-recipe", description="Query recipe and class files of package.") parser.add_argument('packages', nargs='+', help="(Sub-)packages to query") - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing") + addStandardArgs(parser) args = parser.parse_args(argv) @@ -343,22 +287,7 @@ def doInit(argv, bobRoot): def doLsRecipes(argv, bobRoot): parser = argparse.ArgumentParser(prog="bob ls-recipes", description="List all known recipes.") - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") - - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing (default)") + addStandardArgs(parser) group = parser.add_mutually_exclusive_group() group.add_argument('--all', action='store_const', dest='mode', const='all', default='all', diff --git a/pym/bob/cmds/show.py b/pym/bob/cmds/show.py index ee53e529..6c6a70a0 100644 --- a/pym/bob/cmds/show.py +++ b/pym/bob/cmds/show.py @@ -7,7 +7,7 @@ from ..input import RecipeSet from ..tty import colorize, ADDED, DELETED, DEFAULT, ADDED_HIGHLIGHT, DELETED_HIGHLIGHT from ..utils import asHexStr -from .helpers import processDefines, dumpYaml, dumpJson, dumpFlat +from .helpers import processDefines, addStandardArgs, dumpYaml, dumpJson, dumpFlat import argparse import difflib import json @@ -225,21 +225,7 @@ def doShow(argv, bobRoot): parser = argparse.ArgumentParser(prog="bob show", description="Show properties of one or more packages.") parser.add_argument('packages', nargs='+', help="(Sub-)packages to query") - parser.add_argument('-D', default=[], action='append', dest="defines", - help="Override default environment variable") - parser.add_argument('-c', dest="configFile", default=[], action='append', - help="Use config File") - group = parser.add_mutually_exclusive_group() - group.add_argument('--sandbox', action='store_true', default=False, - help="Enable sandboxing") - group.add_argument('--slim-sandbox', action='store_false', dest='sandbox', - help="Enable slim sandboxing") - group.add_argument('--dev-sandbox', action='store_true', dest='sandbox', - help="Enable development sandboxing") - group.add_argument('--strict-sandbox', action='store_true', dest='sandbox', - help="Enable strict sandboxing") - group.add_argument('--no-sandbox', action='store_false', dest='sandbox', - help="Disable sandboxing") + addStandardArgs(parser) group = parser.add_argument_group('output', "Appearance and content of output") group.add_argument('--show-empty', action='store_true', default=False, diff --git a/pym/bob/errors.py b/pym/bob/errors.py index 94960bf7..9eefe1b7 100644 --- a/pym/bob/errors.py +++ b/pym/bob/errors.py @@ -6,6 +6,34 @@ from .tty import colorize class BobError(Exception): + """Base class of all errors raised by Bob. + + A ``BobError`` carries a human readable description of what went wrong + (the ``slogan``), an optional stack of locations that led to the error + and an optional hint on how to resolve the issue. Bob catches these + exceptions at the top level, prints them nicely formatted to the user and + aborts with the given ``returncode``. + + Plugins may raise a ``BobError`` (or, preferably, one of its more + specific subclasses :class:`bob.errors.ParseError` and + :class:`bob.errors.BuildError`) to signal an error condition to the user. + + :param slogan: Human readable description of the error. + :type slogan: str + :param kind: Short prefix that is shown in front of the slogan, e.g. + ``"Parse"`` or ``"Build"``. If ``None`` a generic "Error" prefix is + used. + :type kind: str | None + :param stackSlogan: Caption that is shown in front of ``stack``, e.g. + "Processing stack" or "Failed package". + :type stackSlogan: str + :param help: Additional hint that is appended to the error message. + :type help: str + :param returncode: Process exit code that Bob shall use if this error + propagates to the top level uncaught. + :type returncode: int + """ + def __init__(self, slogan, kind=None, stackSlogan="", help="", returncode=1): self.kind = (kind + " error: ") if kind is not None else "Error: " self.slogan = slogan @@ -23,22 +51,63 @@ def __str__(self): return ret class ParseError(BobError): + """Error while parsing recipes, classes or other configuration input. + + Raise this exception to signal that some input could not be parsed or is + otherwise invalid, e.g. from within a + :meth:`bob.input.PluginProperty.validate` implementation or a string + function. + + :param slogan: Human readable description of the error. + :type slogan: str + """ + def __init__(self, slogan, *args, **kwargs): BobError.__init__(self, slogan, "Parse", "Processing stack", *args, **kwargs) def pushFrame(self, frame): + """Add a location to the processing stack. + + Called while the error propagates up through nested recipe or class + includes to record where it was raised. The stack is shown with the + outermost frame first. + + :param frame: Name of the file, recipe or class that was being + processed. + :type frame: str + """ if not self.stack or (self.stack[0] != frame): self.stack.insert(0, frame) def setPath(self, path): + """Set the file that caused the error. + + :param path: Path of the offending file. + :type path: str + """ self.stackSlogan = "Offending file" self.stack = [path] class BuildError(BobError): + """Error during the execution of a package build step. + + Raise this exception, e.g. from a plugin hook, to signal that building a + package failed. + + :param slogan: Human readable description of the error. + :type slogan: str + """ + def __init__(self, slogan, *args, **kwargs): BobError.__init__(self, slogan, "Build", "Failed package", *args, **kwargs) def setStack(self, stack): + """Set the package stack of the package that failed. + + :param stack: Package path stack, e.g. as returned by + :meth:`bob.input.Package.getStack`, of the failed package. + :type stack: list[str] + """ if not self.stack: self.stack = stack[:] diff --git a/pym/bob/input.py b/pym/bob/input.py index 2830f377..85bfc4ee 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -3269,6 +3269,15 @@ def wrapper(data): raise schema.SchemaError(message) return wrapper +def isPluginPropertyClass(cls): + return isinstance(cls, type) and issubclass(cls, PluginProperty) + +def isPluginStateClass(cls): + return isinstance(cls, type) and issubclass(cls, PluginState) + +def isUpperCaseName(name): + return isinstance(name, str) and not name[:1].islower() + class LayersConfig: def __init__(self): @@ -3333,6 +3342,47 @@ class RecipeSet: schema.Optional('max_depth') : int, }) + PLUGIN_MANIFEST_SCHEMA = schema.Schema({ + 'apiVersion' : str, + schema.Optional('hooks') : schema.Schema({ + str : schema.Schema(wrapValidator(callable, "hook must be callable!")) + }), + schema.Optional('projectGenerators') : schema.Schema({ + str : schema.Or( + schema.Schema(wrapValidator(callable, "generator must be callable!")), + { + 'func' : schema.Schema(wrapValidator(callable, "generator 'func' must be callable!")), + schema.Optional('query') : bool, + }) + }), + schema.Optional('commands') : schema.Schema({ + str : schema.Or( + schema.Schema(wrapValidator(callable, "command must be callable!")), + { + 'func' : schema.Schema(wrapValidator(callable, "command 'func' must be callable!")), + schema.Optional('help') : str, + }) + }), + schema.Optional('properties') : schema.Schema({ + schema.Schema(wrapValidator(isUpperCaseName, + "property name must be a string and must not start lower case!")) : + schema.Schema(wrapValidator(isPluginPropertyClass, "property has wrong type!")) + }), + schema.Optional('state') : schema.Schema({ + schema.Schema(wrapValidator(isUpperCaseName, + "state tracker name must be a string and must not start lower case!")) : + schema.Schema(wrapValidator(isPluginStateClass, "state tracker has wrong type!")) + }), + schema.Optional('stringFunctions') : schema.Schema({ + str : schema.Schema(wrapValidator(callable, "string function must be callable!")) + }), + schema.Optional('settings') : schema.Schema({ + schema.Schema(wrapValidator(isUpperCaseName, + "settings name must be a string and must not start lower case!")) : + PluginSetting + }), + }) + # We do not support the "import" SCM for layers. It just makes no sense. # Also, all SCMs lack the "dir" and "if" attributes. LAYERS_SCM_SCHEMA = ScmValidator({ @@ -3484,6 +3534,7 @@ def __init__(self): self.__scmOverrides = [] self.__hooks = {} self.__projectGenerators = {} + self.__commands = {} self.__configFiles = [] self.__properties = {} self.__states = {} @@ -3677,26 +3728,27 @@ def __loadPlugin(self, mangledName, fileName, name): manifest = mod.manifest except AttributeError: raise ParseError("Plugin '"+fileName+"' did not define 'manifest'!") - apiVersion = manifest.get('apiVersion') + + # Check the API version first so that a plugin requiring a newer Bob + # gets the proper "too old" error instead of a confusing schema + # error about manifest keys that this Bob version does not know yet. + apiVersion = manifest.get('apiVersion') if isinstance(manifest, dict) else None if apiVersion is None: raise ParseError("Plugin '"+fileName+"' did not define 'apiVersion'!") if compareVersion(BOB_VERSION, apiVersion) < 0: raise ParseError("Your Bob is too old. Plugin '"+fileName+"' requires at least version "+apiVersion+"!") toolsAbiBreak = compareVersion(apiVersion, "0.15") < 0 + try: + manifest = self.PLUGIN_MANIFEST_SCHEMA.validate(manifest) + except schema.SchemaError as e: + raise ParseError("Plugin '"+fileName+"': "+str(e)) + hooks = manifest.get('hooks', {}) - if not isinstance(hooks, dict): - raise ParseError("Plugin '"+fileName+"': 'hooks' has wrong type!") for (hook, fun) in hooks.items(): - if not isinstance(hook, str): - raise ParseError("Plugin '"+fileName+"': hook name must be a string!") - if not callable(fun): - raise ParseError("Plugin '"+fileName+"': "+hook+": hook must be callable!") self.__hooks.setdefault(hook, []).append((fun, apiVersion)) projectGenerators = manifest.get('projectGenerators', {}) - if not isinstance(projectGenerators, dict): - raise ParseError("Plugin '"+fileName+"': 'projectGenerators' has wrong type!") if projectGenerators: if compareVersion(apiVersion, "0.16.1.dev33") < 0: # cut off extra argument for old generators @@ -3710,32 +3762,25 @@ def __loadPlugin(self, mangledName, fileName, name): } self.__projectGenerators.update(projectGenerators) + commands = manifest.get('commands', {}) + if commands and compareVersion(apiVersion, "1.2.1.dev1") < 0: + raise ParseError("Plugin '"+fileName+"': 'commands' requires at least apiVersion 1.2.1.dev1!") + for (i, j) in commands.items(): + entry = j if isinstance(j, dict) else {'func' : j} + if i in self.__commands: + raise ParseError("Plugin '"+fileName+"': command '"+i+"' already defined by other plugin!") + self.__commands[i] = entry + properties = manifest.get('properties', {}) - if not isinstance(properties, dict): - raise ParseError("Plugin '"+fileName+"': 'properties' has wrong type!") if properties: self.__pluginPropDeps += pluginStat - for (i,j) in properties.items(): - if not isinstance(i, str): - raise ParseError("Plugin '"+fileName+"': property name must be a string!") - if i[:1].islower(): - raise ParseError(f"Plugin '{fileName}': property '{i}' must not start lower case!") - if not issubclass(j, PluginProperty): - raise ParseError("Plugin '"+fileName+"': property '" +i+"' has wrong type!") + for i in properties: if i in self.__properties: raise ParseError("Plugin '"+fileName+"': property '" +i+"' already defined by other plugin!") self.__properties.update(properties) states = manifest.get('state', {}) - if not isinstance(states, dict): - raise ParseError("Plugin '"+fileName+"': 'states' has wrong type!") - for (i,j) in states.items(): - if not isinstance(i, str): - raise ParseError("Plugin '"+fileName+"': state tracker name must be a string!") - if i[:1].islower(): - raise ParseError(f"Plugin '{fileName}': state tracker '{i}' must not start lower case!") - if not issubclass(j, PluginState): - raise ParseError("Plugin '"+fileName+"': state tracker '" +i+"' has wrong type!") + for i in states: if i in self.__states: raise ParseError("Plugin '"+fileName+"': state tracker '" +i+"' already defined by other plugin!") if states and toolsAbiBreak: @@ -3743,11 +3788,7 @@ def __loadPlugin(self, mangledName, fileName, name): self.__states.update(states) funs = manifest.get('stringFunctions', {}) - if not isinstance(funs, dict): - raise ParseError("Plugin '"+fileName+"': 'stringFunctions' has wrong type!") - for (i,j) in funs.items(): - if not isinstance(i, str): - raise ParseError("Plugin '"+fileName+"': string function name must be a string!") + for i in funs: if i in self.__stringFunctions: raise ParseError("Plugin '"+fileName+"': string function '" +i+"' already defined by other plugin!") if funs and toolsAbiBreak: @@ -3755,17 +3796,9 @@ def __loadPlugin(self, mangledName, fileName, name): self.__stringFunctions.update(funs) settings = manifest.get('settings', {}) - if not isinstance(settings, dict): - raise ParseError("Plugin '"+fileName+"': 'settings' has wrong type!") if settings: self.__pluginSettingsDeps += pluginStat - for (i,j) in settings.items(): - if not isinstance(i, str): - raise ParseError("Plugin '"+fileName+"': settings name must be a string!") - if i[:1].islower(): - raise ParseError("Plugin '"+fileName+"': settings name must not start lower case!") - if not isinstance(j, PluginSetting): - raise ParseError("Plugin '"+fileName+"': setting '"+i+"' has wrong type!") + for i in settings: if i in self.__settings: raise ParseError("Plugin '"+fileName+"': setting '"+i+"' already defined by other plugin!") self.__settings.update(settings) @@ -3792,6 +3825,9 @@ def getHookStack(self, name): def getProjectGenerators(self): return self.__projectGenerators + def getCommands(self): + return self.__commands + def envWhiteList(self): """The set of all white listed environment variables @@ -3872,24 +3908,22 @@ def loadBinary(self, path): def loadYaml(self, path, schema, default={}, preValidate=lambda x: None): return self.__cache.loadYaml(path, schema, default, preValidate) - def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot=""): - if not recipesRoot and os.path.isfile(".bob-project"): - try: - with open(".bob-project") as f: - recipesRoot = f.read() - except OSError as e: - raise ParseError("Broken project link: " + str(e)) - recipesDir = os.path.join(recipesRoot, "recipes") - if not os.path.isdir(recipesDir): - raise ParseError("No recipes directory found in " + recipesDir) - self.__projectRoot = recipesRoot or os.getcwd() + def parse(self, envOverrides={}, platform=getPlatformString(), recipesRoot="", command=None): self.__cache.open() try: - self.__parse(envOverrides, platform, recipesRoot) + self.__parseConfigs(platform, recipesRoot, command) + self.__parseRecipes(envOverrides) finally: self.__cache.close() - def __parse(self, envOverrides, platform, recipesRoot=""): + def parseConfigs(self, platform=getPlatformString(), recipesRoot="", command=None): + self.__cache.open() + try: + self.__parseConfigs(platform, recipesRoot, command) + finally: + self.__cache.close() + + def __parseConfigs(self, platform, recipesRoot, command): if platform not in ('cygwin', 'darwin', 'linux', 'msys', 'win32'): raise ParseError("Invalid platform: " + platform) self.__platform = platform @@ -3899,6 +3933,18 @@ def __parse(self, envOverrides, platform, recipesRoot=""): self.__pluginSettingsDeps = b'' self.__createSchemas() + # Find actual project root + if not recipesRoot and os.path.isfile(".bob-project"): + try: + with open(".bob-project") as f: + recipesRoot = f.read() + except OSError as e: + raise ParseError("Broken project link: " + str(e)) + recipesDir = os.path.join(recipesRoot, "recipes") + if not os.path.isdir(recipesDir): + raise ParseError("No recipes directory found in " + recipesDir) + self.__projectRoot = recipesRoot or os.getcwd() + # global user config(s) if not DEBUG['ngd']: self.__parseUserConfig("/etc/bobdefault.yaml") @@ -3906,7 +3952,12 @@ def __parse(self, envOverrides, platform, recipesRoot=""): os.path.join(os.path.expanduser("~"), '.config')), 'bob', 'default.yaml')) # Begin with root layer - allLayers = self.__parseLayer(LayerSpec(""), "9999", recipesRoot, None) + self.__allLayers = self.__parseLayer(LayerSpec(""), "9999", recipesRoot, None) + + # If a specific command is requested, verify that it's available. + if command is not None and command not in self.__commands: + raise BobError(f"{command}: unknown command! Use 'bob -h' for help.", + returncode=2) # Add string functions added after 1.0. We did not reserve a namespace # and we better not break existing recipes. @@ -3920,9 +3971,21 @@ def __parse(self, envOverrides, platform, recipesRoot=""): else: self.__stringFunctions.update(EXTRA_STRING_FUNS) + # Out-of-tree builds may have a dedicated default.yaml + if recipesRoot: + self.__parseUserConfig("default.yaml") + + # config files overrule everything else + for c in self.__configFiles: + c = str(c) + ".yaml" + if not os.path.isfile(c): + raise ParseError("Config file {} does not exist!".format(c)) + self.__parseUserConfig(c) + + def __parseRecipes(self, envOverrides): # Parse all recipes and classes of all layers. Need to be done last # because only by now we have loaded all plugins. - for layer, rootDir, scriptLanguage in allLayers: + for layer, rootDir, scriptLanguage in self.__allLayers: classesDir = os.path.join(rootDir, 'classes') for root, dirnames, filenames in os.walk(classesDir): for path in fnmatch.filter(filenames, "[!.]*.yaml"): @@ -3960,17 +4023,6 @@ def __parse(self, envOverrides, platform, recipesRoot=""): e.setPath(os.path.join(root, path)) raise - # Out-of-tree builds may have a dedicated default.yaml - if recipesRoot: - self.__parseUserConfig("default.yaml") - - # config files overrule everything else - for c in self.__configFiles: - c = str(c) + ".yaml" - if not os.path.isfile(c): - raise ParseError("Config file {} does not exist!".format(c)) - self.__parseUserConfig(c) - # calculate start environment osEnv = Env(os.environ) osEnv.setFuns(self.__stringFunctions) @@ -3978,7 +4030,7 @@ def __parse(self, envOverrides, platform, recipesRoot=""): self.__defaultEnv.items() }) env.setFuns(self.__stringFunctions) env.update(envOverrides) - env["BOB_HOST_PLATFORM"] = platform + env["BOB_HOST_PLATFORM"] = self.__platform self.__rootEnv = env # resolve recipes and their classes diff --git a/pym/bob/pathspec.py b/pym/bob/pathspec.py index 02bb2bbd..019eeaf8 100644 --- a/pym/bob/pathspec.py +++ b/pym/bob/pathspec.py @@ -894,10 +894,26 @@ def close(self): self.__graph = None def getAliases(self): + """Get all defined path aliases. + + Aliases are substituted for the first path element of a relative + location path. See :ref:`manpage-bobpaths-aliases` for details. + + :return: List of alias names. + :rtype: list[str] + """ return list(self.__aliases.keys()) def getRootPackage(self): - """Get virtual root package.""" + """Get virtual root package. + + The root package is a synthetic package that has all root recipes as + its direct dependencies. It is the starting point of every absolute + location path. + + :return: The virtual root package. + :rtype: bob.input.Package + """ if self.__root is None: self.__root = self.__generator() return self.__root @@ -912,10 +928,22 @@ def queryTreePath(self, path, queryAll=False): return self.__findResultNodes(self.__getGraphRoot(), nodes, valid, queryAll) def queryPackagePath(self, path, queryAll=False): - """Execute query and return bob.input.Package objects. - - Setting 'queryAll' to True will return all alternate paths to a result - package instead of only the first one. + """Execute a package query and return the matching packages. + + The ``path`` is parsed and evaluated as a :ref:`package path + `. Any alias present as the first path element of a + relative path is substituted before the query is executed. + + :param path: Package path query. + :type path: str + :param queryAll: If a package is reachable through more than one + path, return it once for every path that matches instead of only + the first one. + :type queryAll: bool + :return: All packages that matched the query. + :rtype: list[bob.input.Package] + :raises bob.errors.BobError: The query could not be parsed or + evaluated. """ (nodes, valid) = self.__query(path) return self.__findResultPackages(self.__getGraphRoot(), self.getRootPackage(), nodes, valid, queryAll) diff --git a/pym/bob/scripts.py b/pym/bob/scripts.py index 674c89c8..f58cb429 100644 --- a/pym/bob/scripts.py +++ b/pym/bob/scripts.py @@ -40,7 +40,7 @@ def __graph(*args, **kwargs): def __help(*args, **kwargs): from .cmds.help import doHelp - doHelp(availableCommands.keys(), *args, **kwargs) + doHelp(availableCommands, *args, **kwargs) return 0 def __init(*args, **kwargs): @@ -111,6 +111,31 @@ def __lsrecipes(*args, **kwargs): doLsRecipes(*args, **kwargs) return 0 +def __runPluginCommand(command, argv, bobRoot): + """Try to dispatch to a plugin provided command. + + Parses the standard -D/-c/sandbox arguments and generates the package + graph before invoking the plugin. Returns None if no plugin registered + 'command'. + """ + from .input import RecipeSet + from .cmds.helpers import processDefines, addStandardArgs + + parser = argparse.ArgumentParser(add_help=False) + addStandardArgs(parser) + args, remainder = parser.parse_known_args(argv) + + defines = processDefines(args.defines) + recipes = RecipeSet() + recipes.setConfigFiles(args.configFile) + recipes.parse(defines, command=command) + + entry = recipes.getCommands().get(command) + + packages = recipes.generatePackages(lambda s, m: "unused", args.sandbox) + ret = entry['func'](packages, remainder, bobRoot) + return ret if ret is not None else 0 + availableCommands = { "archive" : ('hl', __archive, "Manage binary artifact archives"), "build" : ('hl', __build, "Build (sub-)packages in release mode"), @@ -244,14 +269,15 @@ def cmd(): parser.print_help() return 0 + if args.directory is not None: + for i in args.directory: + try: + os.chdir(i) + except OSError as e: + print("bob -C: unable to change directory:", str(e), file=sys.stderr) + return 1 + if args.command in availableCommands: - if args.directory is not None: - for i in args.directory: - try: - os.chdir(i) - except OSError as e: - print("bob -C: unable to change directory:", str(e), file=sys.stderr) - return 1 cmd = availableCommands[args.command][1] if DEBUG['prof']: import cProfile, pstats @@ -267,8 +293,7 @@ def cmd(): ret = cmd(args.args, bobRoot) return ret else: - print("Don't know what to do for '{}'. Use 'bob -h' for help".format(args.command), file=sys.stderr) - return 2 + return __runPluginCommand(args.command, args.args, bobRoot) try: ret = catchErrors(cmd) diff --git a/test/black-box/plugin-commands/config.yaml b/test/black-box/plugin-commands/config.yaml new file mode 100644 index 00000000..432402d3 --- /dev/null +++ b/test/black-box/plugin-commands/config.yaml @@ -0,0 +1,3 @@ +bobMinimumVersion: "1.2" +plugins: + - "command" diff --git a/test/black-box/plugin-commands/default.yaml b/test/black-box/plugin-commands/default.yaml new file mode 100644 index 00000000..cb175a30 --- /dev/null +++ b/test/black-box/plugin-commands/default.yaml @@ -0,0 +1,2 @@ +alias: + foo: root diff --git a/test/black-box/plugin-commands/output-plain.txt b/test/black-box/plugin-commands/output-plain.txt new file mode 100644 index 00000000..81024cd3 --- /dev/null +++ b/test/black-box/plugin-commands/output-plain.txt @@ -0,0 +1,4 @@ +ROOT: ['root'] +ALIASES: ['foo'] +PACKAGE: root FOO= sandbox=False +ARGV: ['root', '--additional', 'options'] diff --git a/test/black-box/plugin-commands/output-sandbox.txt b/test/black-box/plugin-commands/output-sandbox.txt new file mode 100644 index 00000000..9a0e3c42 --- /dev/null +++ b/test/black-box/plugin-commands/output-sandbox.txt @@ -0,0 +1,4 @@ +ROOT: ['root'] +ALIASES: ['foo'] +PACKAGE: root FOO=bar sandbox=True +ARGV: ['root', '--additional', 'options'] diff --git a/test/black-box/plugin-commands/plugins/command.py b/test/black-box/plugin-commands/plugins/command.py new file mode 100644 index 00000000..853502a2 --- /dev/null +++ b/test/black-box/plugin-commands/plugins/command.py @@ -0,0 +1,24 @@ +def doHello(packages, argv, bobRoot): + path = argv[0] if argv else "" + + root = packages.getRootPackage() + print("ROOT:", [s.getPackage().getName() for s in root.getDirectDepSteps()]) + + print("ALIASES:", sorted(packages.getAliases())) + + for p in packages.queryPackagePath(path): + print(f"PACKAGE: {p.getName()}" + f" FOO={p.getPackageStep().getEnv().get('FOO', '')}" + f" sandbox={p.getPackageStep().getSandbox() is not None}") + + print("ARGV:", argv) + +manifest = { + 'apiVersion' : "1.2.1.dev1", + 'commands' : { + 'hello' : { + 'func' : doHello, + 'help' : "Example plugin command", + }, + }, +} diff --git a/test/black-box/plugin-commands/recipes/root.yaml b/test/black-box/plugin-commands/recipes/root.yaml new file mode 100644 index 00000000..989e892e --- /dev/null +++ b/test/black-box/plugin-commands/recipes/root.yaml @@ -0,0 +1,9 @@ +root: True + +depends: + - name: sandbox + use: [sandbox] + +packageVars: [FOO] +packageScript: | + echo "ok" > result.txt diff --git a/test/black-box/plugin-commands/recipes/sandbox.yaml b/test/black-box/plugin-commands/recipes/sandbox.yaml new file mode 100644 index 00000000..3fbdffa7 --- /dev/null +++ b/test/black-box/plugin-commands/recipes/sandbox.yaml @@ -0,0 +1,14 @@ +# Empty sandbox that mounts the whole host +provideSandbox: + paths: ["/usr/local/bin", "/usr/bin", "/bin"] + mount: + - /bin + - /etc + - /lib + - /opt + - /run + - /usr + - /var + + - ["/lib32", "/lib32", [nofail]] + - ["/lib64", "/lib64", [nofail]] diff --git a/test/black-box/plugin-commands/run.sh b/test/black-box/plugin-commands/run.sh new file mode 100755 index 00000000..10c8ca73 --- /dev/null +++ b/test/black-box/plugin-commands/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e +source "$(dirname "$0")/../../test-lib.sh" "../../.." +cleanup + +# The plugin provided 'hello' command is dispatched like any built-in command +# and receives the generated package graph. Smoke test getRootPackage(), +# queryPackagePath() and getAliases() on the passed PackageSet. +run_bob hello root --additional options | tee log-cmd.txt +diff -u output-plain.txt log-cmd.txt + +# Standard arguments (-D, -c, sandbox mode) are consumed by Bob and must not +# reach the plugin's argv. +run_bob hello -D FOO=bar root --additional options --sandbox | tee log-cmd.txt +diff -u output-sandbox.txt log-cmd.txt + +# An unknown command must still yield the usual error, not a crash. +expect_fail --code=2 run_bob nosuchcommand