diff --git a/coriolis/osmorphing/manager.py b/coriolis/osmorphing/manager.py index 679e88650..757bc949f 100644 --- a/coriolis/osmorphing/manager.py +++ b/coriolis/osmorphing/manager.py @@ -1,6 +1,7 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +import copy import itertools from oslo_config import cfg @@ -16,10 +17,20 @@ from coriolis import schemas +CLOUDBASE_INIT_PLUGINS_OPT = 'cloudbase_init_plugins' + opts = [ cfg.IntOpt('default_osmorphing_operation_timeout', help='Number of seconds to wait for a pending SSH or WinRM ' - 'command before the socket times out.') + 'command before the socket times out.'), + cfg.ListOpt( + CLOUDBASE_INIT_PLUGINS_OPT, + default=None, + help='Cloudbase-Init plugin class names written into Windows guests ' + 'during OS morphing. When unset, each destination provider uses ' + 'its own plugin list. When set, this list replaces the provider ' + 'default. A per-transfer destination_environment value of the ' + 'same name takes precedence over this option.'), ] proxy_opts = [ @@ -43,6 +54,125 @@ LOG = logging.getLogger(__name__) +CLOUDBASE_INIT_PLUGINS_SCHEMA = { + "type": "array", + "items": { + "type": "string" + }, + "title": "Cloudbase-Init Plugins", + "description": ( + "Cloudbase-Init plugins written into Windows guests during OS " + "morphing. When unset, coriolis.conf cloudbase_init_plugins is used " + "if set, otherwise the destination provider default."), +} + + +def _inject_cloudbase_init_plugins_property(object_schema): + props = object_schema.get("properties") + if not isinstance(props, dict): + return + if CLOUDBASE_INIT_PLUGINS_OPT in props: + return + props[CLOUDBASE_INIT_PLUGINS_OPT] = copy.deepcopy( + CLOUDBASE_INIT_PLUGINS_SCHEMA) + + +def inject_cloudbase_init_plugins_schema(schema): + """Add cloudbase_init_plugins to a provider target-environment schema.""" + if not isinstance(schema, dict): + return schema + schema = copy.deepcopy(schema) + _inject_cloudbase_init_plugins_property(schema) + for key in ("oneOf", "anyOf"): + for alt in schema.get(key) or []: + if isinstance(alt, dict): + _inject_cloudbase_init_plugins_property(alt) + return schema + + +def get_cloudbase_init_plugins_destination_option(): + """Return the UI destination option for Cloudbase-Init plugins.""" + from coriolis.osmorphing import windows as base_windows + plugins = list(base_windows.CLOUDBASE_INIT_DEFAULT_PLUGINS) + values = [] + seen = set() + for plugin in plugins: + seen.add(plugin) + values.append({ + "id": plugin, + "name": plugin.rsplit(".", 1)[-1], + }) + default = CONF.cloudbase_init_plugins + if default: + default = list(default) + for plugin in default: + if plugin in seen: + continue + seen.add(plugin) + values.append({ + "id": plugin, + "name": plugin.rsplit(".", 1)[-1], + }) + else: + default = plugins + return { + "name": CLOUDBASE_INIT_PLUGINS_OPT, + "values": values, + "config_default": default, + } + + +def filter_cloudbase_init_plugins_option_names(option_names): + """Remove the injected option so destination providers do not reject it.""" + if not isinstance(option_names, (list, tuple, set)): + return option_names + return [ + name for name in option_names if name != CLOUDBASE_INIT_PLUGINS_OPT] + + +def inject_cloudbase_init_plugins_option(options, option_names=None): + """Append the plugins destination option when the provider omitted it. + + The UI omits the options query. The API then passes ``{}``. + Treat ``{}`` and ``None`` as a request for all destination options. + Skip only for a non-empty name list that does not include this option. + Skip for mock sentinels in tests. + """ + if isinstance(option_names, (list, tuple, set)): + if option_names and CLOUDBASE_INIT_PLUGINS_OPT not in option_names: + return options + elif option_names: + return options + options = list(options or []) + for option in options: + if (isinstance(option, dict) and + option.get("name") == CLOUDBASE_INIT_PLUGINS_OPT): + return options + options.append(get_cloudbase_init_plugins_destination_option()) + return options + + +def apply_cloudbase_init_plugins_override( + osmorphing_info, target_environment): + """Copy ``cloudbase_init_plugins`` from the transfer destination + environment into osmorphing_parameters when present (API override). + """ + if not isinstance(osmorphing_info, dict): + return osmorphing_info or {} + if not isinstance(target_environment, dict): + return osmorphing_info + plugins = target_environment.get(CLOUDBASE_INIT_PLUGINS_OPT) + if not plugins: + return osmorphing_info + osmorphing_info = dict(osmorphing_info) + params = dict(osmorphing_info.get('osmorphing_parameters') or {}) + params[CLOUDBASE_INIT_PLUGINS_OPT] = plugins + osmorphing_info['osmorphing_parameters'] = params + LOG.info( + "Applying Cloudbase-Init plugins override from destination " + "environment: %s", plugins) + return osmorphing_info + def _get_proxy_settings(): return { diff --git a/coriolis/osmorphing/windows.py b/coriolis/osmorphing/windows.py index 653afad3f..d563b3ab7 100644 --- a/coriolis/osmorphing/windows.py +++ b/coriolis/osmorphing/windows.py @@ -9,15 +9,18 @@ import re import uuid +from oslo_config import cfg from oslo_log import log as logging from packaging import version from coriolis import constants from coriolis import exception from coriolis.osmorphing import base +from coriolis.osmorphing import manager as osmorphing_manager from coriolis.osmorphing.osdetect import windows as windows_osdetect from coriolis import utils +CONF = cfg.CONF LOG = logging.getLogger(__name__) WINDOWS_CLIENT_IDENTIFIER = windows_osdetect.WINDOWS_CLIENT_IDENTIFIER @@ -499,6 +502,71 @@ def _write_local_script(self, base_dir, script_path, priority=50): script_path, remote_script_path) + def _parse_cloudbase_init_plugins(self, plugins): + """Return a list of plugin class names, or None if unset. + + A CSV string is split on commas. Tokens are stripped. + Empty tokens are skipped. + """ + if plugins is None: + return None + if isinstance(plugins, str): + text = plugins.strip() + if not text: + return [] + parsed = [] + for raw_token in text.split(","): + token = raw_token.strip() + if not token: + continue + parsed.append(token) + return parsed + if not isinstance(plugins, list): + raise exception.CoriolisException( + "Invalid plugins parameter. Must be list.") + parsed = [] + for item in plugins: + if not isinstance(item, str): + raise exception.CoriolisException( + "Invalid plugins parameter. Must be list.") + token = item.strip() + if token: + parsed.append(token) + return parsed + + def _resolve_cloudbase_init_plugins(self, plugins=None): + """Return the Cloudbase-Init plugin list to write into the guest. + + Resolved in this order: + + 1. osmorphing_parameters ``cloudbase_init_plugins`` + 2. coriolis.conf ``cloudbase_init_plugins`` + 3. provider-supplied ``plugins`` argument + 4. ``CLOUDBASE_INIT_DEFAULT_PLUGINS`` + """ + param_plugins = self._parse_cloudbase_init_plugins( + self._osmorphing_parameters.get( + osmorphing_manager.CLOUDBASE_INIT_PLUGINS_OPT)) + if param_plugins is not None: + LOG.info( + "Using Cloudbase-Init plugins from OS morphing parameters: %s", + param_plugins) + return param_plugins + + conf_plugins = self._parse_cloudbase_init_plugins( + CONF.cloudbase_init_plugins) + if conf_plugins is not None: + LOG.info( + "Using Cloudbase-Init plugins from coriolis.conf: %s", + conf_plugins) + return conf_plugins + + provider_plugins = self._parse_cloudbase_init_plugins(plugins) + if provider_plugins is not None: + return provider_plugins + + return list(CLOUDBASE_INIT_DEFAULT_PLUGINS) + def _write_cloudbase_init_conf(self, cloudbaseinit_base_dir, local_base_dir, com_port="COM1", metadata_services=None, plugins=None, @@ -506,11 +574,7 @@ def _write_cloudbase_init_conf(self, cloudbaseinit_base_dir, if metadata_services is None: metadata_services = CLOUDBASE_INIT_DEFAULT_METADATA_SVCS - if plugins is None: - plugins = CLOUDBASE_INIT_DEFAULT_PLUGINS - elif type(plugins) is not list: - raise exception.CoriolisException( - "Invalid plugins parameter. Must be list.") + plugins = self._resolve_cloudbase_init_plugins(plugins) LOG.info("Writing Cloudbase-Init configuration files") conf_dir = "%s\\conf" % cloudbaseinit_base_dir diff --git a/coriolis/schemas/os_morphing_resources_schema.json b/coriolis/schemas/os_morphing_resources_schema.json index 7591a8aa0..9d6969d88 100644 --- a/coriolis/schemas/os_morphing_resources_schema.json +++ b/coriolis/schemas/os_morphing_resources_schema.json @@ -57,6 +57,13 @@ "retain_user_credentials": { "type": "boolean", "default": false + }, + "cloudbase_init_plugins": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Cloudbase-Init plugin class names for Windows guests. Overrides coriolis.conf and provider defaults when set." } } } diff --git a/coriolis/tasks/osmorphing_tasks.py b/coriolis/tasks/osmorphing_tasks.py index cfd8bbe4c..380a84d26 100644 --- a/coriolis/tasks/osmorphing_tasks.py +++ b/coriolis/tasks/osmorphing_tasks.py @@ -150,6 +150,10 @@ def _run(self, ctxt, instance, origin, destination, task_info, "'osmorphing_info'. Defaulting to %s", destination["type"], os_morphing_info) + os_morphing_info = ( + osmorphing_manager.apply_cloudbase_init_plugins_override( + os_morphing_info, target_environment)) + return { "os_morphing_resources": os_morphing_resources, "osmorphing_connection_info": osmorphing_connection_info, diff --git a/coriolis/tests/osmorphing/test_manager.py b/coriolis/tests/osmorphing/test_manager.py index 8231ee5f9..7ff0f0e64 100644 --- a/coriolis/tests/osmorphing/test_manager.py +++ b/coriolis/tests/osmorphing/test_manager.py @@ -363,3 +363,87 @@ def test_morph_image_dismount_os_exception( mock.sentinel.destination_provider, mock.sentinel.connection_info, self.osmorphing_info, self._mock_user_scripts, self.event_handler) + + def test_apply_cloudbase_init_plugins_override_from_target_env(self): + osmorphing_info = { + "os_type": "windows", + "osmorphing_parameters": {"set_dhcp": True}, + } + target_environment = { + "cloudbase_init_plugins": [ + "cloudbaseinit.plugins.common.mtu.MTUPlugin"]} + result = manager.apply_cloudbase_init_plugins_override( + osmorphing_info, target_environment) + self.assertEqual( + result["osmorphing_parameters"]["cloudbase_init_plugins"], + target_environment["cloudbase_init_plugins"]) + self.assertTrue(result["osmorphing_parameters"]["set_dhcp"]) + self.assertNotIn( + "cloudbase_init_plugins", + osmorphing_info["osmorphing_parameters"]) + + def test_apply_cloudbase_init_plugins_override_noop(self): + osmorphing_info = {"os_type": "windows"} + result = manager.apply_cloudbase_init_plugins_override( + osmorphing_info, {"zone": "zone1"}) + self.assertIs(result, osmorphing_info) + + def test_inject_cloudbase_init_plugins_schema_simple(self): + schema = { + "type": "object", + "properties": {"zone": {"type": "string"}}, + "additionalProperties": False, + } + result = manager.inject_cloudbase_init_plugins_schema(schema) + self.assertIn( + manager.CLOUDBASE_INIT_PLUGINS_OPT, result["properties"]) + self.assertNotIn( + manager.CLOUDBASE_INIT_PLUGINS_OPT, schema["properties"]) + + def test_inject_cloudbase_init_plugins_schema_oneof(self): + schema = { + "oneOf": [ + {"properties": {"migr_network": {"type": "string"}}}, + {"properties": {"network_map": {"type": "object"}}}, + ] + } + result = manager.inject_cloudbase_init_plugins_schema(schema) + for alt in result["oneOf"]: + self.assertIn( + manager.CLOUDBASE_INIT_PLUGINS_OPT, alt["properties"]) + + def test_inject_cloudbase_init_plugins_option_appends(self): + options = [{"name": "zone", "values": []}] + result = manager.inject_cloudbase_init_plugins_option(options) + names = [opt["name"] for opt in result] + self.assertIn(manager.CLOUDBASE_INIT_PLUGINS_OPT, names) + self.assertEqual(options, [{"name": "zone", "values": []}]) + + def test_inject_cloudbase_init_plugins_option_skips_duplicate(self): + options = [{ + "name": manager.CLOUDBASE_INIT_PLUGINS_OPT, + "values": ["already-set"], + }] + result = manager.inject_cloudbase_init_plugins_option(options) + self.assertEqual(1, len(result)) + self.assertEqual(["already-set"], result[0]["values"]) + + def test_inject_cloudbase_init_plugins_option_respects_names(self): + options = [{"name": "zone", "values": []}] + result = manager.inject_cloudbase_init_plugins_option( + options, option_names=["zone"]) + self.assertEqual(options, result) + + def test_inject_cloudbase_init_plugins_option_empty_dict(self): + options = [{"name": "zone", "values": []}] + result = manager.inject_cloudbase_init_plugins_option( + options, option_names={}) + names = [opt["name"] for opt in result] + self.assertIn(manager.CLOUDBASE_INIT_PLUGINS_OPT, names) + + def test_filter_cloudbase_init_plugins_option_names(self): + result = manager.filter_cloudbase_init_plugins_option_names( + ["zone", manager.CLOUDBASE_INIT_PLUGINS_OPT, "import_node"]) + self.assertEqual(["zone", "import_node"], result) + self.assertEqual( + {}, manager.filter_cloudbase_init_plugins_option_names({})) diff --git a/coriolis/tests/osmorphing/test_windows.py b/coriolis/tests/osmorphing/test_windows.py index 87616f115..931c2c7d6 100644 --- a/coriolis/tests/osmorphing/test_windows.py +++ b/coriolis/tests/osmorphing/test_windows.py @@ -544,11 +544,58 @@ def test__write_cloudbase_init_conf( mock_write_local_script.assert_called_once_with( 'C:\\Cloudbase-Init', mocked_full_path, priority=10) + def test__resolve_cloudbase_init_plugins_provider_default(self): + provider_plugins = [ + 'cloudbaseinit.plugins.common.mtu.MTUPlugin'] + result = self.morphing_tools._resolve_cloudbase_init_plugins( + provider_plugins) + self.assertEqual(result, provider_plugins) + + def test__resolve_cloudbase_init_plugins_core_default(self): + result = self.morphing_tools._resolve_cloudbase_init_plugins() + self.assertEqual(result, windows.CLOUDBASE_INIT_DEFAULT_PLUGINS) + + def test__resolve_cloudbase_init_plugins_conf_override(self): + conf_plugins = [ + 'cloudbaseinit.plugins.common.localscripts.LocalScriptsPlugin'] + windows.CONF.set_override('cloudbase_init_plugins', conf_plugins) + self.addCleanup( + windows.CONF.clear_override, 'cloudbase_init_plugins') + result = self.morphing_tools._resolve_cloudbase_init_plugins( + ['cloudbaseinit.plugins.common.mtu.MTUPlugin']) + self.assertEqual(result, conf_plugins) + + def test__resolve_cloudbase_init_plugins_api_override(self): + api_plugins = [ + 'cloudbaseinit.plugins.common.userdata.UserDataPlugin'] + windows.CONF.set_override( + 'cloudbase_init_plugins', + ['cloudbaseinit.plugins.common.mtu.MTUPlugin']) + self.addCleanup( + windows.CONF.clear_override, 'cloudbase_init_plugins') + self.morphing_tools._osmorphing_parameters = { + "cloudbase_init_plugins": api_plugins} + result = self.morphing_tools._resolve_cloudbase_init_plugins( + ['ignored.provider.Plugin']) + self.assertEqual(result, api_plugins) + + def test__resolve_cloudbase_init_plugins_csv_string(self): + csv_plugins = ( + "cloudbaseinit.plugins.common.mtu.MTUPlugin, " + "cloudbaseinit.plugins.common.localscripts.LocalScriptsPlugin") + self.morphing_tools._osmorphing_parameters = { + "cloudbase_init_plugins": csv_plugins} + result = self.morphing_tools._resolve_cloudbase_init_plugins() + self.assertEqual( + result, + ['cloudbaseinit.plugins.common.mtu.MTUPlugin', + 'cloudbaseinit.plugins.common.localscripts.LocalScriptsPlugin']) + @mock.patch.object(windows.utils, 'write_winrm_file') @mock.patch.object(windows.BaseWindowsMorphingTools, '_write_local_script') def test__write_cloudbase_init_conf_with_exception( self, mock_write_local_script, mock_write_winrm_file): - plugins = "invalid plugins" + plugins = {"invalid": "plugins"} self.assertRaises( exception.CoriolisException, diff --git a/coriolis/tests/tasks/test_osmorphing_tasks.py b/coriolis/tests/tasks/test_osmorphing_tasks.py index 3b8ceb300..a86c2401e 100644 --- a/coriolis/tests/tasks/test_osmorphing_tasks.py +++ b/coriolis/tests/tasks/test_osmorphing_tasks.py @@ -116,6 +116,41 @@ def _get_result(*args): mock_marshal_conn_info.assert_called_once_with( import_info.get('osmorphing_connection_info')) + @mock.patch('coriolis.providers.factory.get_provider') + @mock.patch('coriolis.tasks.base.get_connection_info') + @mock.patch('coriolis.schemas.validate_value') + @mock.patch('coriolis.tasks.base.marshal_migr_conn_info') + def test__run_applies_cloudbase_init_plugins_from_target_environment( + self, mock_marshal_conn_info, mock_validate_value, + mock_get_conn_info, mock_get_provider): + plugins = ['cloudbaseinit.plugins.common.mtu.MTUPlugin'] + import_info = { + "os_morphing_resources": {"res1": "id1"}, + "osmorphing_connection_info": {"info1": "secret1"}, + "osmorphing_info": { + "os_type": "windows", + "osmorphing_parameters": {"set_dhcp": True}, + }, + } + prov_fun = mock_get_provider.return_value.deploy_os_morphing_resources + prov_fun.return_value = import_info + task_info = { + "target_environment": {"cloudbase_init_plugins": plugins}, + "instance_deployment_info": {}, + } + destination = mock.MagicMock() + + result = self.task_runner._run( + mock.sentinel.ctxt, mock.sentinel.instance, mock.sentinel.origin, + destination, task_info, mock.sentinel.event_handler) + + self.assertEqual( + result["osmorphing_info"]["osmorphing_parameters"][ + "cloudbase_init_plugins"], + plugins) + self.assertTrue( + result["osmorphing_info"]["osmorphing_parameters"]["set_dhcp"]) + class DeleteOSMorphingResourcesTaskTestCase(test_base.CoriolisBaseTestCase): diff --git a/coriolis/worker/rpc/server.py b/coriolis/worker/rpc/server.py index b9fd171b4..945d87331 100644 --- a/coriolis/worker/rpc/server.py +++ b/coriolis/worker/rpc/server.py @@ -22,6 +22,7 @@ from coriolis import context from coriolis import exception from coriolis.minion_manager.rpc import client as rpc_minion_manager_client +from coriolis.osmorphing import manager as osmorphing_manager from coriolis.providers import factory as providers_factory from coriolis import schemas from coriolis import service @@ -396,8 +397,15 @@ def get_endpoint_destination_options( secret_connection_info = utils.get_secret_connection_info( ctxt, connection_info) + provider_option_names = ( + osmorphing_manager.filter_cloudbase_init_plugins_option_names( + option_names)) options = provider.get_target_environment_options( - ctxt, secret_connection_info, env=env, option_names=option_names) + ctxt, secret_connection_info, env=env, + option_names=provider_option_names) + + options = osmorphing_manager.inject_cloudbase_init_plugins_option( + options, option_names=option_names) schemas.validate_value( options, schemas.CORIOLIS_DESTINATION_ENVIRONMENT_OPTIONS_SCHEMA) @@ -530,6 +538,9 @@ def validate_endpoint_target_environment( provider = providers_factory.get_provider( platform_name, constants.PROVIDER_TYPE_OS_MORPHING, None) target_env_schema = provider.get_target_environment_schema() + target_env_schema = ( + osmorphing_manager.inject_cloudbase_init_plugins_schema( + target_env_schema)) is_valid = True message = None @@ -642,6 +653,8 @@ def get_provider_schemas(self, ctxt, platform_name, provider_type): if provider_type == constants.PROVIDER_TYPE_TRANSFER_IMPORT: schema = provider.get_target_environment_schema() + schema = osmorphing_manager.inject_cloudbase_init_plugins_schema( + schema) schemas["destination_environment_schema"] = schema if provider_type == constants.PROVIDER_TYPE_TRANSFER_EXPORT: diff --git a/etc/coriolis/coriolis.conf b/etc/coriolis/coriolis.conf index 5d172f54d..8f7a1f0f1 100644 --- a/etc/coriolis/coriolis.conf +++ b/etc/coriolis/coriolis.conf @@ -2,6 +2,9 @@ log_dir=/tmp log_file=coriolis.log messaging_transport_url=rabbit://coriolis:Passw0rd@127.0.0.1:5672/ +# Comma-separated Cloudbase-Init plugin class names for Windows OS morphing. +# When unset, each destination provider uses its own plugin list. +# cloudbase_init_plugins = cloudbaseinit.plugins.common.mtu.MTUPlugin,cloudbaseinit.plugins.windows.ntpclient.NTPClientPlugin [keystone_authtoken] auth_type = password