From 85841f46631ec8dfb3ddc548df14fc35e0147ac0 Mon Sep 17 00:00:00 2001 From: crountre Date: Fri, 4 May 2018 17:24:20 -0500 Subject: [PATCH 01/52] Start of variable generation --- .../tool/util/variable_file_helper.py | 254 ++++++++++++++++++ .../wlsdeploy/util/model_search_helper.py | 76 ++++++ .../deploy/messages/wlsdeploy_rb.properties | 10 + .../test/python/variable_file_helper_test.py | 83 ++++++ .../test/resources/variable_insertion.yaml | 123 +++++++++ 5 files changed, 546 insertions(+) create mode 100644 core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py create mode 100644 core/src/main/python/wlsdeploy/util/model_search_helper.py create mode 100644 core/src/test/python/variable_file_helper_test.py create mode 100644 core/src/test/resources/variable_insertion.yaml diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py new file mode 100644 index 0000000000..0e2f9f5fa2 --- /dev/null +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -0,0 +1,254 @@ +""" +Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. +The Universal Permissive License (UPL), Version 1.0 +""" +import copy +import os + +import java.lang.IllegalArgumentException as IllegalArgumentException +import java.lang.StringBuilder as StringBuilder +import wlsdeploy.util.dictionary_utils as dictionary_utils +import wlsdeploy.util.model as model_sections +from wlsdeploy.aliases.location_context import LocationContext +from wlsdeploy.logging.platform_logger import PlatformLogger +from wlsdeploy.util.model_translator import FileToPython +from wlsdeploy.aliases.aliases import Aliases +from wlsdeploy.tool.util.alias_helper import AliasHelper +from wlsdeploy.aliases.wlst_modes import WlstModes + +VARIABLE_HELPER_FILE_NAME = 'model_variable_helper.json' +VARIABLE_HELPER_FILE_NAME_ARG = 'variable_helper_file_name' +VARIABLE_HELPER_DICTIONARY_ARG = 'variable_helper_dictionary' +CUSTOM_KEYWORD = 'CUSTOM' + +_keyword_to_file_map = { + 'PORT': 'port.properties', + 'TOPOLOGY_NAME': 'name.properties', + 'CREDENTIALS': 'credentials.properties', + 'URL': 'url.properties' +} +_class_name = 'variable_file_helper' +_logger = PlatformLogger('wlsdeploy.util') + + +class VariableFileHelper(object): + + def __init__(self, model, model_context=None, version=None): + self.__model = model + if version is not None: + self.__aliases = Aliases(model_context, WlstModes.OFFLINE, version, None) + else: + self.__aliases = Aliases(model_context) + self.__variable_file = None + + def replace_variables_file(self, variable_file_location, **kwargs): + """ + Replace attribute values with variables and generate a variable file. + The variable replacement is driven from the values in the model variable helper file. + This file can either contain the name of a replacement file, or a list of pre-defined + keywords for canned replacement files. + :param model: the model to be massaged + :param variable_file_location: location and name to store the generated variable file, else default + :param kwargs: arguments used to override default for variable processing, typically used in test situations + :return: True if any variable was inserted, False otherwise + """ + _method_name = 'insert_variables' + _logger.entering(variable_file_location, class_name=_class_name, method_name=_method_name) + + variables_inserted = False + + self.__open_variable_file(variable_file_location) + if not self.__variable_file: + _logger.warning('WLSDPLY-19404', variable_file_location) + return variables_inserted + + if VARIABLE_HELPER_DICTIONARY_ARG in kwargs: + variables_dictionary = kwargs[VARIABLE_HELPER_DICTIONARY_ARG] + else: + variable_helper_location = os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'lib', VARIABLE_HELPER_FILE_NAME) + if VARIABLE_HELPER_FILE_NAME_ARG in kwargs: + variable_helper_location = kwargs[VARIABLE_HELPER_FILE_NAME_ARG] + variables_dictionary = _load_variables_dictionary(variable_helper_location) + + variables_file_dictionary = self.replace_variables_dictionary(variables_dictionary) + # now persist the dictionary even if empty + + self.__variable_file.close() + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) + return variables_inserted + + def replace_variables_dictionary(self, variables_dictionary): + """ + Takes a variable keyword dictionary and returns a variables for file in a dictionary + :param variables_dictionary: + :return: + """ + variable_file_entries = dict() + if variables_dictionary: + file_list = [] + if not dictionary_utils.is_empty_dictionary_element(variables_dictionary, CUSTOM_KEYWORD): + file_list.append(variables_dictionary[CUSTOM_KEYWORD]) + _logger.fine('WLSDPLY-19401', file_list[0]) + else: + for keyword in variables_dictionary: + if keyword in _keyword_to_file_map: + file_list.append(_keyword_to_file_map[keyword]) + elif keyword != CUSTOM_KEYWORD: + _logger.warning('WLSDPLY-19403', keyword) + for file_name in file_list: + replacement_list = _load_replacement_list(file_name) + if replacement_list: + entries = self.process_variable_replacement(replacement_list) + if entries: + # log that a set of entries was returned for the keywood + variable_file_entries.append(entries) + return variable_file_entries + + def process_variable_replacement(self, replacement_list): + _method_name = '__process_variable_replacement' + _logger.entering(class_name=_class_name, method_name=_method_name) + variable_dict = dict() + print 'replacement_list=', replacement_list + if replacement_list: + topology = self.__model[model_sections.get_model_topology_key()] + if 'Name' in topology: + domain_name = topology['Name'] + else: + domain_name = 'mydomain' + location = LocationContext() + domain_token = self.__aliases.get_name_token(location) + location.add_name_token(domain_token, domain_name) + for replacement_entry in replacement_list: + entries_dict = self.__process_variable(location, replacement_entry) + if len(entries_dict) > 0: + variable_dict.update(entries_dict) + + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_dict) + return variable_dict + + def __process_variable(self, location, replacement): + + variable_dict = dict() + + def _traverse_variables(model_section, mbean_list, attribute, variable_name=StringBuilder()): + print 'mbean_list=', mbean_list + if mbean_list: + mbean = mbean_list.pop(0) + if mbean in model_section: + next_model_section = model_section[mbean] + _append_to_variable_name(variable_name, mbean.lower()) + location.append_location(mbean) + name_token = self.__aliases.get_name_token(location) + if self.__aliases.supports_multiple_mbean_instances(location): + for mbean_name in next_model_section: + continue_mbean_list = copy.copy(mbean_list) + location.add_name_token(name_token, mbean_name) + continue_variable_name = StringBuilder(variable_name.toString()) + _append_to_variable_name(continue_variable_name, mbean_name) + _traverse_variables(next_model_section[mbean_name], continue_mbean_list, attribute, + continue_variable_name) + location.remove_name_token(name_token) + else: + _traverse_variables(next_model_section, mbean_list, attribute, variable_name) + location.pop_location() + else: + print 'invalid mbean in mbean_list ', mbean, ' : ', model_section + return False + else: + if attribute in model_section: + value = model_section[attribute] + if type(value) != str or not value.startswith('@@PROP:'): + _append_to_variable_name(variable_name, attribute.lower()) + str_var = variable_name.toString() + model_section[attribute] = '@@PROP:%s@@' % str_var + variable_dict[str_var] = value + else: + print 'attribute not in model' + _logger.finer('attribute not in the model') + return True + + section, attribute, segment = _split_section(replacement) + print section, ', ', attribute, ', ', segment + if section and section in self.__model: + start_mbean_list, attribute = _split_attribute_path(attribute) + print start_mbean_list, ', ', attribute + _traverse_variables(self.__model[section], start_mbean_list, attribute) + print variable_dict + return variable_dict + + def __open_variable_file(self, variable_file_name): + _method_name = '_open_variable_file' + _logger.entering(variable_file_name, class_name=_class_name, method_name=_method_name) + if variable_file_name and os.path.isfile(variable_file_name): + try: + self.__variable_file = open(variable_file_name, 'w') + except OSError, oe: + _logger.warning('WLSDPLY-19405', variable_file_name, str(oe), class_name=_class_name, + method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name) + + +def _append_to_variable_name(builder, value): + if builder.length() > 0: + builder.append('-') + builder.append(value) + + +def _load_replacement_list(replacement_file_name): + _method_name = '_load_replacement_dictionary' + _logger.entering(replacement_file_name, class_name=_class_name, method_name=_method_name) + replacement_list = [] + if os.path.isfile(replacement_file_name): + replacement_file = open(replacement_file_name, 'r') + entry = replacement_file.readline() + while entry: + if entry != '\n': + replacement_list.append(entry) + entry = replacement_file.readline() + + _logger.exiting(class_name=_class_name, method_name=_method_name, result=replacement_list) + return replacement_list + + +def _load_variables_dictionary(variable_helper_location): + _method_name = '_load_variables_dictionary' + variables_dictionary = None + if os.path.isfile(variable_helper_location): + try: + variables_dictionary = FileToPython(variable_helper_location).parse() + _logger.fine('WLSDPLY-19400', variable_helper_location, class_name=_class_name, + method_name=_method_name) + except IllegalArgumentException, ia: + _logger.warning('WLSDPLY-19402', variable_helper_location, ia.getLocalizedMessage(), + class_name=_class_name, method_name=_method_name) + return variables_dictionary + + +def _split_section(attribute_path): + """ + Split the section from the attribute path. + :param attribute_path: + :return: + """ + split_list = attribute_path.split(':', 1) + section = None + attribute = None + segment = None + if len(split_list) == 2 and split_list[0] in model_sections.get_model_top_level_keys(): + section = split_list[0] + attribute_split_list = split_list[1].split('${') + attribute = attribute_split_list[0] + if len(attribute_split_list) == 2: + segment = attribute_split_list[1][:len(attribute_split_list[1])-1] + return section, attribute, segment + + +def _split_attribute_path(attribute_path): + mbean_list = attribute_path.split('.') + attribute = None + if len(mbean_list) > 0: + attribute = mbean_list.pop() + return mbean_list, attribute + + + diff --git a/core/src/main/python/wlsdeploy/util/model_search_helper.py b/core/src/main/python/wlsdeploy/util/model_search_helper.py new file mode 100644 index 0000000000..1dc0936b42 --- /dev/null +++ b/core/src/main/python/wlsdeploy/util/model_search_helper.py @@ -0,0 +1,76 @@ +""" +Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. +The Universal Permissive License (UPL), Version 1.0 +""" +from wlsdeploy.logging.platform_logger import PlatformLogger +import wlsdeploy.util.model as model_sections + +_class_name = 'ModelSearchHelper' +_logger = PlatformLogger('wlsdeploy.util') + + +class ModelSearchHelper(object): + """ + Class functions to facilitate searching the model. + """ + + def __init__(self, model): + """ + Encapsulate the model on which to perform searches. + :param model: to encapsulate + """ + self.__model = model + + def locate_attribute_value(self, section, attribute_path): + """ + Locate and return the dictionary entry for the attribute and value using the provided search pattern. +
:[....].[.${search pattern}] + where section is a section of the model, and the mbean is the path of mbeans that contains the attribute and + search pattern is a key=value pair within the attribute's value. + The model is traversed down to the + :param section: + :param attribute_path: + :return: + """ + _method_name = 'locate_attribute_value' + _logger.entering(attribute_path, class_name=_class_name, method_name=_method_name) + if section in model_sections.get_model_top_level_keys(): + model_section = self.__model[section] + mbean_list, attribute_name = _split_attribute_path(attribute_path) + for entry in mbean_list: + if entry in model_section: + model_section = model_section[entry] + else: + _logger.warning('WLSDPLY-19406', entry, attribute_path, section) + break + + + + _logger.exiting(class_name=_class_name, method_name=_method_name) + + +def _split_section(attribute_path): + """ + Split the section from the attribute path. + :param attribute_path: + :return: + """ + split_list = attribute_path.split(':', 1) + section = None + attribute = None + segment = None + if len(split_list) == 2 and split_list[0] in model_sections.get_model_top_level_keys(): + section = split_list[0] + attribute_split_list = split_list[1].split('${') + attribute = attribute_split_list[0] + if len(attribute_split_list) == 2: + segment = attribute_split_list[1][:len(attribute_split_list[1])-1] + return section, attribute, segment + + +def _split_attribute_path(attribute_path): + mbean_list = attribute_path.split('.') + attribute = None + if len(mbean_list) > 0: + attribute = mbean_list.pop() + return mbean_list, attribute diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 8ca4565528..563bfddf75 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1098,6 +1098,16 @@ WLSDPLY-19305=Failed to extract domain library {0} because it does not exist in WLSDPLY-19306=Unable to extract domain library {0} from archive file {1}: {2} WLSDPLY-19307=Unable to extract classpath libraries from archive file {0} to domain directory {1}: {2} +# wlsdeploy/tool/util/variable_file_helper.py +WLSDPLY-19400=model variable replacement helper file found at location {0} +WLSDPLY-19401=Variable replacement using custom list loaded from variable helper file {0} +WLSDPLY-19402=Unable to read and parse variables helper file {0} : {1} +WLSDPLY-19403=Ignoring unknown keyword {0} found in variable replacement helper +WLSDPLY-19404=An invalid variable file location {0} was provided to variable helper. Variable replacement \ + will not continue +WLSDPLY-19405=Unable to open the variable file {0} for writing : {1} +WLSDPLY-19406=Invalid attribute part {0} in path {1) for section {2} + # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} WLSDPLY-20001={0} did not create the domain because validation failed diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py new file mode 100644 index 0000000000..5d8e1884d5 --- /dev/null +++ b/core/src/test/python/variable_file_helper_test.py @@ -0,0 +1,83 @@ +""" +Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. +The Universal Permissive License (UPL), Version 1.0 +""" +import unittest +print unittest +from wlsdeploy.util.model_translator import FileToPython +from wlsdeploy.tool.util.variable_file_helper import VariableFileHelper + + +class VariableFileHelperTest(unittest.TestCase): + _resources_dir = '../../test-classes' + _model_file = _resources_dir + '/variable_insertion.yaml' + + def setUp(self): + self.name = VariableFileHelperTest + self._model = FileToPython(self._model_file).parse() + self._helper = VariableFileHelper(self._model, None, '12.2.1.3') + + def testSingleVariableReplacement(self): + replacement_list = ['topology:Machine.NodeManager.ListenAddress'] + expected = dict() + expected['machine-machine1-nodemanager-listenaddress'] = '127.0.0.1' + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + + def testMultiplesReplacement(self): + expected = dict() + expected['server-AdminServer-listenport'] = 9001 + expected['server-AdminServer-ssl-listenport'] = 9002 + expected['server-m2-listenport'] = 9005 + expected['server-m1-listenport'] = 9003 + expected['server-m1-ssl-listenport'] = 9004 + expected['server-m2-ssl-listenport'] = 9006 + expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-connectionurl']\ + = 't3://my.other.cluster:7001' + expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-' \ + 'foreigndestination-MyRemoteQ-localjndiname'] = 'jms/remoteQ' + #'resources:JMSSystemResource.JmsResource.ForeignDestination.LocalJNDIName', + replacement_list = ['topology:Server.ListenPort', + 'resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL', + 'resources:JMSSystemResource.JmsResource.ForeignServer.ForeignDestination.LocalJNDIName', + 'topology:Server.SSL.ListenPort'] + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + + def testInvalidMBeanNameNoException(self): + expected = dict() + replacement_list = 'resources:JmsSystemResource.Notes' + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + + def testInvalidAttributeName(self): + expected = dict() + replacement_list = 'topology:Server.listenaddress' + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + + def testInvalidSection(self): + expected = dict() + replacement_list = 'topologies:Server.ListenAddress' + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + + def testDomainAttributeReplacementAndModel(self): + expected = dict() + expected['notes'] = 'Test note replacement' + expected_replacement = '@@PROP:notes@@' + replacement_list = ['topology:Notes'] + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + self.assertEqual(expected_replacement, self._model['topology']['Notes']) + + def _compare_to_expected_dictionary(self, expected, actual): + self.assertEqual(len(expected), len(actual), + 'Not the same number of entries : expected=' + str(len(expected)) + ', actual=' + str( + len(actual))) + for k,v in actual.iteritems(): + self.assertEqual(True, k in expected and v == expected[k], 'Actual item not in expected ' + str(k) + + ' : ' + str(v) + ' expected=' + str(expected)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/core/src/test/resources/variable_insertion.yaml b/core/src/test/resources/variable_insertion.yaml new file mode 100644 index 0000000000..20bf189936 --- /dev/null +++ b/core/src/test/resources/variable_insertion.yaml @@ -0,0 +1,123 @@ +domainInfo: + ServerStartMode: prod + AdminUserName: weblogic + AdminPassword: '--FIX-ME--' +topology: + Name: DemoDomain + AdminServerName: AdminServer + DomainVersion: 12.2.1.3.0 + Notes: 'Test note replacement' + Machine: + machine1: + NodeManager: + ListenAddress: 127.0.0.1 + Notes: The only node manager + Notes: The only machine + Server: + m2: + ListenAddress: 127.0.0.1 + Machine: machine1 + Cluster: mycluster + SSL: + Enabled: true + ListenPort: 9006 + ListenPort: 9005 + m1: + ListenAddress: 127.0.0.1 + Machine: machine1 + Cluster: mycluster + SSL: + Enabled: true + ListenPort: 9004 + ListenPort: 9003 + AdminServer: + ListenAddress: 127.0.0.1 + Machine: machine1 + SSL: + Enabled: true + ListenPort: 9002 + ListenPort: 9001 +resources: + MailSession: + MyMailSession: + Properties: + mail.store.protocol: imap + mail.imap.port: 993L + mail.smtp.ssl.enable: True + mail.imap.auth: True + mail.transport.protocol: smtp + mail.imap.ssl.enable: True + mail.smtp.host: stbeehive.oracle.com + mail.smtp.starttls.enable: True + mail.smtp.port: 465L + mail.smtp.auth: True + mail.imap.host: stbeehive.oracle.com + mail.imap.starttls.enable: True + SessionUsername: 'john.smith@oracle.com' + JNDIName: mail/MyMailSession + Target: mycluster + SessionPasswordEncrypted: '--FIX ME--' + 'MailSession-0': + Properties: + mail.imap.port: 996L + mail.smtp.port: 465L + mail.imap.host: stbeehive.oracle.com + mail.smtp.host: stbeehive.oracle.com + mail.store.protocol: imap + SessionUsername: weblogic + JNDIName: javamail/oracle + Target: mycluster + SessionPasswordEncrypted: '--FIX ME--' + JMSSystemResource: + MyJmsModule: + JmsResource: + ForeignServer: + MyForeignServer: + ForeignDestination: + MyRemoteQ: + RemoteJndiName: jms/myQ + LocalJNDIName: jms/remoteQ + ForeignConnectionFactory: + MyRemoteCF: + RemoteJndiName: jms/myCF + LocalJNDIName: jms/remoteCF + ConnectionURL: 't3://my.other.cluster:7001' + JNDIProperty: + java.naming.security.principal: + Key: java.naming.security.principal + Value: weblogic + bar: + Key: bar + Value: True + JNDIPropertiesCredentialEncrypted: '--FIX ME--' + JDBCSystemResource: + Database1: + DescriptorFileName: 'jdbc/Database1-3195-jdbc.xml' + Target: mycluster + JdbcResource: + JDBCDriverParams: + Properties: + oracle.jdbc.ReadTimeout: + Value: 30000 + user: + Value: jshum + oracle.net.CONNECT_TIMEOUT: + Value: 5000 + URL: 'jdbc:oracle:thin:@//den00chv.us.oracle.com:1521/PDBORCL' + PasswordEncrypted: '--FIX ME--' + DriverName: oracle.jdbc.xa.client.OracleXADataSource + Database2: + DescriptorFileName: 'jdbc/Database2-3194-jdbc.xml' + Target: mycluster + JdbcResource: + JDBCDriverParams: + Properties: + oracle.jdbc.ReadTimeout: + Value: 30000 + user: + Value: jshum + oracle.net.CONNECT_TIMEOUT: + Value: 5000 + URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=slc05til.us.oracle.com)(PORT=1521)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' + PasswordEncrypted: '--FIX ME--' + DriverName: oracle.jdbc.xa.client.OracleXADataSource From 9f89cea0949cba4e22f91f67b7193341b258b442 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 8 May 2018 09:45:26 -0500 Subject: [PATCH 02/52] temporary changes will not run --- .../tool/util/variable_file_helper.py | 129 +++++++++++------- .../deploy/messages/wlsdeploy_rb.properties | 1 + .../test/python/variable_file_helper_test.py | 80 ++++++++--- .../src/test/resources/credentials.properties | 2 + core/src/test/resources/custom.properties | 2 + core/src/test/resources/port.properties | 2 + core/src/test/resources/url.properties | 1 + .../resources/variable_helper_custom.json | 7 + .../resources/variable_helper_keyword.json | 8 ++ .../test/resources/variable_insertion.yaml | 3 + 10 files changed, 163 insertions(+), 72 deletions(-) create mode 100644 core/src/test/resources/credentials.properties create mode 100644 core/src/test/resources/custom.properties create mode 100644 core/src/test/resources/port.properties create mode 100644 core/src/test/resources/url.properties create mode 100644 core/src/test/resources/variable_helper_custom.json create mode 100644 core/src/test/resources/variable_helper_keyword.json diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index 0e2f9f5fa2..a8a8041265 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -6,19 +6,21 @@ import os import java.lang.IllegalArgumentException as IllegalArgumentException -import java.lang.StringBuilder as StringBuilder + +import oracle.weblogic.deploy.util.VariableException as VariableException + import wlsdeploy.util.dictionary_utils as dictionary_utils import wlsdeploy.util.model as model_sections +import wlsdeploy.util.variables as variables +from wlsdeploy.aliases.aliases import Aliases from wlsdeploy.aliases.location_context import LocationContext +from wlsdeploy.aliases.wlst_modes import WlstModes from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.util.model_translator import FileToPython -from wlsdeploy.aliases.aliases import Aliases -from wlsdeploy.tool.util.alias_helper import AliasHelper -from wlsdeploy.aliases.wlst_modes import WlstModes VARIABLE_HELPER_FILE_NAME = 'model_variable_helper.json' +VARIABLE_HELPER_PATH_NAME_ARG = 'variable_helper_path_name' VARIABLE_HELPER_FILE_NAME_ARG = 'variable_helper_file_name' -VARIABLE_HELPER_DICTIONARY_ARG = 'variable_helper_dictionary' CUSTOM_KEYWORD = 'CUSTOM' _keyword_to_file_map = { @@ -43,14 +45,15 @@ def __init__(self, model, model_context=None, version=None): def replace_variables_file(self, variable_file_location, **kwargs): """ - Replace attribute values with variables and generate a variable file. + Replace attribute values with variables and generate a variable dictionary. The variable replacement is driven from the values in the model variable helper file. This file can either contain the name of a replacement file, or a list of pre-defined keywords for canned replacement files. - :param model: the model to be massaged + Return the variable dictionary with the variable name inserted into the model, and the value + that the inserted variable replaced. :param variable_file_location: location and name to store the generated variable file, else default :param kwargs: arguments used to override default for variable processing, typically used in test situations - :return: True if any variable was inserted, False otherwise + :return: variable dictionary containing """ _method_name = 'insert_variables' _logger.entering(variable_file_location, class_name=_class_name, method_name=_method_name) @@ -62,27 +65,42 @@ def replace_variables_file(self, variable_file_location, **kwargs): _logger.warning('WLSDPLY-19404', variable_file_location) return variables_inserted - if VARIABLE_HELPER_DICTIONARY_ARG in kwargs: - variables_dictionary = kwargs[VARIABLE_HELPER_DICTIONARY_ARG] + variable_helper_file_name = VARIABLE_HELPER_FILE_NAME + if VARIABLE_HELPER_FILE_NAME_ARG in kwargs: + variable_helper_file_name = kwargs[VARIABLE_HELPER_FILE_NAME_ARG] + if VARIABLE_HELPER_PATH_NAME_ARG in kwargs: + variable_helper_location_path = kwargs[VARIABLE_HELPER_PATH_NAME_ARG] + variable_helper_location_file = os.path.join(variable_helper_location_path, variable_helper_file_name) else: - variable_helper_location = os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'lib', VARIABLE_HELPER_FILE_NAME) - if VARIABLE_HELPER_FILE_NAME_ARG in kwargs: - variable_helper_location = kwargs[VARIABLE_HELPER_FILE_NAME_ARG] - variables_dictionary = _load_variables_dictionary(variable_helper_location) + variable_helper_location_path = os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'lib') + variable_helper_location_file = os.path.join(variable_helper_location_path, variable_helper_file_name) + file_map = dict() + for key, value in _keyword_to_file_map.iteritems(): + file_map[key] = os.path.join(variable_helper_location_path, value) + print file_map - variables_file_dictionary = self.replace_variables_dictionary(variables_dictionary) - # now persist the dictionary even if empty + variables_helper_dictionary = _load_variables_dictionary(variable_helper_location_file) + print 'variables_helper_dictionary=', variables_helper_dictionary + variables_file_dictionary = self.replace_variables_dictionary(file_map, variables_helper_dictionary) + if variables_file_dictionary: + variables_inserted = True + + for entry in variables_file_dictionary: + self.__variable_file.write(entry) + + # now persist the dictionary even if empty self.__variable_file.close() _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) return variables_inserted - def replace_variables_dictionary(self, variables_dictionary): + def replace_variables_dictionary(self, keyword_map, variables_dictionary): """ Takes a variable keyword dictionary and returns a variables for file in a dictionary :param variables_dictionary: :return: """ + print 'keyword map=', keyword_map variable_file_entries = dict() if variables_dictionary: file_list = [] @@ -91,24 +109,28 @@ def replace_variables_dictionary(self, variables_dictionary): _logger.fine('WLSDPLY-19401', file_list[0]) else: for keyword in variables_dictionary: - if keyword in _keyword_to_file_map: - file_list.append(_keyword_to_file_map[keyword]) + print 'find keyword in var dict ', keyword + if keyword in keyword_map: + file_list.append(keyword_map[keyword]) elif keyword != CUSTOM_KEYWORD: + print 'keyword not understood ', keyword _logger.warning('WLSDPLY-19403', keyword) + print 'file_list=', file_list for file_name in file_list: replacement_list = _load_replacement_list(file_name) + print 'replacement_list=', replacement_list if replacement_list: entries = self.process_variable_replacement(replacement_list) if entries: # log that a set of entries was returned for the keywood - variable_file_entries.append(entries) + variable_file_entries.update(entries) + print variable_file_entries return variable_file_entries def process_variable_replacement(self, replacement_list): _method_name = '__process_variable_replacement' _logger.entering(class_name=_class_name, method_name=_method_name) variable_dict = dict() - print 'replacement_list=', replacement_list if replacement_list: topology = self.__model[model_sections.get_model_topology_key()] if 'Name' in topology: @@ -130,26 +152,21 @@ def __process_variable(self, location, replacement): variable_dict = dict() - def _traverse_variables(model_section, mbean_list, attribute, variable_name=StringBuilder()): - print 'mbean_list=', mbean_list + def _traverse_variables(model_section, mbean_list, attribute): if mbean_list: mbean = mbean_list.pop(0) if mbean in model_section: next_model_section = model_section[mbean] - _append_to_variable_name(variable_name, mbean.lower()) location.append_location(mbean) name_token = self.__aliases.get_name_token(location) if self.__aliases.supports_multiple_mbean_instances(location): for mbean_name in next_model_section: continue_mbean_list = copy.copy(mbean_list) location.add_name_token(name_token, mbean_name) - continue_variable_name = StringBuilder(variable_name.toString()) - _append_to_variable_name(continue_variable_name, mbean_name) - _traverse_variables(next_model_section[mbean_name], continue_mbean_list, attribute, - continue_variable_name) + _traverse_variables(next_model_section[mbean_name], continue_mbean_list, attribute) location.remove_name_token(name_token) else: - _traverse_variables(next_model_section, mbean_list, attribute, variable_name) + _traverse_variables(next_model_section, mbean_list, attribute) location.pop_location() else: print 'invalid mbean in mbean_list ', mbean, ' : ', model_section @@ -158,22 +175,19 @@ def _traverse_variables(model_section, mbean_list, attribute, variable_name=Stri if attribute in model_section: value = model_section[attribute] if type(value) != str or not value.startswith('@@PROP:'): - _append_to_variable_name(variable_name, attribute.lower()) - str_var = variable_name.toString() - model_section[attribute] = '@@PROP:%s@@' % str_var - variable_dict[str_var] = value + # change this to be a loaded thing + var_name, var_value = self.__insert_variable(location, attribute, value) + model_section[attribute] = '@@PROP:%s@@' % var_name + variable_dict[var_name] = var_value else: print 'attribute not in model' _logger.finer('attribute not in the model') return True - section, attribute, segment = _split_section(replacement) - print section, ', ', attribute, ', ', segment + section, attr, segment = _split_section(replacement) if section and section in self.__model: - start_mbean_list, attribute = _split_attribute_path(attribute) - print start_mbean_list, ', ', attribute - _traverse_variables(self.__model[section], start_mbean_list, attribute) - print variable_dict + start_mbean_list, start_attribute = _split_attribute_path(attr) + _traverse_variables(self.__model[section], start_mbean_list, start_attribute) return variable_dict def __open_variable_file(self, variable_file_name): @@ -184,14 +198,17 @@ def __open_variable_file(self, variable_file_name): self.__variable_file = open(variable_file_name, 'w') except OSError, oe: _logger.warning('WLSDPLY-19405', variable_file_name, str(oe), class_name=_class_name, - method_name=_method_name) + method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name) - -def _append_to_variable_name(builder, value): - if builder.length() > 0: - builder.append('-') - builder.append(value) + def __insert_variable(self, location, attribute, value): + path = '' + make_path = self.__aliases.get_model_folder_path(location) + if make_path: + make_path = make_path.split(':') + if len(make_path) > 1 and len(make_path[1]) > 1: + path = make_path[1] + return path + '/' + attribute, value def _load_replacement_list(replacement_file_name): @@ -202,9 +219,12 @@ def _load_replacement_list(replacement_file_name): replacement_file = open(replacement_file_name, 'r') entry = replacement_file.readline() while entry: + print 'entry=', entry if entry != '\n': - replacement_list.append(entry) + replacement_list.append(entry.strip()) entry = replacement_file.readline() + else: + print 'file is not a file', replacement_file_name _logger.exiting(class_name=_class_name, method_name=_method_name, result=replacement_list) return replacement_list @@ -216,11 +236,10 @@ def _load_variables_dictionary(variable_helper_location): if os.path.isfile(variable_helper_location): try: variables_dictionary = FileToPython(variable_helper_location).parse() - _logger.fine('WLSDPLY-19400', variable_helper_location, class_name=_class_name, - method_name=_method_name) + _logger.fine('WLSDPLY-19400', variable_helper_location, class_name=_class_name, method_name=_method_name) except IllegalArgumentException, ia: - _logger.warning('WLSDPLY-19402', variable_helper_location, ia.getLocalizedMessage(), - class_name=_class_name, method_name=_method_name) + _logger.warning('WLSDPLY-19402', variable_helper_location, ia.getLocalizedMessage(), class_name=_class_name, + method_name=_method_name) return variables_dictionary @@ -239,7 +258,7 @@ def _split_section(attribute_path): attribute_split_list = split_list[1].split('${') attribute = attribute_split_list[0] if len(attribute_split_list) == 2: - segment = attribute_split_list[1][:len(attribute_split_list[1])-1] + segment = attribute_split_list[1][:len(attribute_split_list[1]) - 1] return section, attribute, segment @@ -250,5 +269,9 @@ def _split_attribute_path(attribute_path): attribute = mbean_list.pop() return mbean_list, attribute - - +def _write_variables_file(variables_dictionary, variables_file_name): + _method_name = '_write_variables_file' + _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) + try: + variables.write_variables(variables_dictionary, variables_file_name) + except VariableException, ve: diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 563bfddf75..c5d5c9477e 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1107,6 +1107,7 @@ WLSDPLY-19404=An invalid variable file location {0} was provided to variable hel will not continue WLSDPLY-19405=Unable to open the variable file {0} for writing : {1} WLSDPLY-19406=Invalid attribute part {0} in path {1) for section {2} +WLSDPLY-19407=Exception attempting to write variables file {0}. Discarding model changes # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index 5d8e1884d5..c27b389cb0 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -3,14 +3,20 @@ The Universal Permissive License (UPL), Version 1.0 """ import unittest -print unittest -from wlsdeploy.util.model_translator import FileToPython + +import java.io.FileInputStream as FileInputStream +import java.util.Properties as Properties + from wlsdeploy.tool.util.variable_file_helper import VariableFileHelper +from wlsdeploy.util.model_translator import FileToPython class VariableFileHelperTest(unittest.TestCase): _resources_dir = '../../test-classes' _model_file = _resources_dir + '/variable_insertion.yaml' + _variable_file_location = _resources_dir + '/variables.properties' + _variable_helper_keyword = 'variable_helper_keyword.json' + _variable_helper_custom = 'variable_helper_custom.json' def setUp(self): self.name = VariableFileHelperTest @@ -20,22 +26,32 @@ def setUp(self): def testSingleVariableReplacement(self): replacement_list = ['topology:Machine.NodeManager.ListenAddress'] expected = dict() - expected['machine-machine1-nodemanager-listenaddress'] = '127.0.0.1' + expected['/Machine/machine1/NodeManager/ListenAddress'] = '127.0.0.1' actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testMultiplesReplacement(self): expected = dict() - expected['server-AdminServer-listenport'] = 9001 - expected['server-AdminServer-ssl-listenport'] = 9002 - expected['server-m2-listenport'] = 9005 - expected['server-m1-listenport'] = 9003 - expected['server-m1-ssl-listenport'] = 9004 - expected['server-m2-ssl-listenport'] = 9006 - expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-connectionurl']\ + # expected['server-AdminServer-listenport'] = 9001 + # expected['server-AdminServer-ssl-listenport'] = 9002 + # expected['server-m2-listenport'] = 9005 + # expected['server-m1-listenport'] = 9003 + # expected['server-m1-ssl-listenport'] = 9004 + # expected['server-m2-ssl-listenport'] = 9006 + # expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-connectionurl'] \ + # = 't3://my.other.cluster:7001' + # expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-' + # 'foreigndestination-MyRemoteQ-localjndiname'] = 'jms/remoteQ' + expected['/Server/AdminServer/SSL/ListenPort'] = 9002 + expected['/Server/AdminServer/ListenPort'] = 9001 + expected['/Server/m2/ListenPort'] = 9005 + expected['/Server/m1/ListenPort'] = 9003 + expected['/Server/m1/SSL/ListenPort'] = 9004 + expected['/Server/m2/SSL/ListenPort'] = 9006 + expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ = 't3://my.other.cluster:7001' - expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-' \ - 'foreigndestination-MyRemoteQ-localjndiname'] = 'jms/remoteQ' + expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/' + 'ForeignDestination/MyRemoteQ/LocalJNDIName'] = 'jms/remoteQ' #'resources:JMSSystemResource.JmsResource.ForeignDestination.LocalJNDIName', replacement_list = ['topology:Server.ListenPort', 'resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL', @@ -46,38 +62,64 @@ def testMultiplesReplacement(self): def testInvalidMBeanNameNoException(self): expected = dict() - replacement_list = 'resources:JmsSystemResource.Notes' + replacement_list = ['resources:JmsSystemResource.Notes'] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testInvalidAttributeName(self): expected = dict() - replacement_list = 'topology:Server.listenaddress' + replacement_list = ['topology:Server.listenaddress'] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testInvalidSection(self): expected = dict() - replacement_list = 'topologies:Server.ListenAddress' + replacement_list = ['topologies:Server.ListenAddress'] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testDomainAttributeReplacementAndModel(self): expected = dict() - expected['notes'] = 'Test note replacement' - expected_replacement = '@@PROP:notes@@' + expected['/Notes'] = 'Test note replacement' + expected_replacement = '@@PROP:/Notes@@' replacement_list = ['topology:Notes'] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) self.assertEqual(expected_replacement, self._model['topology']['Notes']) + def testWithVariableHelperKeywords(self): + expected = dict() + expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ + = 't3://my.other.cluster:7001' + expected['/Server/AdminServer/ListenPort'] = 9001 + expected['/Server/m2/ListenPort'] = 9005 + expected['/Server/m1/ListenPort'] = 9003 + expected['/Machine/machine1/NodeManager/ListenPort'] = '127.0.0.1' + expected['/Machine/machine1/NodeManager/PasswordEncrypted'] = '--FIX ME--' + expected['/Machine/machine1/NodeManager/UserName'] = 'admin' + inserted = self._helper.replace_variables_file(self._variable_file_location, + variable_helper_path_name=self._resources_dir, + variable_helper_file_name=self._variable_helper_keyword) + self.assertEqual(True, inserted) + actual = self.__read_variable_properties() + self._compare_to_expected_dictionary(expected, actual) + def _compare_to_expected_dictionary(self, expected, actual): self.assertEqual(len(expected), len(actual), 'Not the same number of entries : expected=' + str(len(expected)) + ', actual=' + str( len(actual))) - for k,v in actual.iteritems(): + for k, v in actual.iteritems(): self.assertEqual(True, k in expected and v == expected[k], 'Actual item not in expected ' + str(k) + ' : ' + str(v) + ' expected=' + str(expected)) + def __read_variable_properties(self): + props = Properties() + ins = FileInputStream(self._variable_file_location) + props.load(ins) + print props + ins.close() + return props + + if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/core/src/test/resources/credentials.properties b/core/src/test/resources/credentials.properties new file mode 100644 index 0000000000..4049e70510 --- /dev/null +++ b/core/src/test/resources/credentials.properties @@ -0,0 +1,2 @@ +topology:Machine.NodeManager.PasswordEncrypted +topology:Machine.NodeManager.UserName \ No newline at end of file diff --git a/core/src/test/resources/custom.properties b/core/src/test/resources/custom.properties new file mode 100644 index 0000000000..c915515802 --- /dev/null +++ b/core/src/test/resources/custom.properties @@ -0,0 +1,2 @@ +topology:Notes +appDeployments:Application.SecurityDDModel \ No newline at end of file diff --git a/core/src/test/resources/port.properties b/core/src/test/resources/port.properties new file mode 100644 index 0000000000..1de21fd68d --- /dev/null +++ b/core/src/test/resources/port.properties @@ -0,0 +1,2 @@ +topology:Server.ListenPort +topology:Machine.NodeManager.ListenPort \ No newline at end of file diff --git a/core/src/test/resources/url.properties b/core/src/test/resources/url.properties new file mode 100644 index 0000000000..306a6cd6cc --- /dev/null +++ b/core/src/test/resources/url.properties @@ -0,0 +1 @@ +resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL \ No newline at end of file diff --git a/core/src/test/resources/variable_helper_custom.json b/core/src/test/resources/variable_helper_custom.json new file mode 100644 index 0000000000..4bbf7f268b --- /dev/null +++ b/core/src/test/resources/variable_helper_custom.json @@ -0,0 +1,7 @@ +{ + "CUSTOM": [ + { "file": "C:\Users\crountre\temp\custom_variables.properties" } + ], + "PORT": [ + ] +} \ No newline at end of file diff --git a/core/src/test/resources/variable_helper_keyword.json b/core/src/test/resources/variable_helper_keyword.json new file mode 100644 index 0000000000..a8ebedd963 --- /dev/null +++ b/core/src/test/resources/variable_helper_keyword.json @@ -0,0 +1,8 @@ +{ + "PORT": [ + ], + "URL": [ + ], + "CREDENTIALS": [ + ] +} \ No newline at end of file diff --git a/core/src/test/resources/variable_insertion.yaml b/core/src/test/resources/variable_insertion.yaml index 20bf189936..289521e9a3 100644 --- a/core/src/test/resources/variable_insertion.yaml +++ b/core/src/test/resources/variable_insertion.yaml @@ -11,7 +11,10 @@ topology: machine1: NodeManager: ListenAddress: 127.0.0.1 + ListenPort: 5557 Notes: The only node manager + PasswordEncrypted: '--FIX ME--' + UserName: 'admin' Notes: The only machine Server: m2: From 5d975c560c5492db2e06a6e4108ab61dc1937e28 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 8 May 2018 13:12:01 -0500 Subject: [PATCH 03/52] working first variable helper --- .../tool/util/variable_file_helper.py | 111 +++++++++--------- .../deploy/messages/wlsdeploy_rb.properties | 15 ++- .../test/python/variable_file_helper_test.py | 40 +++---- 3 files changed, 86 insertions(+), 80 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index a8a8041265..4625e983c7 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -8,6 +8,7 @@ import java.lang.IllegalArgumentException as IllegalArgumentException import oracle.weblogic.deploy.util.VariableException as VariableException +import oracle.weblogic.deploy.aliases.AliasException as AliasException import wlsdeploy.util.dictionary_utils as dictionary_utils import wlsdeploy.util.model as model_sections @@ -36,6 +37,8 @@ class VariableFileHelper(object): def __init__(self, model, model_context=None, version=None): + self.__original = model + self.__model = copy.deepcopy(model) self.__model = model if version is not None: self.__aliases = Aliases(model_context, WlstModes.OFFLINE, version, None) @@ -58,13 +61,6 @@ def replace_variables_file(self, variable_file_location, **kwargs): _method_name = 'insert_variables' _logger.entering(variable_file_location, class_name=_class_name, method_name=_method_name) - variables_inserted = False - - self.__open_variable_file(variable_file_location) - if not self.__variable_file: - _logger.warning('WLSDPLY-19404', variable_file_location) - return variables_inserted - variable_helper_file_name = VARIABLE_HELPER_FILE_NAME if VARIABLE_HELPER_FILE_NAME_ARG in kwargs: variable_helper_file_name = kwargs[VARIABLE_HELPER_FILE_NAME_ARG] @@ -77,22 +73,20 @@ def replace_variables_file(self, variable_file_location, **kwargs): file_map = dict() for key, value in _keyword_to_file_map.iteritems(): file_map[key] = os.path.join(variable_helper_location_path, value) - print file_map variables_helper_dictionary = _load_variables_dictionary(variable_helper_location_file) - print 'variables_helper_dictionary=', variables_helper_dictionary - variables_file_dictionary = self.replace_variables_dictionary(file_map, variables_helper_dictionary) - if variables_file_dictionary: - variables_inserted = True + variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location) - for entry in variables_file_dictionary: - self.__variable_file.write(entry) + if variables_inserted: + _logger.info('WLSDPLY-19418', variable_helper_location_file) + return_model = self.__model + else: + _logger.fine('WLSDPLY-19419') + return_model = self.__original - # now persist the dictionary even if empty - self.__variable_file.close() _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) - return variables_inserted + return variables_inserted, return_model def replace_variables_dictionary(self, keyword_map, variables_dictionary): """ @@ -100,31 +94,29 @@ def replace_variables_dictionary(self, keyword_map, variables_dictionary): :param variables_dictionary: :return: """ - print 'keyword map=', keyword_map + _method_name = 'replace_variables_dictionary' + _logger.entering(keyword_map, class_name=_class_name, method_name=_method_name) variable_file_entries = dict() if variables_dictionary: - file_list = [] + file_map = dict() if not dictionary_utils.is_empty_dictionary_element(variables_dictionary, CUSTOM_KEYWORD): - file_list.append(variables_dictionary[CUSTOM_KEYWORD]) - _logger.fine('WLSDPLY-19401', file_list[0]) + file_map[CUSTOM_KEYWORD] = variables_dictionary[CUSTOM_KEYWORD] + _logger.fine('WLSDPLY-19401', file_map[CUSTOM_KEYWORD]) else: for keyword in variables_dictionary: - print 'find keyword in var dict ', keyword if keyword in keyword_map: - file_list.append(keyword_map[keyword]) + _logger.fine('WLSDPLY-1940', keyword, keyword_map[keyword]) + file_map[keyword] = keyword_map[keyword] elif keyword != CUSTOM_KEYWORD: - print 'keyword not understood ', keyword _logger.warning('WLSDPLY-19403', keyword) - print 'file_list=', file_list - for file_name in file_list: + for keyword, file_name in file_map.iteritems(): replacement_list = _load_replacement_list(file_name) - print 'replacement_list=', replacement_list if replacement_list: entries = self.process_variable_replacement(replacement_list) if entries: - # log that a set of entries was returned for the keywood + _logger.finer('WLSDPLY-19413', keyword) variable_file_entries.update(entries) - print variable_file_entries + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_file_entries) return variable_file_entries def process_variable_replacement(self, replacement_list): @@ -156,6 +148,7 @@ def _traverse_variables(model_section, mbean_list, attribute): if mbean_list: mbean = mbean_list.pop(0) if mbean in model_section: + _logger.finest('WLSDPLY-19414', mbean) next_model_section = model_section[mbean] location.append_location(mbean) name_token = self.__aliases.get_name_token(location) @@ -169,7 +162,15 @@ def _traverse_variables(model_section, mbean_list, attribute): _traverse_variables(next_model_section, mbean_list, attribute) location.pop_location() else: - print 'invalid mbean in mbean_list ', mbean, ' : ', model_section + mbean_list = [] + try: + mbean_list = self.__aliases.get_model_subfolder_names(location) + except AliasException: + pass + if mbean in mbean_list: + _logger.finer('WLSDPLY-19416', mbean, replacement, location.get_folder_path()) + else: + _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path()) return False else: if attribute in model_section: @@ -181,7 +182,7 @@ def _traverse_variables(model_section, mbean_list, attribute): variable_dict[var_name] = var_value else: print 'attribute not in model' - _logger.finer('attribute not in the model') + _logger.finer('WLSDPLY-19417', attribute, replacement, location.get_folder_path()) return True section, attr, segment = _split_section(replacement) @@ -190,17 +191,6 @@ def _traverse_variables(model_section, mbean_list, attribute): _traverse_variables(self.__model[section], start_mbean_list, start_attribute) return variable_dict - def __open_variable_file(self, variable_file_name): - _method_name = '_open_variable_file' - _logger.entering(variable_file_name, class_name=_class_name, method_name=_method_name) - if variable_file_name and os.path.isfile(variable_file_name): - try: - self.__variable_file = open(variable_file_name, 'w') - except OSError, oe: - _logger.warning('WLSDPLY-19405', variable_file_name, str(oe), class_name=_class_name, - method_name=_method_name) - _logger.exiting(class_name=_class_name, method_name=_method_name) - def __insert_variable(self, location, attribute, value): path = '' make_path = self.__aliases.get_model_folder_path(location) @@ -208,7 +198,7 @@ def __insert_variable(self, location, attribute, value): make_path = make_path.split(':') if len(make_path) > 1 and len(make_path[1]) > 1: path = make_path[1] - return path + '/' + attribute, value + return path + '/' + attribute, str(value).strip() def _load_replacement_list(replacement_file_name): @@ -216,22 +206,27 @@ def _load_replacement_list(replacement_file_name): _logger.entering(replacement_file_name, class_name=_class_name, method_name=_method_name) replacement_list = [] if os.path.isfile(replacement_file_name): - replacement_file = open(replacement_file_name, 'r') - entry = replacement_file.readline() - while entry: - print 'entry=', entry - if entry != '\n': - replacement_list.append(entry.strip()) + try: + replacement_file = open(replacement_file_name, 'r') entry = replacement_file.readline() + while entry: + if entry != '\n': + _logger.finest('WLSDPLY-19411', entry, replacement_file_name, class_name=_class_name, + method_name=_method_name) + replacement_list.append(entry.strip()) + entry = replacement_file.readline() + except OSError, oe: + _logger.warning('WLDPLY-19409', replacement_file_name, oe.getLocalizedMessage()) else: - print 'file is not a file', replacement_file_name + _logger.warning('WLSDPLY-19410', replacement_file_name) - _logger.exiting(class_name=_class_name, method_name=_method_name, result=replacement_list) + _logger.exiting(class_name=_class_name, method_name=_method_name) return replacement_list def _load_variables_dictionary(variable_helper_location): _method_name = '_load_variables_dictionary' + _logger.entering(variable_helper_location, class_name=_class_name, method_name=_method_name) variables_dictionary = None if os.path.isfile(variable_helper_location): try: @@ -240,6 +235,7 @@ def _load_variables_dictionary(variable_helper_location): except IllegalArgumentException, ia: _logger.warning('WLSDPLY-19402', variable_helper_location, ia.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name) return variables_dictionary @@ -269,9 +265,16 @@ def _split_attribute_path(attribute_path): attribute = mbean_list.pop() return mbean_list, attribute + def _write_variables_file(variables_dictionary, variables_file_name): _method_name = '_write_variables_file' _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) - try: - variables.write_variables(variables_dictionary, variables_file_name) - except VariableException, ve: + written = False + if variables_dictionary: + try: + variables.write_variables(variables_dictionary, variables_file_name) + written = True + except VariableException, ve: + _logger.warning('WLSDPLY-19407', ve.getLocalizedMessage()) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) + return written diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index c5d5c9477e..13cbe1f564 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1107,7 +1107,20 @@ WLSDPLY-19404=An invalid variable file location {0} was provided to variable hel will not continue WLSDPLY-19405=Unable to open the variable file {0} for writing : {1} WLSDPLY-19406=Invalid attribute part {0} in path {1) for section {2} -WLSDPLY-19407=Exception attempting to write variables file {0}. Discarding model changes +WLSDPLY-19407=Exception attempting to write to variables file {0} - discarding model changes with variable insertions \ + : {0} +WLSDPLY-19408=Adding property file {0} to variable helper list for keyword {1} +WLSDPLY-19409=Unable to read and load values for property file {0} : {1} +WLSDPLY-19410=Invalid property file name {0} +WLSDPLY-19411=Adding property {0} for model variable replacement from file {1} +WLSDPLY-19412=Loading keywords from variable helper file {0} +WLSDPLY-19413=Variables were located and inserted in the model for keyword {0} +WLSDPLY-19414=Located mbean {0} in the model file +WLSDPLY-19415=Invalid mbean {0} found in replacement property {1} at location {2} +WLSDPLY-19416=MBean {0} not found in the model for replacement property {1} at location (2} +WLSDPLY-19417=Attribute {0} not found in the model for replacement property {1} at location {2} +WLSDPLY-19418=Variables were inserted into the model and written to the variables file {0} +WLSDPLY-19419=No variables were inserted into the model during variable replacement # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index c27b389cb0..b1298434ed 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -4,9 +4,7 @@ """ import unittest -import java.io.FileInputStream as FileInputStream -import java.util.Properties as Properties - +import wlsdeploy.util.variables as variables from wlsdeploy.tool.util.variable_file_helper import VariableFileHelper from wlsdeploy.util.model_translator import FileToPython @@ -42,12 +40,12 @@ def testMultiplesReplacement(self): # = 't3://my.other.cluster:7001' # expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-' # 'foreigndestination-MyRemoteQ-localjndiname'] = 'jms/remoteQ' - expected['/Server/AdminServer/SSL/ListenPort'] = 9002 - expected['/Server/AdminServer/ListenPort'] = 9001 - expected['/Server/m2/ListenPort'] = 9005 - expected['/Server/m1/ListenPort'] = 9003 - expected['/Server/m1/SSL/ListenPort'] = 9004 - expected['/Server/m2/SSL/ListenPort'] = 9006 + expected['/Server/AdminServer/SSL/ListenPort'] = '9002' + expected['/Server/AdminServer/ListenPort'] = '9001' + expected['/Server/m2/ListenPort'] = '9005' + expected['/Server/m1/ListenPort'] = '9003' + expected['/Server/m1/SSL/ListenPort'] = '9004' + expected['/Server/m2/SSL/ListenPort'] = '9006' expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ = 't3://my.other.cluster:7001' expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/' @@ -91,17 +89,17 @@ def testWithVariableHelperKeywords(self): expected = dict() expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ = 't3://my.other.cluster:7001' - expected['/Server/AdminServer/ListenPort'] = 9001 - expected['/Server/m2/ListenPort'] = 9005 - expected['/Server/m1/ListenPort'] = 9003 - expected['/Machine/machine1/NodeManager/ListenPort'] = '127.0.0.1' + expected['/Server/AdminServer/ListenPort'] = '9001' + expected['/Server/m2/ListenPort'] = '9005' + expected['/Server/m1/ListenPort'] = '9003' + expected['/Machine/machine1/NodeManager/ListenPort'] = '5557' expected['/Machine/machine1/NodeManager/PasswordEncrypted'] = '--FIX ME--' expected['/Machine/machine1/NodeManager/UserName'] = 'admin' - inserted = self._helper.replace_variables_file(self._variable_file_location, + inserted, model = self._helper.replace_variables_file(self._variable_file_location, variable_helper_path_name=self._resources_dir, variable_helper_file_name=self._variable_helper_keyword) self.assertEqual(True, inserted) - actual = self.__read_variable_properties() + actual = variables.load_variables(self._variable_file_location) self._compare_to_expected_dictionary(expected, actual) def _compare_to_expected_dictionary(self, expected, actual): @@ -109,16 +107,8 @@ def _compare_to_expected_dictionary(self, expected, actual): 'Not the same number of entries : expected=' + str(len(expected)) + ', actual=' + str( len(actual))) for k, v in actual.iteritems(): - self.assertEqual(True, k in expected and v == expected[k], 'Actual item not in expected ' + str(k) + - ' : ' + str(v) + ' expected=' + str(expected)) - - def __read_variable_properties(self): - props = Properties() - ins = FileInputStream(self._variable_file_location) - props.load(ins) - print props - ins.close() - return props + self.assertEqual(True, k in expected and v == expected[k], 'Actual item not in expected ' + k + + ' : ' + v + ' expected=' + str(expected)) if __name__ == '__main__': From 4cadcd688690e7363a9506e9b2eefc49b011e038 Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 10 May 2018 14:23:57 -0500 Subject: [PATCH 04/52] push dicover implementation --- core/src/main/python/discover.py | 37 ++++- .../tool/util/variable_file_helper.py | 151 ++++++++++++------ .../main/python/wlsdeploy/util/path_utils.py | 12 ++ .../deploy/messages/wlsdeploy_rb.properties | 8 +- .../test/python/variable_file_helper_test.py | 14 +- .../resources/variable_helper_keyword.json | 1 + installer/src/main/etc/credentials.properties | 4 + installer/src/main/etc/host.properties | 0 installer/src/main/etc/port.properties | 0 installer/src/main/etc/url.properties | 0 10 files changed, 165 insertions(+), 62 deletions(-) create mode 100644 installer/src/main/etc/credentials.properties create mode 100644 installer/src/main/etc/host.properties create mode 100644 installer/src/main/etc/port.properties create mode 100644 installer/src/main/etc/url.properties diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index 4e8c834cd8..14d993818c 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -30,6 +30,7 @@ from wlsdeploy.exception import exception_helper from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.util import getcreds +from wlsdeploy.tool.util.variable_file_helper import VariableFileHelper from wlsdeploy.tool.discover import discoverer from wlsdeploy.tool.discover.deployments_discoverer import DeploymentsDiscoverer from wlsdeploy.tool.discover.domain_info_discoverer import DomainInfoDiscoverer @@ -38,6 +39,7 @@ from wlsdeploy.tool.discover.topology_discoverer import TopologyDiscoverer from wlsdeploy.tool.validate.validator import Validator from wlsdeploy.tool.util import filter_helper +from wlsdeploy.util import path_utils from wlsdeploy.util import wlst_helper from wlsdeploy.util import model_translator from wlsdeploy.util.cla_utils import CommandLineArgUtil @@ -58,8 +60,8 @@ __optional_arguments = [ # Used by shell script to locate WLST - CommandLineArgUtil.DOMAIN_TYPE_SWITCH, CommandLineArgUtil.MODEL_FILE_SWITCH, + CommandLineArgUtil.DOMAIN_TYPE_SWITCH, CommandLineArgUtil.ADMIN_URL_SWITCH, CommandLineArgUtil.ADMIN_USER_SWITCH, CommandLineArgUtil.ADMIN_PASS_SWITCH @@ -368,15 +370,36 @@ def __check_and_customize_model(model, model_context): if filter_helper.apply_filters(model.get_model(), "discover"): __logger.info('WLSDPLY-06014', _class_name=_class_name, method_name=_method_name) - + default_variable_file = __get_default_variable_file(model_context) + inserted, variable_model, variable_file_name = VariableFileHelper(model.get_model(), model_context, WebLogicHelper( + __logger).get_actual_weblogic_version()).replace_variables_file(variable_file_name=default_variable_file) + if default_variable_file: + default_variable_file = os.path.join(path_utils.get_pathname_from_path(model_context.get_archive_file_name()), + default_variable_file + '.properties') + inserted, variable_model, variable_file_name = VariableFileHelper(model.get_model(), model_context, WebLogicHelper( + __logger).get_actual_weblogic_version()).replace_variables_file(variable_file_name=default_variable_file) + if inserted: + model = Model(variable_model) try: validator = Validator(model_context, wlst_mode=__wlst_mode) # no variables are generated by the discover tool - validator.validate_in_tool_mode(model.get_model(), variables_file_name=None, - archive_file_name=model_context.get_archive_file_name()) + validator.validate_in_tool_mode(model.get_model(), variables_file_name=variable_file_name, + archive_file_name=model_context.get_archive_file_name()) except ValidateException, ex: __logger.warning('WLSDPLY-06015', ex.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) + return model + + +def __get_default_variable_file(model_context): + extract_file_name = model_context.get_model_file() + if not extract_file_name: + extract_file_name = model_context.get_archive_file_name() + default_variable_file = path_utils.get_filename_no_ext_from_path(extract_file_name) + if default_variable_file: + default_variable_file = os.path.join(path_utils.get_pathname_from_path(extract_file_name), + default_variable_file + '.properties') + return default_variable_file def __log_and_exit(exit_code, _class_name, _method_name): @@ -433,13 +456,13 @@ def main(args): error=ex, class_name=_class_name, method_name=_method_name) __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) - __check_and_customize_model(model, model_context) - + model = __check_and_customize_model(model, model_context) try: __persist_model(model, model_context) + except TranslateException, ex: __logger.severe('WLSDPLY-06012', _program_name, model_context.get_archive_file_name(), ex.getLocalizedMessage(), - error=ex, class_name=_class_name, method_name=_method_name) + error=ex, class_name=_class_name, method_name=_method_name) __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) __close_archive(model_context) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index 4625e983c7..2a134232ba 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -4,6 +4,7 @@ """ import copy import os +import re import java.lang.IllegalArgumentException as IllegalArgumentException @@ -22,8 +23,11 @@ VARIABLE_HELPER_FILE_NAME = 'model_variable_helper.json' VARIABLE_HELPER_PATH_NAME_ARG = 'variable_helper_path_name' VARIABLE_HELPER_FILE_NAME_ARG = 'variable_helper_file_name' +VARIABLE_FILE_NAME_ARG = 'variable_file_name' +VARIABLE_FILE_NAME = 'variables.properties' CUSTOM_KEYWORD = 'CUSTOM' +_segment_pattern = re.compile("\\[[\w.-]+\\]$") _keyword_to_file_map = { 'PORT': 'port.properties', 'TOPOLOGY_NAME': 'name.properties', @@ -37,16 +41,14 @@ class VariableFileHelper(object): def __init__(self, model, model_context=None, version=None): - self.__original = model - self.__model = copy.deepcopy(model) + self.__original = copy.deepcopy(model) self.__model = model - if version is not None: + if version: self.__aliases = Aliases(model_context, WlstModes.OFFLINE, version, None) else: self.__aliases = Aliases(model_context) - self.__variable_file = None - def replace_variables_file(self, variable_file_location, **kwargs): + def replace_variables_file(self, **kwargs): """ Replace attribute values with variables and generate a variable dictionary. The variable replacement is driven from the values in the model variable helper file. @@ -59,34 +61,38 @@ def replace_variables_file(self, variable_file_location, **kwargs): :return: variable dictionary containing """ _method_name = 'insert_variables' - _logger.entering(variable_file_location, class_name=_class_name, method_name=_method_name) - - variable_helper_file_name = VARIABLE_HELPER_FILE_NAME - if VARIABLE_HELPER_FILE_NAME_ARG in kwargs: - variable_helper_file_name = kwargs[VARIABLE_HELPER_FILE_NAME_ARG] - if VARIABLE_HELPER_PATH_NAME_ARG in kwargs: - variable_helper_location_path = kwargs[VARIABLE_HELPER_PATH_NAME_ARG] - variable_helper_location_file = os.path.join(variable_helper_location_path, variable_helper_file_name) - else: - variable_helper_location_path = os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'lib') - variable_helper_location_file = os.path.join(variable_helper_location_path, variable_helper_file_name) - file_map = dict() - for key, value in _keyword_to_file_map.iteritems(): - file_map[key] = os.path.join(variable_helper_location_path, value) + _logger.entering(class_name=_class_name, method_name=_method_name) + variable_helper_location_file = _get_variable_helper_file_name(**kwargs) variables_helper_dictionary = _load_variables_dictionary(variable_helper_location_file) - variables_file_dictionary = self.replace_variables_dictionary(file_map, variables_helper_dictionary) - variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location) - if variables_inserted: - _logger.info('WLSDPLY-19418', variable_helper_location_file) - return_model = self.__model - else: - _logger.fine('WLSDPLY-19419') - return_model = self.__original + variables_inserted = False + return_model = dict() + variable_file_location = None + if variables_helper_dictionary: + variable_file_location = _get_variable_file_name(variables_helper_dictionary, **kwargs) + if not variable_file_location: + _logger.warning('WLSDPLY-19420', variable_helper_location_file, class_name=_class_name, + method_name=_method_name) + else: + file_map = dict() + variable_keywords_location_path = _get_keyword_files_location(**kwargs) + for key, value in _keyword_to_file_map.iteritems(): + file_map[key] = os.path.join(variable_keywords_location_path, value) + + variables_file_dictionary = self.replace_variables_dictionary(file_map, variables_helper_dictionary) + variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location) + if variables_inserted: + _logger.info('WLSDPLY-19418', variable_file_location, class_name=_class_name, + method_name=_method_name) + return_model = self.__model + else: + _logger.fine('WLSDPLY-19419', class_name=_class_name, method_name=_method_name) + return_model = self.__original + variable_file_location = None _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) - return variables_inserted, return_model + return variables_inserted, return_model, variable_file_location def replace_variables_dictionary(self, keyword_map, variables_dictionary): """ @@ -95,33 +101,32 @@ def replace_variables_dictionary(self, keyword_map, variables_dictionary): :return: """ _method_name = 'replace_variables_dictionary' - _logger.entering(keyword_map, class_name=_class_name, method_name=_method_name) + _logger.entering(variables_dictionary, class_name=_class_name, method_name=_method_name) variable_file_entries = dict() if variables_dictionary: file_map = dict() if not dictionary_utils.is_empty_dictionary_element(variables_dictionary, CUSTOM_KEYWORD): file_map[CUSTOM_KEYWORD] = variables_dictionary[CUSTOM_KEYWORD] - _logger.fine('WLSDPLY-19401', file_map[CUSTOM_KEYWORD]) + _logger.fine('WLSDPLY-19401', file_map[CUSTOM_KEYWORD], class_name=_class_name, + method_name=_method_name) else: for keyword in variables_dictionary: if keyword in keyword_map: - _logger.fine('WLSDPLY-1940', keyword, keyword_map[keyword]) file_map[keyword] = keyword_map[keyword] elif keyword != CUSTOM_KEYWORD: - _logger.warning('WLSDPLY-19403', keyword) + _logger.warning('WLSDPLY-19403', keyword, class_name=_class_name, method_name=_method_name) for keyword, file_name in file_map.iteritems(): replacement_list = _load_replacement_list(file_name) if replacement_list: entries = self.process_variable_replacement(replacement_list) if entries: - _logger.finer('WLSDPLY-19413', keyword) + _logger.finer('WLSDPLY-19413', keyword, class_name=_class_name, method_name=_method_name) variable_file_entries.update(entries) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_file_entries) return variable_file_entries def process_variable_replacement(self, replacement_list): _method_name = '__process_variable_replacement' - _logger.entering(class_name=_class_name, method_name=_method_name) variable_dict = dict() if replacement_list: topology = self.__model[model_sections.get_model_topology_key()] @@ -137,18 +142,18 @@ def process_variable_replacement(self, replacement_list): if len(entries_dict) > 0: variable_dict.update(entries_dict) - _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_dict) return variable_dict def __process_variable(self, location, replacement): - + _method_name = '__process_variable' + _logger.entering(replacement, class_name=_class_name, method_name=_method_name) variable_dict = dict() def _traverse_variables(model_section, mbean_list, attribute): if mbean_list: mbean = mbean_list.pop(0) if mbean in model_section: - _logger.finest('WLSDPLY-19414', mbean) + _logger.finest('WLSDPLY-19414', mbean, class_name=_class_name, method_name=_method_name) next_model_section = model_section[mbean] location.append_location(mbean) name_token = self.__aliases.get_name_token(location) @@ -163,14 +168,25 @@ def _traverse_variables(model_section, mbean_list, attribute): location.pop_location() else: mbean_list = [] - try: - mbean_list = self.__aliases.get_model_subfolder_names(location) - except AliasException: - pass + if len(location.get_folder_path()) > 1: + try: + mbean_list = self.__aliases.get_model_subfolder_names(location) + except AliasException, ae: + _logger.fine('AliasException {0}', ae.getLocalizedMessage()) + pass + else: + try: + mbean_list = self.__aliases.get_model_top_level_folder_names() + except AliasException, ae: + _logger.fine('AliasException {0}', ae.getLocalizedMessage()) + pass + _logger.fine('The mbean list from get model subfolder is {0}', mbean_list) if mbean in mbean_list: - _logger.finer('WLSDPLY-19416', mbean, replacement, location.get_folder_path()) + _logger.finer('WLSDPLY-19416', mbean, replacement, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) else: - _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path()) + _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) return False else: if attribute in model_section: @@ -181,14 +197,17 @@ def _traverse_variables(model_section, mbean_list, attribute): model_section[attribute] = '@@PROP:%s@@' % var_name variable_dict[var_name] = var_value else: - print 'attribute not in model' - _logger.finer('WLSDPLY-19417', attribute, replacement, location.get_folder_path()) + _logger.finer('WLSDPLY-19417', attribute, replacement, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) return True section, attr, segment = _split_section(replacement) if section and section in self.__model: start_mbean_list, start_attribute = _split_attribute_path(attr) _traverse_variables(self.__model[section], start_mbean_list, start_attribute) + else: + _logger.finer('WLSDPLY-19423', section, replacement) + _logger.exiting(class_name=_class_name, method_name=_method_name) return variable_dict def __insert_variable(self, location, attribute, value): @@ -201,6 +220,36 @@ def __insert_variable(self, location, attribute, value): return path + '/' + attribute, str(value).strip() +def _get_variable_file_name(variables_helper_dictionary, **kwargs): + if VARIABLE_FILE_NAME_ARG in variables_helper_dictionary: + variable_file_location = variables_helper_dictionary[VARIABLE_FILE_NAME_ARG] + del variables_helper_dictionary[VARIABLE_FILE_NAME_ARG] + _logger.finer('WLSDPLY-19422', variable_file_location) + elif VARIABLE_FILE_NAME_ARG in kwargs: + variable_file_location = kwargs[VARIABLE_FILE_NAME_ARG] + _logger.finer('WLSDPLY-19421', variable_file_location) + else: + variable_file_location = None + return variable_file_location + + +def _get_variable_helper_file_name(**kwargs): + variable_helper_file_name = VARIABLE_HELPER_FILE_NAME + if VARIABLE_HELPER_FILE_NAME_ARG in kwargs: + variable_helper_file_name = kwargs[VARIABLE_HELPER_FILE_NAME_ARG] + if VARIABLE_HELPER_PATH_NAME_ARG in kwargs: + return os.path.join(kwargs[VARIABLE_HELPER_PATH_NAME_ARG], variable_helper_file_name) + else: + return os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'lib', variable_helper_file_name) + + +def _get_keyword_files_location(**kwargs): + if VARIABLE_HELPER_PATH_NAME_ARG in kwargs: + return kwargs[VARIABLE_HELPER_PATH_NAME_ARG] + else: + return os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'etc') + + def _load_replacement_list(replacement_file_name): _method_name = '_load_replacement_dictionary' _logger.entering(replacement_file_name, class_name=_class_name, method_name=_method_name) @@ -216,9 +265,10 @@ def _load_replacement_list(replacement_file_name): replacement_list.append(entry.strip()) entry = replacement_file.readline() except OSError, oe: - _logger.warning('WLDPLY-19409', replacement_file_name, oe.getLocalizedMessage()) + _logger.warning('WLDPLY-19409', replacement_file_name, oe.getLocalizedMessage(), class_name=_class_name, + method_name=_method_name) else: - _logger.warning('WLSDPLY-19410', replacement_file_name) + _logger.warning('WLSDPLY-19410', replacement_file_name, class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name) return replacement_list @@ -239,6 +289,10 @@ def _load_variables_dictionary(variable_helper_location): return variables_dictionary +def split_attribute_segment(attribute_path): + return _segment_pattern.split(attribute_path) + + def _split_section(attribute_path): """ Split the section from the attribute path. @@ -275,6 +329,7 @@ def _write_variables_file(variables_dictionary, variables_file_name): variables.write_variables(variables_dictionary, variables_file_name) written = True except VariableException, ve: - _logger.warning('WLSDPLY-19407', ve.getLocalizedMessage()) + _logger.warning('WLSDPLY-19407', variables_file_name, ve.getLocalizedMessage(), class_name=_class_name, + method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) return written diff --git a/core/src/main/python/wlsdeploy/util/path_utils.py b/core/src/main/python/wlsdeploy/util/path_utils.py index b7675f8239..119fcfdfb5 100644 --- a/core/src/main/python/wlsdeploy/util/path_utils.py +++ b/core/src/main/python/wlsdeploy/util/path_utils.py @@ -105,6 +105,18 @@ def get_filename_from_path(file_path): return file_name +def get_pathname_from_path(file_path): + """ + Return the path without the file name from the existing file path. + :param file_path: file path + :return: path without file name or None if a file name is not present or the file or file path does not exist + """ + file_name = None + if not JStringUtils.isEmpty(file_path) and (os.path.exists(file_path) is False or os.path.isfile(file_path)): + file_path, _ = os.path.split(file_path) + return file_path + + def get_filename_no_ext_from_path(file_path): """ Return the filename with the extension stripped off from the provided file path. diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 13cbe1f564..7566c01dca 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -245,7 +245,7 @@ WLSDPLY-01720=to_boolean() method called with non-boolean value {0} so returning # wlsdeploy/util/variables.py WLSDPLY-01730=Failed to load variables file {0}: {1} -WLSDPLY-01731=Variables updated after encryption +WLSDPLY-01731=Variables updated WLSDPLY-01732=Variable {0} is not found in properties file WLSDPLY-01733=Variable file {0} cannot be read: {1} WLSDPLY-01734=No value in variable file {0} @@ -1108,7 +1108,7 @@ WLSDPLY-19404=An invalid variable file location {0} was provided to variable hel WLSDPLY-19405=Unable to open the variable file {0} for writing : {1} WLSDPLY-19406=Invalid attribute part {0} in path {1) for section {2} WLSDPLY-19407=Exception attempting to write to variables file {0} - discarding model changes with variable insertions \ - : {0} + : {1} WLSDPLY-19408=Adding property file {0} to variable helper list for keyword {1} WLSDPLY-19409=Unable to read and load values for property file {0} : {1} WLSDPLY-19410=Invalid property file name {0} @@ -1121,6 +1121,10 @@ WLSDPLY-19416=MBean {0} not found in the model for replacement property {1} at l WLSDPLY-19417=Attribute {0} not found in the model for replacement property {1} at location {2} WLSDPLY-19418=Variables were inserted into the model and written to the variables file {0} WLSDPLY-19419=No variables were inserted into the model during variable replacement +WLSDPLY-19420=Variable helper file was found at location {0} but no variable properties file name was provided +WLSDPLY-19421=Variables property file name {0} extracted from model helper json file +WLSDPLY-19422=Variables property file name {0} was passed as an argument to the variable helper +WLSDPLY-19423=Variables helper did not find section {0} in the model and will bypass replacement for {1} # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index b1298434ed..4632e453f0 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -5,14 +5,15 @@ import unittest import wlsdeploy.util.variables as variables +import wlsdeploy.tool.util.variable_file_helper as variable_file_helper from wlsdeploy.tool.util.variable_file_helper import VariableFileHelper from wlsdeploy.util.model_translator import FileToPython class VariableFileHelperTest(unittest.TestCase): _resources_dir = '../../test-classes' + _variable_file = _resources_dir + '/variables.properties' _model_file = _resources_dir + '/variable_insertion.yaml' - _variable_file_location = _resources_dir + '/variables.properties' _variable_helper_keyword = 'variable_helper_keyword.json' _variable_helper_custom = 'variable_helper_custom.json' @@ -21,6 +22,9 @@ def setUp(self): self._model = FileToPython(self._model_file).parse() self._helper = VariableFileHelper(self._model, None, '12.2.1.3') + def testSplitPattern(self): + print variable_file_helper.split_attribute_segment('resources:this.that.[other]') + def testSingleVariableReplacement(self): replacement_list = ['topology:Machine.NodeManager.ListenAddress'] expected = dict() @@ -95,11 +99,11 @@ def testWithVariableHelperKeywords(self): expected['/Machine/machine1/NodeManager/ListenPort'] = '5557' expected['/Machine/machine1/NodeManager/PasswordEncrypted'] = '--FIX ME--' expected['/Machine/machine1/NodeManager/UserName'] = 'admin' - inserted, model = self._helper.replace_variables_file(self._variable_file_location, - variable_helper_path_name=self._resources_dir, - variable_helper_file_name=self._variable_helper_keyword) + inserted, model, variable_file_name = self._helper.replace_variables_file( + variable_helper_path_name=self._resources_dir, variable_helper_file_name=self._variable_helper_keyword) self.assertEqual(True, inserted) - actual = variables.load_variables(self._variable_file_location) + self.assertEqual(self._variable_file, variable_file_name) + actual = variables.load_variables(self._variable_file) self._compare_to_expected_dictionary(expected, actual) def _compare_to_expected_dictionary(self, expected, actual): diff --git a/core/src/test/resources/variable_helper_keyword.json b/core/src/test/resources/variable_helper_keyword.json index a8ebedd963..963634be8b 100644 --- a/core/src/test/resources/variable_helper_keyword.json +++ b/core/src/test/resources/variable_helper_keyword.json @@ -1,4 +1,5 @@ { + "variable_file_name": "../../test-classes/variables.properties", "PORT": [ ], "URL": [ diff --git a/installer/src/main/etc/credentials.properties b/installer/src/main/etc/credentials.properties new file mode 100644 index 0000000000..17ea9d564f --- /dev/null +++ b/installer/src/main/etc/credentials.properties @@ -0,0 +1,4 @@ +resources:ForeignJNDIProvider.User +resources:ForeignJNDIProvider.PasswordEncrypted +resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.PasswordEncrypted +resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.PasswordEncrypted diff --git a/installer/src/main/etc/host.properties b/installer/src/main/etc/host.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/installer/src/main/etc/port.properties b/installer/src/main/etc/port.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/installer/src/main/etc/url.properties b/installer/src/main/etc/url.properties new file mode 100644 index 0000000000..e69de29bb2 From c7e6dd2c2f48dca40fc5fccb2c81fde474bed7b3 Mon Sep 17 00:00:00 2001 From: crountre Date: Fri, 11 May 2018 12:29:09 -0500 Subject: [PATCH 05/52] Add feature to replace portion of dictionary value --- .../tool/util/variable_file_helper.py | 114 ++++++++++++++---- .../deploy/messages/wlsdeploy_rb.properties | 3 + .../test/python/variable_file_helper_test.py | 26 +++- core/src/test/resources/port.properties | 2 +- core/src/test/resources/url.properties | 2 +- 5 files changed, 119 insertions(+), 28 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index 2a134232ba..50e32e9330 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -148,8 +148,9 @@ def __process_variable(self, location, replacement): _method_name = '__process_variable' _logger.entering(replacement, class_name=_class_name, method_name=_method_name) variable_dict = dict() + section, start_mbean_list, attribute, segment, segment_name, replace_if_nosegment = _split_property(replacement) - def _traverse_variables(model_section, mbean_list, attribute): + def _traverse_variables(model_section, mbean_list): if mbean_list: mbean = mbean_list.pop(0) if mbean in model_section: @@ -161,10 +162,10 @@ def _traverse_variables(model_section, mbean_list, attribute): for mbean_name in next_model_section: continue_mbean_list = copy.copy(mbean_list) location.add_name_token(name_token, mbean_name) - _traverse_variables(next_model_section[mbean_name], continue_mbean_list, attribute) + _traverse_variables(next_model_section[mbean_name], continue_mbean_list) location.remove_name_token(name_token) else: - _traverse_variables(next_model_section, mbean_list, attribute) + _traverse_variables(next_model_section, mbean_list) location.pop_location() else: mbean_list = [] @@ -190,34 +191,57 @@ def _traverse_variables(model_section, mbean_list, attribute): return False else: if attribute in model_section: - value = model_section[attribute] - if type(value) != str or not value.startswith('@@PROP:'): - # change this to be a loaded thing - var_name, var_value = self.__insert_variable(location, attribute, value) - model_section[attribute] = '@@PROP:%s@@' % var_name - variable_dict[var_name] = var_value + # change this to be a loaded thing + variable_value = model_section[attribute] + if type(variable_value) != str or not variable_value.startswith('@@PROP:'): + variable_name = self.__format_variable_name(location, attribute) + attribute_value = '@@PROP:%s@@' % variable_name + if segment and type(variable_value) == str: + segment_attribute_value = attribute_value + segment_variable_name = variable_name + if segment_name: + segment_variable_name += segment_name + segment_attribute_value = '@@PROP:%s@@' % segment_variable_name + replaced, attr_value, var_replaced = _replace_segment(segment, variable_value, + segment_attribute_value) + if replaced: + variable_name = segment_variable_name + variable_value = var_replaced + attribute_value = attr_value + elif replace_if_nosegment: + variable_value = str(variable_value) + else: + _logger.finer('WLSDPLY-19424', segment, attribute, replacement, + location.get_folder_path, class_name=_class_name, + method_name=_method_name) + variable_value = None + else: + variable_value = str(variable_value) + if variable_value: + _logger.fine('WLSDPLY-19425', attribute_value, attribute, variable_name, variable_value, + class_name=_class_name, method_name=_method_name) + model_section[attribute] = attribute_value + variable_dict[variable_name] = variable_value else: _logger.finer('WLSDPLY-19417', attribute, replacement, location.get_folder_path(), class_name=_class_name, method_name=_method_name) return True - section, attr, segment = _split_section(replacement) if section and section in self.__model: - start_mbean_list, start_attribute = _split_attribute_path(attr) - _traverse_variables(self.__model[section], start_mbean_list, start_attribute) + _traverse_variables(self.__model[section], start_mbean_list) else: _logger.finer('WLSDPLY-19423', section, replacement) _logger.exiting(class_name=_class_name, method_name=_method_name) return variable_dict - def __insert_variable(self, location, attribute, value): + def __format_variable_name(self, location, attribute): path = '' make_path = self.__aliases.get_model_folder_path(location) if make_path: make_path = make_path.split(':') if len(make_path) > 1 and len(make_path[1]) > 1: path = make_path[1] - return path + '/' + attribute, str(value).strip() + return path + '/' + attribute def _get_variable_file_name(variables_helper_dictionary, **kwargs): @@ -289,27 +313,38 @@ def _load_variables_dictionary(variable_helper_location): return variables_dictionary -def split_attribute_segment(attribute_path): - return _segment_pattern.split(attribute_path) +def _replace_segment(segment, variable_value, attribute_value): + replaced = False + replaced_value = None + replacement_string = None + pattern = re.compile(segment) + matcher = pattern.search(variable_value) + if matcher: + replaced = True + replaced_value = variable_value[matcher.start():matcher.end()] + replacement_string = pattern.sub(attribute_value, variable_value) + return replaced, replacement_string, replaced_value -def _split_section(attribute_path): +def _split_property(attribute_path): """ Split the section from the attribute path. :param attribute_path: :return: """ - split_list = attribute_path.split(':', 1) section = None attribute = None + mbean_list = None segment = None + segment_name = None + if_notfound_replace_full = None + split_list = attribute_path.split(':', 1) if len(split_list) == 2 and split_list[0] in model_sections.get_model_top_level_keys(): section = split_list[0] - attribute_split_list = split_list[1].split('${') - attribute = attribute_split_list[0] - if len(attribute_split_list) == 2: - segment = attribute_split_list[1][:len(attribute_split_list[1]) - 1] - return section, attribute, segment + segment, segment_name, if_notfound_replace_full = _find_segment(split_list[1]) + attribute_path = _split_from_segment(split_list[1]) + mbean_list, attribute = _split_attribute_path(attribute_path) + return section, mbean_list, attribute, segment, segment_name, if_notfound_replace_full def _split_attribute_path(attribute_path): @@ -320,6 +355,39 @@ def _split_attribute_path(attribute_path): return mbean_list, attribute +def _split_from_segment(attribute_path): + return re.split('[\[\(].+[\]\)]$', attribute_path)[0] + + +def _find_segment(attribute_path): + segment = None + segment_name = None + if_notfound_replace_full = None + matcher = re.search('[\[\(].+[\]\)]$', attribute_path) + if matcher: + if attribute_path[matcher.start()] == '(': + if_notfound_replace_full = True + else: + if_notfound_replace_full = False + segment = attribute_path[matcher.start()+1:matcher.end()-1] + matcher = re.search('\w*:', segment) + if matcher: + segment_name = segment[matcher.start():matcher.end()-1] + segment = segment[matcher.end():] + return segment, segment_name, if_notfound_replace_full + + +def _find_special_name(mbean): + mbean_name_list = [] + matcher = re.search('(?<=\[).+(?=\])', mbean) + if matcher: + mbean_name = mbean[matcher.start():matcher.end()] + for entry in mbean_name.split[',']: + if entry: + mbean_name_list.append(entry) + return mbean_name_list + + def _write_variables_file(variables_dictionary, variables_file_name): _method_name = '_write_variables_file' _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 7566c01dca..0ff7dc76e9 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1125,6 +1125,9 @@ WLSDPLY-19420=Variable helper file was found at location {0} but no variable pro WLSDPLY-19421=Variables property file name {0} extracted from model helper json file WLSDPLY-19422=Variables property file name {0} was passed as an argument to the variable helper WLSDPLY-19423=Variables helper did not find section {0} in the model and will bypass replacement for {1} +WLSDPLY-19424=Segment {0} for attribute {1} not found in model for replacement property {2} at location {3} +WLSDPLY-19425=Variable {0} inserted into the model for attribute {1} and variable name {2} and value \ + {3} inserted into variable properties # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index 4632e453f0..b5e0b4ce26 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -22,9 +22,6 @@ def setUp(self): self._model = FileToPython(self._model_file).parse() self._helper = VariableFileHelper(self._model, None, '12.2.1.3') - def testSplitPattern(self): - print variable_file_helper.split_attribute_segment('resources:this.that.[other]') - def testSingleVariableReplacement(self): replacement_list = ['topology:Machine.NodeManager.ListenAddress'] expected = dict() @@ -89,6 +86,29 @@ def testDomainAttributeReplacementAndModel(self): self._compare_to_expected_dictionary(expected, actual) self.assertEqual(expected_replacement, self._model['topology']['Notes']) + def testWithSegment(self): + expected = dict() + expected['/JDBCSystemResource/Database1/JdbcResource/JDBCDriverParams/URL'] = \ + 'jdbc:oracle:thin:@//den00chv.us.oracle.com:1521/PDBORCL' + expected['/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLHost'] = \ + 'slc05til.us.oracle.com' + expected['/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLPort'] = \ + '1521' + replacement_list = [ + 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(Port:(?<=PORT=)[\w.-]+(?=\)))', + 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(Host:(?<=HOST=)[\w.-]+(?=\)))'] + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + db2 = 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)' \ + '(HOST=@@PROP:/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLHost@@)' \ + '(PORT=@@PROP:/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLPort@@)))' \ + '(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' + db1 = '@@PROP:/JDBCSystemResource/Database1/JdbcResource/JDBCDriverParams/URL@@' + self.assertEqual(db2, self._model['resources']['JDBCSystemResource']['Database2']['JdbcResource'][ + 'JDBCDriverParams']['URL']) + self.assertEqual(db1, self._model['resources']['JDBCSystemResource']['Database1']['JdbcResource'][ + 'JDBCDriverParams']['URL']) + def testWithVariableHelperKeywords(self): expected = dict() expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ diff --git a/core/src/test/resources/port.properties b/core/src/test/resources/port.properties index 1de21fd68d..f8c40ae603 100644 --- a/core/src/test/resources/port.properties +++ b/core/src/test/resources/port.properties @@ -1,2 +1,2 @@ topology:Server.ListenPort -topology:Machine.NodeManager.ListenPort \ No newline at end of file +topology:Machine.NodeManager.ListenPort diff --git a/core/src/test/resources/url.properties b/core/src/test/resources/url.properties index 306a6cd6cc..44036df22c 100644 --- a/core/src/test/resources/url.properties +++ b/core/src/test/resources/url.properties @@ -1 +1 @@ -resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL \ No newline at end of file +resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL From 88c66bc6c52b9c20792bc2ea1b646aa2f8c941fa Mon Sep 17 00:00:00 2001 From: crountre Date: Fri, 11 May 2018 15:03:01 -0500 Subject: [PATCH 06/52] match segment in value that is dictionary --- .../tool/util/variable_file_helper.py | 40 +++++++++++++------ .../test/python/variable_file_helper_test.py | 31 ++++++++++++++ .../test/resources/variable_insertion.yaml | 4 ++ 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index 50e32e9330..85e4f8c373 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -196,25 +196,39 @@ def _traverse_variables(model_section, mbean_list): if type(variable_value) != str or not variable_value.startswith('@@PROP:'): variable_name = self.__format_variable_name(location, attribute) attribute_value = '@@PROP:%s@@' % variable_name - if segment and type(variable_value) == str: + if segment: segment_attribute_value = attribute_value segment_variable_name = variable_name if segment_name: segment_variable_name += segment_name segment_attribute_value = '@@PROP:%s@@' % segment_variable_name - replaced, attr_value, var_replaced = _replace_segment(segment, variable_value, - segment_attribute_value) - if replaced: - variable_name = segment_variable_name - variable_value = var_replaced - attribute_value = attr_value - elif replace_if_nosegment: - variable_value = str(variable_value) - else: - _logger.finer('WLSDPLY-19424', segment, attribute, replacement, - location.get_folder_path, class_name=_class_name, - method_name=_method_name) + if isinstance(variable_value, dict): + segment_dict = variable_value variable_value = None + for entry in segment_dict: + if segment_dict != str or not segment_dict[entry].startswith('@@PROP:'): + matcher = re.search(segment, entry) + if matcher: + variable_value = str(segment_dict[entry]) + segment_dict[entry] = segment_attribute_value + variable_name = segment_variable_name + attribute_value = segment_dict + elif isinstance(variable_value, list): + print ' need to refactor first' + else: + variable_value = str(variable_value) + replaced, attr_value, var_replaced = _replace_segment(segment, variable_value, + segment_attribute_value) + if replaced: + variable_name = segment_variable_name + variable_value = var_replaced + attribute_value = attr_value + elif not replace_if_nosegment: + _logger.finer('WLSDPLY-19424', segment, attribute, replacement, + location.get_folder_path, class_name=_class_name, + method_name=_method_name) + variable_value = None + else: variable_value = str(variable_value) if variable_value: diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index b5e0b4ce26..1688e4c922 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -109,6 +109,37 @@ def testWithSegment(self): self.assertEqual(db1, self._model['resources']['JDBCSystemResource']['Database1']['JdbcResource'][ 'JDBCDriverParams']['URL']) + def testWithSegmentInDictionary(self): + expected = dict() + expected['/MailSession/MailSession-0/PropertiesSmtpHost'] = 'stbeehive.oracle.com' + expected['/MailSession/MyMailSession/PropertiesSmtpHost'] = 'stbeehive.oracle.com' + expected['/MailSession/MailSession-0/PropertiesImapHost'] = 'stbeehive.oracle.com' + expected['/MailSession/MyMailSession/PropertiesImapHost'] = 'stbeehive.oracle.com' + replacement_list = ["resources:MailSession.Properties[SmtpHost:mail.smtp.host]", + "resources:MailSession.Properties[ImapHost:mail.imap.host]"] + actual = self._helper.process_variable_replacement(replacement_list) + print 'actual=', actual + self._compare_to_expected_dictionary(expected, actual) + self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesSmtpHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) + self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesImapHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) + + def testWithSegmentInDictionaryAndAPattern(self): + expected = dict() + expected['/MailSession/MyMailSession/PropertiesHost'] = 'stbeehive.oracle.com' + expected['/MailSession/MailSession-0/PropertiesHost'] = 'stbeehive.oracle.com' + replacement_list = ['resources:MailSession.Properties[Host:(?<=\w.)host]'] + actual = self._helper.process_variable_replacement(replacement_list) + print 'actual2=', actual + self._compare_to_expected_dictionary(expected, actual) + self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) + self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.host']) + self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) + def testWithVariableHelperKeywords(self): expected = dict() expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ diff --git a/core/src/test/resources/variable_insertion.yaml b/core/src/test/resources/variable_insertion.yaml index 289521e9a3..d5bb797bec 100644 --- a/core/src/test/resources/variable_insertion.yaml +++ b/core/src/test/resources/variable_insertion.yaml @@ -24,6 +24,9 @@ topology: SSL: Enabled: true ListenPort: 9006 + ServerStart: + Arguments: '-Doracle.net.tns_admin=/etc -DANTLR_USE_DIRECT_CLASS_LOADING=true -DANTLR_USE_DIRECT_CLASS_LOADING=true -Djava.awt.headless=true -Dhttp.webdir.enable=false -Duser.timezone=Europe/Zurich -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom -Dweblogic.data.canTransferAnyFile=true' + ClassPath: 'wlsdeploy/classpathLibraries/WlsAttributeNameMapper-2.0.jar' ListenPort: 9005 m1: ListenAddress: 127.0.0.1 @@ -44,6 +47,7 @@ resources: MailSession: MyMailSession: Properties: + mail.host: stbeehive.oracle.com mail.store.protocol: imap mail.imap.port: 993L mail.smtp.ssl.enable: True From 4d29bf073d017b317be2db079c38707848e41f8f Mon Sep 17 00:00:00 2001 From: crountre Date: Mon, 14 May 2018 15:22:39 -0500 Subject: [PATCH 07/52] refactoring code --- .../tool/util/variable_file_helper.py | 210 ++++++++++++------ .../deploy/messages/wlsdeploy_rb.properties | 8 +- .../test/python/variable_file_helper_test.py | 2 - 3 files changed, 149 insertions(+), 71 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index 85e4f8c373..826cce496a 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -168,74 +168,19 @@ def _traverse_variables(model_section, mbean_list): _traverse_variables(next_model_section, mbean_list) location.pop_location() else: - mbean_list = [] - if len(location.get_folder_path()) > 1: - try: - mbean_list = self.__aliases.get_model_subfolder_names(location) - except AliasException, ae: - _logger.fine('AliasException {0}', ae.getLocalizedMessage()) - pass - else: - try: - mbean_list = self.__aliases.get_model_top_level_folder_names() - except AliasException, ae: - _logger.fine('AliasException {0}', ae.getLocalizedMessage()) - pass - _logger.fine('The mbean list from get model subfolder is {0}', mbean_list) - if mbean in mbean_list: - _logger.finer('WLSDPLY-19416', mbean, replacement, location.get_folder_path(), - class_name=_class_name, method_name=_method_name) - else: - _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path(), - class_name=_class_name, method_name=_method_name) + self._log_mbean_not_found(mbean, replacement, location) return False else: if attribute in model_section: # change this to be a loaded thing - variable_value = model_section[attribute] - if type(variable_value) != str or not variable_value.startswith('@@PROP:'): - variable_name = self.__format_variable_name(location, attribute) - attribute_value = '@@PROP:%s@@' % variable_name - if segment: - segment_attribute_value = attribute_value - segment_variable_name = variable_name - if segment_name: - segment_variable_name += segment_name - segment_attribute_value = '@@PROP:%s@@' % segment_variable_name - if isinstance(variable_value, dict): - segment_dict = variable_value - variable_value = None - for entry in segment_dict: - if segment_dict != str or not segment_dict[entry].startswith('@@PROP:'): - matcher = re.search(segment, entry) - if matcher: - variable_value = str(segment_dict[entry]) - segment_dict[entry] = segment_attribute_value - variable_name = segment_variable_name - attribute_value = segment_dict - elif isinstance(variable_value, list): - print ' need to refactor first' - else: - variable_value = str(variable_value) - replaced, attr_value, var_replaced = _replace_segment(segment, variable_value, - segment_attribute_value) - if replaced: - variable_name = segment_variable_name - variable_value = var_replaced - attribute_value = attr_value - elif not replace_if_nosegment: - _logger.finer('WLSDPLY-19424', segment, attribute, replacement, - location.get_folder_path, class_name=_class_name, - method_name=_method_name) - variable_value = None - - else: - variable_value = str(variable_value) - if variable_value: - _logger.fine('WLSDPLY-19425', attribute_value, attribute, variable_name, variable_value, - class_name=_class_name, method_name=_method_name) - model_section[attribute] = attribute_value - variable_dict[variable_name] = variable_value + if segment: + variable_name, variable_value = self._process_segment(model_section, attribute, segment, + segment_name, replace_if_nosegment, + location) + else: + variable_name, variable_value = self._process_attribute(model_section, attribute, location) + if variable_value: + variable_dict[variable_name] = variable_value else: _logger.finer('WLSDPLY-19417', attribute, replacement, location.get_folder_path(), class_name=_class_name, method_name=_method_name) @@ -257,6 +202,136 @@ def __format_variable_name(self, location, attribute): path = make_path[1] return path + '/' + attribute + def __format_variable_name_segment(self, location, attribute, segment_name): + path = '' + make_path = self.__aliases.get_model_folder_path(location) + if make_path: + make_path = make_path.split(':') + if len(make_path) > 1 and len(make_path[1]) > 1: + path = make_path[1] + variable_name = path + '/' + attribute + if segment_name: + variable_name += segment_name + return variable_name + + def _process_attribute(self, model, attribute, location): + _method_name = '_process_attribute' + _logger.entering(attribute, str(location), class_name=_class_name, method_name=_method_name) + variable_name = None + variable_value = None + attribute_value = model[attribute] + if type(attribute_value) != str or not attribute_value.startswith('@@PROP:'): + variable_name = self.__format_variable_name(location, attribute) + variable_value = str(model[attribute]) + model[attribute] = '@@PROP:%s@@' % variable_name + else: + _logger.finer('WLSDPLY-19426', attribute_value, attribute, str(location), class_name=_class_name, + method_name=_method_name) + + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) + return variable_name, variable_value + + def _process_segment(self, model, attribute, segment, segment_name, replace_if_nosegment, location): + if isinstance(model[attribute], dict): + return self._process_segment_in_dictionary(attribute, model[attribute], segment, segment_name, location) + elif isinstance(model[attribute], list): + return self._process_segment_in_list(attribute, model[attribute], segment, segment_name, location) + else: + return self._process_segment_string(model, attribute, segment, segment_name, + replace_if_nosegment, location) + + def _process_segment_string(self, model, attribute, segment, segment_name, replace_if_nosegment, location): + _method_name = '_process_segment_string' + _logger.entering(attribute, segment, segment_name, replace_if_nosegment, str(location)) + variable_name = None + variable_value = None + search_string = model[attribute] + if (type(search_string) != str) or (not search_string.startswith('@@PROP:')): + variable_name = self.__format_variable_name_segment(location, attribute, segment_name) + attribute_value, variable_value = _replace_segment(segment, str(search_string), + '@@PROP:%s@@' % variable_name) + if variable_value: + _logger.finer('WLSDPLY-19429', attribute, attribute_value, class_name=_class_name, + method_name=_method_name) + if variable_value.startswith('@@PROP:'): + variable_value = None + else: + model[attribute] = attribute_value + elif replace_if_nosegment: + variable_value = search_string + variable_name = self.__format_variable_name(location, attribute) + model[attribute] = '@@PROP:%s@@' % variable_name + _logger.finer('WLSDPLY-19430', attribute, model[attribute], class_name=_class_name, + method_name=_method_name) + else: + _logger.finer('WLSDPLY-19424', segment, attribute, model[attribute], + location.get_folder_path, class_name=_class_name, + method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) + return variable_name, variable_value + + def _process_segment_in_list(self, attribute_name, attribute_list, segment, segment_name, location): + _method_name = '_process_segment_in_dictionary' + _logger.entering(attribute_name, attribute_list, segment, str(location), class_name=_class_name, + method_name=_method_name) + variable_name = self.__format_variable_name_segment(location, attribute_name, segment_name) + variable_value = None + replacement = '@@PROP:%s@@' % variable_name + for entry in attribute_list: + if type(entry) != str or not entry.startswith('@@PROP:'): + attribute_value = str(entry) + matcher = re.search(segment, attribute_value) + if matcher: + _logger.finer('WLSDPLY-19428', attribute_name, replacement, class_name=_class_name, + method_name=_method_name) + variable_value = str(entry) + attribute_list[entry] = replacement + # don't break, continue replacing any in dictionary, return the last variable value found + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) + return variable_name, variable_value + + def _process_segment_in_dictionary(self, attribute_name, attribute_dict, segment, segment_name, location): + _method_name = '_process_segment_in_dictionary' + _logger.entering(attribute_name, attribute_dict, segment, str(location), class_name=_class_name, + method_name=_method_name) + variable_name = self.__format_variable_name_segment(location, attribute_name, segment_name) + variable_value = None + replacement = '@@PROP:%s@@' % variable_name + for entry in attribute_dict: + if type(attribute_dict[entry]) != str or not attribute_dict[entry].startswith('@@PROP:'): + matcher = re.search(segment, entry) + if matcher: + _logger.finer('WLSDPLY-19427', attribute_name, replacement, class_name=_class_name, + method_name=_method_name) + variable_value = str(attribute_dict[entry]) + attribute_dict[entry] = replacement + # don't break, continue replacing any in dictionary, return the last variable value found + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) + return variable_name, variable_value + + def _log_mbean_not_found(self, mbean, replacement, location): + _method_name = '_log_mbean_not_found' + mbean_list = [] + if len(location.get_folder_path()) > 1: + try: + mbean_list = self.__aliases.get_model_subfolder_names(location) + except AliasException, ae: + _logger.fine('AliasException {0}', ae.getLocalizedMessage()) + pass + else: + try: + mbean_list = self.__aliases.get_model_top_level_folder_names() + except AliasException, ae: + _logger.fine('AliasException {0}', ae.getLocalizedMessage()) + pass + _logger.fine('The mbean list from get model subfolder is {0}', mbean_list) + if mbean in mbean_list: + _logger.finer('WLSDPLY-19416', mbean, replacement, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) + else: + _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) + def _get_variable_file_name(variables_helper_dictionary, **kwargs): if VARIABLE_FILE_NAME_ARG in variables_helper_dictionary: @@ -328,16 +403,15 @@ def _load_variables_dictionary(variable_helper_location): def _replace_segment(segment, variable_value, attribute_value): - replaced = False replaced_value = None - replacement_string = None + replacement_string = variable_value pattern = re.compile(segment) matcher = pattern.search(variable_value) if matcher: - replaced = True replaced_value = variable_value[matcher.start():matcher.end()] + replacement_string = pattern.sub(attribute_value, variable_value) - return replaced, replacement_string, replaced_value + return replacement_string, replaced_value def _split_property(attribute_path): diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 0ff7dc76e9..f27778ba99 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1125,9 +1125,15 @@ WLSDPLY-19420=Variable helper file was found at location {0} but no variable pro WLSDPLY-19421=Variables property file name {0} extracted from model helper json file WLSDPLY-19422=Variables property file name {0} was passed as an argument to the variable helper WLSDPLY-19423=Variables helper did not find section {0} in the model and will bypass replacement for {1} -WLSDPLY-19424=Segment {0} for attribute {1} not found in model for replacement property {2} at location {3} +WLSDPLY-19424=Segment {0} for attribute {1} not found in model value {2} at location {3} WLSDPLY-19425=Variable {0} inserted into the model for attribute {1} and variable name {2} and value \ {3} inserted into variable properties +WLSDPLY-19426=Variable {0} was already inserted into model for attribute {1} at location {2} +WLSDPLY-19427=Segment found in dictionary for attribute {0}; inserting variable replacement {1} into dictionary value +WLSDPLY-19428=Segment found in list for attribute {0} has been replaced by variable {1} +WLSDPLY-19429=Segment was found in attribute {0} and variable has been inserted into string {1} +WLSDPLY-19430=Segment was not found in attribute {0} value so entire variable value was replaced by {1} + # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index 1688e4c922..1fd6f672a9 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -118,7 +118,6 @@ def testWithSegmentInDictionary(self): replacement_list = ["resources:MailSession.Properties[SmtpHost:mail.smtp.host]", "resources:MailSession.Properties[ImapHost:mail.imap.host]"] actual = self._helper.process_variable_replacement(replacement_list) - print 'actual=', actual self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesSmtpHost@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) @@ -131,7 +130,6 @@ def testWithSegmentInDictionaryAndAPattern(self): expected['/MailSession/MailSession-0/PropertiesHost'] = 'stbeehive.oracle.com' replacement_list = ['resources:MailSession.Properties[Host:(?<=\w.)host]'] actual = self._helper.process_variable_replacement(replacement_list) - print 'actual2=', actual self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) From bf84469cd70dd36fafdb13954fedec53a0cc523e Mon Sep 17 00:00:00 2001 From: crountre Date: Mon, 14 May 2018 17:48:46 -0500 Subject: [PATCH 08/52] add segments in list --- .../tool/util/variable_file_helper.py | 90 +++++++++++-------- .../test/python/variable_file_helper_test.py | 8 ++ .../test/resources/variable_insertion.yaml | 16 ++++ 3 files changed, 76 insertions(+), 38 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index 826cce496a..00d7468016 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -216,14 +216,14 @@ def __format_variable_name_segment(self, location, attribute, segment_name): def _process_attribute(self, model, attribute, location): _method_name = '_process_attribute' - _logger.entering(attribute, str(location), class_name=_class_name, method_name=_method_name) + _logger.entering(attribute, location.get_folder_path(), class_name=_class_name, method_name=_method_name) variable_name = None variable_value = None attribute_value = model[attribute] - if type(attribute_value) != str or not attribute_value.startswith('@@PROP:'): + if not _already_property(attribute_value): variable_name = self.__format_variable_name(location, attribute) variable_value = str(model[attribute]) - model[attribute] = '@@PROP:%s@@' % variable_name + model[attribute] = _format_as_property(variable_name) else: _logger.finer('WLSDPLY-19426', attribute_value, attribute, str(location), class_name=_class_name, method_name=_method_name) @@ -234,7 +234,7 @@ def _process_attribute(self, model, attribute, location): def _process_segment(self, model, attribute, segment, segment_name, replace_if_nosegment, location): if isinstance(model[attribute], dict): return self._process_segment_in_dictionary(attribute, model[attribute], segment, segment_name, location) - elif isinstance(model[attribute], list): + elif type(model[attribute]) == list: return self._process_segment_in_list(attribute, model[attribute], segment, segment_name, location) else: return self._process_segment_string(model, attribute, segment, segment_name, @@ -243,50 +243,56 @@ def _process_segment(self, model, attribute, segment, segment_name, replace_if_n def _process_segment_string(self, model, attribute, segment, segment_name, replace_if_nosegment, location): _method_name = '_process_segment_string' _logger.entering(attribute, segment, segment_name, replace_if_nosegment, str(location)) - variable_name = None - variable_value = None - search_string = model[attribute] - if (type(search_string) != str) or (not search_string.startswith('@@PROP:')): - variable_name = self.__format_variable_name_segment(location, attribute, segment_name) - attribute_value, variable_value = _replace_segment(segment, str(search_string), - '@@PROP:%s@@' % variable_name) - if variable_value: - _logger.finer('WLSDPLY-19429', attribute, attribute_value, class_name=_class_name, - method_name=_method_name) - if variable_value.startswith('@@PROP:'): - variable_value = None - else: - model[attribute] = attribute_value - elif replace_if_nosegment: - variable_value = search_string + attribute_value, variable_name, variable_value = self._find_segment_in_string(attribute, model[attribute], + segment, segment_name, location) + if variable_value: + _logger.finer('WLSDPLY-19429', attribute, attribute_value, class_name=_class_name, + method_name=_method_name) + model[attribute] = attribute_value + elif replace_if_nosegment: + check_value = model[attribute] + if not _already_property(check_value): + variable_value = check_value variable_name = self.__format_variable_name(location, attribute) - model[attribute] = '@@PROP:%s@@' % variable_name + model[attribute] = _format_as_property(variable_name) _logger.finer('WLSDPLY-19430', attribute, model[attribute], class_name=_class_name, method_name=_method_name) - else: - _logger.finer('WLSDPLY-19424', segment, attribute, model[attribute], - location.get_folder_path, class_name=_class_name, - method_name=_method_name) + else: + _logger.finer('WLSDPLY-19424', segment, attribute, model[attribute], + location.get_folder_path, class_name=_class_name, + method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value + def _find_segment_in_string(self, attribute, attribute_value, segment, segment_name, location): + variable_name = None + variable_value = None + if not _already_property(attribute_value): + variable_name = self.__format_variable_name_segment(location, attribute, segment_name) + attribute_value, variable_value = _replace_segment(segment, str(attribute_value), + _format_as_property(variable_name)) + return attribute_value, variable_name, variable_value + def _process_segment_in_list(self, attribute_name, attribute_list, segment, segment_name, location): _method_name = '_process_segment_in_dictionary' _logger.entering(attribute_name, attribute_list, segment, str(location), class_name=_class_name, method_name=_method_name) - variable_name = self.__format_variable_name_segment(location, attribute_name, segment_name) + variable_name = None variable_value = None - replacement = '@@PROP:%s@@' % variable_name + idx = 0 for entry in attribute_list: - if type(entry) != str or not entry.startswith('@@PROP:'): - attribute_value = str(entry) - matcher = re.search(segment, attribute_value) - if matcher: - _logger.finer('WLSDPLY-19428', attribute_name, replacement, class_name=_class_name, - method_name=_method_name) - variable_value = str(entry) - attribute_list[entry] = replacement - # don't break, continue replacing any in dictionary, return the last variable value found + attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, + segment, segment_name, + location) + if seg_var_value: + _logger.finer('WLSDPLY-19429', attribute_name, attribute_value, class_name=_class_name, + method_name=_method_name) + attribute_list[idx] = attribute_value + variable_name = seg_var_name + variable_value = seg_var_value + + idx += 1 + # don't break, continue replacing any in dictionary, return the last variable value found _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value @@ -296,9 +302,9 @@ def _process_segment_in_dictionary(self, attribute_name, attribute_dict, segment method_name=_method_name) variable_name = self.__format_variable_name_segment(location, attribute_name, segment_name) variable_value = None - replacement = '@@PROP:%s@@' % variable_name + replacement = _format_as_property(variable_name) for entry in attribute_dict: - if type(attribute_dict[entry]) != str or not attribute_dict[entry].startswith('@@PROP:'): + if not _already_property(attribute_dict[entry]): matcher = re.search(segment, entry) if matcher: _logger.finer('WLSDPLY-19427', attribute_name, replacement, class_name=_class_name, @@ -414,6 +420,14 @@ def _replace_segment(segment, variable_value, attribute_value): return replacement_string, replaced_value +def _already_property(check_string): + return type(check_string) == str and check_string.startswith('@@PROP:') + + +def _format_as_property(prop_name): + return '@@PROP:%s@@' % prop_name + + def _split_property(attribute_path): """ Split the section from the attribute path. diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index 1fd6f672a9..fab3a8092f 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -138,6 +138,14 @@ def testWithSegmentInDictionaryAndAPattern(self): self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) + def testWithSegmentInList(self): + expected = dict() + expected['/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/weblogic.management.' + 'runtime.ServerRuntimeMBean/HarvestedAttribute'] = 'OracleHome' + replacement_list = ['resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute[OracleHome]'] + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + def testWithVariableHelperKeywords(self): expected = dict() expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ diff --git a/core/src/test/resources/variable_insertion.yaml b/core/src/test/resources/variable_insertion.yaml index d5bb797bec..71e5bfcb89 100644 --- a/core/src/test/resources/variable_insertion.yaml +++ b/core/src/test/resources/variable_insertion.yaml @@ -128,3 +128,19 @@ resources: URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=slc05til.us.oracle.com)(PORT=1521)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' PasswordEncrypted: '--FIX ME--' DriverName: oracle.jdbc.xa.client.OracleXADataSource + WLDFSystemResource: + MyWldfModule: + WLDFResource: + Target: mycluster + Harvester: + SamplePeriod: 6000000 + HarvestedType: + weblogic.management.runtime.ServerRuntimeMBean: + Enabled: true + Namespace: ServerRuntime + HarvestedInstance: [ 'com.bea:Name=AdminServer,Type=ServerRuntime', 'com.bea:Name=m1,Type=ServerRuntime' ] + HarvestedAttribute: [ CurrentDirectory, MiddlewareHome, OracleHome, ServerClasspath, ServerStartupTime ] + weblogic.management.runtime.PersistentStoreRuntimeMBean: + Namespace: DomainRuntime + HarvestedInstance: [ ] + HarvestedAttribute: [ AllocatedIoBufferBytes ] From bf4005ee8d5af7c8c2c81837c36edf8c52e083f6 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 15 May 2018 15:10:01 -0500 Subject: [PATCH 09/52] add mbean name list --- .../tool/util/variable_file_helper.py | 32 ++++++----- .../deploy/messages/wlsdeploy_rb.properties | 3 +- .../test/python/variable_file_helper_test.py | 55 +++++++++++++++++-- 3 files changed, 68 insertions(+), 22 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py index 00d7468016..c6a1823800 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py @@ -153,13 +153,16 @@ def __process_variable(self, location, replacement): def _traverse_variables(model_section, mbean_list): if mbean_list: mbean = mbean_list.pop(0) + mbean, mbean_name_list = _find_special_name(mbean) if mbean in model_section: _logger.finest('WLSDPLY-19414', mbean, class_name=_class_name, method_name=_method_name) next_model_section = model_section[mbean] location.append_location(mbean) name_token = self.__aliases.get_name_token(location) - if self.__aliases.supports_multiple_mbean_instances(location): - for mbean_name in next_model_section: + if not mbean_name_list and self.__aliases.supports_multiple_mbean_instances(location): + mbean_name_list = next_model_section + if mbean_name_list: + for mbean_name in mbean_name_list: continue_mbean_list = copy.copy(mbean_list) location.add_name_token(name_token, mbean_name) _traverse_variables(next_model_section[mbean_name], continue_mbean_list) @@ -281,9 +284,8 @@ def _process_segment_in_list(self, attribute_name, attribute_list, segment, segm variable_value = None idx = 0 for entry in attribute_list: - attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, - segment, segment_name, - location) + attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, segment, + segment_name, location) if seg_var_value: _logger.finer('WLSDPLY-19429', attribute_name, attribute_value, class_name=_class_name, method_name=_method_name) @@ -446,6 +448,7 @@ def _split_property(attribute_path): segment, segment_name, if_notfound_replace_full = _find_segment(split_list[1]) attribute_path = _split_from_segment(split_list[1]) mbean_list, attribute = _split_attribute_path(attribute_path) + _logger.finest('WLSDPLY-19431', section, mbean_list, attribute, segment, segment_name, if_notfound_replace_full) return section, mbean_list, attribute, segment, segment_name, if_notfound_replace_full @@ -472,22 +475,21 @@ def _find_segment(attribute_path): else: if_notfound_replace_full = False segment = attribute_path[matcher.start()+1:matcher.end()-1] - matcher = re.search('\w*:', segment) + matcher = re.search('(?<=`)\w*(?=`)', segment) if matcher: - segment_name = segment[matcher.start():matcher.end()-1] - segment = segment[matcher.end():] + segment_name = segment[matcher.start():matcher.end()] + segment = segment[matcher.end()+1:] return segment, segment_name, if_notfound_replace_full def _find_special_name(mbean): + mbean_name = mbean mbean_name_list = [] - matcher = re.search('(?<=\[).+(?=\])', mbean) - if matcher: - mbean_name = mbean[matcher.start():matcher.end()] - for entry in mbean_name.split[',']: - if entry: - mbean_name_list.append(entry) - return mbean_name_list + name_list = re.split('[\{).+\}]', mbean) + if name_list and len(name_list) > 1: + mbean_name = name_list[0] + mbean_name_list = name_list[1].split(',') + return mbean_name, mbean_name_list def _write_variables_file(variables_dictionary, variables_file_name): diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index f27778ba99..53170e86eb 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1133,7 +1133,8 @@ WLSDPLY-19427=Segment found in dictionary for attribute {0}; inserting variable WLSDPLY-19428=Segment found in list for attribute {0} has been replaced by variable {1} WLSDPLY-19429=Segment was found in attribute {0} and variable has been inserted into string {1} WLSDPLY-19430=Segment was not found in attribute {0} value so entire variable value was replaced by {1} - +WLSDPLY-19431=Model section {0} mbean list {1} attribute {2} search for segment with {3} use segment name {4} \ + and replace with full value if segment not found is {5} # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index fab3a8092f..648ff98357 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -95,8 +95,8 @@ def testWithSegment(self): expected['/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLPort'] = \ '1521' replacement_list = [ - 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(Port:(?<=PORT=)[\w.-]+(?=\)))', - 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(Host:(?<=HOST=)[\w.-]+(?=\)))'] + 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Port`(?<=PORT=)[\w.-]+(?=\)))', + 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Host`(?<=HOST=)[\w.-]+(?=\)))'] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) db2 = 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)' \ @@ -115,8 +115,8 @@ def testWithSegmentInDictionary(self): expected['/MailSession/MyMailSession/PropertiesSmtpHost'] = 'stbeehive.oracle.com' expected['/MailSession/MailSession-0/PropertiesImapHost'] = 'stbeehive.oracle.com' expected['/MailSession/MyMailSession/PropertiesImapHost'] = 'stbeehive.oracle.com' - replacement_list = ["resources:MailSession.Properties[SmtpHost:mail.smtp.host]", - "resources:MailSession.Properties[ImapHost:mail.imap.host]"] + replacement_list = ["resources:MailSession.Properties[`SmtpHost`mail.smtp.host]", + "resources:MailSession.Properties[`ImapHost`mail.imap.host]"] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesSmtpHost@@', @@ -128,7 +128,7 @@ def testWithSegmentInDictionaryAndAPattern(self): expected = dict() expected['/MailSession/MyMailSession/PropertiesHost'] = 'stbeehive.oracle.com' expected['/MailSession/MailSession-0/PropertiesHost'] = 'stbeehive.oracle.com' - replacement_list = ['resources:MailSession.Properties[Host:(?<=\w.)host]'] + replacement_list = ['resources:MailSession.Properties[`Host`(?<=\w.)host]'] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', @@ -142,9 +142,52 @@ def testWithSegmentInList(self): expected = dict() expected['/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/weblogic.management.' 'runtime.ServerRuntimeMBean/HarvestedAttribute'] = 'OracleHome' - replacement_list = ['resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute[OracleHome]'] + replacement_list = [ + 'resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute[OracleHome]'] + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ + 'HarvestedType']['weblogic.management.runtime.ServerRuntimeMBean']['HarvestedAttribute'] + found = False + for entry in list: + if entry == '@@PROP:/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/' \ + 'weblogic.management.runtime.ServerRuntimeMBean/HarvestedAttribute@@': + found = True + break + self.assertEqual(True, found) + + def testWithSegmentInStringInList(self): + expected = dict() + expected['/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/weblogic.management.' + 'runtime.ServerRuntimeMBean/HarvestedInstanceManagedServer'] = 'm1' + replacement_list = [ + 'resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance[`ManagedServer`m1]'] + actual = self._helper.process_variable_replacement(replacement_list) + self._compare_to_expected_dictionary(expected, actual) + list = \ + self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester']['HarvestedType'][ + 'weblogic.management.runtime.ServerRuntimeMBean']['HarvestedInstance'] + found = False + for entry in list: + if entry == 'com.bea:Name=@@PROP:/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/' \ + 'weblogic.management.runtime.ServerRuntimeMBean/HarvestedInstanceManagedServer@@' \ + ',Type=ServerRuntime': + found = True + break + self.assertEqual(True, found) + + def testWithNameInMBeanSingle(self): + expected = dict() + expected['/Server/m2/ServerStart/Arguments'] = '/etc' + replacement_list = ['topology:Server{m2}.ServerStart.Arguments[(?<=-Doracle.net.tns_admin=)[\w\\/._:]+]'] actual = self._helper.process_variable_replacement(replacement_list) self._compare_to_expected_dictionary(expected, actual) + arg = '-Doracle.net.tns_admin=@@PROP:/Server/m2/ServerStart/Arguments@@ ' \ + '-DANTLR_USE_DIRECT_CLASS_LOADING=true ' \ + '-DANTLR_USE_DIRECT_CLASS_LOADING=true -Djava.awt.headless=true -Dhttp.webdir.enable=false ' \ + '-Duser.timezone=Europe/Zurich -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom ' \ + '-Dweblogic.data.canTransferAnyFile=true' + self.assertEqual(arg, self._model['topology']['Server']['m2']['ServerStart']['Arguments']) def testWithVariableHelperKeywords(self): expected = dict() From b16845a2abf08635c06100d8ed98287039bcba51 Mon Sep 17 00:00:00 2001 From: crountre Date: Wed, 16 May 2018 10:25:57 -0500 Subject: [PATCH 10/52] add properties --- installer/src/main/etc/credentials.properties | 25 ++++++++++++++++++- installer/src/main/etc/port.properties | 2 ++ installer/src/main/etc/url.properties | 2 ++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/installer/src/main/etc/credentials.properties b/installer/src/main/etc/credentials.properties index 17ea9d564f..8a3ffbb6b6 100644 --- a/installer/src/main/etc/credentials.properties +++ b/installer/src/main/etc/credentials.properties @@ -1,4 +1,27 @@ resources:ForeignJNDIProvider.User resources:ForeignJNDIProvider.PasswordEncrypted resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.PasswordEncrypted -resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.PasswordEncrypted +resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.Properties{user}.Value +resources:JDBCSystemResource.JdbcResource.JDBCOracleParams.OnsWalletPasswordEncrypted +resources:JMSBridgeDestination.UserName +resources:JMSBridgeDestination.UserPasswordEncrypted +resources:JMSSystemResource.JmsResource.ForeignServer.ForeignConnectionFactory.Username +resources:JMSSystemResource.JmsResource.ForeignServer.ForeignConnectionFactory.PasswordEncrypted +resources:JMSSystemResource.JmsResource.ForeignServer.JNDIPropertiesCredentialEncrypted +resources:JMSSystemResource.JmsResource.SAFRemoteContext.SAFLoginContext.PasswordEncrypted +resources:JMSSystemResource.JmsResource.SAFRemoteContext.SAFLoginContext.Username +topology:Machine.NodeManager.PasswordEncrypted +topology:Machine.NodeManager.UserName +topology:UnixMachine.NodeManager.PasswordEncrypted +topology:UnixMachine.NodeManager.UserName +resources:MailSession.SessionUsername +resources:MailSession.SessionPasswordEncrypted +resources:MailSession.Properties[`User`(?<=\w.)user] +resources:MailSession.Properties[`Password`(?<=\w.)password] +resources:Partition.ForeignJndiProviderOverride.User +resources:Partition.ForeignJndiProviderOverride.PasswordEncrypted +resources:Partition.JdbcSystemResourceOverride.PasswordEncrypted +resources:Partition.JdbcSystemResourceOverride.JdbcPropertyOverride{user}.Value +resources:Partition.JmsSystemResourceOverride.ForeignServer.ForeignConnectionFactory.Username +resources:Partition.JmsSystemResourceOverride.ForeignServer.ForeignConnectionFactory.PasswordEncrypted +resources:JmsSystemResourceOverride.ForeignServer.JndiPropertiesCredentialEncrypted diff --git a/installer/src/main/etc/port.properties b/installer/src/main/etc/port.properties index e69de29bb2..2d49e2fb46 100644 --- a/installer/src/main/etc/port.properties +++ b/installer/src/main/etc/port.properties @@ -0,0 +1,2 @@ +resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Port`(?<=PORT=)[\w.-]+(?=\))) +resources:MailSession.Properties[`Port`(?<=\w.)port] diff --git a/installer/src/main/etc/url.properties b/installer/src/main/etc/url.properties index e69de29bb2..9edbeac747 100644 --- a/installer/src/main/etc/url.properties +++ b/installer/src/main/etc/url.properties @@ -0,0 +1,2 @@ +resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Host`(?<=HOST=)[\w.-]+(?=\))) +resources:MailSession.Properties[`Host`(?<=\w.)host] \ No newline at end of file From 424d6716dc68ccc027069f6ee1aadb1c284e8a56 Mon Sep 17 00:00:00 2001 From: crountre Date: Wed, 16 May 2018 10:50:18 -0500 Subject: [PATCH 11/52] rename to injector --- core/src/main/python/discover.py | 6 ++-- ...le_file_helper.py => variable_injector.py} | 20 ++++++------- .../deploy/messages/wlsdeploy_rb.properties | 2 +- .../test/python/variable_file_helper_test.py | 30 +++++++++---------- 4 files changed, 29 insertions(+), 29 deletions(-) rename core/src/main/python/wlsdeploy/tool/util/{variable_file_helper.py => variable_injector.py} (97%) diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index 14d993818c..d80e91728b 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -30,7 +30,7 @@ from wlsdeploy.exception import exception_helper from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.util import getcreds -from wlsdeploy.tool.util.variable_file_helper import VariableFileHelper +from wlsdeploy.tool.util.variable_injector import VariableFileHelper from wlsdeploy.tool.discover import discoverer from wlsdeploy.tool.discover.deployments_discoverer import DeploymentsDiscoverer from wlsdeploy.tool.discover.domain_info_discoverer import DomainInfoDiscoverer @@ -372,12 +372,12 @@ def __check_and_customize_model(model, model_context): default_variable_file = __get_default_variable_file(model_context) inserted, variable_model, variable_file_name = VariableFileHelper(model.get_model(), model_context, WebLogicHelper( - __logger).get_actual_weblogic_version()).replace_variables_file(variable_file_name=default_variable_file) + __logger).get_actual_weblogic_version()).inject_variables_keyword_file(variable_file_name=default_variable_file) if default_variable_file: default_variable_file = os.path.join(path_utils.get_pathname_from_path(model_context.get_archive_file_name()), default_variable_file + '.properties') inserted, variable_model, variable_file_name = VariableFileHelper(model.get_model(), model_context, WebLogicHelper( - __logger).get_actual_weblogic_version()).replace_variables_file(variable_file_name=default_variable_file) + __logger).get_actual_weblogic_version()).inject_variables_keyword_file(variable_file_name=default_variable_file) if inserted: model = Model(variable_model) try: diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py similarity index 97% rename from core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py rename to core/src/main/python/wlsdeploy/tool/util/variable_injector.py index c6a1823800..5719ef2df0 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_file_helper.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -48,7 +48,7 @@ def __init__(self, model, model_context=None, version=None): else: self.__aliases = Aliases(model_context) - def replace_variables_file(self, **kwargs): + def inject_variables_keyword_file(self, **kwargs): """ Replace attribute values with variables and generate a variable dictionary. The variable replacement is driven from the values in the model variable helper file. @@ -60,7 +60,7 @@ def replace_variables_file(self, **kwargs): :param kwargs: arguments used to override default for variable processing, typically used in test situations :return: variable dictionary containing """ - _method_name = 'insert_variables' + _method_name = 'inject_variables_keyword_file' _logger.entering(class_name=_class_name, method_name=_method_name) variable_helper_location_file = _get_variable_helper_file_name(**kwargs) @@ -80,7 +80,7 @@ def replace_variables_file(self, **kwargs): for key, value in _keyword_to_file_map.iteritems(): file_map[key] = os.path.join(variable_keywords_location_path, value) - variables_file_dictionary = self.replace_variables_dictionary(file_map, variables_helper_dictionary) + variables_file_dictionary = self.inject_variables_keyword_dictionary(file_map, variables_helper_dictionary) variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location) if variables_inserted: _logger.info('WLSDPLY-19418', variable_file_location, class_name=_class_name, @@ -94,13 +94,13 @@ def replace_variables_file(self, **kwargs): _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) return variables_inserted, return_model, variable_file_location - def replace_variables_dictionary(self, keyword_map, variables_dictionary): + def inject_variables_keyword_dictionary(self, keyword_map, variables_dictionary): """ Takes a variable keyword dictionary and returns a variables for file in a dictionary :param variables_dictionary: :return: """ - _method_name = 'replace_variables_dictionary' + _method_name = 'inject_variables_keyword_dictionary' _logger.entering(variables_dictionary, class_name=_class_name, method_name=_method_name) variable_file_entries = dict() if variables_dictionary: @@ -118,15 +118,15 @@ def replace_variables_dictionary(self, keyword_map, variables_dictionary): for keyword, file_name in file_map.iteritems(): replacement_list = _load_replacement_list(file_name) if replacement_list: - entries = self.process_variable_replacement(replacement_list) + entries = self.inject_variables(replacement_list) if entries: _logger.finer('WLSDPLY-19413', keyword, class_name=_class_name, method_name=_method_name) variable_file_entries.update(entries) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_file_entries) return variable_file_entries - def process_variable_replacement(self, replacement_list): - _method_name = '__process_variable_replacement' + def inject_variables(self, replacement_list): + _method_name = 'inject_variables' variable_dict = dict() if replacement_list: topology = self.__model[model_sections.get_model_topology_key()] @@ -138,13 +138,13 @@ def process_variable_replacement(self, replacement_list): domain_token = self.__aliases.get_name_token(location) location.add_name_token(domain_token, domain_name) for replacement_entry in replacement_list: - entries_dict = self.__process_variable(location, replacement_entry) + entries_dict = self.__inject_variable(location, replacement_entry) if len(entries_dict) > 0: variable_dict.update(entries_dict) return variable_dict - def __process_variable(self, location, replacement): + def __inject_variable(self, location, replacement): _method_name = '__process_variable' _logger.entering(replacement, class_name=_class_name, method_name=_method_name) variable_dict = dict() diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 53170e86eb..2e1f6f72cc 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1098,7 +1098,7 @@ WLSDPLY-19305=Failed to extract domain library {0} because it does not exist in WLSDPLY-19306=Unable to extract domain library {0} from archive file {1}: {2} WLSDPLY-19307=Unable to extract classpath libraries from archive file {0} to domain directory {1}: {2} -# wlsdeploy/tool/util/variable_file_helper.py +# wlsdeploy/tool/util/variable_injector.py WLSDPLY-19400=model variable replacement helper file found at location {0} WLSDPLY-19401=Variable replacement using custom list loaded from variable helper file {0} WLSDPLY-19402=Unable to read and parse variables helper file {0} : {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py index 648ff98357..ba402d65ca 100644 --- a/core/src/test/python/variable_file_helper_test.py +++ b/core/src/test/python/variable_file_helper_test.py @@ -5,8 +5,8 @@ import unittest import wlsdeploy.util.variables as variables -import wlsdeploy.tool.util.variable_file_helper as variable_file_helper -from wlsdeploy.tool.util.variable_file_helper import VariableFileHelper +import wlsdeploy.tool.util.variable_injector as variable_file_helper +from wlsdeploy.tool.util.variable_injector import VariableFileHelper from wlsdeploy.util.model_translator import FileToPython @@ -26,7 +26,7 @@ def testSingleVariableReplacement(self): replacement_list = ['topology:Machine.NodeManager.ListenAddress'] expected = dict() expected['/Machine/machine1/NodeManager/ListenAddress'] = '127.0.0.1' - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testMultiplesReplacement(self): @@ -56,25 +56,25 @@ def testMultiplesReplacement(self): 'resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL', 'resources:JMSSystemResource.JmsResource.ForeignServer.ForeignDestination.LocalJNDIName', 'topology:Server.SSL.ListenPort'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testInvalidMBeanNameNoException(self): expected = dict() replacement_list = ['resources:JmsSystemResource.Notes'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testInvalidAttributeName(self): expected = dict() replacement_list = ['topology:Server.listenaddress'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testInvalidSection(self): expected = dict() replacement_list = ['topologies:Server.ListenAddress'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) def testDomainAttributeReplacementAndModel(self): @@ -82,7 +82,7 @@ def testDomainAttributeReplacementAndModel(self): expected['/Notes'] = 'Test note replacement' expected_replacement = '@@PROP:/Notes@@' replacement_list = ['topology:Notes'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) self.assertEqual(expected_replacement, self._model['topology']['Notes']) @@ -97,7 +97,7 @@ def testWithSegment(self): replacement_list = [ 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Port`(?<=PORT=)[\w.-]+(?=\)))', 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Host`(?<=HOST=)[\w.-]+(?=\)))'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) db2 = 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)' \ '(HOST=@@PROP:/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLHost@@)' \ @@ -117,7 +117,7 @@ def testWithSegmentInDictionary(self): expected['/MailSession/MyMailSession/PropertiesImapHost'] = 'stbeehive.oracle.com' replacement_list = ["resources:MailSession.Properties[`SmtpHost`mail.smtp.host]", "resources:MailSession.Properties[`ImapHost`mail.imap.host]"] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesSmtpHost@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) @@ -129,7 +129,7 @@ def testWithSegmentInDictionaryAndAPattern(self): expected['/MailSession/MyMailSession/PropertiesHost'] = 'stbeehive.oracle.com' expected['/MailSession/MailSession-0/PropertiesHost'] = 'stbeehive.oracle.com' replacement_list = ['resources:MailSession.Properties[`Host`(?<=\w.)host]'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) @@ -144,7 +144,7 @@ def testWithSegmentInList(self): 'runtime.ServerRuntimeMBean/HarvestedAttribute'] = 'OracleHome' replacement_list = [ 'resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute[OracleHome]'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ 'HarvestedType']['weblogic.management.runtime.ServerRuntimeMBean']['HarvestedAttribute'] @@ -162,7 +162,7 @@ def testWithSegmentInStringInList(self): 'runtime.ServerRuntimeMBean/HarvestedInstanceManagedServer'] = 'm1' replacement_list = [ 'resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance[`ManagedServer`m1]'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) list = \ self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester']['HarvestedType'][ @@ -180,7 +180,7 @@ def testWithNameInMBeanSingle(self): expected = dict() expected['/Server/m2/ServerStart/Arguments'] = '/etc' replacement_list = ['topology:Server{m2}.ServerStart.Arguments[(?<=-Doracle.net.tns_admin=)[\w\\/._:]+]'] - actual = self._helper.process_variable_replacement(replacement_list) + actual = self._helper.inject_variables(replacement_list) self._compare_to_expected_dictionary(expected, actual) arg = '-Doracle.net.tns_admin=@@PROP:/Server/m2/ServerStart/Arguments@@ ' \ '-DANTLR_USE_DIRECT_CLASS_LOADING=true ' \ @@ -199,7 +199,7 @@ def testWithVariableHelperKeywords(self): expected['/Machine/machine1/NodeManager/ListenPort'] = '5557' expected['/Machine/machine1/NodeManager/PasswordEncrypted'] = '--FIX ME--' expected['/Machine/machine1/NodeManager/UserName'] = 'admin' - inserted, model, variable_file_name = self._helper.replace_variables_file( + inserted, model, variable_file_name = self._helper.inject_variables_keyword_file( variable_helper_path_name=self._resources_dir, variable_helper_file_name=self._variable_helper_keyword) self.assertEqual(True, inserted) self.assertEqual(self._variable_file, variable_file_name) From 7b59fa5549493429eb23a20391d78550509e2f81 Mon Sep 17 00:00:00 2001 From: crountre Date: Wed, 16 May 2018 17:52:45 -0500 Subject: [PATCH 12/52] first redesign --- .../wlsdeploy/tool/util/variable_injector.py | 492 +++++++++--------- .../deploy/messages/wlsdeploy_rb.properties | 15 +- .../test/python/variable_file_helper_test.py | 219 -------- .../src/test/python/variable_injector_test.py | 232 +++++++++ core/src/test/resources/credentials.json | 4 + .../src/test/resources/credentials.properties | 2 - core/src/test/resources/custom.json | 4 + core/src/test/resources/custom.properties | 2 - core/src/test/resources/port.json | 4 + core/src/test/resources/port.properties | 2 - core/src/test/resources/url.json | 3 + core/src/test/resources/url.properties | 1 - .../resources/variable_helper_custom.json | 7 - .../resources/variable_injector_custom.json | 4 + ...rd.json => variable_injector_keyword.json} | 9 +- 15 files changed, 510 insertions(+), 490 deletions(-) delete mode 100644 core/src/test/python/variable_file_helper_test.py create mode 100644 core/src/test/python/variable_injector_test.py create mode 100644 core/src/test/resources/credentials.json delete mode 100644 core/src/test/resources/credentials.properties create mode 100644 core/src/test/resources/custom.json delete mode 100644 core/src/test/resources/custom.properties create mode 100644 core/src/test/resources/port.json delete mode 100644 core/src/test/resources/port.properties create mode 100644 core/src/test/resources/url.json delete mode 100644 core/src/test/resources/url.properties delete mode 100644 core/src/test/resources/variable_helper_custom.json create mode 100644 core/src/test/resources/variable_injector_custom.json rename core/src/test/resources/{variable_helper_keyword.json => variable_injector_keyword.json} (55%) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 5719ef2df0..7b24eca45f 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -8,32 +8,44 @@ import java.lang.IllegalArgumentException as IllegalArgumentException -import oracle.weblogic.deploy.util.VariableException as VariableException import oracle.weblogic.deploy.aliases.AliasException as AliasException +import oracle.weblogic.deploy.json.JsonException +import oracle.weblogic.deploy.util.VariableException as VariableException -import wlsdeploy.util.dictionary_utils as dictionary_utils import wlsdeploy.util.model as model_sections import wlsdeploy.util.variables as variables from wlsdeploy.aliases.aliases import Aliases from wlsdeploy.aliases.location_context import LocationContext from wlsdeploy.aliases.wlst_modes import WlstModes +from wlsdeploy.aliases.validation_codes import ValidationCodes +from wlsdeploy.json.json_translator import JsonToPython from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.util.model_translator import FileToPython -VARIABLE_HELPER_FILE_NAME = 'model_variable_helper.json' -VARIABLE_HELPER_PATH_NAME_ARG = 'variable_helper_path_name' -VARIABLE_HELPER_FILE_NAME_ARG = 'variable_helper_file_name' +VARIABLE_INJECTOR_FILE_NAME = 'model_variable_injector.json' +VARIABLE_KEYWORDS_FILE_NAME = 'variable_keywords.json' +VARIABLE_INJECTOR_PATH_NAME_ARG = 'variable_injector_path_name' +VARIABLE_KEYWORDS_PATH_NAME_ARG = 'variable_keywords_path_name' +VARIABLE_INJECTOR_FILE_NAME_ARG = 'variable_injector_file_name' +VARIABLE_KEYWORDS_FILE_NAME_ARG = 'variable_keywords_file_name' VARIABLE_FILE_NAME_ARG = 'variable_file_name' -VARIABLE_FILE_NAME = 'variables.properties' +VARIABLE_FILE_NAME = 'variables.json' +# custom keyword in model injector file CUSTOM_KEYWORD = 'CUSTOM' - +KEYWORD_FILES = 'file-list' +# location for model injector file, keyword file and injector files +DEFAULT_FILE_LOCATION = 'lib' +# should these injector json keywords be included in the keyword file +REGEXP = 'regexp' +SUFFIX = 'suffix' +FORCE = 'force' + +VARIABLE_SEP = '.' +SUFFIX_SEP = '--' +_variable_sep_pattern = re.compile('/') + +_wlsdeploy_location = os.environ.get('WLSDEPLOY_HOME') _segment_pattern = re.compile("\\[[\w.-]+\\]$") -_keyword_to_file_map = { - 'PORT': 'port.properties', - 'TOPOLOGY_NAME': 'name.properties', - 'CREDENTIALS': 'credentials.properties', - 'URL': 'url.properties' -} _class_name = 'variable_file_helper' _logger = PlatformLogger('wlsdeploy.util') @@ -42,7 +54,15 @@ class VariableFileHelper(object): def __init__(self, model, model_context=None, version=None): self.__original = copy.deepcopy(model) - self.__model = model + self.__model = dict() + # consolidate all the model sections into one for speedier search + # currently don't want to do variable injection in the domainInfo section of the model + if model_sections.get_model_topology_key() in model: + self.__model.update(model[model_sections.get_model_topology_key()]) + elif model_sections.get_model_resources_key() in model: + self.__model.update(model_sections.get_model_resources_key()) + elif model_sections.get_model_deployments_key() in model: + self.__model.update(model_sections.get_model_deployments_key()) if version: self.__aliases = Aliases(model_context, WlstModes.OFFLINE, version, None) else: @@ -56,31 +76,30 @@ def inject_variables_keyword_file(self, **kwargs): keywords for canned replacement files. Return the variable dictionary with the variable name inserted into the model, and the value that the inserted variable replaced. - :param variable_file_location: location and name to store the generated variable file, else default :param kwargs: arguments used to override default for variable processing, typically used in test situations :return: variable dictionary containing """ _method_name = 'inject_variables_keyword_file' _logger.entering(class_name=_class_name, method_name=_method_name) - variable_helper_location_file = _get_variable_helper_file_name(**kwargs) - variables_helper_dictionary = _load_variables_dictionary(variable_helper_location_file) + variable_injector_location_file = _get_variable_injector_file_name(**kwargs) + variables_injector_dictionary = _load_variables_file(variable_injector_location_file) + variable_keywords_location_file = _get_variable_keywords_file_name(**kwargs) + keywords_dictionary = _load_keywords_file(variable_keywords_location_file) variables_inserted = False return_model = dict() variable_file_location = None - if variables_helper_dictionary: - variable_file_location = _get_variable_file_name(variables_helper_dictionary, **kwargs) + if variables_injector_dictionary and keywords_dictionary: + injector_file_list = _create_injector_file_list(variables_injector_dictionary, keywords_dictionary, + _get_keyword_files_location(**kwargs)) + return_model = dict() + variable_file_location = _get_variable_file_name(variables_injector_dictionary, **kwargs) if not variable_file_location: - _logger.warning('WLSDPLY-19420', variable_helper_location_file, class_name=_class_name, + _logger.warning('WLSDPLY-19420', variable_injector_location_file, class_name=_class_name, method_name=_method_name) else: - file_map = dict() - variable_keywords_location_path = _get_keyword_files_location(**kwargs) - for key, value in _keyword_to_file_map.iteritems(): - file_map[key] = os.path.join(variable_keywords_location_path, value) - - variables_file_dictionary = self.inject_variables_keyword_dictionary(file_map, variables_helper_dictionary) + variables_file_dictionary = self.inject_variables_keyword_dictionary(injector_file_list) variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location) if variables_inserted: _logger.info('WLSDPLY-19418', variable_file_location, class_name=_class_name, @@ -94,75 +113,64 @@ def inject_variables_keyword_file(self, **kwargs): _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) return variables_inserted, return_model, variable_file_location - def inject_variables_keyword_dictionary(self, keyword_map, variables_dictionary): + def inject_variables_keyword_dictionary(self, injector_file_list): """ Takes a variable keyword dictionary and returns a variables for file in a dictionary - :param variables_dictionary: + :param injector_file_list: :return: """ _method_name = 'inject_variables_keyword_dictionary' - _logger.entering(variables_dictionary, class_name=_class_name, method_name=_method_name) - variable_file_entries = dict() - if variables_dictionary: - file_map = dict() - if not dictionary_utils.is_empty_dictionary_element(variables_dictionary, CUSTOM_KEYWORD): - file_map[CUSTOM_KEYWORD] = variables_dictionary[CUSTOM_KEYWORD] - _logger.fine('WLSDPLY-19401', file_map[CUSTOM_KEYWORD], class_name=_class_name, - method_name=_method_name) - else: - for keyword in variables_dictionary: - if keyword in keyword_map: - file_map[keyword] = keyword_map[keyword] - elif keyword != CUSTOM_KEYWORD: - _logger.warning('WLSDPLY-19403', keyword, class_name=_class_name, method_name=_method_name) - for keyword, file_name in file_map.iteritems(): - replacement_list = _load_replacement_list(file_name) - if replacement_list: - entries = self.inject_variables(replacement_list) - if entries: - _logger.finer('WLSDPLY-19413', keyword, class_name=_class_name, method_name=_method_name) - variable_file_entries.update(entries) - _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_file_entries) - return variable_file_entries - - def inject_variables(self, replacement_list): - _method_name = 'inject_variables' + _logger.entering(injector_file_list, class_name=_class_name, method_name=_method_name) + variables_dictionary = dict() + for filename in injector_file_list: + injector_dictionary = _load_injector_file(filename) + entries = self.inject_variables(injector_dictionary) + if entries: + _logger.finer('WLSDPLY-19413', filename, class_name=_class_name, method_name=_method_name) + variables_dictionary.update(entries) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_dictionary) + return variables_dictionary + + def inject_variables(self, injector_dictionary): + """ + Iterate through the injector dictionary that was loaded from the file for the model + injector file keyword. + :param injector_dictionary: + :return: variable dictionary containing the variable string and model value entries + """ variable_dict = dict() - if replacement_list: - topology = self.__model[model_sections.get_model_topology_key()] - if 'Name' in topology: - domain_name = topology['Name'] - else: - domain_name = 'mydomain' + if injector_dictionary: location = LocationContext() domain_token = self.__aliases.get_name_token(location) - location.add_name_token(domain_token, domain_name) - for replacement_entry in replacement_list: - entries_dict = self.__inject_variable(location, replacement_entry) + location.add_name_token(domain_token, 'fakedomain') + for injector in injector_dictionary: + entries_dict = self.__inject_variable(location, injector) if len(entries_dict) > 0: variable_dict.update(entries_dict) return variable_dict - def __inject_variable(self, location, replacement): - _method_name = '__process_variable' - _logger.entering(replacement, class_name=_class_name, method_name=_method_name) + def __inject_variable(self, location, injector): + _method_name = '__inject_variable' + _logger.entering(injector, class_name=_class_name, method_name=_method_name) variable_dict = dict() - section, start_mbean_list, attribute, segment, segment_name, replace_if_nosegment = _split_property(replacement) + start_mbean_list, attribute = _split_injector(injector) def _traverse_variables(model_section, mbean_list): if mbean_list: mbean = mbean_list.pop(0) - mbean, mbean_name_list = _find_special_name(mbean) + # mbean, mbean_name_list = _find_special_name(mbean) if mbean in model_section: _logger.finest('WLSDPLY-19414', mbean, class_name=_class_name, method_name=_method_name) next_model_section = model_section[mbean] location.append_location(mbean) name_token = self.__aliases.get_name_token(location) - if not mbean_name_list and self.__aliases.supports_multiple_mbean_instances(location): - mbean_name_list = next_model_section - if mbean_name_list: - for mbean_name in mbean_name_list: + # if not mbean_name_list and self.__aliases.supports_multiple_mbean_instances(location): + # mbean_name_list = next_model_section + # if mbean_name_list: + # for mbean_name in mbean_name_list: + if self.__aliases.supports_multiple_mbean_instances(location): + for mbean_name in next_model_section: continue_mbean_list = copy.copy(mbean_list) location.add_name_token(name_token, mbean_name) _traverse_variables(next_model_section[mbean_name], continue_mbean_list) @@ -171,29 +179,20 @@ def _traverse_variables(model_section, mbean_list): _traverse_variables(next_model_section, mbean_list) location.pop_location() else: - self._log_mbean_not_found(mbean, replacement, location) + self._log_mbean_not_found(mbean, injector[0], location) return False else: if attribute in model_section: - # change this to be a loaded thing - if segment: - variable_name, variable_value = self._process_segment(model_section, attribute, segment, - segment_name, replace_if_nosegment, - location) - else: - variable_name, variable_value = self._process_attribute(model_section, attribute, location) + variable_name, variable_value = self._variable_info(model_section, attribute, location, injector) if variable_value: variable_dict[variable_name] = variable_value else: - _logger.finer('WLSDPLY-19417', attribute, replacement, location.get_folder_path(), + _logger.finer('WLSDPLY-19417', attribute, injector, location.get_folder_path(), class_name=_class_name, method_name=_method_name) return True - if section and section in self.__model: - _traverse_variables(self.__model[section], start_mbean_list) - else: - _logger.finer('WLSDPLY-19423', section, replacement) - _logger.exiting(class_name=_class_name, method_name=_method_name) + _traverse_variables(self.__model, start_mbean_list) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_dict) return variable_dict def __format_variable_name(self, location, attribute): @@ -203,23 +202,24 @@ def __format_variable_name(self, location, attribute): make_path = make_path.split(':') if len(make_path) > 1 and len(make_path[1]) > 1: path = make_path[1] - return path + '/' + attribute + path = path[1:] + VARIABLE_SEP + attribute + _variable_sep_pattern.sub(VARIABLE_SEP, path) + return path - def __format_variable_name_segment(self, location, attribute, segment_name): - path = '' - make_path = self.__aliases.get_model_folder_path(location) - if make_path: - make_path = make_path.split(':') - if len(make_path) > 1 and len(make_path[1]) > 1: - path = make_path[1] - variable_name = path + '/' + attribute - if segment_name: - variable_name += segment_name - return variable_name + def __format_variable_name_segment(self, location, attribute, suffix): + path = self.__format_variable_name(location, attribute) + return path + SUFFIX_SEP + suffix + + def _variable_info(self, model, attribute, location, injector): + if REGEXP in injector: + return self._process_regexp(model, attribute, location, injector) + else: + return self._process_attribute(model, attribute, location, injector) - def _process_attribute(self, model, attribute, location): + def _process_attribute(self, model, attribute, location, injector): _method_name = '_process_attribute' - _logger.entering(attribute, location.get_folder_path(), class_name=_class_name, method_name=_method_name) + _logger.entering(attribute, location.get_folder_path(), injector, class_name=_class_name, + method_name=_method_name) variable_name = None variable_value = None attribute_value = model[attribute] @@ -234,58 +234,62 @@ def _process_attribute(self, model, attribute, location): _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value - def _process_segment(self, model, attribute, segment, segment_name, replace_if_nosegment, location): + def _process_regexp(self, model, attribute, location, injector): + regexp = injector[REGEXP] + suffix = None + if SUFFIX in injector: + suffix = injector[SUFFIX] if isinstance(model[attribute], dict): - return self._process_segment_in_dictionary(attribute, model[attribute], segment, segment_name, location) + return self._process_regexp_dictionary(attribute, model[attribute], location, regexp, suffix) elif type(model[attribute]) == list: - return self._process_segment_in_list(attribute, model[attribute], segment, segment_name, location) + return self._process_regexp_list(attribute, model[attribute], location, regexp, suffix) else: - return self._process_segment_string(model, attribute, segment, segment_name, - replace_if_nosegment, location) + return self._process_regexp_string(model, attribute, location, regexp, suffix) - def _process_segment_string(self, model, attribute, segment, segment_name, replace_if_nosegment, location): - _method_name = '_process_segment_string' - _logger.entering(attribute, segment, segment_name, replace_if_nosegment, str(location)) + def _process_regexp_string(self, model, attribute, location, regexp, suffix): + _method_name = '_process_regexp_string' + _logger.entering(attribute, location.get_folder_path(), regexp, suffix, class_name=_class_name, + method_name=_method_name) attribute_value, variable_name, variable_value = self._find_segment_in_string(attribute, model[attribute], - segment, segment_name, location) + regexp, suffix, location) if variable_value: _logger.finer('WLSDPLY-19429', attribute, attribute_value, class_name=_class_name, method_name=_method_name) model[attribute] = attribute_value - elif replace_if_nosegment: - check_value = model[attribute] - if not _already_property(check_value): - variable_value = check_value - variable_name = self.__format_variable_name(location, attribute) - model[attribute] = _format_as_property(variable_name) - _logger.finer('WLSDPLY-19430', attribute, model[attribute], class_name=_class_name, - method_name=_method_name) + # elif replace_if_nosegment: + # check_value = model[attribute] + # if not _already_property(check_value): + # variable_value = check_value + # variable_name = self.__format_variable_name(location, attribute) + # model[attribute] = _format_as_property(variable_name) + # _logger.finer('WLSDPLY-19430', attribute, model[attribute], class_name=_class_name, + # method_name=_method_name) else: - _logger.finer('WLSDPLY-19424', segment, attribute, model[attribute], + _logger.finer('WLSDPLY-19424', regexp, attribute, model[attribute], location.get_folder_path, class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value - def _find_segment_in_string(self, attribute, attribute_value, segment, segment_name, location): + def _find_segment_in_string(self, attribute, attribute_value, regexp, suffix, location): variable_name = None variable_value = None if not _already_property(attribute_value): - variable_name = self.__format_variable_name_segment(location, attribute, segment_name) - attribute_value, variable_value = _replace_segment(segment, str(attribute_value), + variable_name = self.__format_variable_name_segment(location, attribute, suffix) + attribute_value, variable_value = _replace_segment(regexp, str(attribute_value), _format_as_property(variable_name)) return attribute_value, variable_name, variable_value - def _process_segment_in_list(self, attribute_name, attribute_list, segment, segment_name, location): - _method_name = '_process_segment_in_dictionary' - _logger.entering(attribute_name, attribute_list, segment, str(location), class_name=_class_name, - method_name=_method_name) + def _process_regexp_list(self, attribute_name, attribute_list, regexp, location, suffix): + _method_name = '_process_regexp_list' + _logger.entering(attribute_name, attribute_list, regexp, location.get_folder_path(), suffix, + class_name=_class_name, method_name=_method_name) variable_name = None variable_value = None idx = 0 for entry in attribute_list: - attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, segment, - segment_name, location) + attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, regexp, + suffix, location) if seg_var_value: _logger.finer('WLSDPLY-19429', attribute_name, attribute_value, class_name=_class_name, method_name=_method_name) @@ -298,16 +302,16 @@ def _process_segment_in_list(self, attribute_name, attribute_list, segment, segm _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value - def _process_segment_in_dictionary(self, attribute_name, attribute_dict, segment, segment_name, location): - _method_name = '_process_segment_in_dictionary' - _logger.entering(attribute_name, attribute_dict, segment, str(location), class_name=_class_name, - method_name=_method_name) - variable_name = self.__format_variable_name_segment(location, attribute_name, segment_name) + def _process_regexp_dictionary(self, attribute_name, attribute_dict, location, regexp, suffix): + _method_name = '_process_regexp_dictionary' + _logger.entering(attribute_name, attribute_dict, location.get_folder_path(), regexp, suffix, + class_name=_class_name, method_name=_method_name) + variable_name = self.__format_variable_name_segment(location, attribute_name, suffix) variable_value = None replacement = _format_as_property(variable_name) for entry in attribute_dict: if not _already_property(attribute_dict[entry]): - matcher = re.search(segment, entry) + matcher = re.search(suffix, entry) if matcher: _logger.finer('WLSDPLY-19427', attribute_name, replacement, class_name=_class_name, method_name=_method_name) @@ -319,32 +323,24 @@ def _process_segment_in_dictionary(self, attribute_name, attribute_dict, segment def _log_mbean_not_found(self, mbean, replacement, location): _method_name = '_log_mbean_not_found' - mbean_list = [] - if len(location.get_folder_path()) > 1: - try: - mbean_list = self.__aliases.get_model_subfolder_names(location) - except AliasException, ae: - _logger.fine('AliasException {0}', ae.getLocalizedMessage()) - pass + code = ValidationCodes.INVALID + try: + code, __ = self.__aliases.is_valid_model_folder_name(location, mbean) + except AliasException, ae: + _logger.fine('AliasException {0}', ae.getLocalizedMessage()) + pass + if code == ValidationCodes.INVALID: + _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) else: - try: - mbean_list = self.__aliases.get_model_top_level_folder_names() - except AliasException, ae: - _logger.fine('AliasException {0}', ae.getLocalizedMessage()) - pass - _logger.fine('The mbean list from get model subfolder is {0}', mbean_list) - if mbean in mbean_list: _logger.finer('WLSDPLY-19416', mbean, replacement, location.get_folder_path(), class_name=_class_name, method_name=_method_name) - else: - _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path(), - class_name=_class_name, method_name=_method_name) -def _get_variable_file_name(variables_helper_dictionary, **kwargs): - if VARIABLE_FILE_NAME_ARG in variables_helper_dictionary: - variable_file_location = variables_helper_dictionary[VARIABLE_FILE_NAME_ARG] - del variables_helper_dictionary[VARIABLE_FILE_NAME_ARG] +def _get_variable_file_name(variables_injector_dictionary, **kwargs): + if VARIABLE_FILE_NAME_ARG in variables_injector_dictionary: + variable_file_location = variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] + del variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] _logger.finer('WLSDPLY-19422', variable_file_location) elif VARIABLE_FILE_NAME_ARG in kwargs: variable_file_location = kwargs[VARIABLE_FILE_NAME_ARG] @@ -354,66 +350,111 @@ def _get_variable_file_name(variables_helper_dictionary, **kwargs): return variable_file_location -def _get_variable_helper_file_name(**kwargs): - variable_helper_file_name = VARIABLE_HELPER_FILE_NAME - if VARIABLE_HELPER_FILE_NAME_ARG in kwargs: - variable_helper_file_name = kwargs[VARIABLE_HELPER_FILE_NAME_ARG] - if VARIABLE_HELPER_PATH_NAME_ARG in kwargs: - return os.path.join(kwargs[VARIABLE_HELPER_PATH_NAME_ARG], variable_helper_file_name) +def _get_variable_injector_file_name(**kwargs): + variable_injector_file_name = VARIABLE_INJECTOR_FILE_NAME + if VARIABLE_INJECTOR_FILE_NAME_ARG in kwargs: + variable_injector_file_name = kwargs[VARIABLE_INJECTOR_FILE_NAME_ARG] + if VARIABLE_INJECTOR_PATH_NAME_ARG in kwargs: + return os.path.join(kwargs[VARIABLE_INJECTOR_PATH_NAME_ARG], variable_injector_file_name) else: - return os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'lib', variable_helper_file_name) + return os.path.join(_wlsdeploy_location, DEFAULT_FILE_LOCATION, variable_injector_file_name) -def _get_keyword_files_location(**kwargs): - if VARIABLE_HELPER_PATH_NAME_ARG in kwargs: - return kwargs[VARIABLE_HELPER_PATH_NAME_ARG] +def _get_variable_keywords_file_name(**kwargs): + variable_keywords_file_name = VARIABLE_KEYWORDS_FILE_NAME + if VARIABLE_KEYWORDS_FILE_NAME_ARG in kwargs: + variable_keywords_file_name = kwargs[VARIABLE_KEYWORDS_FILE_NAME_ARG] + if VARIABLE_KEYWORDS_PATH_NAME_ARG in kwargs: + return os.path.join(kwargs[VARIABLE_KEYWORDS_PATH_NAME_ARG], variable_keywords_file_name) else: - return os.path.join(os.environ.get('WLSDEPLOY_HOME'), 'etc') + return os.path.join(_wlsdeploy_location, DEFAULT_FILE_LOCATION, variable_keywords_file_name) -def _load_replacement_list(replacement_file_name): - _method_name = '_load_replacement_dictionary' - _logger.entering(replacement_file_name, class_name=_class_name, method_name=_method_name) - replacement_list = [] - if os.path.isfile(replacement_file_name): +def _load_variables_file(variable_injector_location): + _method_name = '_load_variables_dictionary' + _logger.entering(variable_injector_location, class_name=_class_name, method_name=_method_name) + variables_dictionary = None + if os.path.isfile(variable_injector_location): try: - replacement_file = open(replacement_file_name, 'r') - entry = replacement_file.readline() - while entry: - if entry != '\n': - _logger.finest('WLSDPLY-19411', entry, replacement_file_name, class_name=_class_name, - method_name=_method_name) - replacement_list.append(entry.strip()) - entry = replacement_file.readline() - except OSError, oe: - _logger.warning('WLDPLY-19409', replacement_file_name, oe.getLocalizedMessage(), class_name=_class_name, - method_name=_method_name) - else: - _logger.warning('WLSDPLY-19410', replacement_file_name, class_name=_class_name, method_name=_method_name) - - _logger.exiting(class_name=_class_name, method_name=_method_name) - return replacement_list + variables_dictionary = FileToPython(variable_injector_location).parse() + _logger.fine('WLSDPLY-19400', variable_injector_location, class_name=_class_name, method_name=_method_name) + except IllegalArgumentException, ia: + _logger.warning('WLSDPLY-19402', variable_injector_location, ia.getLocalizedMessage(), + class_name=_class_name, method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_dictionary) + return variables_dictionary -def _load_variables_dictionary(variable_helper_location): - _method_name = '_load_variables_dictionary' - _logger.entering(variable_helper_location, class_name=_class_name, method_name=_method_name) - variables_dictionary = None - if os.path.isfile(variable_helper_location): +def _load_keywords_file(variable_keywords_location): + _method_name = '_load_keywords_dictionary' + _logger.entering(variable_keywords_location, class_name=_class_name, method_name=_method_name) + keywords_dictionary = None + if os.path.isfile(variable_keywords_location): try: - variables_dictionary = FileToPython(variable_helper_location).parse() - _logger.fine('WLSDPLY-19400', variable_helper_location, class_name=_class_name, method_name=_method_name) + keywords_dictionary = FileToPython(variable_keywords_location).parse() + _logger.fine('WLSDPLY-19432', variable_keywords_location, class_name=_class_name, method_name=_method_name) except IllegalArgumentException, ia: - _logger.warning('WLSDPLY-19402', variable_helper_location, ia.getLocalizedMessage(), class_name=_class_name, + _logger.warning('WLSDPLY-19433', variable_keywords_location, ia.getLocalizedMessage(), + class_name=_class_name, method_name=_method_name) + + _logger.exiting(class_name=_class_name, method_name=_method_name, result=keywords_dictionary) + return keywords_dictionary + + +def _create_injector_file_list(variables_dictionary, keyword_dictionary, injector_path): + _method_name = '_create_file_dictionary' + injector_file_list = [] + if CUSTOM_KEYWORD in variables_dictionary: + if KEYWORD_FILES in variables_dictionary[CUSTOM_KEYWORD]: + file_list = variables_dictionary[CUSTOM_KEYWORD][KEYWORD_FILES] + if type(file_list) != list: + file_list = file_list.split(',') + for filename in file_list: + injector_file_list.append(filename) + else: + _logger.info('WLSDPLY-19434', class_name=_class_name, method_name=_method_name) + del variables_dictionary[CUSTOM_KEYWORD] + for keyword in variables_dictionary: + if keyword in keyword_dictionary: + filename = keyword_dictionary[keyword] + if filename and filename not in injector_file_list: + if not os.path.isabs(filename): + filename = os.path.join(injector_path, filename) + injector_file_list.append(filename) + _logger.finer('WLSDPLY-19408', filename, keyword) + else: + _logger.warning('WLSDPLY-19403', keyword, class_name=_class_name, method_name=_method_name) + return injector_file_list + + +def _get_keyword_files_location(**kwargs): + if VARIABLE_INJECTOR_PATH_NAME_ARG in kwargs: + return kwargs[VARIABLE_INJECTOR_PATH_NAME_ARG] + else: + return _wlsdeploy_location + + +def _load_injector_file(injector_file_name): + _method_name = '_load_injector_file' + _logger.entering(injector_file_name, class_name=_class_name, method_name=_method_name) + injector_dictionary = dict() + if os.path.isfile(injector_file_name): + try: + injector_dictionary = JsonToPython(injector_file_name).parse() + except oracle.weblogic.deploy.json, je: + _logger.warning('WLDPLY-19409', injector_file_name, je.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) + else: + _logger.warning('WLSDPLY-19410', injector_file_name, class_name=_class_name, method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name) - return variables_dictionary + return injector_dictionary -def _replace_segment(segment, variable_value, attribute_value): +def _replace_segment(regexp, variable_value, attribute_value): replaced_value = None replacement_string = variable_value - pattern = re.compile(segment) + pattern = re.compile(regexp) matcher = pattern.search(variable_value) if matcher: replaced_value = variable_value[matcher.start():matcher.end()] @@ -430,56 +471,17 @@ def _format_as_property(prop_name): return '@@PROP:%s@@' % prop_name -def _split_property(attribute_path): +def _split_injector(injector_path): """ - Split the section from the attribute path. - :param attribute_path: - :return: + Split the injector path into an mbean list and an attribute name from the injector path string + :param injector_path: + :return: attribute name:mbean list of mbean folder nodes """ - section = None - attribute = None - mbean_list = None - segment = None - segment_name = None - if_notfound_replace_full = None - split_list = attribute_path.split(':', 1) - if len(split_list) == 2 and split_list[0] in model_sections.get_model_top_level_keys(): - section = split_list[0] - segment, segment_name, if_notfound_replace_full = _find_segment(split_list[1]) - attribute_path = _split_from_segment(split_list[1]) - mbean_list, attribute = _split_attribute_path(attribute_path) - _logger.finest('WLSDPLY-19431', section, mbean_list, attribute, segment, segment_name, if_notfound_replace_full) - return section, mbean_list, attribute, segment, segment_name, if_notfound_replace_full - - -def _split_attribute_path(attribute_path): - mbean_list = attribute_path.split('.') - attribute = None - if len(mbean_list) > 0: - attribute = mbean_list.pop() - return mbean_list, attribute - - -def _split_from_segment(attribute_path): - return re.split('[\[\(].+[\]\)]$', attribute_path)[0] - - -def _find_segment(attribute_path): - segment = None - segment_name = None - if_notfound_replace_full = None - matcher = re.search('[\[\(].+[\]\)]$', attribute_path) - if matcher: - if attribute_path[matcher.start()] == '(': - if_notfound_replace_full = True - else: - if_notfound_replace_full = False - segment = attribute_path[matcher.start()+1:matcher.end()-1] - matcher = re.search('(?<=`)\w*(?=`)', segment) - if matcher: - segment_name = segment[matcher.start():matcher.end()] - segment = segment[matcher.end()+1:] - return segment, segment_name, if_notfound_replace_full + attr = None + ml = injector_path.split('.') + if len(ml) > 0: + attr = ml.pop() + return ml, attr def _find_special_name(mbean): diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 2e1f6f72cc..735570d177 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1099,22 +1099,22 @@ WLSDPLY-19306=Unable to extract domain library {0} from archive file {1}: {2} WLSDPLY-19307=Unable to extract classpath libraries from archive file {0} to domain directory {1}: {2} # wlsdeploy/tool/util/variable_injector.py -WLSDPLY-19400=model variable replacement helper file found at location {0} +WLSDPLY-19400=Model variable injector values loaded from location {0} WLSDPLY-19401=Variable replacement using custom list loaded from variable helper file {0} WLSDPLY-19402=Unable to read and parse variables helper file {0} : {1} -WLSDPLY-19403=Ignoring unknown keyword {0} found in variable replacement helper +WLSDPLY-19403=Ignoring unknown keyword {0} found in model variable injector file WLSDPLY-19404=An invalid variable file location {0} was provided to variable helper. Variable replacement \ will not continue WLSDPLY-19405=Unable to open the variable file {0} for writing : {1} WLSDPLY-19406=Invalid attribute part {0} in path {1) for section {2} WLSDPLY-19407=Exception attempting to write to variables file {0} - discarding model changes with variable insertions \ : {1} -WLSDPLY-19408=Adding property file {0} to variable helper list for keyword {1} +WLSDPLY-19408=Adding file {0} to injector file list for keyword {1} WLSDPLY-19409=Unable to read and load values for property file {0} : {1} WLSDPLY-19410=Invalid property file name {0} WLSDPLY-19411=Adding property {0} for model variable replacement from file {1} WLSDPLY-19412=Loading keywords from variable helper file {0} -WLSDPLY-19413=Variables were located and inserted in the model for keyword {0} +WLSDPLY-19413=Variables were located and inserted in the model using injector file {0} WLSDPLY-19414=Located mbean {0} in the model file WLSDPLY-19415=Invalid mbean {0} found in replacement property {1} at location {2} WLSDPLY-19416=MBean {0} not found in the model for replacement property {1} at location (2} @@ -1133,8 +1133,11 @@ WLSDPLY-19427=Segment found in dictionary for attribute {0}; inserting variable WLSDPLY-19428=Segment found in list for attribute {0} has been replaced by variable {1} WLSDPLY-19429=Segment was found in attribute {0} and variable has been inserted into string {1} WLSDPLY-19430=Segment was not found in attribute {0} value so entire variable value was replaced by {1} -WLSDPLY-19431=Model section {0} mbean list {1} attribute {2} search for segment with {3} use segment name {4} \ - and replace with full value if segment not found is {5} +WLSDPLY-19431=MBean {0} from injector path represents a named MBean folder at location {1} +WLSDPLY-19432=Variable keywords file loaded from location {0} +WLSDPLY-19433=Unable to read and parse variable keywords file {0} : {1} +WLSDPLY-19434=CUSTOM keyword was found in variables injector list but no files were included. Skipping the CUSTOM \ + keyword # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_file_helper_test.py b/core/src/test/python/variable_file_helper_test.py deleted file mode 100644 index ba402d65ca..0000000000 --- a/core/src/test/python/variable_file_helper_test.py +++ /dev/null @@ -1,219 +0,0 @@ -""" -Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. -The Universal Permissive License (UPL), Version 1.0 -""" -import unittest - -import wlsdeploy.util.variables as variables -import wlsdeploy.tool.util.variable_injector as variable_file_helper -from wlsdeploy.tool.util.variable_injector import VariableFileHelper -from wlsdeploy.util.model_translator import FileToPython - - -class VariableFileHelperTest(unittest.TestCase): - _resources_dir = '../../test-classes' - _variable_file = _resources_dir + '/variables.properties' - _model_file = _resources_dir + '/variable_insertion.yaml' - _variable_helper_keyword = 'variable_helper_keyword.json' - _variable_helper_custom = 'variable_helper_custom.json' - - def setUp(self): - self.name = VariableFileHelperTest - self._model = FileToPython(self._model_file).parse() - self._helper = VariableFileHelper(self._model, None, '12.2.1.3') - - def testSingleVariableReplacement(self): - replacement_list = ['topology:Machine.NodeManager.ListenAddress'] - expected = dict() - expected['/Machine/machine1/NodeManager/ListenAddress'] = '127.0.0.1' - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - - def testMultiplesReplacement(self): - expected = dict() - # expected['server-AdminServer-listenport'] = 9001 - # expected['server-AdminServer-ssl-listenport'] = 9002 - # expected['server-m2-listenport'] = 9005 - # expected['server-m1-listenport'] = 9003 - # expected['server-m1-ssl-listenport'] = 9004 - # expected['server-m2-ssl-listenport'] = 9006 - # expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-connectionurl'] \ - # = 't3://my.other.cluster:7001' - # expected['jmssystemresource-MyJmsModule-jmsresource-foreignserver-MyForeignServer-' - # 'foreigndestination-MyRemoteQ-localjndiname'] = 'jms/remoteQ' - expected['/Server/AdminServer/SSL/ListenPort'] = '9002' - expected['/Server/AdminServer/ListenPort'] = '9001' - expected['/Server/m2/ListenPort'] = '9005' - expected['/Server/m1/ListenPort'] = '9003' - expected['/Server/m1/SSL/ListenPort'] = '9004' - expected['/Server/m2/SSL/ListenPort'] = '9006' - expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ - = 't3://my.other.cluster:7001' - expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/' - 'ForeignDestination/MyRemoteQ/LocalJNDIName'] = 'jms/remoteQ' - #'resources:JMSSystemResource.JmsResource.ForeignDestination.LocalJNDIName', - replacement_list = ['topology:Server.ListenPort', - 'resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL', - 'resources:JMSSystemResource.JmsResource.ForeignServer.ForeignDestination.LocalJNDIName', - 'topology:Server.SSL.ListenPort'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - - def testInvalidMBeanNameNoException(self): - expected = dict() - replacement_list = ['resources:JmsSystemResource.Notes'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - - def testInvalidAttributeName(self): - expected = dict() - replacement_list = ['topology:Server.listenaddress'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - - def testInvalidSection(self): - expected = dict() - replacement_list = ['topologies:Server.ListenAddress'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - - def testDomainAttributeReplacementAndModel(self): - expected = dict() - expected['/Notes'] = 'Test note replacement' - expected_replacement = '@@PROP:/Notes@@' - replacement_list = ['topology:Notes'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - self.assertEqual(expected_replacement, self._model['topology']['Notes']) - - def testWithSegment(self): - expected = dict() - expected['/JDBCSystemResource/Database1/JdbcResource/JDBCDriverParams/URL'] = \ - 'jdbc:oracle:thin:@//den00chv.us.oracle.com:1521/PDBORCL' - expected['/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLHost'] = \ - 'slc05til.us.oracle.com' - expected['/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLPort'] = \ - '1521' - replacement_list = [ - 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Port`(?<=PORT=)[\w.-]+(?=\)))', - 'resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Host`(?<=HOST=)[\w.-]+(?=\)))'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - db2 = 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)' \ - '(HOST=@@PROP:/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLHost@@)' \ - '(PORT=@@PROP:/JDBCSystemResource/Database2/JdbcResource/JDBCDriverParams/URLPort@@)))' \ - '(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' - db1 = '@@PROP:/JDBCSystemResource/Database1/JdbcResource/JDBCDriverParams/URL@@' - self.assertEqual(db2, self._model['resources']['JDBCSystemResource']['Database2']['JdbcResource'][ - 'JDBCDriverParams']['URL']) - self.assertEqual(db1, self._model['resources']['JDBCSystemResource']['Database1']['JdbcResource'][ - 'JDBCDriverParams']['URL']) - - def testWithSegmentInDictionary(self): - expected = dict() - expected['/MailSession/MailSession-0/PropertiesSmtpHost'] = 'stbeehive.oracle.com' - expected['/MailSession/MyMailSession/PropertiesSmtpHost'] = 'stbeehive.oracle.com' - expected['/MailSession/MailSession-0/PropertiesImapHost'] = 'stbeehive.oracle.com' - expected['/MailSession/MyMailSession/PropertiesImapHost'] = 'stbeehive.oracle.com' - replacement_list = ["resources:MailSession.Properties[`SmtpHost`mail.smtp.host]", - "resources:MailSession.Properties[`ImapHost`mail.imap.host]"] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesSmtpHost@@', - self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) - self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesImapHost@@', - self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) - - def testWithSegmentInDictionaryAndAPattern(self): - expected = dict() - expected['/MailSession/MyMailSession/PropertiesHost'] = 'stbeehive.oracle.com' - expected['/MailSession/MailSession-0/PropertiesHost'] = 'stbeehive.oracle.com' - replacement_list = ['resources:MailSession.Properties[`Host`(?<=\w.)host]'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', - self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) - self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', - self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.host']) - self.assertEqual('@@PROP:/MailSession/MyMailSession/PropertiesHost@@', - self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) - - def testWithSegmentInList(self): - expected = dict() - expected['/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/weblogic.management.' - 'runtime.ServerRuntimeMBean/HarvestedAttribute'] = 'OracleHome' - replacement_list = [ - 'resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute[OracleHome]'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ - 'HarvestedType']['weblogic.management.runtime.ServerRuntimeMBean']['HarvestedAttribute'] - found = False - for entry in list: - if entry == '@@PROP:/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/' \ - 'weblogic.management.runtime.ServerRuntimeMBean/HarvestedAttribute@@': - found = True - break - self.assertEqual(True, found) - - def testWithSegmentInStringInList(self): - expected = dict() - expected['/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/weblogic.management.' - 'runtime.ServerRuntimeMBean/HarvestedInstanceManagedServer'] = 'm1' - replacement_list = [ - 'resources:WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance[`ManagedServer`m1]'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - list = \ - self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester']['HarvestedType'][ - 'weblogic.management.runtime.ServerRuntimeMBean']['HarvestedInstance'] - found = False - for entry in list: - if entry == 'com.bea:Name=@@PROP:/WLDFSystemResource/MyWldfModule/WLDFResource/Harvester/HarvestedType/' \ - 'weblogic.management.runtime.ServerRuntimeMBean/HarvestedInstanceManagedServer@@' \ - ',Type=ServerRuntime': - found = True - break - self.assertEqual(True, found) - - def testWithNameInMBeanSingle(self): - expected = dict() - expected['/Server/m2/ServerStart/Arguments'] = '/etc' - replacement_list = ['topology:Server{m2}.ServerStart.Arguments[(?<=-Doracle.net.tns_admin=)[\w\\/._:]+]'] - actual = self._helper.inject_variables(replacement_list) - self._compare_to_expected_dictionary(expected, actual) - arg = '-Doracle.net.tns_admin=@@PROP:/Server/m2/ServerStart/Arguments@@ ' \ - '-DANTLR_USE_DIRECT_CLASS_LOADING=true ' \ - '-DANTLR_USE_DIRECT_CLASS_LOADING=true -Djava.awt.headless=true -Dhttp.webdir.enable=false ' \ - '-Duser.timezone=Europe/Zurich -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom ' \ - '-Dweblogic.data.canTransferAnyFile=true' - self.assertEqual(arg, self._model['topology']['Server']['m2']['ServerStart']['Arguments']) - - def testWithVariableHelperKeywords(self): - expected = dict() - expected['/JMSSystemResource/MyJmsModule/JmsResource/ForeignServer/MyForeignServer/ConnectionURL'] \ - = 't3://my.other.cluster:7001' - expected['/Server/AdminServer/ListenPort'] = '9001' - expected['/Server/m2/ListenPort'] = '9005' - expected['/Server/m1/ListenPort'] = '9003' - expected['/Machine/machine1/NodeManager/ListenPort'] = '5557' - expected['/Machine/machine1/NodeManager/PasswordEncrypted'] = '--FIX ME--' - expected['/Machine/machine1/NodeManager/UserName'] = 'admin' - inserted, model, variable_file_name = self._helper.inject_variables_keyword_file( - variable_helper_path_name=self._resources_dir, variable_helper_file_name=self._variable_helper_keyword) - self.assertEqual(True, inserted) - self.assertEqual(self._variable_file, variable_file_name) - actual = variables.load_variables(self._variable_file) - self._compare_to_expected_dictionary(expected, actual) - - def _compare_to_expected_dictionary(self, expected, actual): - self.assertEqual(len(expected), len(actual), - 'Not the same number of entries : expected=' + str(len(expected)) + ', actual=' + str( - len(actual))) - for k, v in actual.iteritems(): - self.assertEqual(True, k in expected and v == expected[k], 'Actual item not in expected ' + k + - ' : ' + v + ' expected=' + str(expected)) - - -if __name__ == '__main__': - unittest.main() diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py new file mode 100644 index 0000000000..0d1fde3d94 --- /dev/null +++ b/core/src/test/python/variable_injector_test.py @@ -0,0 +1,232 @@ +""" +Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. +The Universal Permissive License (UPL), Version 1.0 +""" +import unittest + +import wlsdeploy.util.variables as variables +import wlsdeploy.tool.util.variable_injector as variable_injector +from wlsdeploy.tool.util.variable_injector import VariableFileHelper +from wlsdeploy.util.model_translator import FileToPython + + +class VariableFileHelperTest(unittest.TestCase): + _resources_dir = '../../test-classes' + _variable_file = _resources_dir + '/variables.properties' + _model_file = _resources_dir + '/variable_insertion.yaml' + _variable_helper_keyword = 'variable_injector_keyword.json' + _variable_helper_custom = 'variable_injector_custom.json' + + def setUp(self): + self.name = VariableFileHelperTest + self._model = FileToPython(self._model_file).parse() + self._helper = VariableFileHelper(self._model, None, '12.2.1.3') + + def testSingleVariableReplacement(self): + replacement_dict = dict() + replacement_dict['Machine.NodeManager.ListenAddress'] = dict() + expected = dict() + expected['Machine.machine1.NodeManager.ListenAddress'] = '127.0.0.1' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testMultiplesReplacement(self): + expected = dict() + expected['Server.AdminServer.SSL.ListenPort'] = '9002' + expected['Server.AdminServer.ListenPort'] = '9001' + expected['Server.m2.ListenPort'] = '9005' + expected['Server.m1.ListenPort'] = '9003' + expected['Server.m1.SSL.ListenPort'] = '9004' + expected['Server.m2.SSL.ListenPort'] = '9006' + expected['JMSSystemResource.MyJmsModule.JmsResource.ForeignServer.MyForeignServer.ConnectionURL'] \ + = 't3://my.other.cluster:7001' + expected['JMSSystemResource.MyJmsModule.JmsResource.ForeignServer.MyForeignServer.' + 'ForeignDestination.MyRemoteQ.LocalJNDIName'] = 'jms.remoteQ' + replacement_dict = dict() + replacement_dict['Server.ListenPort'] = dict() + replacement_dict['JMSSystemResource.JmsResource.ForeignServer.ConnectionURL'] = dict() + replacement_dict['JMSSystemResource.JmsResource.ForeignServer.ForeignDestination.LocalJNDIName'] = dict() + replacement_dict['Server.SSL.ListenPort'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testInvalidMBeanNameNoException(self): + expected = dict() + replacement_dict = dict() + replacement_dict['JmsSystemResource.Notes'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testInvalidAttributeName(self): + expected = dict() + replacement_dict = dict() + replacement_dict['Server.listenaddress'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testInvalidSection(self): + expected = dict() + replacement_dict = dict() + replacement_dict['Server.ListenAddress'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testDomainAttributeReplacementAndModel(self): + expected = dict() + expected['Notes'] = 'Test note replacement' + expected_replacement = '@@PROP:Notes@@' + replacement_dict = dict() + replacement_dict['Notes'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + self.assertEqual(expected_replacement, self._model['topology']['Notes']) + + def testWithSegment(self): + expected = dict() + expected['JDBCSystemResource.Database2.JdbcResource.JDBCDriverParams.URL--Host'] = \ + 'slc05til.us.oracle.com' + expected['JDBCSystemResource.Database2.JdbcResource.JDBCDriverParams.URL--Port'] = \ + '1521' + replacement_dict = dict() + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'] = dict() + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][ + variable_injector.REGEXP] = '(?<=PORT=)[\w.-]+(?=\)))' + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][variable_injector.SUFFIX] = 'Port' + + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'] = dict() + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][ + variable_injector.REGEXP] = '(?<=HOST=)[\w.-]+(?=\)))' + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][variable_injector.SUFFIX] = 'Host' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + db2 = 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)' \ + '(HOST=@@PROP:JDBCSystemResource.Database2.JdbcResource.JDBCDriverParams.URL--Host@@)' \ + '(PORT=@@PROP:JDBCSystemResource.Database2.JdbcResource.JDBCDriverParams.URL--Port@@)))' \ + '(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' + db1 = 'jdbc:oracle:thin:@//den00chv.us.oracle.com:1521/PDBORCL' + self.assertEqual(db2, self._model['resources']['JDBCSystemResource']['Database2']['JdbcResource'][ + 'JDBCDriverParams']['URL']) + self.assertEqual(db1, self._model['resources']['JDBCSystemResource']['Database1']['JdbcResource'][ + 'JDBCDriverParams']['URL']) + + def testWithSegmentInDictionary(self): + expected = dict() + expected['MailSession.MailSession-0.Properties--SmtpHost'] = 'stbeehive.oracle.com' + expected['MailSession.MyMailSession.Properties--SmtpHost'] = 'stbeehive.oracle.com' + expected['MailSession.MailSession-0.Properties--ImapHost'] = 'stbeehive.oracle.com' + expected['MailSession.MyMailSession.Properties--ImapHost'] = 'stbeehive.oracle.com' + replacement_dict = dict() + replacement_dict['MailSession.Properties'] = dict() + replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = 'mail.smtp.host' + replacement_dict['MailSession.Properties'][variable_injector.SUFFIX] = 'SmtpHost' + replacement_dict['MailSession.Properties'] = dict() + replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = 'mail.imap.host' + replacement_dict['MailSession.Properties'][variable_injector.SUFFIX] = 'ImapHost' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + self.assertEqual('@@PROP:MailSession.MyMailSession.Properties--SmtpHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) + self.assertEqual('@@PROP:MailSession.MyMailSession.Properties--ImapHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) + + def testWithSegmentInDictionaryAndAPattern(self): + expected = dict() + expected['MailSession.MyMailSession.Properties--Host'] = 'stbeehive.oracle.com' + expected['MailSession.MailSession-0.Properties--Host'] = 'stbeehive.oracle.com' + replacement_dict = dict() + replacement_dict['MailSession.Properties'] = dict() + replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = '(?<=\w.)host]' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + self.assertEqual('@@PROP:MailSession.MyMailSession.Properties--Host@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) + self.assertEqual('@@PROP:MailSession.MyMailSession.PropertiesHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.host']) + self.assertEqual('@@PROP:MailSession.MyMailSession.PropertiesHost@@', + self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) + + def testWithSegmentInList(self): + expected = dict() + expected['WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.weblogic.management.' + 'runtime.ServerRuntimeMBean.HarvestedAttribute'] = 'OracleHome' + replacement_dict = dict() + replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute'] = dict() + replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute'][ + variable_injector.REGEXP] = 'OracleHome' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ + 'HarvestedType']['weblogic.management.runtime.ServerRuntimeMBean']['HarvestedAttribute'] + found = False + for entry in list: + if entry == '@@PROP:WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.' \ + 'weblogic.management.runtime.ServerRuntimeMBean.HarvestedAttribute@@': + found = True + break + self.assertEqual(True, found) + + def testWithSegmentInStringInList(self): + expected = dict() + expected['WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.weblogic.management.' + 'runtime.ServerRuntimeMBean.HarvestedInstanceManagedServer'] = 'm1' + replacement_dict = dict() + replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance'] = dict() + replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance'][ + variable_injector.REGEXP] = 'm1' + replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance'][ + variable_injector.SUFFIX] = 'ManagedServer' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + list = \ + self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester']['HarvestedType'][ + 'weblogic.management.runtime.ServerRuntimeMBean']['HarvestedInstance'] + found = False + for entry in list: + if entry == 'com.bea:Name=@@PROP:WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.' \ + 'weblogic.management.runtime.ServerRuntimeMBean.HarvestedInstanceManagedServer@@' \ + ',Type=ServerRuntime': + found = True + break + self.assertEqual(True, found) + # + # def testWithNameInMBeanSingle(self): + # expected = dict() + # expected['Server.m2.ServerStart.Arguments'] = '/etc' + # replacement_list = ['topology:Server{m2}.ServerStart.Arguments[(?<=-Doracle.net.tns_admin=)[\w\\/._:]+]'] + # actual = self._helper.inject_variables(replacement_list) + # self._compare_to_expected_dictionary(expected, actual) + # arg = '-Doracle.net.tns_admin=@@PROP:Server.m2.ServerStart.Arguments@@ ' \ + # '-DANTLR_USE_DIRECT_CLASS_LOADING=true ' \ + # '-DANTLR_USE_DIRECT_CLASS_LOADING=true -Djava.awt.headless=true -Dhttp.webdir.enable=false ' \ + # '-Duser.timezone=Europe/Zurich -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom ' \ + # '-Dweblogic.data.canTransferAnyFile=true' + # self.assertEqual(arg, self._model['topology']['Server']['m2']['ServerStart']['Arguments']) + + def testWithVariableHelperKeywords(self): + expected = dict() + expected['JMSSystemResource.MyJmsModule.JmsResource.ForeignServer.MyForeignServer.ConnectionURL'] \ + = 't3://my.other.cluster:7001' + expected['Server.AdminServer.ListenPort'] = '9001' + expected['Server.m2.ListenPort'] = '9005' + expected['Server.m1.ListenPort'] = '9003' + expected['Machine.machine1.NodeManager.ListenPort'] = '5557' + expected['Machine.machine1.NodeManager.PasswordEncrypted'] = '--FIX ME--' + expected['Machine.machine1.NodeManager.UserName'] = 'admin' + inserted, model, variable_file_name = self._helper.inject_variables_keyword_file( + variable_helper_path_name=self._resources_dir, variable_helper_file_name=self._variable_helper_keyword) + self.assertEqual(True, inserted) + self.assertEqual(self._variable_file, variable_file_name) + actual = variables.load_variables(self._variable_file) + self._compare_to_expected_dictionary(expected, actual) + + def _compare_to_expected_dictionary(self, expected, actual): + self.assertEqual(len(expected), len(actual), + 'Not the same number of entries : expected=' + str(len(expected)) + ', actual=' + str( + len(actual))) + for k, v in actual.iteritems(): + self.assertEqual(True, k in expected and v == expected[k], 'Actual item not in expected ' + k + + ' : ' + v + ' expected=' + str(expected)) + + +if __name__ == '__main__': + unittest.main() diff --git a/core/src/test/resources/credentials.json b/core/src/test/resources/credentials.json new file mode 100644 index 0000000000..e777cc2bef --- /dev/null +++ b/core/src/test/resources/credentials.json @@ -0,0 +1,4 @@ +{ + "Machine.NodeManager.PasswordEncrypted": {}, + "Machine.NodeManager.UserName": {} +} diff --git a/core/src/test/resources/credentials.properties b/core/src/test/resources/credentials.properties deleted file mode 100644 index 4049e70510..0000000000 --- a/core/src/test/resources/credentials.properties +++ /dev/null @@ -1,2 +0,0 @@ -topology:Machine.NodeManager.PasswordEncrypted -topology:Machine.NodeManager.UserName \ No newline at end of file diff --git a/core/src/test/resources/custom.json b/core/src/test/resources/custom.json new file mode 100644 index 0000000000..d36362f3e3 --- /dev/null +++ b/core/src/test/resources/custom.json @@ -0,0 +1,4 @@ +{ + Notes: {}, + Application.SecurityDDModel: {} +} \ No newline at end of file diff --git a/core/src/test/resources/custom.properties b/core/src/test/resources/custom.properties deleted file mode 100644 index c915515802..0000000000 --- a/core/src/test/resources/custom.properties +++ /dev/null @@ -1,2 +0,0 @@ -topology:Notes -appDeployments:Application.SecurityDDModel \ No newline at end of file diff --git a/core/src/test/resources/port.json b/core/src/test/resources/port.json new file mode 100644 index 0000000000..9d89ef5024 --- /dev/null +++ b/core/src/test/resources/port.json @@ -0,0 +1,4 @@ +{ + Server.ListenPort: {}, + Machine.NodeManager.ListenPort: {} +} diff --git a/core/src/test/resources/port.properties b/core/src/test/resources/port.properties deleted file mode 100644 index f8c40ae603..0000000000 --- a/core/src/test/resources/port.properties +++ /dev/null @@ -1,2 +0,0 @@ -topology:Server.ListenPort -topology:Machine.NodeManager.ListenPort diff --git a/core/src/test/resources/url.json b/core/src/test/resources/url.json new file mode 100644 index 0000000000..d604bdca8e --- /dev/null +++ b/core/src/test/resources/url.json @@ -0,0 +1,3 @@ +{ + JMSSystemResource.JmsResource.ForeignServer.ConnectionURL: {} +} diff --git a/core/src/test/resources/url.properties b/core/src/test/resources/url.properties deleted file mode 100644 index 44036df22c..0000000000 --- a/core/src/test/resources/url.properties +++ /dev/null @@ -1 +0,0 @@ -resources:JMSSystemResource.JmsResource.ForeignServer.ConnectionURL diff --git a/core/src/test/resources/variable_helper_custom.json b/core/src/test/resources/variable_helper_custom.json deleted file mode 100644 index 4bbf7f268b..0000000000 --- a/core/src/test/resources/variable_helper_custom.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "CUSTOM": [ - { "file": "C:\Users\crountre\temp\custom_variables.properties" } - ], - "PORT": [ - ] -} \ No newline at end of file diff --git a/core/src/test/resources/variable_injector_custom.json b/core/src/test/resources/variable_injector_custom.json new file mode 100644 index 0000000000..da12fa1944 --- /dev/null +++ b/core/src/test/resources/variable_injector_custom.json @@ -0,0 +1,4 @@ +{ + "CUSTOM": { "file": "C:\Users\crountre\temp\custom_variables.properties" }, + "PORT": {} +} \ No newline at end of file diff --git a/core/src/test/resources/variable_helper_keyword.json b/core/src/test/resources/variable_injector_keyword.json similarity index 55% rename from core/src/test/resources/variable_helper_keyword.json rename to core/src/test/resources/variable_injector_keyword.json index 963634be8b..51e9bc34c3 100644 --- a/core/src/test/resources/variable_helper_keyword.json +++ b/core/src/test/resources/variable_injector_keyword.json @@ -1,9 +1,6 @@ { "variable_file_name": "../../test-classes/variables.properties", - "PORT": [ - ], - "URL": [ - ], - "CREDENTIALS": [ - ] + "PORT": {}, + "URL": {}, + "CREDENTIALS": {} } \ No newline at end of file From 80b4c263667d03de30ece5d9afaaca5600853596 Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 17 May 2018 16:18:07 -0500 Subject: [PATCH 13/52] redesign working order --- .../wlsdeploy/tool/util/variable_injector.py | 205 ++++++++++++------ .../deploy/messages/wlsdeploy_rb.properties | 1 + .../src/test/python/variable_injector_test.py | 70 +++--- core/src/test/resources/custom.json | 4 +- core/src/test/resources/keywords.json | 6 + core/src/test/resources/port.json | 4 +- core/src/test/resources/url.json | 2 +- 7 files changed, 185 insertions(+), 107 deletions(-) create mode 100644 core/src/test/resources/keywords.json diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 7b24eca45f..e3d72af9a6 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -37,7 +37,8 @@ DEFAULT_FILE_LOCATION = 'lib' # should these injector json keywords be included in the keyword file REGEXP = 'regexp' -SUFFIX = 'suffix' +REGEXP_SUFFIX = 'suffix' +REGEXP_PATTERN = 'pattern' FORCE = 'force' VARIABLE_SEP = '.' @@ -54,15 +55,11 @@ class VariableFileHelper(object): def __init__(self, model, model_context=None, version=None): self.__original = copy.deepcopy(model) - self.__model = dict() - # consolidate all the model sections into one for speedier search - # currently don't want to do variable injection in the domainInfo section of the model - if model_sections.get_model_topology_key() in model: - self.__model.update(model[model_sections.get_model_topology_key()]) - elif model_sections.get_model_resources_key() in model: - self.__model.update(model_sections.get_model_resources_key()) - elif model_sections.get_model_deployments_key() in model: - self.__model.update(model_sections.get_model_deployments_key()) + self.__model = model + + self.__section_keys = model_sections.get_model_top_level_keys() + self.__section_keys.remove(model_sections.get_model_domain_info_key()) + if version: self.__aliases = Aliases(model_context, WlstModes.OFFLINE, version, None) else: @@ -91,10 +88,10 @@ def inject_variables_keyword_file(self, **kwargs): return_model = dict() variable_file_location = None if variables_injector_dictionary and keywords_dictionary: + variable_file_location = _get_variable_file_name(variables_injector_dictionary, **kwargs) injector_file_list = _create_injector_file_list(variables_injector_dictionary, keywords_dictionary, _get_keyword_files_location(**kwargs)) - return_model = dict() - variable_file_location = _get_variable_file_name(variables_injector_dictionary, **kwargs) + return_model = self.__original if not variable_file_location: _logger.warning('WLSDPLY-19420', variable_injector_location_file, class_name=_class_name, method_name=_method_name) @@ -107,7 +104,6 @@ def inject_variables_keyword_file(self, **kwargs): return_model = self.__model else: _logger.fine('WLSDPLY-19419', class_name=_class_name, method_name=_method_name) - return_model = self.__original variable_file_location = None _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) @@ -143,14 +139,14 @@ def inject_variables(self, injector_dictionary): location = LocationContext() domain_token = self.__aliases.get_name_token(location) location.add_name_token(domain_token, 'fakedomain') - for injector in injector_dictionary: - entries_dict = self.__inject_variable(location, injector) + for injector, injector_values in injector_dictionary.iteritems(): + entries_dict = self.__inject_variable(location, injector, injector_values) if len(entries_dict) > 0: variable_dict.update(entries_dict) return variable_dict - def __inject_variable(self, location, injector): + def __inject_variable(self, location, injector, injector_values): _method_name = '__inject_variable' _logger.entering(injector, class_name=_class_name, method_name=_method_name) variable_dict = dict() @@ -179,79 +175,109 @@ def _traverse_variables(model_section, mbean_list): _traverse_variables(next_model_section, mbean_list) location.pop_location() else: - self._log_mbean_not_found(mbean, injector[0], location) + self._log_mbean_not_found(mbean, injector, location) return False else: if attribute in model_section: - variable_name, variable_value = self._variable_info(model_section, attribute, location, injector) - if variable_value: - variable_dict[variable_name] = variable_value + returned_dict = self._variable_info(model_section, attribute, location, injector, + injector_values) + if returned_dict: + variable_dict.update(returned_dict) else: _logger.finer('WLSDPLY-19417', attribute, injector, location.get_folder_path(), class_name=_class_name, method_name=_method_name) return True - _traverse_variables(self.__model, start_mbean_list) + section = self.__model + if start_mbean_list: + # Find out in what section is the mbean top folder so can move to that section in the model + for entry in self.__section_keys: + if entry in self.__model and start_mbean_list[0] in self.__model[entry]: + section = self.__model[entry] + break + else: + # This is a domain attribute + section = self.__model[model_sections.get_model_topology_key()] + # if it wasn't found, will log appropriately in the called method + # This also will allow someone to put the section in the injector string + _traverse_variables(section, start_mbean_list) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_dict) + return variable_dict def __format_variable_name(self, location, attribute): - path = '' + variable_name = attribute make_path = self.__aliases.get_model_folder_path(location) if make_path: make_path = make_path.split(':') if len(make_path) > 1 and len(make_path[1]) > 1: - path = make_path[1] - path = path[1:] + VARIABLE_SEP + attribute - _variable_sep_pattern.sub(VARIABLE_SEP, path) - return path + variable_name = make_path[1] + variable_name = variable_name[1:] + VARIABLE_SEP + attribute + variable_name = variable_name.replace('/', '.') + return variable_name def __format_variable_name_segment(self, location, attribute, suffix): path = self.__format_variable_name(location, attribute) - return path + SUFFIX_SEP + suffix + if suffix: + return path + SUFFIX_SEP + suffix + return path - def _variable_info(self, model, attribute, location, injector): - if REGEXP in injector: - return self._process_regexp(model, attribute, location, injector) + def _variable_info(self, model, attribute, location, injector, injector_values): + # add code here to put in model if force in injector values + if REGEXP in injector_values: + return self._process_regexp(model, attribute, location, injector_values[REGEXP]) else: - return self._process_attribute(model, attribute, location, injector) + return self._process_attribute(model, attribute, location) - def _process_attribute(self, model, attribute, location, injector): + def _process_attribute(self, model, attribute, location): _method_name = '_process_attribute' - _logger.entering(attribute, location.get_folder_path(), injector, class_name=_class_name, + _logger.entering(attribute, location.get_folder_path(), class_name=_class_name, method_name=_method_name) + variable_dict = dict() variable_name = None variable_value = None attribute_value = model[attribute] if not _already_property(attribute_value): variable_name = self.__format_variable_name(location, attribute) - variable_value = str(model[attribute]) + variable_value = str(attribute_value) model[attribute] = _format_as_property(variable_name) else: _logger.finer('WLSDPLY-19426', attribute_value, attribute, str(location), class_name=_class_name, method_name=_method_name) - + if variable_value: + variable_dict[variable_name] = variable_value _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) - return variable_name, variable_value + return variable_dict - def _process_regexp(self, model, attribute, location, injector): - regexp = injector[REGEXP] - suffix = None - if SUFFIX in injector: - suffix = injector[SUFFIX] + def _process_regexp(self, model, attribute, location, regexp_list): if isinstance(model[attribute], dict): - return self._process_regexp_dictionary(attribute, model[attribute], location, regexp, suffix) + return self._process_patterns_dictionary(attribute, model[attribute], location, regexp_list) elif type(model[attribute]) == list: - return self._process_regexp_list(attribute, model[attribute], location, regexp, suffix) + return self._process_patterns_list(attribute, model[attribute], location, regexp_list) else: - return self._process_regexp_string(model, attribute, location, regexp, suffix) + return self._process_patterns_string(model, attribute, location, regexp_list) + + def _process_patterns_string(self, model, attribute, location, regexp_list): + variable_dict = dict() + for dictionary in regexp_list: + pattern = None + suffix = None + if REGEXP_PATTERN in dictionary: + pattern = dictionary[REGEXP_PATTERN] + if REGEXP_SUFFIX in dictionary: + suffix = dictionary[REGEXP_SUFFIX] + variable_name, variable_value = self._process_pattern_string(model, attribute, location, pattern, suffix) + if variable_value: + variable_dict[variable_name] = variable_value + return variable_dict - def _process_regexp_string(self, model, attribute, location, regexp, suffix): + def _process_pattern_string(self, model, attribute, location, pattern, suffix): _method_name = '_process_regexp_string' - _logger.entering(attribute, location.get_folder_path(), regexp, suffix, class_name=_class_name, + _logger.entering(attribute, location.get_folder_path(), pattern, suffix, class_name=_class_name, method_name=_method_name) attribute_value, variable_name, variable_value = self._find_segment_in_string(attribute, model[attribute], - regexp, suffix, location) + location, pattern, suffix) if variable_value: _logger.finer('WLSDPLY-19429', attribute, attribute_value, class_name=_class_name, method_name=_method_name) @@ -265,31 +291,46 @@ def _process_regexp_string(self, model, attribute, location, regexp, suffix): # _logger.finer('WLSDPLY-19430', attribute, model[attribute], class_name=_class_name, # method_name=_method_name) else: - _logger.finer('WLSDPLY-19424', regexp, attribute, model[attribute], + _logger.finer('WLSDPLY-19424', pattern, attribute, model[attribute], location.get_folder_path, class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value - def _find_segment_in_string(self, attribute, attribute_value, regexp, suffix, location): + def _find_segment_in_string(self, attribute, attribute_value, location, pattern, suffix): variable_name = None variable_value = None if not _already_property(attribute_value): variable_name = self.__format_variable_name_segment(location, attribute, suffix) - attribute_value, variable_value = _replace_segment(regexp, str(attribute_value), + attribute_value, variable_value = _replace_segment(pattern, str(attribute_value), _format_as_property(variable_name)) return attribute_value, variable_name, variable_value - def _process_regexp_list(self, attribute_name, attribute_list, regexp, location, suffix): + def _process_patterns_list(self, attribute, attribute_value, location, regexp_list): + variable_dict = dict() + for dictionary in regexp_list: + pattern = None + suffix = None + if REGEXP_PATTERN in dictionary: + pattern = dictionary[REGEXP_PATTERN] + if REGEXP_SUFFIX in dictionary: + suffix = dictionary[REGEXP_SUFFIX] + variable_name, variable_value = self._process_pattern_list(attribute, attribute_value, location, pattern, + suffix) + if variable_value: + variable_dict[variable_name] = variable_value + return variable_dict + + def _process_pattern_list(self, attribute_name, attribute_list, location, pattern, suffix): _method_name = '_process_regexp_list' - _logger.entering(attribute_name, attribute_list, regexp, location.get_folder_path(), suffix, + _logger.entering(attribute_name, attribute_list, location.get_folder_path(), pattern, suffix, class_name=_class_name, method_name=_method_name) variable_name = None variable_value = None idx = 0 for entry in attribute_list: - attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, regexp, - suffix, location) + attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, location, + pattern, suffix) if seg_var_value: _logger.finer('WLSDPLY-19429', attribute_name, attribute_value, class_name=_class_name, method_name=_method_name) @@ -302,21 +343,38 @@ def _process_regexp_list(self, attribute_name, attribute_list, regexp, location, _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value - def _process_regexp_dictionary(self, attribute_name, attribute_dict, location, regexp, suffix): + def _process_patterns_dictionary(self, attribute, attribute_dict, location, regexp_list): + variable_dict = dict() + for dictionary in regexp_list: + pattern = None + suffix = None + if REGEXP_PATTERN in dictionary: + pattern = dictionary[REGEXP_PATTERN] + if REGEXP_SUFFIX in dictionary: + suffix = dictionary[REGEXP_SUFFIX] + variable_name, variable_value = self._process_pattern_dictionary(attribute, attribute_dict, location, + pattern, suffix) + if variable_value: + variable_dict[variable_name] = variable_value + return variable_dict + + def _process_pattern_dictionary(self, attribute_name, attribute_dict, location, regexp, suffix): _method_name = '_process_regexp_dictionary' _logger.entering(attribute_name, attribute_dict, location.get_folder_path(), regexp, suffix, class_name=_class_name, method_name=_method_name) variable_name = self.__format_variable_name_segment(location, attribute_name, suffix) variable_value = None - replacement = _format_as_property(variable_name) - for entry in attribute_dict: - if not _already_property(attribute_dict[entry]): - matcher = re.search(suffix, entry) - if matcher: - _logger.finer('WLSDPLY-19427', attribute_name, replacement, class_name=_class_name, - method_name=_method_name) - variable_value = str(attribute_dict[entry]) - attribute_dict[entry] = replacement + pattern = _compile_pattern(regexp) + if pattern: + replacement = _format_as_property(variable_name) + for entry in attribute_dict: + if not _already_property(attribute_dict[entry]): + matcher = pattern.search(entry) + if matcher: + _logger.finer('WLSDPLY-19427', attribute_name, replacement, class_name=_class_name, + method_name=_method_name) + variable_value = str(attribute_dict[entry]) + attribute_dict[entry] = replacement # don't break, continue replacing any in dictionary, return the last variable value found _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value @@ -454,15 +512,24 @@ def _load_injector_file(injector_file_name): def _replace_segment(regexp, variable_value, attribute_value): replaced_value = None replacement_string = variable_value - pattern = re.compile(regexp) - matcher = pattern.search(variable_value) - if matcher: - replaced_value = variable_value[matcher.start():matcher.end()] + pattern = _compile_pattern(regexp) + if pattern: + matcher = pattern.search(variable_value) + if matcher: + replaced_value = variable_value[matcher.start():matcher.end()] - replacement_string = pattern.sub(attribute_value, variable_value) + replacement_string = pattern.sub(attribute_value, variable_value) return replacement_string, replaced_value +def _compile_pattern(pattern): + try: + return re.compile(pattern) + except Exception, e: + _logger.warning('WLSDPLY-19435', pattern, e) + return None + + def _already_property(check_string): return type(check_string) == str and check_string.startswith('@@PROP:') diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 735570d177..37af5c685f 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1138,6 +1138,7 @@ WLSDPLY-19432=Variable keywords file loaded from location {0} WLSDPLY-19433=Unable to read and parse variable keywords file {0} : {1} WLSDPLY-19434=CUSTOM keyword was found in variables injector list but no files were included. Skipping the CUSTOM \ keyword +WLSDPLY-19435=Invalid Regular expression pattern {0} : {1} # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index 0d1fde3d94..b102f62553 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -14,8 +14,9 @@ class VariableFileHelperTest(unittest.TestCase): _resources_dir = '../../test-classes' _variable_file = _resources_dir + '/variables.properties' _model_file = _resources_dir + '/variable_insertion.yaml' - _variable_helper_keyword = 'variable_injector_keyword.json' - _variable_helper_custom = 'variable_injector_custom.json' + _variable_injector_keyword = 'variable_injector_keyword.json' + _variable_injector_custom = 'variable_injector_custom.json' + _keywords_file = 'keywords.json' def setUp(self): self.name = VariableFileHelperTest @@ -41,7 +42,7 @@ def testMultiplesReplacement(self): expected['JMSSystemResource.MyJmsModule.JmsResource.ForeignServer.MyForeignServer.ConnectionURL'] \ = 't3://my.other.cluster:7001' expected['JMSSystemResource.MyJmsModule.JmsResource.ForeignServer.MyForeignServer.' - 'ForeignDestination.MyRemoteQ.LocalJNDIName'] = 'jms.remoteQ' + 'ForeignDestination.MyRemoteQ.LocalJNDIName'] = 'jms/remoteQ' replacement_dict = dict() replacement_dict['Server.ListenPort'] = dict() replacement_dict['JMSSystemResource.JmsResource.ForeignServer.ConnectionURL'] = dict() @@ -64,13 +65,6 @@ def testInvalidAttributeName(self): actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) - def testInvalidSection(self): - expected = dict() - replacement_dict = dict() - replacement_dict['Server.ListenAddress'] = dict() - actual = self._helper.inject_variables(replacement_dict) - self._compare_to_expected_dictionary(expected, actual) - def testDomainAttributeReplacementAndModel(self): expected = dict() expected['Notes'] = 'Test note replacement' @@ -89,14 +83,14 @@ def testWithSegment(self): '1521' replacement_dict = dict() replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'] = dict() - replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][ - variable_injector.REGEXP] = '(?<=PORT=)[\w.-]+(?=\)))' - replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][variable_injector.SUFFIX] = 'Port' - - replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'] = dict() - replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][ - variable_injector.REGEXP] = '(?<=HOST=)[\w.-]+(?=\)))' - replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][variable_injector.SUFFIX] = 'Host' + list_entry1 = dict() + list_entry1[variable_injector.REGEXP_PATTERN] = '(?<=PORT=)[\w.-]+(?=\))' + list_entry1[variable_injector.REGEXP_SUFFIX] = 'Port' + list_entry2 = dict() + list_entry2[variable_injector.REGEXP_PATTERN] = '(?<=HOST=)[\w.-]+(?=\))' + list_entry2[variable_injector.REGEXP_SUFFIX] = 'Host' + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.URL'][variable_injector.REGEXP] = [ + list_entry1, list_entry2] actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) db2 = 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)' \ @@ -117,11 +111,13 @@ def testWithSegmentInDictionary(self): expected['MailSession.MyMailSession.Properties--ImapHost'] = 'stbeehive.oracle.com' replacement_dict = dict() replacement_dict['MailSession.Properties'] = dict() - replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = 'mail.smtp.host' - replacement_dict['MailSession.Properties'][variable_injector.SUFFIX] = 'SmtpHost' - replacement_dict['MailSession.Properties'] = dict() - replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = 'mail.imap.host' - replacement_dict['MailSession.Properties'][variable_injector.SUFFIX] = 'ImapHost' + list_entry1 = dict() + list_entry1[variable_injector.REGEXP_PATTERN] = 'mail.smtp.host' + list_entry1[variable_injector.REGEXP_SUFFIX] = 'SmtpHost' + list_entry2 = dict() + list_entry2[variable_injector.REGEXP_PATTERN] = 'mail.imap.host' + list_entry2[variable_injector.REGEXP_SUFFIX] = 'ImapHost' + replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = [list_entry1, list_entry2] actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:MailSession.MyMailSession.Properties--SmtpHost@@', @@ -135,14 +131,17 @@ def testWithSegmentInDictionaryAndAPattern(self): expected['MailSession.MailSession-0.Properties--Host'] = 'stbeehive.oracle.com' replacement_dict = dict() replacement_dict['MailSession.Properties'] = dict() - replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = '(?<=\w.)host]' + list_entry = dict() + list_entry[variable_injector.REGEXP_PATTERN] = '(?<=\w.)host' + list_entry[variable_injector.REGEXP_SUFFIX] = 'Host' + replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = [list_entry] actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) self.assertEqual('@@PROP:MailSession.MyMailSession.Properties--Host@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.imap.host']) - self.assertEqual('@@PROP:MailSession.MyMailSession.PropertiesHost@@', + self.assertEqual('@@PROP:MailSession.MyMailSession.Properties--Host@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.host']) - self.assertEqual('@@PROP:MailSession.MyMailSession.PropertiesHost@@', + self.assertEqual('@@PROP:MailSession.MyMailSession.Properties--Host@@', self._model['resources']['MailSession']['MyMailSession']['Properties']['mail.smtp.host']) def testWithSegmentInList(self): @@ -151,8 +150,10 @@ def testWithSegmentInList(self): 'runtime.ServerRuntimeMBean.HarvestedAttribute'] = 'OracleHome' replacement_dict = dict() replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute'] = dict() + list_entry = dict() + list_entry[variable_injector.REGEXP_PATTERN] = 'OracleHome' replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedAttribute'][ - variable_injector.REGEXP] = 'OracleHome' + variable_injector.REGEXP] = [list_entry] actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ @@ -168,13 +169,14 @@ def testWithSegmentInList(self): def testWithSegmentInStringInList(self): expected = dict() expected['WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.weblogic.management.' - 'runtime.ServerRuntimeMBean.HarvestedInstanceManagedServer'] = 'm1' + 'runtime.ServerRuntimeMBean.HarvestedInstance--ManagedServer'] = 'm1' replacement_dict = dict() replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance'] = dict() + list_entry = dict() + list_entry[variable_injector.REGEXP_PATTERN] = 'm1' + list_entry[variable_injector.REGEXP_SUFFIX] = 'ManagedServer' replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance'][ - variable_injector.REGEXP] = 'm1' - replacement_dict['WLDFSystemResource.WLDFResource.Harvester.HarvestedType.HarvestedInstance'][ - variable_injector.SUFFIX] = 'ManagedServer' + variable_injector.REGEXP] = [list_entry] actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) list = \ @@ -183,7 +185,7 @@ def testWithSegmentInStringInList(self): found = False for entry in list: if entry == 'com.bea:Name=@@PROP:WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.' \ - 'weblogic.management.runtime.ServerRuntimeMBean.HarvestedInstanceManagedServer@@' \ + 'weblogic.management.runtime.ServerRuntimeMBean.HarvestedInstance--ManagedServer@@' \ ',Type=ServerRuntime': found = True break @@ -213,7 +215,9 @@ def testWithVariableHelperKeywords(self): expected['Machine.machine1.NodeManager.PasswordEncrypted'] = '--FIX ME--' expected['Machine.machine1.NodeManager.UserName'] = 'admin' inserted, model, variable_file_name = self._helper.inject_variables_keyword_file( - variable_helper_path_name=self._resources_dir, variable_helper_file_name=self._variable_helper_keyword) + variable_injector_path_name=self._resources_dir, + variable_injector_file_name=self._variable_injector_keyword, + variable_keywords_path_name=self._resources_dir, variable_keywords_file_name=self._keywords_file) self.assertEqual(True, inserted) self.assertEqual(self._variable_file, variable_file_name) actual = variables.load_variables(self._variable_file) diff --git a/core/src/test/resources/custom.json b/core/src/test/resources/custom.json index d36362f3e3..742836bce9 100644 --- a/core/src/test/resources/custom.json +++ b/core/src/test/resources/custom.json @@ -1,4 +1,4 @@ { - Notes: {}, - Application.SecurityDDModel: {} + "Notes": {}, + "Application.SecurityDDModel": {} } \ No newline at end of file diff --git a/core/src/test/resources/keywords.json b/core/src/test/resources/keywords.json new file mode 100644 index 0000000000..f4d86c0ceb --- /dev/null +++ b/core/src/test/resources/keywords.json @@ -0,0 +1,6 @@ +{ + "PORT": "port.json", + "HOST": "host.json", + "URL": "url.json", + "CREDENTIALS": "credentials.json" +} \ No newline at end of file diff --git a/core/src/test/resources/port.json b/core/src/test/resources/port.json index 9d89ef5024..c5b6d0355f 100644 --- a/core/src/test/resources/port.json +++ b/core/src/test/resources/port.json @@ -1,4 +1,4 @@ { - Server.ListenPort: {}, - Machine.NodeManager.ListenPort: {} + "Server.ListenPort": {}, + "Machine.NodeManager.ListenPort": {} } diff --git a/core/src/test/resources/url.json b/core/src/test/resources/url.json index d604bdca8e..9159c1ace3 100644 --- a/core/src/test/resources/url.json +++ b/core/src/test/resources/url.json @@ -1,3 +1,3 @@ { - JMSSystemResource.JmsResource.ForeignServer.ConnectionURL: {} + "JMSSystemResource.JmsResource.ForeignServer.ConnectionURL": {} } From 087a4e6743674ca8f78d3c61c83b17d53c06fa3f Mon Sep 17 00:00:00 2001 From: crountre Date: Fri, 18 May 2018 12:27:46 -0500 Subject: [PATCH 14/52] add name list check --- .../wlsdeploy/tool/util/variable_injector.py | 119 ++++++++++-------- .../deploy/messages/wlsdeploy_rb.properties | 32 +++-- .../src/test/python/variable_injector_test.py | 31 +++-- .../test/resources/variable_insertion.yaml | 4 +- 4 files changed, 100 insertions(+), 86 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index e3d72af9a6..78c49093fb 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -43,10 +43,9 @@ VARIABLE_SEP = '.' SUFFIX_SEP = '--' -_variable_sep_pattern = re.compile('/') +_find_special_names_pattern = re.compile('[\[).+\]]') _wlsdeploy_location = os.environ.get('WLSDEPLOY_HOME') -_segment_pattern = re.compile("\\[[\w.-]+\\]$") _class_name = 'variable_file_helper' _logger = PlatformLogger('wlsdeploy.util') @@ -85,17 +84,16 @@ def inject_variables_keyword_file(self, **kwargs): keywords_dictionary = _load_keywords_file(variable_keywords_location_file) variables_inserted = False - return_model = dict() + return_model = self.__original variable_file_location = None if variables_injector_dictionary and keywords_dictionary: variable_file_location = _get_variable_file_name(variables_injector_dictionary, **kwargs) - injector_file_list = _create_injector_file_list(variables_injector_dictionary, keywords_dictionary, - _get_keyword_files_location(**kwargs)) - return_model = self.__original if not variable_file_location: _logger.warning('WLSDPLY-19420', variable_injector_location_file, class_name=_class_name, method_name=_method_name) else: + injector_file_list = _create_injector_file_list(variables_injector_dictionary, keywords_dictionary, + _get_keyword_files_location(**kwargs)) variables_file_dictionary = self.inject_variables_keyword_dictionary(injector_file_list) variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location) if variables_inserted: @@ -155,22 +153,27 @@ def __inject_variable(self, location, injector, injector_values): def _traverse_variables(model_section, mbean_list): if mbean_list: mbean = mbean_list.pop(0) - # mbean, mbean_name_list = _find_special_name(mbean) + mbean, mbean_name_list = _find_special_name(mbean) + _logger.finer('WLSDPLY-19423', mbean, location.get_folder_path(), class_name=_class_name, + method_name=_method_name) if mbean in model_section: _logger.finest('WLSDPLY-19414', mbean, class_name=_class_name, method_name=_method_name) next_model_section = model_section[mbean] location.append_location(mbean) name_token = self.__aliases.get_name_token(location) - # if not mbean_name_list and self.__aliases.supports_multiple_mbean_instances(location): - # mbean_name_list = next_model_section - # if mbean_name_list: - # for mbean_name in mbean_name_list: - if self.__aliases.supports_multiple_mbean_instances(location): - for mbean_name in next_model_section: - continue_mbean_list = copy.copy(mbean_list) - location.add_name_token(name_token, mbean_name) - _traverse_variables(next_model_section[mbean_name], continue_mbean_list) - location.remove_name_token(name_token) + if not mbean_name_list: + if self.__aliases.supports_multiple_mbean_instances(location): + mbean_name_list = next_model_section + else: + _logger.fine('WLSDPLY-19406', mbean_name_list, attribute, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) + if mbean_name_list: + for mbean_name in mbean_name_list: + if mbean_name in next_model_section: + continue_mbean_list = copy.copy(mbean_list) + location.add_name_token(name_token, mbean_name) + _traverse_variables(next_model_section[mbean_name], continue_mbean_list) + location.remove_name_token(name_token) else: _traverse_variables(next_model_section, mbean_list) location.pop_location() @@ -179,8 +182,7 @@ def _traverse_variables(model_section, mbean_list): return False else: if attribute in model_section: - returned_dict = self._variable_info(model_section, attribute, location, injector, - injector_values) + returned_dict = self._variable_info(model_section, attribute, location, injector_values) if returned_dict: variable_dict.update(returned_dict) else: @@ -191,8 +193,9 @@ def _traverse_variables(model_section, mbean_list): section = self.__model if start_mbean_list: # Find out in what section is the mbean top folder so can move to that section in the model + mbean, __ = _find_special_name(start_mbean_list[0]) for entry in self.__section_keys: - if entry in self.__model and start_mbean_list[0] in self.__model[entry]: + if entry in self.__model and mbean in self.__model[entry]: section = self.__model[entry] break else: @@ -223,7 +226,7 @@ def __format_variable_name_segment(self, location, attribute, suffix): return path + SUFFIX_SEP + suffix return path - def _variable_info(self, model, attribute, location, injector, injector_values): + def _variable_info(self, model, attribute, location, injector_values): # add code here to put in model if force in injector values if REGEXP in injector_values: return self._process_regexp(model, attribute, location, injector_values[REGEXP]) @@ -240,8 +243,10 @@ def _process_attribute(self, model, attribute, location): attribute_value = model[attribute] if not _already_property(attribute_value): variable_name = self.__format_variable_name(location, attribute) - variable_value = str(attribute_value) + variable_value = _format_variable_value(attribute_value) model[attribute] = _format_as_property(variable_name) + _logger.fine('WLSDPLY-19425', variable_name, attribute_value, attribute, variable_value, + class_name=_class_name, method_name=_method_name) else: _logger.finer('WLSDPLY-19426', attribute_value, attribute, str(location), class_name=_class_name, method_name=_method_name) @@ -267,7 +272,7 @@ def _process_patterns_string(self, model, attribute, location, regexp_list): pattern = dictionary[REGEXP_PATTERN] if REGEXP_SUFFIX in dictionary: suffix = dictionary[REGEXP_SUFFIX] - variable_name, variable_value = self._process_pattern_string(model, attribute, location, pattern, suffix) + variable_name, variable_value = self._process_pattern_string(model, attribute, location, pattern, suffix) if variable_value: variable_dict[variable_name] = variable_value return variable_dict @@ -279,8 +284,8 @@ def _process_pattern_string(self, model, attribute, location, pattern, suffix): attribute_value, variable_name, variable_value = self._find_segment_in_string(attribute, model[attribute], location, pattern, suffix) if variable_value: - _logger.finer('WLSDPLY-19429', attribute, attribute_value, class_name=_class_name, - method_name=_method_name) + _logger.finer('WLSDPLY-19429', variable_name, attribute_value, attribute, variable_value, + class_name=_class_name, method_name=_method_name) model[attribute] = attribute_value # elif replace_if_nosegment: # check_value = model[attribute] @@ -302,7 +307,7 @@ def _find_segment_in_string(self, attribute, attribute_value, location, pattern, variable_value = None if not _already_property(attribute_value): variable_name = self.__format_variable_name_segment(location, attribute, suffix) - attribute_value, variable_value = _replace_segment(pattern, str(attribute_value), + attribute_value, variable_value = _replace_segment(pattern, _format_variable_value(attribute_value), _format_as_property(variable_name)) return attribute_value, variable_name, variable_value @@ -332,7 +337,7 @@ def _process_pattern_list(self, attribute_name, attribute_list, location, patter attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, location, pattern, suffix) if seg_var_value: - _logger.finer('WLSDPLY-19429', attribute_name, attribute_value, class_name=_class_name, + _logger.finer('WLSDPLY-19428', variable_name, attribute_name, variable_value, class_name=_class_name, method_name=_method_name) attribute_list[idx] = attribute_value variable_name = seg_var_name @@ -373,7 +378,7 @@ def _process_pattern_dictionary(self, attribute_name, attribute_dict, location, if matcher: _logger.finer('WLSDPLY-19427', attribute_name, replacement, class_name=_class_name, method_name=_method_name) - variable_value = str(attribute_dict[entry]) + variable_value = _format_variable_value(attribute_dict[entry]) attribute_dict[entry] = replacement # don't break, continue replacing any in dictionary, return the last variable value found _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) @@ -450,9 +455,9 @@ def _load_keywords_file(variable_keywords_location): if os.path.isfile(variable_keywords_location): try: keywords_dictionary = FileToPython(variable_keywords_location).parse() - _logger.fine('WLSDPLY-19432', variable_keywords_location, class_name=_class_name, method_name=_method_name) + _logger.finer('WLSDPLY-19404', variable_keywords_location, class_name=_class_name, method_name=_method_name) except IllegalArgumentException, ia: - _logger.warning('WLSDPLY-19433', variable_keywords_location, ia.getLocalizedMessage(), + _logger.warning('WLSDPLY-19405', variable_keywords_location, ia.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=keywords_dictionary) @@ -464,13 +469,12 @@ def _create_injector_file_list(variables_dictionary, keyword_dictionary, injecto injector_file_list = [] if CUSTOM_KEYWORD in variables_dictionary: if KEYWORD_FILES in variables_dictionary[CUSTOM_KEYWORD]: - file_list = variables_dictionary[CUSTOM_KEYWORD][KEYWORD_FILES] - if type(file_list) != list: - file_list = file_list.split(',') - for filename in file_list: - injector_file_list.append(filename) + injector_file_list = variables_dictionary[CUSTOM_KEYWORD][KEYWORD_FILES] + if type(injector_file_list) != list: + injector_file_list = injector_file_list.split(',') + _logger.fine('WLSDPLY-19401', injector_file_list) else: - _logger.info('WLSDPLY-19434', class_name=_class_name, method_name=_method_name) + _logger.info('WLSDPLY-19412', class_name=_class_name, method_name=_method_name) del variables_dictionary[CUSTOM_KEYWORD] for keyword in variables_dictionary: if keyword in keyword_dictionary: @@ -509,6 +513,30 @@ def _load_injector_file(injector_file_name): return injector_dictionary +def _write_variables_file(variables_dictionary, variables_file_name): + _method_name = '_write_variables_file' + _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) + written = False + if variables_dictionary: + try: + variables.write_variables(variables_dictionary, variables_file_name) + written = True + except VariableException, ve: + _logger.warning('WLSDPLY-19407', variables_file_name, ve.getLocalizedMessage(), class_name=_class_name, + method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) + return written + + +def _format_variable_value(value): + if type(value) == bool: + if value: + return 'True' + return 'False' + else: + return str(value) + + def _replace_segment(regexp, variable_value, attribute_value): replaced_value = None replacement_string = variable_value @@ -526,7 +554,7 @@ def _compile_pattern(pattern): try: return re.compile(pattern) except Exception, e: - _logger.warning('WLSDPLY-19435', pattern, e) + _logger.warning('WLSDPLY-19411', pattern, e) return None @@ -554,23 +582,8 @@ def _split_injector(injector_path): def _find_special_name(mbean): mbean_name = mbean mbean_name_list = [] - name_list = re.split('[\{).+\}]', mbean) + name_list = _find_special_names_pattern.split(mbean) if name_list and len(name_list) > 1: mbean_name = name_list[0] mbean_name_list = name_list[1].split(',') return mbean_name, mbean_name_list - - -def _write_variables_file(variables_dictionary, variables_file_name): - _method_name = '_write_variables_file' - _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) - written = False - if variables_dictionary: - try: - variables.write_variables(variables_dictionary, variables_file_name) - written = True - except VariableException, ve: - _logger.warning('WLSDPLY-19407', variables_file_name, ve.getLocalizedMessage(), class_name=_class_name, - method_name=_method_name) - _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) - return written diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 37af5c685f..1cfd96d933 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1100,20 +1100,20 @@ WLSDPLY-19307=Unable to extract classpath libraries from archive file {0} to dom # wlsdeploy/tool/util/variable_injector.py WLSDPLY-19400=Model variable injector values loaded from location {0} -WLSDPLY-19401=Variable replacement using custom list loaded from variable helper file {0} +WLSDPLY-19401=Variable replacement using custom file list {0} WLSDPLY-19402=Unable to read and parse variables helper file {0} : {1} WLSDPLY-19403=Ignoring unknown keyword {0} found in model variable injector file -WLSDPLY-19404=An invalid variable file location {0} was provided to variable helper. Variable replacement \ - will not continue -WLSDPLY-19405=Unable to open the variable file {0} for writing : {1} -WLSDPLY-19406=Invalid attribute part {0} in path {1) for section {2} +WLSDPLY-19404=Variable keywords file loaded from location {0} +WLSDPLY-19405=Unable to read and parse variable keywords file {0} : {1} +WLSDPLY-19406=Selecting only the entries from the specified mbean name list {0} for attribute {1} at location {2} WLSDPLY-19407=Exception attempting to write to variables file {0} - discarding model changes with variable insertions \ : {1} WLSDPLY-19408=Adding file {0} to injector file list for keyword {1} WLSDPLY-19409=Unable to read and load values for property file {0} : {1} WLSDPLY-19410=Invalid property file name {0} -WLSDPLY-19411=Adding property {0} for model variable replacement from file {1} -WLSDPLY-19412=Loading keywords from variable helper file {0} +WLSDPLY-19411=Invalid Regular expression pattern {0} : {1} +WLSDPLY-19412=CUSTOM keyword was found in variables injector list but no files were included. Skipping the CUSTOM \ + keyword WLSDPLY-19413=Variables were located and inserted in the model using injector file {0} WLSDPLY-19414=Located mbean {0} in the model file WLSDPLY-19415=Invalid mbean {0} found in replacement property {1} at location {2} @@ -1121,24 +1121,20 @@ WLSDPLY-19416=MBean {0} not found in the model for replacement property {1} at l WLSDPLY-19417=Attribute {0} not found in the model for replacement property {1} at location {2} WLSDPLY-19418=Variables were inserted into the model and written to the variables file {0} WLSDPLY-19419=No variables were inserted into the model during variable replacement -WLSDPLY-19420=Variable helper file was found at location {0} but no variable properties file name was provided +WLSDPLY-19420=Variable injector file was found at location {0} but no variable properties file name was provided WLSDPLY-19421=Variables property file name {0} extracted from model helper json file WLSDPLY-19422=Variables property file name {0} was passed as an argument to the variable helper -WLSDPLY-19423=Variables helper did not find section {0} in the model and will bypass replacement for {1} +WLSDPLY-19423=MBean {0} from injector path represents a named MBean folder at location {1} WLSDPLY-19424=Segment {0} for attribute {1} not found in model value {2} at location {3} -WLSDPLY-19425=Variable {0} inserted into the model for attribute {1} and variable name {2} and value \ +WLSDPLY-19425=Variable {0} inserted into the model {1} for attribute {2} and value \ {3} inserted into variable properties WLSDPLY-19426=Variable {0} was already inserted into model for attribute {1} at location {2} WLSDPLY-19427=Segment found in dictionary for attribute {0}; inserting variable replacement {1} into dictionary value -WLSDPLY-19428=Segment found in list for attribute {0} has been replaced by variable {1} -WLSDPLY-19429=Segment was found in attribute {0} and variable has been inserted into string {1} +WLSDPLY-19428=Variable {0} was inserted into the model list for attribute {1} and value {2} has been inserted \ + into the variable file +WLSDPLY-19429=Variable {0} was inserted into the model string {1} for attribute {2} and value {3} has been \ + inserted into the variable file WLSDPLY-19430=Segment was not found in attribute {0} value so entire variable value was replaced by {1} -WLSDPLY-19431=MBean {0} from injector path represents a named MBean folder at location {1} -WLSDPLY-19432=Variable keywords file loaded from location {0} -WLSDPLY-19433=Unable to read and parse variable keywords file {0} : {1} -WLSDPLY-19434=CUSTOM keyword was found in variables injector list but no files were included. Skipping the CUSTOM \ - keyword -WLSDPLY-19435=Invalid Regular expression pattern {0} : {1} # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index b102f62553..0c8559abda 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -190,19 +190,24 @@ def testWithSegmentInStringInList(self): found = True break self.assertEqual(True, found) - # - # def testWithNameInMBeanSingle(self): - # expected = dict() - # expected['Server.m2.ServerStart.Arguments'] = '/etc' - # replacement_list = ['topology:Server{m2}.ServerStart.Arguments[(?<=-Doracle.net.tns_admin=)[\w\\/._:]+]'] - # actual = self._helper.inject_variables(replacement_list) - # self._compare_to_expected_dictionary(expected, actual) - # arg = '-Doracle.net.tns_admin=@@PROP:Server.m2.ServerStart.Arguments@@ ' \ - # '-DANTLR_USE_DIRECT_CLASS_LOADING=true ' \ - # '-DANTLR_USE_DIRECT_CLASS_LOADING=true -Djava.awt.headless=true -Dhttp.webdir.enable=false ' \ - # '-Duser.timezone=Europe/Zurich -Djava.net.preferIPv4Stack=true -Djava.security.egd=file:/dev/./urandom ' \ - # '-Dweblogic.data.canTransferAnyFile=true' - # self.assertEqual(arg, self._model['topology']['Server']['m2']['ServerStart']['Arguments']) + + def testWithMBeanName(self): + expected = dict() + expected['JDBCSystemResource.Database2.JdbcResource.JDBCDriverParams.Properties.user.Value'] = 'sys as dba' + expected['JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.Properties.user.Value'] = 'admin' + replacement_dict = dict() + replacement_dict['JDBCSystemResource.JdbcResource.JDBCDriverParams.Properties[user].Value'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testWithListMBeanName(self): + expected = dict() + expected['Server.m1.SSL.Enabled'] = 'True' + expected['Server.m2.SSL.Enabled'] = 'True' + replacement_dict = dict() + replacement_dict['Server[m1,m2].SSL.Enabled'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) def testWithVariableHelperKeywords(self): expected = dict() diff --git a/core/src/test/resources/variable_insertion.yaml b/core/src/test/resources/variable_insertion.yaml index 71e5bfcb89..378b602fdb 100644 --- a/core/src/test/resources/variable_insertion.yaml +++ b/core/src/test/resources/variable_insertion.yaml @@ -107,7 +107,7 @@ resources: oracle.jdbc.ReadTimeout: Value: 30000 user: - Value: jshum + Value: admin oracle.net.CONNECT_TIMEOUT: Value: 5000 URL: 'jdbc:oracle:thin:@//den00chv.us.oracle.com:1521/PDBORCL' @@ -122,7 +122,7 @@ resources: oracle.jdbc.ReadTimeout: Value: 30000 user: - Value: jshum + Value: sys as dba oracle.net.CONNECT_TIMEOUT: Value: 5000 URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=slc05til.us.oracle.com)(PORT=1521)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' From 13eedafe3e5bf8525197e9c43943c93cdd446b47 Mon Sep 17 00:00:00 2001 From: crountre Date: Mon, 21 May 2018 14:22:20 -0500 Subject: [PATCH 15/52] fixes for real testing --- core/src/main/python/discover.py | 13 +- .../wlsdeploy/tool/util/variable_injector.py | 69 ++++++--- .../deploy/messages/wlsdeploy_rb.properties | 7 +- .../src/test/python/variable_injector_test.py | 4 +- installer/src/assembly/zip.xml | 14 ++ installer/src/main/etc/credentials.properties | 27 ---- .../src/main/lib/injectors/credentials.json | 143 ++++++++++++++++++ .../injectors/host.json} | 0 .../injectors/port.json} | 0 .../url.properties => lib/injectors/url.json} | 0 installer/src/main/lib/variable_keywords.json | 6 + .../main/samples/model_variable_injector.json | 3 + 12 files changed, 222 insertions(+), 64 deletions(-) delete mode 100644 installer/src/main/etc/credentials.properties create mode 100644 installer/src/main/lib/injectors/credentials.json rename installer/src/main/{etc/host.properties => lib/injectors/host.json} (100%) rename installer/src/main/{etc/port.properties => lib/injectors/port.json} (100%) rename installer/src/main/{etc/url.properties => lib/injectors/url.json} (100%) create mode 100644 installer/src/main/lib/variable_keywords.json create mode 100644 installer/src/main/samples/model_variable_injector.json diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index d80e91728b..4736e4948c 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -30,7 +30,7 @@ from wlsdeploy.exception import exception_helper from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.util import getcreds -from wlsdeploy.tool.util.variable_injector import VariableFileHelper +from wlsdeploy.tool.util.variable_injector import VariableInjector from wlsdeploy.tool.discover import discoverer from wlsdeploy.tool.discover.deployments_discoverer import DeploymentsDiscoverer from wlsdeploy.tool.discover.domain_info_discoverer import DomainInfoDiscoverer @@ -370,14 +370,9 @@ def __check_and_customize_model(model, model_context): if filter_helper.apply_filters(model.get_model(), "discover"): __logger.info('WLSDPLY-06014', _class_name=_class_name, method_name=_method_name) - default_variable_file = __get_default_variable_file(model_context) - inserted, variable_model, variable_file_name = VariableFileHelper(model.get_model(), model_context, WebLogicHelper( - __logger).get_actual_weblogic_version()).inject_variables_keyword_file(variable_file_name=default_variable_file) - if default_variable_file: - default_variable_file = os.path.join(path_utils.get_pathname_from_path(model_context.get_archive_file_name()), - default_variable_file + '.properties') - inserted, variable_model, variable_file_name = VariableFileHelper(model.get_model(), model_context, WebLogicHelper( - __logger).get_actual_weblogic_version()).inject_variables_keyword_file(variable_file_name=default_variable_file) + inserted, variable_model, variable_file_name = VariableInjector(model.get_model(), model_context, WebLogicHelper( + __logger).get_actual_weblogic_version()).inject_variables_keyword_file( + variable_file_name=__get_default_variable_file(model_context)) if inserted: model = Model(variable_model) try: diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 78c49093fb..f111289d36 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -9,7 +9,8 @@ import java.lang.IllegalArgumentException as IllegalArgumentException import oracle.weblogic.deploy.aliases.AliasException as AliasException -import oracle.weblogic.deploy.json.JsonException +import oracle.weblogic.deploy.json.JsonException as JsonException +import oracle.weblogic.deploy.util.PyOrderedDict as PyOrderedDict import oracle.weblogic.deploy.util.VariableException as VariableException import wlsdeploy.util.model as model_sections @@ -20,7 +21,6 @@ from wlsdeploy.aliases.validation_codes import ValidationCodes from wlsdeploy.json.json_translator import JsonToPython from wlsdeploy.logging.platform_logger import PlatformLogger -from wlsdeploy.util.model_translator import FileToPython VARIABLE_INJECTOR_FILE_NAME = 'model_variable_injector.json' VARIABLE_KEYWORDS_FILE_NAME = 'variable_keywords.json' @@ -28,6 +28,7 @@ VARIABLE_KEYWORDS_PATH_NAME_ARG = 'variable_keywords_path_name' VARIABLE_INJECTOR_FILE_NAME_ARG = 'variable_injector_file_name' VARIABLE_KEYWORDS_FILE_NAME_ARG = 'variable_keywords_file_name' +VARIABLE_INJECTOR_FILES_PATH_ARG = 'variable_injector_files_path_name' VARIABLE_FILE_NAME_ARG = 'variable_file_name' VARIABLE_FILE_NAME = 'variables.json' # custom keyword in model injector file @@ -35,6 +36,7 @@ KEYWORD_FILES = 'file-list' # location for model injector file, keyword file and injector files DEFAULT_FILE_LOCATION = 'lib' +INJECTORS_LOCATION = 'injectors' # should these injector json keywords be included in the keyword file REGEXP = 'regexp' REGEXP_SUFFIX = 'suffix' @@ -44,13 +46,16 @@ VARIABLE_SEP = '.' SUFFIX_SEP = '--' _find_special_names_pattern = re.compile('[\[).+\]]') +_fake_name_marker = 'fakename' +_fake_name_replacement = re.compile('.' + _fake_name_marker) +_white_space_replacement = re.compile('\s') _wlsdeploy_location = os.environ.get('WLSDEPLOY_HOME') -_class_name = 'variable_file_helper' +_class_name = 'variable_injector' _logger = PlatformLogger('wlsdeploy.util') -class VariableFileHelper(object): +class VariableInjector(object): def __init__(self, model, model_context=None, version=None): self.__original = copy.deepcopy(model) @@ -115,7 +120,7 @@ def inject_variables_keyword_dictionary(self, injector_file_list): """ _method_name = 'inject_variables_keyword_dictionary' _logger.entering(injector_file_list, class_name=_class_name, method_name=_method_name) - variables_dictionary = dict() + variables_dictionary = PyOrderedDict() for filename in injector_file_list: injector_dictionary = _load_injector_file(filename) entries = self.inject_variables(injector_dictionary) @@ -136,7 +141,7 @@ def inject_variables(self, injector_dictionary): if injector_dictionary: location = LocationContext() domain_token = self.__aliases.get_name_token(location) - location.add_name_token(domain_token, 'fakedomain') + location.add_name_token(domain_token, _fake_name_marker) for injector, injector_values in injector_dictionary.iteritems(): entries_dict = self.__inject_variable(location, injector, injector_values) if len(entries_dict) > 0: @@ -164,6 +169,8 @@ def _traverse_variables(model_section, mbean_list): if not mbean_name_list: if self.__aliases.supports_multiple_mbean_instances(location): mbean_name_list = next_model_section + else: + self._check_name_token(location, name_token) else: _logger.fine('WLSDPLY-19406', mbean_name_list, attribute, location.get_folder_path(), class_name=_class_name, method_name=_method_name) @@ -195,7 +202,7 @@ def _traverse_variables(model_section, mbean_list): # Find out in what section is the mbean top folder so can move to that section in the model mbean, __ = _find_special_name(start_mbean_list[0]) for entry in self.__section_keys: - if entry in self.__model and mbean in self.__model[entry]: + if entry in self.__model and mbean in self.__model[entry]: section = self.__model[entry] break else: @@ -211,14 +218,17 @@ def _traverse_variables(model_section, mbean_list): def __format_variable_name(self, location, attribute): variable_name = attribute - make_path = self.__aliases.get_model_folder_path(location) + make_path = None + try: + make_path = self.__aliases.get_model_folder_path(location) + except AliasException, ae: + _logger.warning('WLSDPLY-19431', str(location), attribute, ae.getLocalizedMessage()) if make_path: make_path = make_path.split(':') if len(make_path) > 1 and len(make_path[1]) > 1: variable_name = make_path[1] variable_name = variable_name[1:] + VARIABLE_SEP + attribute - variable_name = variable_name.replace('/', '.') - return variable_name + return _massage_name(variable_name) def __format_variable_name_segment(self, location, attribute, suffix): path = self.__format_variable_name(location, attribute) @@ -384,6 +394,10 @@ def _process_pattern_dictionary(self, attribute_name, attribute_dict, location, _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value + def _check_name_token(self, location, name_token): + if self.__aliases.requires_unpredictable_single_name_handling(location): + location.add_name_token(name_token, _fake_name_marker) + def _log_mbean_not_found(self, mbean, replacement, location): _method_name = '_log_mbean_not_found' code = ValidationCodes.INVALID @@ -404,10 +418,10 @@ def _get_variable_file_name(variables_injector_dictionary, **kwargs): if VARIABLE_FILE_NAME_ARG in variables_injector_dictionary: variable_file_location = variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] del variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] - _logger.finer('WLSDPLY-19422', variable_file_location) + _logger.finer('WLSDPLY-19421', variable_file_location) elif VARIABLE_FILE_NAME_ARG in kwargs: variable_file_location = kwargs[VARIABLE_FILE_NAME_ARG] - _logger.finer('WLSDPLY-19421', variable_file_location) + _logger.finer('WLSDPLY-19422', variable_file_location) else: variable_file_location = None return variable_file_location @@ -434,30 +448,31 @@ def _get_variable_keywords_file_name(**kwargs): def _load_variables_file(variable_injector_location): - _method_name = '_load_variables_dictionary' + _method_name = '_load_variables_file' _logger.entering(variable_injector_location, class_name=_class_name, method_name=_method_name) variables_dictionary = None if os.path.isfile(variable_injector_location): try: - variables_dictionary = FileToPython(variable_injector_location).parse() + variables_dictionary = JsonToPython(variable_injector_location).parse() _logger.fine('WLSDPLY-19400', variable_injector_location, class_name=_class_name, method_name=_method_name) - except IllegalArgumentException, ia: - _logger.warning('WLSDPLY-19402', variable_injector_location, ia.getLocalizedMessage(), + except (IllegalArgumentException, JsonException), e: + _logger.warning('WLSDPLY-19402', variable_injector_location, e.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_dictionary) return variables_dictionary def _load_keywords_file(variable_keywords_location): - _method_name = '_load_keywords_dictionary' + _method_name = '_load_keywords_file' _logger.entering(variable_keywords_location, class_name=_class_name, method_name=_method_name) keywords_dictionary = None if os.path.isfile(variable_keywords_location): try: - keywords_dictionary = FileToPython(variable_keywords_location).parse() + keywords_dictionary = JsonToPython(variable_keywords_location).parse() _logger.finer('WLSDPLY-19404', variable_keywords_location, class_name=_class_name, method_name=_method_name) - except IllegalArgumentException, ia: - _logger.warning('WLSDPLY-19405', variable_keywords_location, ia.getLocalizedMessage(), + except (IllegalArgumentException, JsonException), e: + _logger.warning('WLSDPLY-19405', variable_keywords_location, e.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=keywords_dictionary) @@ -493,7 +508,7 @@ def _get_keyword_files_location(**kwargs): if VARIABLE_INJECTOR_PATH_NAME_ARG in kwargs: return kwargs[VARIABLE_INJECTOR_PATH_NAME_ARG] else: - return _wlsdeploy_location + return os.path.join(_wlsdeploy_location, DEFAULT_FILE_LOCATION, INJECTORS_LOCATION) def _load_injector_file(injector_file_name): @@ -503,8 +518,8 @@ def _load_injector_file(injector_file_name): if os.path.isfile(injector_file_name): try: injector_dictionary = JsonToPython(injector_file_name).parse() - except oracle.weblogic.deploy.json, je: - _logger.warning('WLDPLY-19409', injector_file_name, je.getLocalizedMessage(), class_name=_class_name, + except (IllegalArgumentException, JsonException), e: + _logger.warning('WLDPLY-19409', injector_file_name, e.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) else: _logger.warning('WLSDPLY-19410', injector_file_name, class_name=_class_name, method_name=_method_name) @@ -537,6 +552,14 @@ def _format_variable_value(value): return str(value) +def _massage_name(variable_name): + if variable_name: + variable_name = variable_name.replace('/', '.') + variable_name = _white_space_replacement.sub('-', variable_name) + variable_name = _fake_name_replacement.sub('', variable_name) + return variable_name + + def _replace_segment(regexp, variable_value, attribute_value): replaced_value = None replacement_string = variable_value diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 1cfd96d933..041c588939 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1101,7 +1101,7 @@ WLSDPLY-19307=Unable to extract classpath libraries from archive file {0} to dom # wlsdeploy/tool/util/variable_injector.py WLSDPLY-19400=Model variable injector values loaded from location {0} WLSDPLY-19401=Variable replacement using custom file list {0} -WLSDPLY-19402=Unable to read and parse variables helper file {0} : {1} +WLSDPLY-19402=Unable to read and parse model variables injector json file {0} : {1} WLSDPLY-19403=Ignoring unknown keyword {0} found in model variable injector file WLSDPLY-19404=Variable keywords file loaded from location {0} WLSDPLY-19405=Unable to read and parse variable keywords file {0} : {1} @@ -1122,8 +1122,8 @@ WLSDPLY-19417=Attribute {0} not found in the model for replacement property {1} WLSDPLY-19418=Variables were inserted into the model and written to the variables file {0} WLSDPLY-19419=No variables were inserted into the model during variable replacement WLSDPLY-19420=Variable injector file was found at location {0} but no variable properties file name was provided -WLSDPLY-19421=Variables property file name {0} extracted from model helper json file -WLSDPLY-19422=Variables property file name {0} was passed as an argument to the variable helper +WLSDPLY-19421=Variables property file name {0} extracted from model injector json file +WLSDPLY-19422=Variables property file name {0} was passed as an argument to the variable injector WLSDPLY-19423=MBean {0} from injector path represents a named MBean folder at location {1} WLSDPLY-19424=Segment {0} for attribute {1} not found in model value {2} at location {3} WLSDPLY-19425=Variable {0} inserted into the model {1} for attribute {2} and value \ @@ -1135,6 +1135,7 @@ WLSDPLY-19428=Variable {0} was inserted into the model list for attribute {1} an WLSDPLY-19429=Variable {0} was inserted into the model string {1} for attribute {2} and value {3} has been \ inserted into the variable file WLSDPLY-19430=Segment was not found in attribute {0} value so entire variable value was replaced by {1} +WLSDPLY-19431=Invalid location {0} for variable injection into attribute {1} : {2} # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index 0c8559abda..ccf4fe38d4 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -6,7 +6,7 @@ import wlsdeploy.util.variables as variables import wlsdeploy.tool.util.variable_injector as variable_injector -from wlsdeploy.tool.util.variable_injector import VariableFileHelper +from wlsdeploy.tool.util.variable_injector import VariableInjector from wlsdeploy.util.model_translator import FileToPython @@ -21,7 +21,7 @@ class VariableFileHelperTest(unittest.TestCase): def setUp(self): self.name = VariableFileHelperTest self._model = FileToPython(self._model_file).parse() - self._helper = VariableFileHelper(self._model, None, '12.2.1.3') + self._helper = VariableInjector(self._model, None, '12.2.1.3') def testSingleVariableReplacement(self): replacement_dict = dict() diff --git a/installer/src/assembly/zip.xml b/installer/src/assembly/zip.xml index f101dcf350..acda1db1f8 100644 --- a/installer/src/assembly/zip.xml +++ b/installer/src/assembly/zip.xml @@ -54,6 +54,20 @@ unix + + src/main/lib + lib + 0750 + unix + + + + src/main/samples + samples + 0750 + unix + + .. . diff --git a/installer/src/main/etc/credentials.properties b/installer/src/main/etc/credentials.properties deleted file mode 100644 index 8a3ffbb6b6..0000000000 --- a/installer/src/main/etc/credentials.properties +++ /dev/null @@ -1,27 +0,0 @@ -resources:ForeignJNDIProvider.User -resources:ForeignJNDIProvider.PasswordEncrypted -resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.PasswordEncrypted -resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.Properties{user}.Value -resources:JDBCSystemResource.JdbcResource.JDBCOracleParams.OnsWalletPasswordEncrypted -resources:JMSBridgeDestination.UserName -resources:JMSBridgeDestination.UserPasswordEncrypted -resources:JMSSystemResource.JmsResource.ForeignServer.ForeignConnectionFactory.Username -resources:JMSSystemResource.JmsResource.ForeignServer.ForeignConnectionFactory.PasswordEncrypted -resources:JMSSystemResource.JmsResource.ForeignServer.JNDIPropertiesCredentialEncrypted -resources:JMSSystemResource.JmsResource.SAFRemoteContext.SAFLoginContext.PasswordEncrypted -resources:JMSSystemResource.JmsResource.SAFRemoteContext.SAFLoginContext.Username -topology:Machine.NodeManager.PasswordEncrypted -topology:Machine.NodeManager.UserName -topology:UnixMachine.NodeManager.PasswordEncrypted -topology:UnixMachine.NodeManager.UserName -resources:MailSession.SessionUsername -resources:MailSession.SessionPasswordEncrypted -resources:MailSession.Properties[`User`(?<=\w.)user] -resources:MailSession.Properties[`Password`(?<=\w.)password] -resources:Partition.ForeignJndiProviderOverride.User -resources:Partition.ForeignJndiProviderOverride.PasswordEncrypted -resources:Partition.JdbcSystemResourceOverride.PasswordEncrypted -resources:Partition.JdbcSystemResourceOverride.JdbcPropertyOverride{user}.Value -resources:Partition.JmsSystemResourceOverride.ForeignServer.ForeignConnectionFactory.Username -resources:Partition.JmsSystemResourceOverride.ForeignServer.ForeignConnectionFactory.PasswordEncrypted -resources:JmsSystemResourceOverride.ForeignServer.JndiPropertiesCredentialEncrypted diff --git a/installer/src/main/lib/injectors/credentials.json b/installer/src/main/lib/injectors/credentials.json new file mode 100644 index 0000000000..5772cc3821 --- /dev/null +++ b/installer/src/main/lib/injectors/credentials.json @@ -0,0 +1,143 @@ +{ + "ForeignJNDIProvider.User": {}, + "ForeignJNDIProvider.PasswordEncrypted": {}, + "JDBCSystemResource.JdbcResource.JDBCDriverParams.PasswordEncrypted": {}, + "JDBCSystemResource.JdbcResource.JDBCDriverParams.Properties[user].Value": {}, + "JDBCSystemResource.JdbcResource.JDBCOracleParams.OnsWalletPasswordEncrypted": {}, + "JMSBridgeDestination.UserName": {}, + "JMSBridgeDestination.UserPasswordEncrypted": {}, + "JMSSystemResource.JmsResource.ForeignServer.ForeignConnectionFactory.Username": {}, + "JMSSystemResource.JmsResource.ForeignServer.ForeignConnectionFactory.PasswordEncrypted": {}, + "JMSSystemResource.JmsResource.ForeignServer.JNDIPropertiesCredentialEncrypted": {}, + "JMSSystemResource.JmsResource.SAFRemoteContext.SAFLoginContext.PasswordEncrypted": {}, + "JMSSystemResource.JmsResource.SAFRemoteContext.SAFLoginContext.Username": {}, + "Machine.NodeManager.PasswordEncrypted": {}, + "Machine.NodeManager.UserName": {}, + "MailSession.SessionUsername": {}, + "MailSession.SessionPasswordEncrypted": {}, + "MailSession.Properties": { + "regexp": [ + { + "pattern": "(?<=\\w.)imap.user", + "suffix": "imap.user" + }, + { + "pattern": "(?<=\\w.)imap.password", + "suffix": "imap.password" + }, + { + "pattern": "(?<=\\w.)pop3.user", + "suffix": "pop3.user" + }, + { + "pattern": "(?<=\\w.)pop3.password", + "suffix": "pop3.password" + }, + { + "pattern": "(?<=\\w.)smtp.user", + "suffix": "smtp.user" + }, + { + "pattern": "(?<=\\w.)smtp.password", + "suffix": "smtp.password" + } + ] + }, + "Partition.ForeignJndiProviderOverride.User": {}, + "Partition.ForeignJndiProviderOverride.PasswordEncrypted": {}, + "Partition.JdbcSystemResourceOverride.PasswordEncrypted": {}, + "Partition.JdbcSystemResourceOverride.JdbcPropertyOverride[user].Value": {}, + "Partition.JmsSystemResourceOverride.ForeignServer.ForeignConnectionFactory.Username": {}, + "Partition.JmsSystemResourceOverride.ForeignServer.ForeignConnectionFactory.PasswordEncrypted": {}, + "Partition.JmsSystemResourceOverride.ForeignServer.JndiPropertiesCredentialEncrypted": {}, + "Partition.MailSessionOverride.SessionUsername": {}, + "Partition.MailSessionOverride.SessionPasswordEncrypted": {}, + "Security.User.Password": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.ActiveDirectoryAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.IPlanetAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.LDAPAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.LDAPX509IdentityAsserter.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.NovellAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OpenLDAPAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleInternetDirectoryAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleUnifiedDirectoryAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleVirtualDirectoryAuthenticator.CredentialEncrypted": {}, + "SecurityConfiguration.Realm.CredentialMapper.PKICredentialMapper.KeyStorePassPhraseEncrypted": {}, + "SecurityConfiguration.Realm.CredentialMapper.SAML2CredentialMapper.SigningKeyPassPhraseEncrypted": {}, + "SecurityConfiguration.Realm.CredentialMapper.SAMLCredentialMapperV2.SigningKeyPassPhraseEncrypted": {}, + "SecurityConfiguration.Realm.RDBMSSecurityStore.JNDIUsername": {}, + "SecurityConfiguration.Realm.RDBMSSecurityStore.JNDIPasswordEncrypted": {}, + "SecurityConfiguration.Realm.RDBMSSecurityStore.Username": {}, + "SecurityConfiguration.Realm.RDBMSSecurityStore.PasswordEncrypted": {}, + "SecurityConfiguration.CredentialEncrypted": {}, + "SecurityConfiguration.NodeManagerUsername": {}, + "SecurityConfiguration.NodeManagerPasswordEncrypted": {}, + "Server.FederationServices.SigningKeyAlias": {}, + "Server.FederationServices.SigningKeyPassPhraseEncrypted": {}, + "Server.FederationServices.SslClientIdentityAlias": {}, + "Server.FederationServices.SslClientIdentityPassPhraseEncrypted": {}, + "Server.NetworkAccessPoint.CustomIdentityKeyStorePassPhraseEncrypted": {}, + "Server.NetworkAccessPoint.CustomPrivateKeyAlias": {}, + "Server.NetworkAccessPoint.CustomPrivateKeyPassPhraseEncrypted": {}, + "Server.NetworkAccessPoint.OutboundPrivateKeyAlias": {}, + "Server.NetworkAccessPoint.OutboundPrivateKeyPassPhrase": {}, + "Server.NetworkAccessPoint.PrivateKeyAlias": {}, + "Server.NetworkAccessPoint.PrivateKeyPassPhrase": {}, + "Server.ServerStart.Username": {}, + "Server.ServerStart.PasswordEncrypted": {}, + "Server.SingleSignOnServices.BasicAuthUsername": {}, + "Server.SingleSignOnServices.BasicAuthPasswordEncrypted": {}, + "Server.SingleSignOnServices.SsoSigningKeyAlias": {}, + "Server.SingleSignOnServices.SsoSigningKeyPassPhraseEncrypted": {}, + "Server.SingleSignOnServices.TransportLayerSecurityKeyAlias": {}, + "Server.SingleSignOnServices.TransportLayerSecurityKeyPassPhraseEncrypted": {}, + "Server.SSL.ClientCertAlias": {}, + "Server.SSL.ClientCertPrivateKeyPassPhraseEncrypted": {}, + "Server.SSL.OutboundPrivateKeyAlias": {}, + "Server.SSL.OutboundPrivateKeyPassPhraseEncrypted": {}, + "Server.SSL.ServerPrivateKeyAlias": {}, + "Server.SSL.ServerPrivateKeyPassPhraseEncrypted": {}, + "Server.DefaultIIOPUser": {}, + "Server.DefaultIIOPPasswordEncrypted": {}, + "Server.DefaultTGIOPUser": {}, + "Server.DefaultTGIOPPasswordEncrypted": {}, + "Server.JavaStandardTrustKeyStorePassPhraseEncrypted": {}, + "Server.SystemPasswordEncrypted": {}, + "ServerTemplate.FederationServices.SigningKeyAlias": {}, + "ServerTemplate.FederationServices.SigningKeyPassPhraseEncrypted": {}, + "ServerTemplate.FederationServices.SslClientIdentityAlias": {}, + "ServerTemplate.FederationServices.SslClientIdentityPassPhraseEncrypted": {}, + "ServerTemplate.NetworkAccessPoint.CustomIdentityKeyStorePassPhraseEncrypted": {}, + "ServerTemplate.NetworkAccessPoint.CustomPrivateKeyAlias": {}, + "ServerTemplate.NetworkAccessPoint.CustomPrivateKeyPassPhraseEncrypted": {}, + "ServerTemplate.NetworkAccessPoint.OutboundPrivateKeyAlias": {}, + "ServerTemplate.NetworkAccessPoint.OutboundPrivateKeyPassPhrase": {}, + "ServerTemplate.NetworkAccessPoint.PrivateKeyAlias": {}, + "ServerTemplate.NetworkAccessPoint.PrivateKeyPassPhrase": {}, + "ServerTemplate.ServerStart.Username": {}, + "ServerTemplate.ServerStart.PasswordEncrypted": {}, + "ServerTemplate.SingleSignOnServices.BasicAuthUsername": {}, + "ServerTemplate.SingleSignOnServices.BasicAuthPasswordEncrypted": {}, + "ServerTemplate.SingleSignOnServices.SsoSigningKeyAlias": {}, + "ServerTemplate.SingleSignOnServices.SsoSigningKeyPassPhraseEncrypted": {}, + "ServerTemplate.SingleSignOnServices.TransportLayerSecurityKeyAlias": {}, + "ServerTemplate.SingleSignOnServices.TransportLayerSecurityKeyPassPhraseEncrypted": {}, + "ServerTemplate.SSL.ClientCertAlias": {}, + "ServerTemplate.SSL.ClientCertPrivateKeyPassPhraseEncrypted": {}, + "ServerTemplate.SSL.OutboundPrivateKeyAlias": {}, + "ServerTemplate.SSL.OutboundPrivateKeyPassPhraseEncrypted": {}, + "ServerTemplate.SSL.ServerPrivateKeyAlias": {}, + "ServerTemplate.SSL.ServerPrivateKeyPassPhraseEncrypted": {}, + "ServerTemplate.DefaultIIOPUser": {}, + "ServerTemplate.DefaultIIOPPasswordEncrypted": {}, + "ServerTemplate.DefaultTGIOPUser": {}, + "ServerTemplate.DefaultTGIOPPasswordEncrypted": {}, + "ServerTemplate.JavaStandardTrustKeyStorePassPhraseEncrypted": {}, + "ServerTemplate.SystemPasswordEncrypted": {}, + "UnixMachine.NodeManager.UserName": {}, + "UnixMachine.NodeManager.PasswordEncrypted": {}, + "WLDFSystemResource.WLDFResource.WatchNotification.RestNotification.HttpAuthenticationUserName": {}, + "WLDFSystemResource.WLDFResource.WatchNotification.RestNotification.HttpAuthenticationPasswordEncrypted": {} + +} + diff --git a/installer/src/main/etc/host.properties b/installer/src/main/lib/injectors/host.json similarity index 100% rename from installer/src/main/etc/host.properties rename to installer/src/main/lib/injectors/host.json diff --git a/installer/src/main/etc/port.properties b/installer/src/main/lib/injectors/port.json similarity index 100% rename from installer/src/main/etc/port.properties rename to installer/src/main/lib/injectors/port.json diff --git a/installer/src/main/etc/url.properties b/installer/src/main/lib/injectors/url.json similarity index 100% rename from installer/src/main/etc/url.properties rename to installer/src/main/lib/injectors/url.json diff --git a/installer/src/main/lib/variable_keywords.json b/installer/src/main/lib/variable_keywords.json new file mode 100644 index 0000000000..be1126ca36 --- /dev/null +++ b/installer/src/main/lib/variable_keywords.json @@ -0,0 +1,6 @@ +{ + "CREDENTIALS": "credentials.json", + "PORT": "port.json", + "HOST": "host.json", + "URL": "url.json" +} \ No newline at end of file diff --git a/installer/src/main/samples/model_variable_injector.json b/installer/src/main/samples/model_variable_injector.json new file mode 100644 index 0000000000..5c4447bdc9 --- /dev/null +++ b/installer/src/main/samples/model_variable_injector.json @@ -0,0 +1,3 @@ +{ + "CREDENTIALS": {} +} From 0f9a6b22534d3b8885ea21cd7fdf4b1ed49c388f Mon Sep 17 00:00:00 2001 From: crountre Date: Mon, 21 May 2018 18:10:40 -0500 Subject: [PATCH 16/52] add property files --- .../wlsdeploy/tool/util/variable_injector.py | 23 +++++-- .../deploy/messages/wlsdeploy_rb.properties | 3 + installer/src/main/lib/injectors/host.json | 55 ++++++++++++++++ installer/src/main/lib/injectors/port.json | 64 ++++++++++++++++++- installer/src/main/lib/injectors/url.json | 26 +++++++- .../src/main/samples/custom_injector.json | 15 +++++ .../main/samples/model_variable_injector.json | 6 +- 7 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 installer/src/main/samples/custom_injector.json diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index f111289d36..14c38c79d3 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -22,6 +22,7 @@ from wlsdeploy.json.json_translator import JsonToPython from wlsdeploy.logging.platform_logger import PlatformLogger +WEBLOGIC_DEPLOY_HOME_TOKEN = '@@WLSDEPLOY@@' VARIABLE_INJECTOR_FILE_NAME = 'model_variable_injector.json' VARIABLE_KEYWORDS_FILE_NAME = 'variable_keywords.json' VARIABLE_INJECTOR_PATH_NAME_ARG = 'variable_injector_path_name' @@ -60,7 +61,7 @@ class VariableInjector(object): def __init__(self, model, model_context=None, version=None): self.__original = copy.deepcopy(model) self.__model = model - + self.__model_context = model_context self.__section_keys = model_sections.get_model_top_level_keys() self.__section_keys.remove(model_sections.get_model_domain_info_key()) @@ -97,6 +98,9 @@ def inject_variables_keyword_file(self, **kwargs): _logger.warning('WLSDPLY-19420', variable_injector_location_file, class_name=_class_name, method_name=_method_name) else: + _logger.info('WLSDPLY-19433', variable_injector_location_file, class_name=_class_name, + method_name=_method_name) + variable_file_location = self._replace_tokens(variable_file_location) injector_file_list = _create_injector_file_list(variables_injector_dictionary, keywords_dictionary, _get_keyword_files_location(**kwargs)) variables_file_dictionary = self.inject_variables_keyword_dictionary(injector_file_list) @@ -106,8 +110,11 @@ def inject_variables_keyword_file(self, **kwargs): method_name=_method_name) return_model = self.__model else: - _logger.fine('WLSDPLY-19419', class_name=_class_name, method_name=_method_name) + _logger.info('WLSDPLY-19419', class_name=_class_name, method_name=_method_name) variable_file_location = None + else: + _logger.info('WLSDPLY-19432', variable_injector_location_file, class_name=_class_name, + method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) return variables_inserted, return_model, variable_file_location @@ -122,7 +129,7 @@ def inject_variables_keyword_dictionary(self, injector_file_list): _logger.entering(injector_file_list, class_name=_class_name, method_name=_method_name) variables_dictionary = PyOrderedDict() for filename in injector_file_list: - injector_dictionary = _load_injector_file(filename) + injector_dictionary = _load_injector_file(self._replace_tokens(filename)) entries = self.inject_variables(injector_dictionary) if entries: _logger.finer('WLSDPLY-19413', filename, class_name=_class_name, method_name=_method_name) @@ -398,6 +405,14 @@ def _check_name_token(self, location, name_token): if self.__aliases.requires_unpredictable_single_name_handling(location): location.add_name_token(name_token, _fake_name_marker) + def _replace_tokens(self, path_string): + result = path_string + if path_string.startswith(WEBLOGIC_DEPLOY_HOME_TOKEN): + result = path_string.replace(WEBLOGIC_DEPLOY_HOME_TOKEN, _wlsdeploy_location) + elif self.__model_context: + result = self.__model_context.tokenize_path(path_string) + return result + def _log_mbean_not_found(self, mbean, replacement, location): _method_name = '_log_mbean_not_found' code = ValidationCodes.INVALID @@ -577,7 +592,7 @@ def _compile_pattern(pattern): try: return re.compile(pattern) except Exception, e: - _logger.warning('WLSDPLY-19411', pattern, e) + _logger.warning('WLSDPLY-19411', pattern, e, class_name=_class_name, method_name='_compile_pattern') return None diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 4c5e9d2219..a7e335d2eb 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1143,6 +1143,9 @@ WLSDPLY-19429=Variable {0} was inserted into the model string {1} for attribute inserted into the variable file WLSDPLY-19430=Segment was not found in attribute {0} value so entire variable value was replaced by {1} WLSDPLY-19431=Invalid location {0} for variable injection into attribute {1} : {2} +WLSDPLY-19432=No model variable injector file {0} +WLSDPLY-19433=Will inject variable replacement strings into model using keyword directions found in model \ + variable injector file {0} # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} diff --git a/installer/src/main/lib/injectors/host.json b/installer/src/main/lib/injectors/host.json index e69de29bb2..734ec668c4 100644 --- a/installer/src/main/lib/injectors/host.json +++ b/installer/src/main/lib/injectors/host.json @@ -0,0 +1,55 @@ +{ + "Cluster.FrontendHost": {}, + "Cluster.ClusterAddress": {}, + "Cluster.MulticastAddress": {}, + "Cluster.RemoteClusterAddress": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceAddressProvider.CoherenceSocketAddress.Address": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.CoherenceClusterWellKnownAddress.ListenAddress": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.MulticastListenAddress": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.UnicastListenAddress": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceFederationParams.RemoteParticipantHost": {}, + "CoherenceClusterSystemResource.FederationRemoteParticipantHost": {}, + "JMSSystemResource.JmsResource.Template.Multicast.MulticastAddress": {}, + "JMSSystemResource.JmsResource.Topic.Multicast.MulticastAddress": {}, + "JMSSystemResource.JmsResource.UniformDistributedTopic.Multicast.MulticastAddress": {}, + "Machine.NodeManager.ListenAddress": {}, + "Machine.Address": {}, + "NMProperties.ListenAddress": {}, + "SecurityConfiguration.CertRevoc.CrlCacheTypeLdapHostname": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.ActiveDirectoryAuthenticator.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.IPlanetAuthenticator.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.LDAPAuthenticator.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.LDAPX509IdentityAsserter.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.NovellAuthenticator.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OpenLDAPAuthenticator.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleInternetDirectoryAuthenticator.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleUnifiedDirectoryAuthenticator.Host": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleVirtualDirectoryAuthenticator.Host": {}, + "Server.CoherenceMemberConfig.UnicastListenAddress": {}, + "Server.COM.NTAuthHost": {}, + "Server.NetworkAccessPoint.ClusterAddress": {}, + "Server.NetworkAccessPoint.ListenAddress": {}, + "Server.NetworkAccessPoint.ProxyAddress": {}, + "Server.NetworkAccessPoint.PublicAddress": {}, + "Server.JTAMigratableTarget.HostingServer": {}, + "Server.WebServer.FrontendHost": {}, + "Server.BuzzAddress": {}, + "Server.InterfaceAddress": {}, + "Server.ListenAddress": {}, + "ServerTemplate.CoherenceMemberConfig.UnicastListenAddress": {}, + "ServerTemplate.COM.NTAuthHost": {}, + "ServerTemplate.NetworkAccessPoint.ClusterAddress": {}, + "ServerTemplate.NetworkAccessPoint.ListenAddress": {}, + "ServerTemplate.NetworkAccessPoint.ProxyAddress": {}, + "ServerTemplate.NetworkAccessPoint.PublicAddress": {}, + "ServerTemplate.JTAMigratableTarget.HostingServer": {}, + "ServerTemplate.WebServer.FrontendHost": {}, + "ServerTemplate.BuzzAddress": {}, + "ServerTemplate.InterfaceAddress": {}, + "ServerTemplate.ListenAddress": {}, + "UnixMachine.NodeManager.ListenAddress": {}, + "VirtualHost.FrontendHost": {}, + "VirtualHost.VirtualHostName": {}, + "VirtualTarget.WebServer.FrontendHost": {}, + "VirtualTarget.HostName": {} +} \ No newline at end of file diff --git a/installer/src/main/lib/injectors/port.json b/installer/src/main/lib/injectors/port.json index 2d49e2fb46..24db7e3a08 100644 --- a/installer/src/main/lib/injectors/port.json +++ b/installer/src/main/lib/injectors/port.json @@ -1,2 +1,62 @@ -resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Port`(?<=PORT=)[\w.-]+(?=\))) -resources:MailSession.Properties[`Port`(?<=\w.)port] +{ + "Cluster.FrontendHTTPPort": {}, + "Cluster.FrontendHTTPSPort": {}, + "Cluster.MulticastPort": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceAddressProvider.CoherenceSocketAddress.Port": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.CoherenceClusterWellKnownAddress.ListenPort": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.ClusterListenPort": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.MulticastListenPort": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.UnicastListenPort": {}, + "CoherenceClusterSystemResource.CoherenceResource.CoherenceFederationParams.RemoteCoherenceClusterListenPort": {}, + "CoherenceClusterSystemResource.FederationRemoteClusterListenPort": {}, + "JMSSystemResource.JmsResource.Template.Multicast.MulticastPort": {}, + "JMSSystemResource.JmsResource.Topic.Multicast.MulticastPort": {}, + "JMSSystemResource.JmsResource.UniformDistributedTopic.Multicast.MulticastPort": {}, + "Machine.NodeManager.ListenPort": {}, + "NMProperties.ListenPort": {}, + "SecurityConfiguration.CertRevoc.CrlCacheTypeLdapPort": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.ActiveDirectoryAuthenticator.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.IPlanetAuthenticator.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.LDAPAuthenticator.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.LDAPX509IdentityAsserter.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.NovellAuthenticator.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OpenLDAPAuthenticator.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleInternetDirectoryAuthenticator.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleUnifiedDirectoryAuthenticator.Port": {}, + "SecurityConfiguration.Realm.AuthenticationProvider.OracleVirtualDirectoryAuthenticator.Port": {}, + "Server.CoherenceMemberConfig.UnicastListenPort": {}, + "Server.NetworkAccessPoint.ListenPort": {}, + "Server.NetworkAccessPoint.ProxyPort": {}, + "Server.NetworkAccessPoint.PublicPort": {}, + "Server.NetworkAccessPoint.SSLListenPort": {}, + "Server.SSL.ListenPort": {}, + "Server.WebServer.FrontendHTTPPort": {}, + "Server.WebServer.FrontendHTTPSPort": {}, + "Server.AdministrationPort": {}, + "Server.AdministrationPortEnabled": {}, + "Server.BuzzPort": {}, + "Server.ListenPort": {}, + "Server.ListenPortEnabled": {}, + "Server.ReplicationPorts": {}, + "ServerTemplate.CoherenceMemberConfig.UnicastListenPort": {}, + "ServerTemplate.NetworkAccessPoint.ListenPort": {}, + "ServerTemplate.NetworkAccessPoint.ProxyPort": {}, + "ServerTemplate.NetworkAccessPoint.PublicPort": {}, + "ServerTemplate.NetworkAccessPoint.SSLListenPort": {}, + "ServerTemplate.SSL.ListenPort": {}, + "ServerTemplate.WebServer.FrontendHTTPPort": {}, + "ServerTemplate.WebServer.FrontendHTTPSPort": {}, + "ServerTemplate.AdministrationPort": {}, + "ServerTemplate.AdministrationPortEnabled": {}, + "ServerTemplate.BuzzPort": {}, + "ServerTemplate.ListenPort": {}, + "ServerTemplate.ListenPortEnabled": {}, + "ServerTemplate.ReplicationPorts": {}, + "UnixMachine.NodeManager.ListenPort": {}, + "VirtualHost.FrontendHTTPPort": {}, + "VirtualHost.FrontendHTTPSPort": {}, + "VirtualTarget.WebServer.FrontendHTTPPort": {}, + "VirtualTarget.WebServer.FrontendHTTPSPort": {}, + "VirtualTarget.ExplicitPort": {}, + "VirtualTarget.PortOffset": {} +} diff --git a/installer/src/main/lib/injectors/url.json b/installer/src/main/lib/injectors/url.json index 9edbeac747..05e256a200 100644 --- a/installer/src/main/lib/injectors/url.json +++ b/installer/src/main/lib/injectors/url.json @@ -1,2 +1,24 @@ -resources:JDBCSystemResource.JdbcResource.JDBCDriverParams.URL(`Host`(?<=HOST=)[\w.-]+(?=\))) -resources:MailSession.Properties[`Host`(?<=\w.)host] \ No newline at end of file +{ + "ForeignJNDIProvider.ProviderUrl": {}, + "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": {}, + "JMSBridgeDestination.ConnectionURL": {}, + "JMSSystemResource.JmsResource.ForeignServer.ConnectionURL": {}, + "JMSSystemResource.JmsResource.SAFRemoteContext.SAFLoginContext.LoginURL": {}, + "Partition.ForeignJndiProviderOverride.ProviderUrl": {}, + "SecurityConfiguration.CertRevoc.CertRevocCa.CrlDpUrl": {}, + "SecurityConfiguration.CertRevoc.CertRevocCa.OcspResponderUrl": {}, + "SecurityConfiguration.Realm.RDBMSSecurityStore.ConnectionURL": {}, + "Server.FederationServices.SourceSiteUrl": {}, + "Server.ServerDebug.BugReportServiceWsdlUrl": {}, + "Server.SingleSignOnServices.DefaultUrl": {}, + "Server.SingleSignOnServices.LoginUrl": {}, + "Server.SingleSignOnServices.OrganizationUrl": {}, + "Server.SingleSignOnServices.PublishedSiteUrl": {}, + "ServerTemplate.FederationServices.SourceSiteUrl": {}, + "ServerTemplate.ServerDebug.BugReportServiceWsdlUrl": {}, + "ServerTemplate.SingleSignOnServices.DefaultUrl": {}, + "ServerTemplate.SingleSignOnServices.LoginUrl": {}, + "ServerTemplate.SingleSignOnServices.OrganizationUrl": {}, + "ServerTemplate.SingleSignOnServices.PublishedSiteUrl": {}, + "WLDFSystemResource.WLDFResource.WatchNotification.RestNotification.EndpointUrl": {} +} \ No newline at end of file diff --git a/installer/src/main/samples/custom_injector.json b/installer/src/main/samples/custom_injector.json new file mode 100644 index 0000000000..8973f10091 --- /dev/null +++ b/installer/src/main/samples/custom_injector.json @@ -0,0 +1,15 @@ +{ + "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": + { + "regexp": [ + { + "pattern": "(?<=PORT=)[\\\w.-]+(?=\\\))", + "suffix": "Port" + }, + { + "pattern": "(?<=HOST=)[\\\w.-]+(?=\\\))", + "suffix": "Host" + } + ] + } +} \ No newline at end of file diff --git a/installer/src/main/samples/model_variable_injector.json b/installer/src/main/samples/model_variable_injector.json index 5c4447bdc9..d419deca0b 100644 --- a/installer/src/main/samples/model_variable_injector.json +++ b/installer/src/main/samples/model_variable_injector.json @@ -1,3 +1,7 @@ { - "CREDENTIALS": {} + "variable_file_name": "@@WLSDEPLOY@@/samples/variables.properties", + "CREDENTIALS": {}, + "CUSTOM": { + "file-list": [ "@@WLSDEPLOY@@/samples/custom_injector.json" ] + } } From 1b2187b94df36ed6a6526af5cc97de2c0976d590 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 22 May 2018 13:52:54 -0500 Subject: [PATCH 17/52] add in keywords and samples --- .../wlsdeploy/tool/util/variable_injector.py | 19 ++++++++++++ .../src/main/lib/injectors/credentials.json | 12 +++---- installer/src/main/lib/injectors/host.json | 7 +++++ .../src/main/lib/injectors/topology.json | 31 +++++++++++++++++++ installer/src/main/lib/variable_keywords.json | 3 +- .../src/main/samples/custom_injector.json | 19 ++++++++++-- 6 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 installer/src/main/lib/injectors/topology.json diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 14c38c79d3..0e71a7891f 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -533,6 +533,7 @@ def _load_injector_file(injector_file_name): if os.path.isfile(injector_file_name): try: injector_dictionary = JsonToPython(injector_file_name).parse() + __temporary_fix(injector_dictionary) except (IllegalArgumentException, JsonException), e: _logger.warning('WLDPLY-19409', injector_file_name, e.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) @@ -625,3 +626,21 @@ def _find_special_name(mbean): mbean_name = name_list[0] mbean_name_list = name_list[1].split(',') return mbean_name, mbean_name_list + + +def __temporary_fix(injector_dictionary): + #this is very dangerous - for now, if you want to escape a backslash, need to do 4 backslash. + _method_name = '__temporary_fix' + for value in injector_dictionary.itervalues(): + if REGEXP in value: + for dict_entry in value[REGEXP]: + if REGEXP_PATTERN in dict_entry: + pattern = dict_entry[REGEXP_PATTERN] + listsplit = re.split('\\\\(?!\\\\)', pattern) + if listsplit: + newpattern = '' + for split in listsplit: + newpattern += split[:len(split)] + dict_entry[REGEXP_PATTERN] = newpattern + _logger.fine('Pattern after temporary fix {0}', dict_entry[REGEXP_PATTERN]) + diff --git a/installer/src/main/lib/injectors/credentials.json b/installer/src/main/lib/injectors/credentials.json index 5772cc3821..b7f8cb3b41 100644 --- a/installer/src/main/lib/injectors/credentials.json +++ b/installer/src/main/lib/injectors/credentials.json @@ -18,27 +18,27 @@ "MailSession.Properties": { "regexp": [ { - "pattern": "(?<=\\w.)imap.user", + "pattern": "mail.imap.user", "suffix": "imap.user" }, { - "pattern": "(?<=\\w.)imap.password", + "pattern": "mail.imap.password", "suffix": "imap.password" }, { - "pattern": "(?<=\\w.)pop3.user", + "pattern": "mail.pop3.user", "suffix": "pop3.user" }, { - "pattern": "(?<=\\w.)pop3.password", + "pattern": "mail.pop3.password", "suffix": "pop3.password" }, { - "pattern": "(?<=\\w.)smtp.user", + "pattern": "mail.smtp.user", "suffix": "smtp.user" }, { - "pattern": "(?<=\\w.)smtp.password", + "pattern": "mail.smtp.password", "suffix": "smtp.password" } ] diff --git a/installer/src/main/lib/injectors/host.json b/installer/src/main/lib/injectors/host.json index 734ec668c4..0761849661 100644 --- a/installer/src/main/lib/injectors/host.json +++ b/installer/src/main/lib/injectors/host.json @@ -9,6 +9,13 @@ "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.UnicastListenAddress": {}, "CoherenceClusterSystemResource.CoherenceResource.CoherenceFederationParams.RemoteParticipantHost": {}, "CoherenceClusterSystemResource.FederationRemoteParticipantHost": {}, + "JDBCSystemResource.JdbcResource.JDBCOracleParams.OnsNodeList": { + "regexp": [ + { + "pattern": "[\w\\\\]" + } + ] + }, "JMSSystemResource.JmsResource.Template.Multicast.MulticastAddress": {}, "JMSSystemResource.JmsResource.Topic.Multicast.MulticastAddress": {}, "JMSSystemResource.JmsResource.UniformDistributedTopic.Multicast.MulticastAddress": {}, diff --git a/installer/src/main/lib/injectors/topology.json b/installer/src/main/lib/injectors/topology.json new file mode 100644 index 0000000000..06e9fb5e98 --- /dev/null +++ b/installer/src/main/lib/injectors/topology.json @@ -0,0 +1,31 @@ +{ + "Cluster.ClusterAddress": {}, + "Cluster.ClusterMessagingMode": {}, + "Cluster.MulticastAddress": {}, + "Cluster.MulticastPort": {}, + "Server.Log.LogFileRotationDir": {}, + "Server.NetworkAccessPoint.HostnameVerificationIgnored": {}, + "Server.NetworkAccessPoint.HostnameVerifier": {}, + "Server.SSL.Enabled": {}, + "Server.SSL.ListenPort": {}, + "Server.ListenAddress": {}, + "Server.ListenPort": {}, + "Server.ListenPortEnabled": {}, + "Server.Machine": {}, + "Server.ServerTemplate": {}, + "Server.SystemPasswordEncrypted": {}, + "Server.TunnelingEnabled": {}, + "Server.UseFusionForLLR": {}, + "Machine.Address": {}, + "Machine.NodeManager.ListenAddress": {}, + "Machine.NodeManager.ListenPort": {}, + "Machine.NodeManager.NodeManagerHome": {}, + "Machine.NodeManager.PasswordEncrypted": {}, + "Machine.NodeManager.UserName": {}, + "UnixMachine.Address": {}, + "UnixMachine.NodeManager.ListenAddress": {}, + "UnixMachine.NodeManager.ListenPort": {}, + "UnixMachine.NodeManager.NodeManagerHome": {}, + "UnixMachine.NodeManager.PasswordEncrypted": {}, + "UnixMachine.NodeManager.UserName": {} +} \ No newline at end of file diff --git a/installer/src/main/lib/variable_keywords.json b/installer/src/main/lib/variable_keywords.json index be1126ca36..49386112f5 100644 --- a/installer/src/main/lib/variable_keywords.json +++ b/installer/src/main/lib/variable_keywords.json @@ -2,5 +2,6 @@ "CREDENTIALS": "credentials.json", "PORT": "port.json", "HOST": "host.json", - "URL": "url.json" + "URL": "url.json", + "TOPOLOGY": "topology.json" } \ No newline at end of file diff --git a/installer/src/main/samples/custom_injector.json b/installer/src/main/samples/custom_injector.json index 8973f10091..acc8349073 100644 --- a/installer/src/main/samples/custom_injector.json +++ b/installer/src/main/samples/custom_injector.json @@ -1,13 +1,28 @@ +//{ +// "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": +// { +// "regexp": [ +// { +// "pattern": "(?<=PORT=)[\w.-]+(?=\))", +// "suffix": "Port" +// }, +// { +// "pattern": "(?<=HOST=)[\w.-]+(?=\))", +// "suffix": "Host" +// } +// ] +// } +//} { "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": { "regexp": [ { - "pattern": "(?<=PORT=)[\\\w.-]+(?=\\\))", + "pattern": "(?<=PORT=)[\\w.-]+(?=\\))", "suffix": "Port" }, { - "pattern": "(?<=HOST=)[\\\w.-]+(?=\\\))", + "pattern": "(?<=HOST=)[\\w.-]+(?=\\))", "suffix": "Host" } ] From 548f79ba60af35ef90a1dba7bf10c0bfdf79a983 Mon Sep 17 00:00:00 2001 From: crountre Date: Wed, 23 May 2018 11:10:52 -0500 Subject: [PATCH 18/52] Issue#86-discover-with-variables --- core/src/main/python/variable_inject.py | 466 ++++++++++++++++++ .../main/python/wlsdeploy/util/cla_utils.py | 88 +++- .../deploy/messages/wlsdeploy_rb.properties | 13 +- installer/src/main/bin/injectVariables.cmd | 363 ++++++++++++++ 4 files changed, 927 insertions(+), 3 deletions(-) create mode 100644 core/src/main/python/variable_inject.py create mode 100644 installer/src/main/bin/injectVariables.cmd diff --git a/core/src/main/python/variable_inject.py b/core/src/main/python/variable_inject.py new file mode 100644 index 0000000000..f44f3431ff --- /dev/null +++ b/core/src/main/python/variable_inject.py @@ -0,0 +1,466 @@ +""" +Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. +The Universal Permissive License (UPL), Version 1.0 + +The entry point for the discoverDomain tool. +""" +import javaos as os +import sys + +from java.io import File +from java.io import IOException +from java.lang import IllegalArgumentException +from java.lang import IllegalStateException +from java.lang import String + +from oracle.weblogic.deploy.aliases import AliasException +from oracle.weblogic.deploy.discover import DiscoverException +from oracle.weblogic.deploy.util import CLAException +from oracle.weblogic.deploy.util import FileUtils +from oracle.weblogic.deploy.util import PyWLSTException +from oracle.weblogic.deploy.util import TranslateException +from oracle.weblogic.deploy.util import WebLogicDeployToolingVersion +from oracle.weblogic.deploy.util import WLSDeployArchive +from oracle.weblogic.deploy.util import WLSDeployArchiveIOException +from oracle.weblogic.deploy.validate import ValidateException + +sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0]))) + +from wlsdeploy.aliases.wlst_modes import WlstModes +from wlsdeploy.exception import exception_helper +from wlsdeploy.logging.platform_logger import PlatformLogger +from wlsdeploy.util import getcreds +from wlsdeploy.tool.util.variable_injector import VariableInjector +from wlsdeploy.tool.discover import discoverer +from wlsdeploy.tool.discover.deployments_discoverer import DeploymentsDiscoverer +from wlsdeploy.tool.discover.domain_info_discoverer import DomainInfoDiscoverer +from wlsdeploy.tool.discover.multi_tenant_discoverer import MultiTenantDiscoverer +from wlsdeploy.tool.discover.resources_discoverer import ResourcesDiscoverer +from wlsdeploy.tool.discover.topology_discoverer import TopologyDiscoverer +from wlsdeploy.tool.validate.validator import Validator +from wlsdeploy.tool.util import filter_helper +from wlsdeploy.util import path_utils +from wlsdeploy.util import wlst_helper +from wlsdeploy.util import model_translator +from wlsdeploy.util.cla_utils import CommandLineArgUtil +from wlsdeploy.util.model import Model +from wlsdeploy.util.model_context import ModelContext +from wlsdeploy.util.weblogic_helper import WebLogicHelper + +_program_name = 'discoverDomain' +_class_name = 'discover' +__logger = PlatformLogger(discoverer.get_discover_logger_name()) +__wlst_mode = WlstModes.OFFLINE + +__required_arguments = [] + +__optional_arguments = [ + CommandLineArgUtil.MODEL_FILE_SWITCH, + CommandLineArgUtil.ARCHIVE_FILE_SWITCH, + CommandLineArgUtil.VARIABLE_INJECTOR_FILE_SWITCH, + CommandLineArgUtil.VARIABLE_KEYWORDS_FILE_SWITCH, + CommandLineArgUtil.VARIABLE_PROPERTIES_FILE_SWITCH, + CommandLineArgUtil.DOMAIN_TYPE_SWITCH +] + + +def __process_args(args): + """ + Process the command-line arguments and prompt the user for any missing information + :param args: the command-line arguments list + :raises CLAException: if an error occurs while validating and processing the command-line arguments + """ + global __wlst_mode + + cla_util = CommandLineArgUtil(_program_name, __required_arguments, __optional_arguments) + required_arg_map, optional_arg_map = cla_util.process_args(args) + + __verify_required_args_present(required_arg_map) + __wlst_mode = __process_online_args(optional_arg_map) + __process_archive_filename_arg(required_arg_map) + + combined_arg_map = optional_arg_map.copy() + combined_arg_map.update(required_arg_map) + + return ModelContext(_program_name, combined_arg_map) + + +def __verify_required_args_present(required_arg_map): + """ + Verify that the required args are present. + :param required_arg_map: the required arguments map + :raises CLAException: if one or more of the required arguments are missing + """ + _method_name = '__verify_required_args_present' + + for req_arg in __required_arguments: + if req_arg not in required_arg_map: + ex = exception_helper.create_cla_exception('WLSDPLY-20005', _program_name, req_arg) + ex.setExitCode(CommandLineArgUtil.USAGE_ERROR_EXIT_CODE) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + return + + +def __process_online_args(optional_arg_map): + """ + Determine if we are discover in online mode and if so, validate/prompt for the necessary parameters. + :param optional_arg_map: the optional arguments map + :return: the WLST mode + :raises CLAException: if an error occurs reading input from the user + """ + _method_name = '__process_online_args' + + mode = WlstModes.OFFLINE + if CommandLineArgUtil.ADMIN_URL_SWITCH in optional_arg_map: + if CommandLineArgUtil.ADMIN_USER_SWITCH not in optional_arg_map: + try: + username = getcreds.getuser('WLSDPLY-06016') + except IOException, ioe: + ex = exception_helper.create_cla_exception('WLSDPLY-06017', ioe.getLocalizedMessage(), error=ioe) + ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + optional_arg_map[CommandLineArgUtil.ADMIN_USER_SWITCH] = username + + if CommandLineArgUtil.ADMIN_PASS_SWITCH not in optional_arg_map: + try: + password = getcreds.getpass('WLSDPLY-06018') + except IOException, ioe: + ex = exception_helper.create_cla_exception('WLSDPLY-06019', ioe.getLocalizedMessage(), error=ioe) + ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + optional_arg_map[CommandLineArgUtil.ADMIN_PASS_SWITCH] = String(password) + + __logger.info('WLSDPLY-06020') + return mode + + +def __process_archive_filename_arg(required_arg_map): + """ + Validate the archive file name and load the archive file object. + :param required_arg_map: the required arguments map + :raises CLAException: if a validation error occurs while loading the archive file object + """ + _method_name = '__process_archive_filename_arg' + + archive_file_name = required_arg_map[CommandLineArgUtil.ARCHIVE_FILE_SWITCH] + try: + archive_file = WLSDeployArchive(archive_file_name) + except (IllegalArgumentException, IllegalStateException), ie: + ex = exception_helper.create_cla_exception('WLSDPLY-06013', _program_name, archive_file_name, + ie.getLocalizedMessage(), error=ie) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + required_arg_map[CommandLineArgUtil.ARCHIVE_FILE] = archive_file + return + + +def __discover(model_context): + """ + Populate the model from the domain. + :param model_context: the model context + :return: the fully-populated model + :raises DiscoverException: if an error occurred while discover the domain + """ + _method_name = '__discover' + + model = Model() + __connect_to_domain(model_context) + try: + DomainInfoDiscoverer(model_context, model.get_model_domain_info(), wlst_mode=__wlst_mode).discover() + TopologyDiscoverer(model_context, model.get_model_topology(), wlst_mode=__wlst_mode).discover() + ResourcesDiscoverer(model_context, model.get_model_resources(), wlst_mode=__wlst_mode).discover() + DeploymentsDiscoverer(model_context, model.get_model_app_deployments(), wlst_mode=__wlst_mode).discover() + __discover_multi_tenant(model, model_context) + except AliasException, ae: + wls_version = WebLogicHelper(__logger).get_actual_weblogic_version() + wlst_mode = WlstModes.from_value(__wlst_mode) + ex = exception_helper.create_discover_exception('WLSDPLY-06000', model_context.get_domain_name(), + model_context.get_domain_home(), wls_version, wlst_mode, + ae.getLocalizedMessage(), error=ae) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + + __disconnect_domain() + return model + + +def __discover_multi_tenant(model, model_context): + """ + Discover the multi-tenant-related parts of the domain, if they exist. + :param model: the model object to populate + :param model_context: the model context object + :raises DiscoverException: if an error occurs during discovery + """ + MultiTenantDiscoverer(model, model_context, wlst_mode=__wlst_mode).discover() + return + + +def __connect_to_domain(model_context): + """ + Connects WLST to the domain by either connecting to the Admin Server or reading the domain from disk. + :param model_context: the model context + :raises DiscoverException: if a WLST error occurs while connecting to or reading the domain + """ + _method_name = '__connect_to_domain' + + __logger.entering(class_name=_class_name, method_name=_method_name) + if __wlst_mode == WlstModes.ONLINE: + try: + wlst_helper.connect(model_context.get_admin_user(), model_context.get_admin_password(), + model_context.get_admin_url()) + except PyWLSTException, wlst_ex: + ex = exception_helper.create_discover_exception('WLSDPLY-06001', model_context.get_admin_url(), + model_context.get_admin_user(), + wlst_ex.getLocalizedMessage(), error=wlst_ex) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + else: + try: + wlst_helper.read_domain(model_context.get_domain_home()) + except PyWLSTException, wlst_ex: + ex = exception_helper.create_discover_exception('WLSDPLY-06002', model_context.get_domain_home(), + wlst_ex.getLocalizedMessage(), error=wlst_ex) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + + __logger.exiting(class_name=_class_name, method_name=_method_name) + return + + +def __clear_archive_file(model_context): + """ + Remove any binaries already in the archive file. + :param model_context: the model context + :raises DiscoverException: if an error occurs while removing the binaries + """ + _method_name = '__clear_archive_file' + __logger.entering(class_name=_class_name, method_name=_method_name) + + archive_file = model_context.get_archive_file() + + if archive_file is None: + de = exception_helper.create_discover_exception('WLSDPLY-06004', model_context.get_archive_file_name()) + __logger.throwing(class_name=_class_name, method_name=_method_name, error=de) + raise de + + try: + archive_file.removeAllBinaries() + except WLSDeployArchiveIOException, wioe: + de = exception_helper.create_discover_exception('WLSDPLY-06005', wioe.getLocalizedMessage()) + __logger.throwing(class_name=_class_name, method_name=_method_name, error=de) + raise de + + return + + +def __close_archive(model_context): + """ + Close the archive object + :param model_context: the model context + """ + _method_name = '__close_archive' + + __logger.entering(_class_name=_class_name, method_name=_method_name) + archive_file = model_context.get_archive_file() + archive_file.close() + __logger.exiting(class_name=_class_name, method_name=_method_name) + return + + +def __disconnect_domain(): + """ + Disconnects WLST from the domain by either disconnecting from the Admin Server or closing the domain read from disk. + :raises DiscoverException: if a WLST error occurred while disconnecting or closing the domain + """ + _method_name = '__disconnect_domain' + + __logger.entering(class_name=_class_name, method_name=_method_name) + if __wlst_mode == WlstModes.ONLINE: + try: + wlst_helper.disconnect() + except PyWLSTException, wlst_ex: + ex = exception_helper.create_discover_exception('WLSDPLY-06006', + wlst_ex.getLocalizedMessage(), error=wlst_ex) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + else: + try: + wlst_helper.close_domain() + except PyWLSTException, wlst_ex: + ex = exception_helper.create_discover_exception('WLSDPLY-06007', + wlst_ex.getLocalizedMessage(), error=wlst_ex) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + + __logger.exiting(class_name=_class_name, method_name=_method_name) + return + + +def __persist_model(model, model_context): + """ + Save the model to the specified model file name or to the archive if the file name was not specified. + :param model: the model to save + :param model_context: the model context + :raises DiscoverException: if an error occurs while create a temporary file for the model + or while adding it to the archive + :raises TranslateException: if an error occurs while serializing the model or writing it to disk + """ + _method_name = '__persist_model' + + __logger.entering(class_name=_class_name, method_name=_method_name) + + add_to_archive = False + model_file_name = model_context.get_model_file() + if model_file_name is None: + add_to_archive = True + try: + domain_name = model_context.get_domain_name() + model_file = File.createTempFile(domain_name, '.yaml').getCanonicalFile() + model_file_name = model_context.get_domain_name() + '.yaml' + except (IllegalArgumentException, IOException), ie: + ex = exception_helper.create_discover_exception('WLSDPLY-06008', ie.getLocalizedMessage(), error=ie) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + else: + model_file = FileUtils.getCanonicalFile(File(model_file_name)) + + try: + model_translator.PythonToFile(model.get_model()).write_to_file(model_file.getAbsolutePath()) + except TranslateException, ex: + # Jython 2.2.1 does not support finally so use this like a finally block... + if add_to_archive and not model_file.delete(): + model_file.deleteOnExit() + raise ex + + if add_to_archive: + try: + archive_file = model_context.get_archive_file() + archive_file.addModel(model_file, model_file_name) + if not model_file.delete(): + model_file.deleteOnExit() + except (WLSDeployArchiveIOException, IllegalArgumentException), arch_ex: + ex = exception_helper.create_discover_exception('WLSDPLY-06009', model_file.getAbsolutePath(), + model_file_name, arch_ex.getLocalizedMessage(), + error=arch_ex) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + if not model_file.delete(): + model_file.deleteOnExit() + raise ex + + __logger.exiting(class_name=_class_name, method_name=_method_name) + return + + +def __check_and_customize_model(model, model_context): + """ + Customize the model dictionary before persisting. Validate the model after customization for informational + purposes. Any validation errors will not stop the discovered model to be persisted. + :param model: completely discovered model + """ + _method_name = '__check_and_customize_model' + __logger.entering(class_name=_class_name, method_name=_method_name) + + if filter_helper.apply_filters(model.get_model(), "discover"): + __logger.info('WLSDPLY-06014', _class_name=_class_name, method_name=_method_name) + + inserted, variable_model, variable_file_name = VariableInjector(model.get_model(), model_context, WebLogicHelper( + __logger).get_actual_weblogic_version()).inject_variables_keyword_file( + variable_file_name=__get_default_variable_file(model_context)) + if inserted: + model = Model(variable_model) + try: + validator = Validator(model_context, wlst_mode=__wlst_mode) + + # no variables are generated by the discover tool + validator.validate_in_tool_mode(model.get_model(), variables_file_name=variable_file_name, + archive_file_name=model_context.get_archive_file_name()) + except ValidateException, ex: + __logger.warning('WLSDPLY-06015', ex.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) + return model + + +def __get_default_variable_file(model_context): + extract_file_name = model_context.get_model_file() + if not extract_file_name: + extract_file_name = model_context.get_archive_file_name() + default_variable_file = path_utils.get_filename_no_ext_from_path(extract_file_name) + if default_variable_file: + default_variable_file = os.path.join(path_utils.get_pathname_from_path(extract_file_name), + default_variable_file + '.properties') + return default_variable_file + + +def __log_and_exit(exit_code, _class_name, _method_name): + """ + Helper method to log the exiting message and call sys.exit() + :param exit_code: the exit code to use + :param _class_name: the class name to pass to the logger + :param _method_name: the method name to pass to the logger + """ + __logger.exiting(result=exit_code, class_name=_class_name, method_name=_method_name) + sys.exit(exit_code) + + +def main(args): + """ + The main entry point for the discoverDomain tool. + + :param args: + :return: + """ + _method_name = 'main' + + __logger.entering(class_name=_class_name, method_name=_method_name) + for index, arg in enumerate(args): + __logger.finer('sys.argv[{0}] = {1}', str(index), str(arg), class_name=_class_name, method_name=_method_name) + + wlst_helper.silence() + + exit_code = CommandLineArgUtil.PROG_OK_EXIT_CODE + + model_context = None + try: + model_context = __process_args(args) + except CLAException, ex: + exit_code = ex.getExitCode() + if exit_code != CommandLineArgUtil.HELP_EXIT_CODE: + __logger.severe('WLSDPLY-20008', _program_name, ex.getLocalizedMessage(), error=ex, + class_name=_class_name, method_name=_method_name) + __log_and_exit(exit_code, _class_name, _method_name) + + try: + __clear_archive_file(model_context) + except DiscoverException, ex: + __logger.severe('WLSDPLY-06010', _program_name, model_context.get_archive_file_name(), + ex.getLocalizedMessage(), error=ex, class_name=_class_name, method_name=_method_name) + __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) + + model = None + try: + model = __discover(model_context) + except DiscoverException, ex: + __logger.severe('WLSDPLY-06011', _program_name, model_context.get_domain_name(), + model_context.get_domain_home(), ex.getLocalizedMessage(), + error=ex, class_name=_class_name, method_name=_method_name) + __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) + + model = __check_and_customize_model(model, model_context) + try: + __persist_model(model, model_context) + + except TranslateException, ex: + __logger.severe('WLSDPLY-06012', _program_name, model_context.get_archive_file_name(), ex.getLocalizedMessage(), + error=ex, class_name=_class_name, method_name=_method_name) + __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) + + __close_archive(model_context) + + __logger.exiting(result=exit_code, class_name=_class_name, method_name=_method_name) + sys.exit(exit_code) + +if __name__ == 'main': + WebLogicDeployToolingVersion.logVersionInfo(_program_name) + main(sys.argv) diff --git a/core/src/main/python/wlsdeploy/util/cla_utils.py b/core/src/main/python/wlsdeploy/util/cla_utils.py index ad955f1c37..cb867bed6e 100644 --- a/core/src/main/python/wlsdeploy/util/cla_utils.py +++ b/core/src/main/python/wlsdeploy/util/cla_utils.py @@ -61,6 +61,10 @@ class CommandLineArgUtil(object): ATTRIBUTES_ONLY_SWITCH = '-attributes_only' FOLDERS_ONLY_SWITCH = '-folders_only' RECURSIVE_SWITCH = '-recursive' + # overrides for the variable injector + VARIABLE_INJECTOR_FILE_SWITCH = '-variable_injector_file' + VARIABLE_KEYWORDS_FILE_SWITCH = '-variable_keywords_file' + VARIABLE_PROPERTIES_FILE_SWITCH = '-variable_properties_file' # a slot to stash the parsed domain typedef dictionary DOMAIN_TYPEDEF = 'domain_typedef' # a slot to stash the archive file object @@ -332,6 +336,33 @@ def process_args(self, args): self._add_arg(key, True) elif self.is_recursive_switch(key): self._add_arg(key, True) + elif self.is_variable_injector_file_key(key): + idx += 1 + if idx < args_len: + full_path = self._validate_variable_injector_file_arg(args[idx]) + self._add_arg(key, full_path, True) + else: + ex = self._get_out_of_args_exception(key) + self._logger.throwing(ex, class_name=self._class_name, method_name=method_name) + raise ex + elif self.is_variable_keywords_file_key(key): + idx += 1 + if idx < args_len: + full_path = self._validate_variable_keywords_file_arg(args[idx]) + self._add_arg(key, full_path, True) + else: + ex = self._get_out_of_args_exception(key) + self._logger.throwing(ex, class_name=self._class_name, method_name=method_name) + raise ex + elif self.is_variable_properties_file_key(key): + idx += 1 + if idx < args_len: + full_path = self._validate_variable_properties_file_arg(args[idx]) + self._add_arg(key, full_path, True) + else: + ex = self._get_out_of_args_exception(key) + self._logger.throwing(ex, class_name=self._class_name, method_name=method_name) + raise ex else: ex = exception_helper.create_cla_exception('WLSDPLY-01601', self._program_name, key) ex.setExitCode(self.USAGE_ERROR_EXIT_CODE) @@ -388,7 +419,6 @@ def _validate_oracle_home_arg(self, value): return oh_name - def get_java_home_key(self): return str(self.JAVA_HOME_SWITCH) @@ -507,7 +537,6 @@ def _validate_wlst_path_arg(self, value): return wlst_path.getAbsolutePath() - def get_admin_url_key(self): return self.ADMIN_URL_SWITCH @@ -863,6 +892,61 @@ def _validate_target_mode_arg(self, value): raise ex return + def get_variable_injector_file_key(self): + return self.VARIABLE_INJECTOR_FILE_SWITCH + + def is_variable_injector_file_key(self, key): + return self.VARIABLE_INJECTOR_FILE_SWITCH == key + + def _validate_variable_injector_file_arg(self, value): + method_name = '_validate_variable_injector_file_arg' + + try: + injector = JFileUtils.validateExistingFile(value) + except JIllegalArgumentException, iae: + ex = exception_helper.create_cla_exception('WLSDPLY-01635', value, iae.getLocalizedMessage(), error=iae) + ex.setExitCode(self.ARG_VALIDATION_ERROR_EXIT_CODE) + self._logger.throwing(ex, class_name=self._class_name, method_name=method_name) + raise ex + return injector.getAbsolutePath() + + def get_variable_keywords_file_key(self): + return self.VARIABLE_KEYWORDS_FILE_SWITCH + + def is_variable_keywords_file_key(self, key): + return self.VARIABLE_KEYWORDS_FILE_SWITCH == key + + def _validate_variable_keywords_file_arg(self, value): + method_name = '_validate_variable_keywords_file_arg' + + try: + keywords = JFileUtils.validateExistingFile(value) + except JIllegalArgumentException, iae: + ex = exception_helper.create_cla_exception('WLSDPLY-01636', value, iae.getLocalizedMessage(), error=iae) + ex.setExitCode(self.ARG_VALIDATION_ERROR_EXIT_CODE) + self._logger.throwing(ex, class_name=self._class_name, method_name=method_name) + raise ex + return keywords.getAbsolutePath() + + # use this argument switch for the injector as the variables file does not have to exist + def get_variable_properties_file_key(self): + return self.VARIABLE_PROPERTIES_FILE_SWITCH + + def is_variable_properties_file_key(self, key): + return self.VARIABLE_PROPERTIES_FILE_SWITCH == key + + def _validate_variable_properties_file_arg(self, value): + method_name = '_validate_variable_properties_file_arg' + + try: + variables = JFileUtils.validateFileName(value) + except JIllegalArgumentException, iae: + ex = exception_helper.create_cla_exception('WLSDPLY-01620', value, iae.getLocalizedMessage(), error=iae) + ex.setExitCode(self.ARG_VALIDATION_ERROR_EXIT_CODE) + self._logger.throwing(ex, class_name=self._class_name, method_name=method_name) + raise ex + return variables.getAbsolutePath() + ########################################################################### # Helper methods # ########################################################################### diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index a7e335d2eb..30164864f3 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -230,6 +230,8 @@ WLSDPLY-01631=Specified target WLST mode {0} is not a valid WLST mode WLSDPLY-01632=Unknown command-line argument {0} for {1}...skipping WLSDPLY-01633=Specified {0} argument {1} does not match the expected pattern: :[/]* WLSDPLY-01634=Specified {0} argument {1} references model section {2} which is not one of the known model sections: {3} +WLSDPLY-01635=Specified Model Variable Injector File {0} is not a valid file : {1} +WLSDPLY-01636=Specified Model Variable Keywords File {0} is not a valid file : {1} # wlsdeploy/util/enum.py WLSDPLY-01700=The value {0} is not a valid value of the Enum type {1} @@ -262,7 +264,7 @@ WLSDPLY-01750=The WebLogic Deploy Tooling {0} version is {1} WLSDPLY-01760=Failed to access key in map: {0} ############################################################################### -# Encrypt Messages (04000 - 04999) # +# Encrypt Messages (04000 - 04499) # ############################################################################### # oracle.weblogic.deploy.encrypt.EncryptionUtils @@ -301,6 +303,15 @@ WLSDPLY-04210={0} encrypted {1} model attributes and wrote them to file {2} WLSDPLY-04211={0} failed to write encrypted model to file {1}: {2} WLSDPLY-04212={0} failed to encrypt the password: {1} +############################################################################### +# Variable Injector Messages (04500 - 04999) # +############################################################################### +# wlsdeploy/tool/variable_inject.py +WLSDPLY-04500=User passphrase was empty or null + +# wlsdeploy/tool/util/variable_injector.py +WLSDPLY-04600=Decryption failed: {0} + ############################################################################### # Validate Messages (05000 - 05999) # ############################################################################### diff --git a/installer/src/main/bin/injectVariables.cmd b/installer/src/main/bin/injectVariables.cmd new file mode 100644 index 0000000000..016c5efd30 --- /dev/null +++ b/installer/src/main/bin/injectVariables.cmd @@ -0,0 +1,363 @@ +@ECHO OFF +@rem ************************************************************************** +@rem injectVariables.cmd +@rem +@rem Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. +@rem The Universal Permissive License (UPL), Version 1.0 +@rem +@rem NAME +@rem injectVariables.cmd - Inject variables into the model. +@rem +@rem DESCRIPTION +@rem This script will inject variable tokens into the model and persist the variables to the +@rem indicated variable file. This can be run against a model that has injected variables. Any +@rem injected variables will not be replaced. If the existing variable file was provided, the +@rem new injected variables will be appended to the file. +@rem +@rem +@rem +@rem This script uses the following command-line arguments directly, the rest +@rem of the arguments are passed down to the underlying python program: +@rem +@rem - -oracle_home The directory of the existing Oracle Home to use. +@rem This directory must exist and it is the caller^'s +@rem responsibility to verify that it does. This +@rem argument is required. +@rem +@rem - -domain_type The type of domain to create. This argument is +@rem is optional. If not specified, it defaults to WLS. +@rem +@rem - -wlst_path The path to the Oracle Home product directory under +@rem which to find the wlst.cmd script. This is only +@rem needed for pre-12.2.1 upper stack products like SOA. +@rem +@rem For example, for SOA 12.1.3, -wlst_path should be +@rem specified as %ORACLE_HOME%\soa +@rem +@rem This script uses the following variables: +@rem +@rem JAVA_HOME - The location of the JDK to use. The caller must set +@rem this variable to a valid Java 7 (or later) JDK. +@rem +@rem WLSDEPLOY_HOME - The location of the WLS Deploy installation. +@rem If the caller sets this, the callers location will be +@rem honored provided it is an existing directory. +@rem Otherwise, the location will be calculated from the +@rem location of this script. +@rem +@rem WLSDEPLOY_PROPERTIES - Extra system properties to pass to WLST. The caller +@rem can use this environment variable to add additional +@rem system properties to the WLST environment. +@rem + +SETLOCAL + +SET WLSDEPLOY_PROGRAM_NAME=validateModel + +SET SCRIPT_PATH=%~dp0 +FOR %%i IN ("%SCRIPT_PATH%") DO SET SCRIPT_PATH=%%~fsi +IF %SCRIPT_PATH:~-1%==\ SET SCRIPT_PATH=%SCRIPT_PATH:~0,-1% + +IF NOT DEFINED WLSDEPLOY_HOME ( + SET WLSDEPLOY_HOME=%SCRIPT_PATH%\.. +) ELSE ( + IF NOT EXIST "%WLSDEPLOY_HOME%" ( + ECHO Specified WLSDEPLOY_HOME of "%WLSDEPLOY_HOME%" does not exist >&2 + SET RETURN_CODE=2 + GOTO exit_script + ) +) +FOR %%i IN ("%WLSDEPLOY_HOME%") DO SET WLSDEPLOY_HOME=%%~fsi +IF %WLSDEPLOY_HOME:~-1%==\ SET WLSDEPLOY_HOME=%WLSDEPLOY_HOME:~0,-1% + +@rem +@rem Make sure that the JAVA_HOME environment variable is set to point to a +@rem JDK 7 or higher JVM (and that it isn't OpenJDK). +@rem +IF NOT DEFINED JAVA_HOME ( + ECHO Please set the JAVA_HOME environment variable to point to a Java 7 installation >&2 + SET RETURN_CODE=2 + GOTO exit_script +) ELSE ( + IF NOT EXIST "%JAVA_HOME%" ( + ECHO Your JAVA_HOME environment variable to points to a non-existent directory: %JAVA_HOME% >&2 + SET RETURN_CODE=2 + GOTO exit_script + ) +) +FOR %%i IN ("%JAVA_HOME%") DO SET JAVA_HOME=%%~fsi +IF %JAVA_HOME:~-1%==\ SET JAVA_HOME=%JAVA_HOME:~0,-1% + +IF EXIST %JAVA_HOME%\bin\java.exe ( + FOR %%i IN ("%JAVA_HOME%\bin\java.exe") DO SET JAVA_EXE=%%~fsi +) ELSE ( + ECHO Java executable does not exist at %JAVA_HOME%\bin\java.exe does not exist >&2 + SET RETURN_CODE=2 + GOTO exit_script +) + +FOR /F %%i IN ('%JAVA_EXE% -version 2^>^&1') DO ( + IF "%%i" == "OpenJDK" ( + ECHO JAVA_HOME %JAVA_HOME% contains OpenJDK^, which is not supported >&2 + SET RETURN_CODE=2 + GOTO exit_script + ) +) + +FOR /F tokens^=2-5^ delims^=.-_^" %%j IN ('%JAVA_EXE% -fullversion 2^>^&1') DO ( + SET "JVM_FULL_VERSION=%%j.%%k.%%l_%%m" + SET "JVM_VERSION=%%k" +) + +IF %JVM_VERSION% LSS 7 ( + ECHO You are using an unsupported JDK version %JVM_FULL_VERSION% >&2 + SET RETURN_CODE=2 + GOTO exit_script +) ELSE ( + ECHO JDK version is %JVM_FULL_VERSION%, setting JAVA_VENDOR to Sun... + SET JAVA_VENDOR=Sun +) + +@rem +@rem Check to see if no args were given and print the usage message +@rem +IF "%~1" == "" ( + SET RETURN_CODE=0 + GOTO usage +) + +@rem +@rem Find the args required to determine the WLST script to run +@rem + +SET ORACLE_HOME= +SET DOMAIN_TYPE= +SET WLST_PATH_DIR= + +:arg_loop +IF "%1" == "-help" ( + SET RETURN_CODE=0 + GOTO usage +) +IF "%1" == "-oracle_home" ( + SET ORACLE_HOME=%2 + SHIFT + GOTO arg_continue +) +IF "%1" == "-domain_type" ( + SET DOMAIN_TYPE=%2 + SHIFT + GOTO arg_continue +) +IF "%1" == "-wlst_path" ( + SET WLST_PATH_DIR=%2 + SHIFT + GOTO arg_continue +) +@REM If none of the above, unknown argument so skip it +:arg_continue +SHIFT +IF NOT "%~1" == "" ( + GOTO arg_loop +) + +@rem +@rem Check for values of required arguments for this script to continue. +@rem The underlying WLST script has other required arguments. +@rem +IF "%ORACLE_HOME%" == "" ( + ECHO Required argument ORACLE_HOME not provided >&2 + SET RETURN_CODE=99 + GOTO usage +) + +@rem +@rem If the WLST_PATH_DIR is specified, validate that it contains the wlst.cmd script +@rem +IF DEFINED WLST_PATH_DIR ( + FOR %%i IN ("%WLST_PATH_DIR%") DO SET WLST_PATH_DIR=%%~fsi + IF NOT EXIST "%WLST_PATH_DIR%" ( + ECHO WLST_PATH_DIR specified does not exist: %WLST_PATH_DIR% >&2 + SET RETURN_CODE=98 + GOTO exit_script + ) + set "WLST=%WLST_PATH_DIR%\common\bin\wlst.cmd" + IF NOT EXIST "%WLST%" ( + ECHO WLST executable %WLST% not found under specified WLST_PATH_DIR %WLST_PATH_DIR% >&2 + SET RETURN_CODE=98 + GOTO exit_script + ) + SET CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + SET WLST_EXT_CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + GOTO found_wlst +) + +@rem +@rem Find the location for wlst.cmd +@rem +SET WLST= +SET USE_JRF_WLST=FALSE +IF DEFINED DOMAIN_TYPE ( + IF "%DOMAIN_TYPE%" == "WLS" ( + SET USE_JRF_WLST=FALSE + GOTO domain_type_recognized + ) + IF "%DOMAIN_TYPE%" == "RestrictedJRF" ( + SET USE_JRF_WLST=TRUE + GOTO domain_type_recognized + ) + IF "%DOMAIN_TYPE%" == "JRF" ( + SET USE_JRF_WLST=TRUE + GOTO domain_type_recognized + ) + ECHO Domain type %DOMAIN_TYPE% not recognized by shell script...assuming JRF is required + SET USE_JRF_WLST=TRUE +) + +:domain_type_recognized +IF "%USE_JRF_WLST%" == "TRUE" ( + IF EXIST "%ORACLE_HOME%\oracle_common\common\bin\wlst.cmd" ( + SET WLST=%ORACLE_HOME%\oracle_common\common\bin\wlst.cmd + SET CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + SET WLST_EXT_CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + GOTO found_wlst + ) +) ELSE ( + IF EXIST "%ORACLE_HOME%\wlserver_10.3\common\bin\wlst.cmd" ( + SET WLST=%ORACLE_HOME%\wlserver_10.3\common\bin\wlst.cmd + SET CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + GOTO found_wlst + ) + IF EXIST "%ORACLE_HOME%\wlserver_12.1\common\bin\wlst.cmd" ( + SET WLST=%ORACLE_HOME%\wlserver_12.1\common\bin\wlst.cmd + SET CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + GOTO found_wlst + ) + IF EXIST "%ORACLE_HOME%\wlserver\common\bin\wlst.cmd" ( + IF EXIST "%ORACLE_HOME%\wlserver\.product.properties" ( + @rem WLS 12.1.2 or WLS 12.1.3 + SET WLST=%ORACLE_HOME%\wlserver\common\bin\wlst.cmd + SET CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + ) ELSE ( + @rem WLS 12.2.1+ + SET WLST=%ORACLE_HOME%\oracle_common\common\bin\wlst.cmd + SET WLST_EXT_CLASSPATH=%WLSDEPLOY_HOME%\lib\weblogic-deploy-core.jar + ) + GOTO found_wlst + ) +) + +IF NOT EXIST "%WLST%" ( + ECHO Unable to locate wlst.cmd script in ORACLE_HOME %ORACLE_HOME% >&2 + SET RETURN_CODE=98 + GOTO exit_script +) +:found_wlst + +SET LOG_CONFIG_CLASS=oracle.weblogic.deploy.logging.WLSDeployLoggingConfig +SET WLST_PROPERTIES=-Dcom.oracle.cie.script.throwException=true +SET "WLST_PROPERTIES=-Djava.util.logging.config.class=%LOG_CONFIG_CLASS% %WLST_PROPERTIES%" +SET "WLST_PROPERTIES=%WLST_PROPERTIES% %WLSDEPLOY_PROPERTIES%" + +IF NOT DEFINED WLSDEPLOY_LOG_PROPERTIES ( + SET WLSDEPLOY_LOG_PROPERTIES=%WLSDEPLOY_HOME%\etc\logging.properties +) +IF NOT DEFINED WLSDEPLOY_LOG_DIRECTORY ( + SET WLSDEPLOY_LOG_DIRECTORY=%WLSDEPLOY_HOME%\logs +) + +ECHO JAVA_HOME = %JAVA_HOME% +ECHO WLST_EXT_CLASSPATH = %WLST_EXT_CLASSPATH% +ECHO CLASSPATH = %CLASSPATH% +ECHO WLST_PROPERTIES = %WLST_PROPERTIES% + +SET PY_SCRIPTS_PATH=%WLSDEPLOY_HOME%\lib\python +ECHO %WLST% %PY_SCRIPTS_PATH%\variable_inject.py %* + +"%WLST%" "%PY_SCRIPTS_PATH%\variable_inject.py" %* + +SET RETURN_CODE=%ERRORLEVEL% +IF "%RETURN_CODE%" == "100" ( + GOTO usage +) +IF "%RETURN_CODE%" == "99" ( + GOTO usage +) +IF "%RETURN_CODE%" == "98" ( + ECHO. + ECHO variableInjector.cmd failed due to a parameter validation error >&2 + GOTO exit_script +) +IF "%RETURN_CODE%" == "2" ( + ECHO. + ECHO variableInjector.cmd failed ^(exit code = %RETURN_CODE%^) + GOTO exit_script +) +IF "%RETURN_CODE%" == "1" ( + ECHO. + ECHO variableInjector.cmd completed but with some issues ^(exit code = %RETURN_CODE%^) >&2 + GOTO exit_script +) +IF "%RETURN_CODE%" == "0" ( + ECHO. + ECHO variableInjector.cmd completed successfully ^(exit code = %RETURN_CODE%^) + GOTO exit_script +) +@rem Unexpected return code so just print the message and exit... +ECHO. +ECHO variableInjector.cmd failed ^(exit code = %RETURN_CODE%^) >&2 +GOTO exit_script + +:usage +ECHO. +ECHO Usage: %~nx0 [-help] +ECHO -oracle_home ^ +ECHO -model_file ^ | -archive_file ^ +ECHO [-variable_injector_file ^] +ECHO [-variable_keywords_file ^] +ECHO [-variable_properties_file ^] +ECHO [-domain_type ^] +ECHO [-wlst_path ^] +ECHO. +ECHO where: +ECHO oracle-home - the existing Oracle Home directory with the correct version for the model +ECHO. +ECHO model-file - the location of the model file in which variables will be injected. +ECHO If not specified, the tool will look for the model +ECHO in the archive file. Either the model_file or the archive_file argument must be provided. +ECHO. +ECHO archive-file - the path to the archive file that contains a model in which the variables +ECHO will be injected. If the model-file argument is used, this argument will be +ECHO ignored. The archive file must contain a valid model. +ECHO. +ECHO variable-injector-file - the location of the variable injector file which contains the variable +ECHO injector keywords for this model injection run. If this argument is not provided, +ECHO the model_variable_injector.json file must exist in the lib directory in the +ECHO WLSDEPLOY_HOME location. +ECHO. +ECHO variable-keywords-file - this argument overrides the INSTALLED version of the allowed variable keywords +ECHO for the variable injector. This argument is for advanced usage only. The installed +ECHO keywords file is located in the lib directory of WLSDEPLOY_HOME location. +ECHO. +ECHO variable-file - the location of the property file in which to store any variable names injected +ECHO into the model. This argument overrides the value in the model injector file. +ECHO If the variable file is not listed in the model injector file, and this command +ECHO line argument is not used, the variable properties will be located and named +ECHO based on the model file or archive file name and location. +ECHO If the variable file exists, new variable values will be appended to the file. +ECHO. +ECHO domain-type - the type of domain (e.g., WLS, JRF). +ECHO Used to locate wlst.cmd if wlst-path not specified +ECHO. +ECHO wlst-path - the Oracle Home subdirectory of the wlst.cmd +ECHO script to use (e.g., ^\soa) +ECHO. + +:exit_script +IF DEFINED USE_CMD_EXIT ( + EXIT %RETURN_CODE% +) ELSE ( + EXIT /B %RETURN_CODE% +) + +ENDLOCAL From 1486e54f854ce401729f96490a0296fe49e10a23 Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 24 May 2018 10:04:43 -0500 Subject: [PATCH 19/52] Add commandline injector tool --- core/src/main/python/discover.py | 21 +- core/src/main/python/variable_inject.py | 375 ++++++------------ .../wlsdeploy/tool/util/variable_injector.py | 133 ++++--- .../main/python/wlsdeploy/util/variables.py | 35 +- .../deploy/messages/wlsdeploy_rb.properties | 15 +- installer/src/main/bin/injectVariables.cmd | 2 +- installer/src/main/bin/injectVariables.sh | 308 ++++++++++++++ installer/src/main/lib/injectors/host.json | 8 +- .../src/main/samples/custom_injector.json | 15 - 9 files changed, 542 insertions(+), 370 deletions(-) create mode 100644 installer/src/main/bin/injectVariables.sh diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index 7d903627b6..500cc6bff4 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -39,7 +39,6 @@ from wlsdeploy.tool.discover.topology_discoverer import TopologyDiscoverer from wlsdeploy.tool.validate.validator import Validator from wlsdeploy.tool.util import filter_helper -from wlsdeploy.util import path_utils from wlsdeploy.util import wlst_helper from wlsdeploy.util import model_translator from wlsdeploy.util.cla_utils import CommandLineArgUtil @@ -137,7 +136,7 @@ def __process_online_args(optional_arg_map): raise ex optional_arg_map[CommandLineArgUtil.ADMIN_PASS_SWITCH] = String(password) - __logger.info('WLSDPLY-06020') + __logger.info('WLSDPLY-06020', class_name=_class_name, method_name=_method_name) return mode @@ -346,7 +345,7 @@ def __persist_model(model, model_context): if not model_file.delete(): model_file.deleteOnExit() except (WLSDeployArchiveIOException, IllegalArgumentException), arch_ex: - ex = exception_helper.create_discover_exception('WLSDPLY-06009', model_file.getAbsolutePath(), + ex = exception_helper.create_discover_exception('WLSDPLY-20023', model_file.getAbsolutePath(), model_file_name, arch_ex.getLocalizedMessage(), error=arch_ex) __logger.throwing(ex, class_name=_class_name, method_name=_method_name) @@ -371,8 +370,7 @@ def __check_and_customize_model(model, model_context): __logger.info('WLSDPLY-06014', _class_name=_class_name, method_name=_method_name) inserted, variable_model, variable_file_name = VariableInjector(model.get_model(), model_context, WebLogicHelper( - __logger).get_actual_weblogic_version()).inject_variables_keyword_file( - variable_file_name=__get_default_variable_file(model_context)) + __logger).get_actual_weblogic_version()).inject_variables_keyword_file() if inserted: model = Model(variable_model) try: @@ -386,17 +384,6 @@ def __check_and_customize_model(model, model_context): return model -def __get_default_variable_file(model_context): - extract_file_name = model_context.get_model_file() - if not extract_file_name: - extract_file_name = model_context.get_archive_file_name() - default_variable_file = path_utils.get_filename_no_ext_from_path(extract_file_name) - if default_variable_file: - default_variable_file = os.path.join(path_utils.get_pathname_from_path(extract_file_name), - default_variable_file + '.properties') - return default_variable_file - - def __log_and_exit(exit_code, _class_name, _method_name): """ Helper method to log the exiting message and call sys.exit() @@ -456,7 +443,7 @@ def main(args): __persist_model(model, model_context) except TranslateException, ex: - __logger.severe('WLSDPLY-06012', _program_name, model_context.get_archive_file_name(), ex.getLocalizedMessage(), + __logger.severe('WLSDPLY-20024', _program_name, model_context.get_archive_file_name(), ex.getLocalizedMessage(), error=ex, class_name=_class_name, method_name=_method_name) __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) diff --git a/core/src/main/python/variable_inject.py b/core/src/main/python/variable_inject.py index f44f3431ff..b689643b34 100644 --- a/core/src/main/python/variable_inject.py +++ b/core/src/main/python/variable_inject.py @@ -4,55 +4,46 @@ The entry point for the discoverDomain tool. """ -import javaos as os +import os import sys from java.io import File -from java.io import IOException from java.lang import IllegalArgumentException from java.lang import IllegalStateException -from java.lang import String - -from oracle.weblogic.deploy.aliases import AliasException -from oracle.weblogic.deploy.discover import DiscoverException from oracle.weblogic.deploy.util import CLAException from oracle.weblogic.deploy.util import FileUtils -from oracle.weblogic.deploy.util import PyWLSTException from oracle.weblogic.deploy.util import TranslateException -from oracle.weblogic.deploy.util import WebLogicDeployToolingVersion from oracle.weblogic.deploy.util import WLSDeployArchive from oracle.weblogic.deploy.util import WLSDeployArchiveIOException -from oracle.weblogic.deploy.validate import ValidateException +from oracle.weblogic.deploy.util import WebLogicDeployToolingVersion + sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0]))) +import wlsdeploy.tool.util.variable_injector as variable_injector from wlsdeploy.aliases.wlst_modes import WlstModes from wlsdeploy.exception import exception_helper from wlsdeploy.logging.platform_logger import PlatformLogger -from wlsdeploy.util import getcreds from wlsdeploy.tool.util.variable_injector import VariableInjector -from wlsdeploy.tool.discover import discoverer -from wlsdeploy.tool.discover.deployments_discoverer import DeploymentsDiscoverer -from wlsdeploy.tool.discover.domain_info_discoverer import DomainInfoDiscoverer -from wlsdeploy.tool.discover.multi_tenant_discoverer import MultiTenantDiscoverer -from wlsdeploy.tool.discover.resources_discoverer import ResourcesDiscoverer -from wlsdeploy.tool.discover.topology_discoverer import TopologyDiscoverer -from wlsdeploy.tool.validate.validator import Validator -from wlsdeploy.tool.util import filter_helper -from wlsdeploy.util import path_utils from wlsdeploy.util import wlst_helper from wlsdeploy.util import model_translator from wlsdeploy.util.cla_utils import CommandLineArgUtil from wlsdeploy.util.model import Model from wlsdeploy.util.model_context import ModelContext +from wlsdeploy.util.model_translator import FileToPython from wlsdeploy.util.weblogic_helper import WebLogicHelper -_program_name = 'discoverDomain' -_class_name = 'discover' -__logger = PlatformLogger(discoverer.get_discover_logger_name()) +_program_name = 'injectVariables' +_class_name = 'variable_inject' +__logger = PlatformLogger('wlsdeploy.tool.util') __wlst_mode = WlstModes.OFFLINE +__kwargs = dict() +__inserted = False +__tmp_model_dir = None -__required_arguments = [] +__required_arguments = [ + CommandLineArgUtil.ORACLE_HOME_SWITCH +] __optional_arguments = [ CommandLineArgUtil.MODEL_FILE_SWITCH, @@ -75,9 +66,10 @@ def __process_args(args): cla_util = CommandLineArgUtil(_program_name, __required_arguments, __optional_arguments) required_arg_map, optional_arg_map = cla_util.process_args(args) - __verify_required_args_present(required_arg_map) - __wlst_mode = __process_online_args(optional_arg_map) - __process_archive_filename_arg(required_arg_map) + __process_model_args(optional_arg_map) + __process_injector_file(optional_arg_map) + __process_keywords_file(optional_arg_map) + __process_properties_file(optional_arg_map) combined_arg_map = optional_arg_map.copy() combined_arg_map.update(required_arg_map) @@ -102,158 +94,97 @@ def __verify_required_args_present(required_arg_map): return -def __process_online_args(optional_arg_map): +def __process_model_args(optional_arg_map): """ - Determine if we are discover in online mode and if so, validate/prompt for the necessary parameters. + Verify that either the model_file or archive_file was provided and exists. + If the model_file was not provided, the archive_file must be provided and the indicated archive file + must contain a model file. Extract the model file if the archive_file was provided. :param optional_arg_map: the optional arguments map - :return: the WLST mode - :raises CLAException: if an error occurs reading input from the user + :raises CLAException: if the arguments are invalid or an error occurs extracting the model from the archive """ - _method_name = '__process_online_args' - - mode = WlstModes.OFFLINE - if CommandLineArgUtil.ADMIN_URL_SWITCH in optional_arg_map: - if CommandLineArgUtil.ADMIN_USER_SWITCH not in optional_arg_map: - try: - username = getcreds.getuser('WLSDPLY-06016') - except IOException, ioe: - ex = exception_helper.create_cla_exception('WLSDPLY-06017', ioe.getLocalizedMessage(), error=ioe) - ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE) - __logger.throwing(ex, class_name=_class_name, method_name=_method_name) - raise ex - optional_arg_map[CommandLineArgUtil.ADMIN_USER_SWITCH] = username + _method_name = '__process_model_args' + global __tmp_model_dir + + if CommandLineArgUtil.MODEL_FILE_SWITCH in optional_arg_map: + model_file_name = optional_arg_map[CommandLineArgUtil.MODEL_FILE_SWITCH] + + try: + FileUtils.validateExistingFile(model_file_name) + except IllegalArgumentException, iae: + ex = exception_helper.create_cla_exception('WLSDPLY-20006', _program_name, model_file_name, + iae.getLocalizedMessage(), error=iae) + ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + elif CommandLineArgUtil.ARCHIVE_FILE_SWITCH in optional_arg_map: + archive_file_name = optional_arg_map[CommandLineArgUtil.ARCHIVE_FILE_SWITCH] - if CommandLineArgUtil.ADMIN_PASS_SWITCH not in optional_arg_map: - try: - password = getcreds.getpass('WLSDPLY-06018') - except IOException, ioe: - ex = exception_helper.create_cla_exception('WLSDPLY-06019', ioe.getLocalizedMessage(), error=ioe) + try: + archive_file = WLSDeployArchive(archive_file_name) + contains_model = archive_file.containsModel() + if not contains_model: + ex = exception_helper.create_cla_exception('WLSDPLY-19603', archive_file_name) ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE) __logger.throwing(ex, class_name=_class_name, method_name=_method_name) raise ex - optional_arg_map[CommandLineArgUtil.ADMIN_PASS_SWITCH] = String(password) - - __logger.info('WLSDPLY-06020') - return mode - - -def __process_archive_filename_arg(required_arg_map): - """ - Validate the archive file name and load the archive file object. - :param required_arg_map: the required arguments map - :raises CLAException: if a validation error occurs while loading the archive file object - """ - _method_name = '__process_archive_filename_arg' - - archive_file_name = required_arg_map[CommandLineArgUtil.ARCHIVE_FILE_SWITCH] - try: - archive_file = WLSDeployArchive(archive_file_name) - except (IllegalArgumentException, IllegalStateException), ie: - ex = exception_helper.create_cla_exception('WLSDPLY-06013', _program_name, archive_file_name, - ie.getLocalizedMessage(), error=ie) + else: + __tmp_model_dir = FileUtils.createTempDirectory(_program_name) + tmp_model_file = \ + FileUtils.fixupFileSeparatorsForJython(archive_file.extractModel(__tmp_model_dir).getAbsolutePath()) + except (IllegalArgumentException, IllegalStateException, WLSDeployArchiveIOException), archex: + ex = exception_helper.create_cla_exception('WLSDPLY-20010', _program_name, archive_file_name, + archex.getLocalizedMessage(), error=archex) + ex.setExitCode(CommandLineArgUtil.ARG_VALIDATION_ERROR_EXIT_CODE) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + optional_arg_map[CommandLineArgUtil.MODEL_FILE_SWITCH] = FileUtils.fixupFileSeparatorsForJython(tmp_model_file) + else: + ex = exception_helper.create_cla_exception('WLSDPLY-20015', _program_name, CommandLineArgUtil.MODEL_FILE_SWITCH, + CommandLineArgUtil.ARCHIVE_FILE_SWITCH) + ex.setExitCode(CommandLineArgUtil.USAGE_ERROR_EXIT_CODE) __logger.throwing(ex, class_name=_class_name, method_name=_method_name) raise ex - required_arg_map[CommandLineArgUtil.ARCHIVE_FILE] = archive_file return -def __discover(model_context): - """ - Populate the model from the domain. - :param model_context: the model context - :return: the fully-populated model - :raises DiscoverException: if an error occurred while discover the domain - """ - _method_name = '__discover' +def __process_injector_file(optional_arg_map): + _method_name = '__process_injector_file' + if CommandLineArgUtil.VARIABLE_INJECTOR_FILE_SWITCH in optional_arg_map: + injector = optional_arg_map[CommandLineArgUtil.VARIABLE_INJECTOR_FILE_SWITCH] + __logger.fine('WLSDPLY-19600', injector, class_name=_class_name, method_name=_method_name) + __kwargs[variable_injector.VARIABLE_INJECTOR_FILE_NAME_ARG] = injector - model = Model() - __connect_to_domain(model_context) - try: - DomainInfoDiscoverer(model_context, model.get_model_domain_info(), wlst_mode=__wlst_mode).discover() - TopologyDiscoverer(model_context, model.get_model_topology(), wlst_mode=__wlst_mode).discover() - ResourcesDiscoverer(model_context, model.get_model_resources(), wlst_mode=__wlst_mode).discover() - DeploymentsDiscoverer(model_context, model.get_model_app_deployments(), wlst_mode=__wlst_mode).discover() - __discover_multi_tenant(model, model_context) - except AliasException, ae: - wls_version = WebLogicHelper(__logger).get_actual_weblogic_version() - wlst_mode = WlstModes.from_value(__wlst_mode) - ex = exception_helper.create_discover_exception('WLSDPLY-06000', model_context.get_domain_name(), - model_context.get_domain_home(), wls_version, wlst_mode, - ae.getLocalizedMessage(), error=ae) - __logger.throwing(ex, class_name=_class_name, method_name=_method_name) - raise ex - __disconnect_domain() - return model +def __process_keywords_file(optional_arg_map): + _method_name = '__process_keywords_file' + if CommandLineArgUtil.VARIABLE_KEYWORDS_FILE_SWITCH in optional_arg_map: + keywords = optional_arg_map[CommandLineArgUtil.VARIABLE_KEYWORDS_FILE_SWITCH] + __logger.fine('WLSDPLY-19601', keywords, class_name=_class_name, method_name=_method_name) + __kwargs[variable_injector.VARIABLE_KEYWORDS_FILE_NAME_ARG] = keywords -def __discover_multi_tenant(model, model_context): - """ - Discover the multi-tenant-related parts of the domain, if they exist. - :param model: the model object to populate - :param model_context: the model context object - :raises DiscoverException: if an error occurs during discovery - """ - MultiTenantDiscoverer(model, model_context, wlst_mode=__wlst_mode).discover() - return - - -def __connect_to_domain(model_context): - """ - Connects WLST to the domain by either connecting to the Admin Server or reading the domain from disk. - :param model_context: the model context - :raises DiscoverException: if a WLST error occurs while connecting to or reading the domain - """ - _method_name = '__connect_to_domain' - - __logger.entering(class_name=_class_name, method_name=_method_name) - if __wlst_mode == WlstModes.ONLINE: - try: - wlst_helper.connect(model_context.get_admin_user(), model_context.get_admin_password(), - model_context.get_admin_url()) - except PyWLSTException, wlst_ex: - ex = exception_helper.create_discover_exception('WLSDPLY-06001', model_context.get_admin_url(), - model_context.get_admin_user(), - wlst_ex.getLocalizedMessage(), error=wlst_ex) - __logger.throwing(ex, class_name=_class_name, method_name=_method_name) - raise ex - else: - try: - wlst_helper.read_domain(model_context.get_domain_home()) - except PyWLSTException, wlst_ex: - ex = exception_helper.create_discover_exception('WLSDPLY-06002', model_context.get_domain_home(), - wlst_ex.getLocalizedMessage(), error=wlst_ex) - __logger.throwing(ex, class_name=_class_name, method_name=_method_name) - raise ex - - __logger.exiting(class_name=_class_name, method_name=_method_name) - return +def __process_properties_file(optional_arg_map): + _method_name = '__process_properties_file' + if CommandLineArgUtil.VARIABLE_PROPERTIES_FILE_SWITCH in optional_arg_map: + properties = optional_arg_map[CommandLineArgUtil.VARIABLE_PROPERTIES_FILE_SWITCH] + __logger.fine('WLSDPLY-19602', properties, class_name=_class_name, method_name=_method_name) + __kwargs[variable_injector.VARIABLE_FILE_NAME_ARG] = properties -def __clear_archive_file(model_context): +def __inject(model, model_context): """ - Remove any binaries already in the archive file. + Inject variables into the model file that is loaded into the model_context. :param model_context: the model context - :raises DiscoverException: if an error occurs while removing the binaries + :return: True if variables were inserted into model: The updated model """ - _method_name = '__clear_archive_file' - __logger.entering(class_name=_class_name, method_name=_method_name) - - archive_file = model_context.get_archive_file() - - if archive_file is None: - de = exception_helper.create_discover_exception('WLSDPLY-06004', model_context.get_archive_file_name()) - __logger.throwing(class_name=_class_name, method_name=_method_name, error=de) - raise de - - try: - archive_file.removeAllBinaries() - except WLSDeployArchiveIOException, wioe: - de = exception_helper.create_discover_exception('WLSDPLY-06005', wioe.getLocalizedMessage()) - __logger.throwing(class_name=_class_name, method_name=_method_name, error=de) - raise de - - return + __kwargs[variable_injector.VARIABLE_FILE_APPEND_ARG] = True + inserted, variable_model, variable_file_name = VariableInjector(model, model_context, + WebLogicHelper( + __logger).get_actual_weblogic_version()). \ + inject_variables_keyword_file(**__kwargs) + if inserted: + model = Model(variable_model) + return inserted, model def __close_archive(model_context): @@ -265,36 +196,8 @@ def __close_archive(model_context): __logger.entering(_class_name=_class_name, method_name=_method_name) archive_file = model_context.get_archive_file() - archive_file.close() - __logger.exiting(class_name=_class_name, method_name=_method_name) - return - - -def __disconnect_domain(): - """ - Disconnects WLST from the domain by either disconnecting from the Admin Server or closing the domain read from disk. - :raises DiscoverException: if a WLST error occurred while disconnecting or closing the domain - """ - _method_name = '__disconnect_domain' - - __logger.entering(class_name=_class_name, method_name=_method_name) - if __wlst_mode == WlstModes.ONLINE: - try: - wlst_helper.disconnect() - except PyWLSTException, wlst_ex: - ex = exception_helper.create_discover_exception('WLSDPLY-06006', - wlst_ex.getLocalizedMessage(), error=wlst_ex) - __logger.throwing(ex, class_name=_class_name, method_name=_method_name) - raise ex - else: - try: - wlst_helper.close_domain() - except PyWLSTException, wlst_ex: - ex = exception_helper.create_discover_exception('WLSDPLY-06007', - wlst_ex.getLocalizedMessage(), error=wlst_ex) - __logger.throwing(ex, class_name=_class_name, method_name=_method_name) - raise ex - + if archive_file: + archive_file.close() __logger.exiting(class_name=_class_name, method_name=_method_name) return @@ -314,19 +217,9 @@ def __persist_model(model, model_context): add_to_archive = False model_file_name = model_context.get_model_file() + model_file = FileUtils.getCanonicalFile(File(model_file_name)) if model_file_name is None: add_to_archive = True - try: - domain_name = model_context.get_domain_name() - model_file = File.createTempFile(domain_name, '.yaml').getCanonicalFile() - model_file_name = model_context.get_domain_name() + '.yaml' - except (IllegalArgumentException, IOException), ie: - ex = exception_helper.create_discover_exception('WLSDPLY-06008', ie.getLocalizedMessage(), error=ie) - __logger.throwing(ex, class_name=_class_name, method_name=_method_name) - raise ex - else: - model_file = FileUtils.getCanonicalFile(File(model_file_name)) - try: model_translator.PythonToFile(model.get_model()).write_to_file(model_file.getAbsolutePath()) except TranslateException, ex: @@ -342,8 +235,9 @@ def __persist_model(model, model_context): if not model_file.delete(): model_file.deleteOnExit() except (WLSDeployArchiveIOException, IllegalArgumentException), arch_ex: - ex = exception_helper.create_discover_exception('WLSDPLY-06009', model_file.getAbsolutePath(), - model_file_name, arch_ex.getLocalizedMessage(), + ex = exception_helper.create_discover_exception('WLSDPLY-20023', _program_name, + model_file.getAbsolutePath(), model_file_name, + arch_ex.getLocalizedMessage(), error=arch_ex) __logger.throwing(ex, class_name=_class_name, method_name=_method_name) if not model_file.delete(): @@ -354,53 +248,14 @@ def __persist_model(model, model_context): return -def __check_and_customize_model(model, model_context): - """ - Customize the model dictionary before persisting. Validate the model after customization for informational - purposes. Any validation errors will not stop the discovered model to be persisted. - :param model: completely discovered model - """ - _method_name = '__check_and_customize_model' - __logger.entering(class_name=_class_name, method_name=_method_name) - - if filter_helper.apply_filters(model.get_model(), "discover"): - __logger.info('WLSDPLY-06014', _class_name=_class_name, method_name=_method_name) - - inserted, variable_model, variable_file_name = VariableInjector(model.get_model(), model_context, WebLogicHelper( - __logger).get_actual_weblogic_version()).inject_variables_keyword_file( - variable_file_name=__get_default_variable_file(model_context)) - if inserted: - model = Model(variable_model) - try: - validator = Validator(model_context, wlst_mode=__wlst_mode) - - # no variables are generated by the discover tool - validator.validate_in_tool_mode(model.get_model(), variables_file_name=variable_file_name, - archive_file_name=model_context.get_archive_file_name()) - except ValidateException, ex: - __logger.warning('WLSDPLY-06015', ex.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) - return model - - -def __get_default_variable_file(model_context): - extract_file_name = model_context.get_model_file() - if not extract_file_name: - extract_file_name = model_context.get_archive_file_name() - default_variable_file = path_utils.get_filename_no_ext_from_path(extract_file_name) - if default_variable_file: - default_variable_file = os.path.join(path_utils.get_pathname_from_path(extract_file_name), - default_variable_file + '.properties') - return default_variable_file - - -def __log_and_exit(exit_code, _class_name, _method_name): +def __log_and_exit(exit_code, class_name, _method_name): """ Helper method to log the exiting message and call sys.exit() :param exit_code: the exit code to use - :param _class_name: the class name to pass to the logger + :param class_name: the class name to pass to the logger :param _method_name: the method name to pass to the logger """ - __logger.exiting(result=exit_code, class_name=_class_name, method_name=_method_name) + __logger.exiting(result=exit_code, class_name=class_name, method_name=_method_name) sys.exit(exit_code) @@ -431,30 +286,24 @@ def main(args): class_name=_class_name, method_name=_method_name) __log_and_exit(exit_code, _class_name, _method_name) + model_file = model_context.get_model_file() try: - __clear_archive_file(model_context) - except DiscoverException, ex: - __logger.severe('WLSDPLY-06010', _program_name, model_context.get_archive_file_name(), - ex.getLocalizedMessage(), error=ex, class_name=_class_name, method_name=_method_name) - __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) + model = FileToPython(model_file, True).parse() + except TranslateException, te: + __logger.severe('WLSDPLY-20009', _program_name, model_file, te.getLocalizedMessage(), error=te, + class_name=_class_name, method_name=_method_name) + sys.exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE) - model = None - try: - model = __discover(model_context) - except DiscoverException, ex: - __logger.severe('WLSDPLY-06011', _program_name, model_context.get_domain_name(), - model_context.get_domain_home(), ex.getLocalizedMessage(), - error=ex, class_name=_class_name, method_name=_method_name) - __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) - - model = __check_and_customize_model(model, model_context) - try: - __persist_model(model, model_context) + inserted, model = __inject(model, model_context) + if inserted: + __logger.info('WLSDPLY-19604', class_name=_class_name, method_name=_method_name) + try: + __persist_model(model, model_context) - except TranslateException, ex: - __logger.severe('WLSDPLY-06012', _program_name, model_context.get_archive_file_name(), ex.getLocalizedMessage(), - error=ex, class_name=_class_name, method_name=_method_name) - __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) + except TranslateException, ex: + __logger.severe('WLSDPLY-20024', _program_name, model_context.get_archive_file_name(), + ex.getLocalizedMessage(), error=ex, class_name=_class_name, method_name=_method_name) + __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) __close_archive(model_context) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 0e71a7891f..b7547f9780 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -32,6 +32,7 @@ VARIABLE_INJECTOR_FILES_PATH_ARG = 'variable_injector_files_path_name' VARIABLE_FILE_NAME_ARG = 'variable_file_name' VARIABLE_FILE_NAME = 'variables.json' +VARIABLE_FILE_APPEND_ARG = 'append_to_variables' # custom keyword in model injector file CUSTOM_KEYWORD = 'CUSTOM' KEYWORD_FILES = 'file-list' @@ -53,7 +54,7 @@ _wlsdeploy_location = os.environ.get('WLSDEPLOY_HOME') _class_name = 'variable_injector' -_logger = PlatformLogger('wlsdeploy.util') +_logger = PlatformLogger('wlsdeploy.tool.util') class VariableInjector(object): @@ -93,27 +94,28 @@ def inject_variables_keyword_file(self, **kwargs): return_model = self.__original variable_file_location = None if variables_injector_dictionary and keywords_dictionary: - variable_file_location = _get_variable_file_name(variables_injector_dictionary, **kwargs) + variable_file_location = self._get_variable_file_name(variables_injector_dictionary, **kwargs) if not variable_file_location: - _logger.warning('WLSDPLY-19420', variable_injector_location_file, class_name=_class_name, + _logger.warning('WLSDPLY-19520', variable_injector_location_file, class_name=_class_name, method_name=_method_name) else: - _logger.info('WLSDPLY-19433', variable_injector_location_file, class_name=_class_name, + _logger.info('WLSDPLY-19533', variable_injector_location_file, class_name=_class_name, method_name=_method_name) variable_file_location = self._replace_tokens(variable_file_location) injector_file_list = _create_injector_file_list(variables_injector_dictionary, keywords_dictionary, _get_keyword_files_location(**kwargs)) variables_file_dictionary = self.inject_variables_keyword_dictionary(injector_file_list) - variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location) + variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location, + _get_append_to_variable_file(**kwargs)) if variables_inserted: - _logger.info('WLSDPLY-19418', variable_file_location, class_name=_class_name, + _logger.info('WLSDPLY-19518', variable_file_location, class_name=_class_name, method_name=_method_name) return_model = self.__model else: - _logger.info('WLSDPLY-19419', class_name=_class_name, method_name=_method_name) + _logger.info('WLSDPLY-19519', class_name=_class_name, method_name=_method_name) variable_file_location = None else: - _logger.info('WLSDPLY-19432', variable_injector_location_file, class_name=_class_name, + _logger.info('WLSDPLY-19532', variable_injector_location_file, class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_inserted) @@ -132,7 +134,7 @@ def inject_variables_keyword_dictionary(self, injector_file_list): injector_dictionary = _load_injector_file(self._replace_tokens(filename)) entries = self.inject_variables(injector_dictionary) if entries: - _logger.finer('WLSDPLY-19413', filename, class_name=_class_name, method_name=_method_name) + _logger.finer('WLSDPLY-19513', filename, class_name=_class_name, method_name=_method_name) variables_dictionary.update(entries) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_dictionary) return variables_dictionary @@ -166,10 +168,10 @@ def _traverse_variables(model_section, mbean_list): if mbean_list: mbean = mbean_list.pop(0) mbean, mbean_name_list = _find_special_name(mbean) - _logger.finer('WLSDPLY-19423', mbean, location.get_folder_path(), class_name=_class_name, + _logger.finer('WLSDPLY-19523', mbean, location.get_folder_path(), class_name=_class_name, method_name=_method_name) if mbean in model_section: - _logger.finest('WLSDPLY-19414', mbean, class_name=_class_name, method_name=_method_name) + _logger.finest('WLSDPLY-19514', mbean, class_name=_class_name, method_name=_method_name) next_model_section = model_section[mbean] location.append_location(mbean) name_token = self.__aliases.get_name_token(location) @@ -179,7 +181,7 @@ def _traverse_variables(model_section, mbean_list): else: self._check_name_token(location, name_token) else: - _logger.fine('WLSDPLY-19406', mbean_name_list, attribute, location.get_folder_path(), + _logger.fine('WLSDPLY-19506', mbean_name_list, attribute, location.get_folder_path(), class_name=_class_name, method_name=_method_name) if mbean_name_list: for mbean_name in mbean_name_list: @@ -200,16 +202,16 @@ def _traverse_variables(model_section, mbean_list): if returned_dict: variable_dict.update(returned_dict) else: - _logger.finer('WLSDPLY-19417', attribute, injector, location.get_folder_path(), + _logger.finer('WLSDPLY-19517', attribute, injector, location.get_folder_path(), class_name=_class_name, method_name=_method_name) return True section = self.__model if start_mbean_list: # Find out in what section is the mbean top folder so can move to that section in the model - mbean, __ = _find_special_name(start_mbean_list[0]) + top_mbean, __ = _find_special_name(start_mbean_list[0]) for entry in self.__section_keys: - if entry in self.__model and mbean in self.__model[entry]: + if entry in self.__model and top_mbean in self.__model[entry]: section = self.__model[entry] break else: @@ -224,12 +226,14 @@ def _traverse_variables(model_section, mbean_list): return variable_dict def __format_variable_name(self, location, attribute): + _method_name = '__format_variable_name' variable_name = attribute make_path = None try: make_path = self.__aliases.get_model_folder_path(location) except AliasException, ae: - _logger.warning('WLSDPLY-19431', str(location), attribute, ae.getLocalizedMessage()) + _logger.warning('WLSDPLY-19531', str(location), attribute, ae.getLocalizedMessage(), class_name=_class_name, + method_name=_method_name) if make_path: make_path = make_path.split(':') if len(make_path) > 1 and len(make_path[1]) > 1: @@ -262,10 +266,10 @@ def _process_attribute(self, model, attribute, location): variable_name = self.__format_variable_name(location, attribute) variable_value = _format_variable_value(attribute_value) model[attribute] = _format_as_property(variable_name) - _logger.fine('WLSDPLY-19425', variable_name, attribute_value, attribute, variable_value, + _logger.fine('WLSDPLY-19525', variable_name, attribute_value, attribute, variable_value, class_name=_class_name, method_name=_method_name) else: - _logger.finer('WLSDPLY-19426', attribute_value, attribute, str(location), class_name=_class_name, + _logger.finer('WLSDPLY-19526', attribute_value, attribute, str(location), class_name=_class_name, method_name=_method_name) if variable_value: variable_dict[variable_name] = variable_value @@ -301,7 +305,7 @@ def _process_pattern_string(self, model, attribute, location, pattern, suffix): attribute_value, variable_name, variable_value = self._find_segment_in_string(attribute, model[attribute], location, pattern, suffix) if variable_value: - _logger.finer('WLSDPLY-19429', variable_name, attribute_value, attribute, variable_value, + _logger.finer('WLSDPLY-19529', variable_name, attribute_value, attribute, variable_value, class_name=_class_name, method_name=_method_name) model[attribute] = attribute_value # elif replace_if_nosegment: @@ -310,10 +314,10 @@ def _process_pattern_string(self, model, attribute, location, pattern, suffix): # variable_value = check_value # variable_name = self.__format_variable_name(location, attribute) # model[attribute] = _format_as_property(variable_name) - # _logger.finer('WLSDPLY-19430', attribute, model[attribute], class_name=_class_name, + # _logger.finer('WLSDPLY-19530', attribute, model[attribute], class_name=_class_name, # method_name=_method_name) else: - _logger.finer('WLSDPLY-19424', pattern, attribute, model[attribute], + _logger.finer('WLSDPLY-19524', pattern, attribute, model[attribute], location.get_folder_path, class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) @@ -354,7 +358,7 @@ def _process_pattern_list(self, attribute_name, attribute_list, location, patter attribute_value, seg_var_name, seg_var_value = self._find_segment_in_string(attribute_name, entry, location, pattern, suffix) if seg_var_value: - _logger.finer('WLSDPLY-19428', variable_name, attribute_name, variable_value, class_name=_class_name, + _logger.finer('WLSDPLY-19528', variable_name, attribute_name, variable_value, class_name=_class_name, method_name=_method_name) attribute_list[idx] = attribute_value variable_name = seg_var_name @@ -393,7 +397,7 @@ def _process_pattern_dictionary(self, attribute_name, attribute_dict, location, if not _already_property(attribute_dict[entry]): matcher = pattern.search(entry) if matcher: - _logger.finer('WLSDPLY-19427', attribute_name, replacement, class_name=_class_name, + _logger.finer('WLSDPLY-19527', attribute_name, replacement, class_name=_class_name, method_name=_method_name) variable_value = _format_variable_value(attribute_dict[entry]) attribute_dict[entry] = replacement @@ -409,8 +413,8 @@ def _replace_tokens(self, path_string): result = path_string if path_string.startswith(WEBLOGIC_DEPLOY_HOME_TOKEN): result = path_string.replace(WEBLOGIC_DEPLOY_HOME_TOKEN, _wlsdeploy_location) - elif self.__model_context: - result = self.__model_context.tokenize_path(path_string) + elif path_string and self.__model_context: + result = self.__model_context.replace_token_string(path_string) return result def _log_mbean_not_found(self, mbean, replacement, location): @@ -419,27 +423,28 @@ def _log_mbean_not_found(self, mbean, replacement, location): try: code, __ = self.__aliases.is_valid_model_folder_name(location, mbean) except AliasException, ae: - _logger.fine('AliasException {0}', ae.getLocalizedMessage()) pass if code == ValidationCodes.INVALID: - _logger.warning('WLSDPLY-19415', mbean, replacement, location.get_folder_path(), + _logger.warning('WLSDPLY-19515', mbean, replacement, location.get_folder_path(), class_name=_class_name, method_name=_method_name) else: - _logger.finer('WLSDPLY-19416', mbean, replacement, location.get_folder_path(), + _logger.finer('WLSDPLY-19516', mbean, replacement, location.get_folder_path(), class_name=_class_name, method_name=_method_name) - - -def _get_variable_file_name(variables_injector_dictionary, **kwargs): - if VARIABLE_FILE_NAME_ARG in variables_injector_dictionary: - variable_file_location = variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] - del variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] - _logger.finer('WLSDPLY-19421', variable_file_location) - elif VARIABLE_FILE_NAME_ARG in kwargs: - variable_file_location = kwargs[VARIABLE_FILE_NAME_ARG] - _logger.finer('WLSDPLY-19422', variable_file_location) - else: - variable_file_location = None - return variable_file_location + + def _get_variable_file_name(self, variables_injector_dictionary, **kwargs): + _method_name = '_get_variable_file_name' + if VARIABLE_FILE_NAME_ARG in kwargs: + variable_file_location = kwargs[VARIABLE_FILE_NAME_ARG] + _logger.finer('WLSDPLY-19522', variable_file_location, class_name=_class_name, method_name=_method_name) + if VARIABLE_FILE_NAME_ARG in variables_injector_dictionary: + del variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] + elif VARIABLE_FILE_NAME_ARG in variables_injector_dictionary: + variable_file_location = variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] + del variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] + _logger.finer('WLSDPLY-19521', variable_file_location, class_name=_class_name, method_name=_method_name) + else: + variable_file_location = variables.get_default_variable_file_name(self.__model_context) + return variable_file_location def _get_variable_injector_file_name(**kwargs): @@ -469,9 +474,9 @@ def _load_variables_file(variable_injector_location): if os.path.isfile(variable_injector_location): try: variables_dictionary = JsonToPython(variable_injector_location).parse() - _logger.fine('WLSDPLY-19400', variable_injector_location, class_name=_class_name, method_name=_method_name) + _logger.fine('WLSDPLY-19500', variable_injector_location, class_name=_class_name, method_name=_method_name) except (IllegalArgumentException, JsonException), e: - _logger.warning('WLSDPLY-19402', variable_injector_location, e.getLocalizedMessage(), + _logger.warning('WLSDPLY-19502', variable_injector_location, e.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_dictionary) @@ -485,9 +490,9 @@ def _load_keywords_file(variable_keywords_location): if os.path.isfile(variable_keywords_location): try: keywords_dictionary = JsonToPython(variable_keywords_location).parse() - _logger.finer('WLSDPLY-19404', variable_keywords_location, class_name=_class_name, method_name=_method_name) + _logger.finer('WLSDPLY-19504', variable_keywords_location, class_name=_class_name, method_name=_method_name) except (IllegalArgumentException, JsonException), e: - _logger.warning('WLSDPLY-19405', variable_keywords_location, e.getLocalizedMessage(), + _logger.warning('WLSDPLY-19505', variable_keywords_location, e.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=keywords_dictionary) @@ -502,9 +507,9 @@ def _create_injector_file_list(variables_dictionary, keyword_dictionary, injecto injector_file_list = variables_dictionary[CUSTOM_KEYWORD][KEYWORD_FILES] if type(injector_file_list) != list: injector_file_list = injector_file_list.split(',') - _logger.fine('WLSDPLY-19401', injector_file_list) + _logger.fine('WLSDPLY-19501', injector_file_list, class_name=_class_name, method_name=_method_name) else: - _logger.info('WLSDPLY-19412', class_name=_class_name, method_name=_method_name) + _logger.info('WLSDPLY-19512', class_name=_class_name, method_name=_method_name) del variables_dictionary[CUSTOM_KEYWORD] for keyword in variables_dictionary: if keyword in keyword_dictionary: @@ -513,9 +518,9 @@ def _create_injector_file_list(variables_dictionary, keyword_dictionary, injecto if not os.path.isabs(filename): filename = os.path.join(injector_path, filename) injector_file_list.append(filename) - _logger.finer('WLSDPLY-19408', filename, keyword) + _logger.finer('WLSDPLY-19508', filename, keyword, class_name=_class_name, method_name=_method_name) else: - _logger.warning('WLSDPLY-19403', keyword, class_name=_class_name, method_name=_method_name) + _logger.warning('WLSDPLY-19503', keyword, class_name=_class_name, method_name=_method_name) return injector_file_list @@ -538,22 +543,29 @@ def _load_injector_file(injector_file_name): _logger.warning('WLDPLY-19409', injector_file_name, e.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) else: - _logger.warning('WLSDPLY-19410', injector_file_name, class_name=_class_name, method_name=_method_name) + _logger.warning('WLSDPLY-19510', injector_file_name, class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name) return injector_dictionary -def _write_variables_file(variables_dictionary, variables_file_name): +def _get_append_to_variable_file(**kwargs): + if VARIABLE_FILE_APPEND_ARG in kwargs: + return kwargs[VARIABLE_FILE_APPEND_ARG] + return False + + +def _write_variables_file(variables_dictionary, variables_file_name, append): _method_name = '_write_variables_file' _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) + written = False if variables_dictionary: try: - variables.write_variables(variables_dictionary, variables_file_name) + variables.write_variables(variables_dictionary, variables_file_name, append) written = True except VariableException, ve: - _logger.warning('WLSDPLY-19407', variables_file_name, ve.getLocalizedMessage(), class_name=_class_name, + _logger.warning('WLSDPLY-19507', variables_file_name, ve.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) return written @@ -583,9 +595,10 @@ def _replace_segment(regexp, variable_value, attribute_value): if pattern: matcher = pattern.search(variable_value) if matcher: - replaced_value = variable_value[matcher.start():matcher.end()] - - replacement_string = pattern.sub(attribute_value, variable_value) + temp_value = variable_value[matcher.start():matcher.end()] + if not _already_property(temp_value): + replacement_string = pattern.sub(attribute_value, variable_value) + replaced_value = temp_value return replacement_string, replaced_value @@ -593,7 +606,7 @@ def _compile_pattern(pattern): try: return re.compile(pattern) except Exception, e: - _logger.warning('WLSDPLY-19411', pattern, e, class_name=_class_name, method_name='_compile_pattern') + _logger.warning('WLSDPLY-19511', pattern, e, class_name=_class_name, method_name='_compile_pattern') return None @@ -629,7 +642,7 @@ def _find_special_name(mbean): def __temporary_fix(injector_dictionary): - #this is very dangerous - for now, if you want to escape a backslash, need to do 4 backslash. + # this is very dangerous - for now, if you want to escape a backslash, need to do 4 backslash. _method_name = '__temporary_fix' for value in injector_dictionary.itervalues(): if REGEXP in value: @@ -642,5 +655,5 @@ def __temporary_fix(injector_dictionary): for split in listsplit: newpattern += split[:len(split)] dict_entry[REGEXP_PATTERN] = newpattern - _logger.fine('Pattern after temporary fix {0}', dict_entry[REGEXP_PATTERN]) - + _logger.fine('Pattern after temporary fix {0}', dict_entry[REGEXP_PATTERN], class_name=_class_name, + method_name=_method_name) diff --git a/core/src/main/python/wlsdeploy/util/variables.py b/core/src/main/python/wlsdeploy/util/variables.py index 9a41a0e2ca..9a7117718e 100644 --- a/core/src/main/python/wlsdeploy/util/variables.py +++ b/core/src/main/python/wlsdeploy/util/variables.py @@ -2,9 +2,12 @@ Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. The Universal Permissive License (UPL), Version 1.0 """ +import os import re +from java.lang import Boolean from java.io import BufferedReader +from java.io import File from java.io import FileInputStream from java.io import FileOutputStream from java.io import FileReader @@ -13,6 +16,7 @@ from oracle.weblogic.deploy.util import PyOrderedDict as OrderedDict +from wlsdeploy.util import path_utils from wlsdeploy.exception import exception_helper from wlsdeploy.logging import platform_logger @@ -54,14 +58,16 @@ def load_variables(file_path): return variable_map -def write_variables(variable_map, file_path): +def write_variables(variable_map, file_path, append=False): """ Write the dictionary of variables to the specified file. :param variable_map: the dictionary of variables :param file_path: the file to which to write the properties + :param append: defaults to False. Append properties to the end of file :raises VariableException if an error occurs while storing the variables in the file """ - method_name = 'write_variables' + _method_name = 'write_variables' + _logger.entering(file_path, append, class_name=_class_name, method_name=_method_name) props = Properties() for key in variable_map: value = variable_map[key] @@ -70,19 +76,40 @@ def write_variables(variable_map, file_path): comment = exception_helper.get_message('WLSDPLY-01731') output_stream = None try: - output_stream = FileOutputStream(file_path) + output_stream = FileOutputStream(File(file_path), Boolean(append)) props.store(output_stream, comment) output_stream.close() except IOException, ioe: ex = exception_helper.create_variable_exception('WLSDPLY-20007', file_path, ioe.getLocalizedMessage(), error=ioe) - _logger.throwing(ex, class_name=_class_name, method_name=method_name) + _logger.throwing(ex, class_name=_class_name, method_name=_method_name) if output_stream is not None: output_stream.close() raise ex + _logger.exiting(class_name=_class_name, method_name=_method_name) return +def get_default_variable_file_name(model_context): + """ + Generate location and file name for the variable file. + If model file is present, use the model file name and location; + else, use the archive file name and location. + :param model_context: contains the model and archive file arguments + :return: location and file name of variable properties file. + """ + _method_name = 'get_default_variable_file_name' + extract_file_name = model_context.get_model_file() + if not extract_file_name: + extract_file_name = model_context.get_archive_file_name() + default_variable_file = path_utils.get_filename_no_ext_from_path(extract_file_name) + if default_variable_file: + default_variable_file = os.path.join(path_utils.get_pathname_from_path(extract_file_name), + default_variable_file + '.properties') + _logger.finer('WLSDPLY-01736', default_variable_file, class_name=_class_name, method_name=_method_name) + return default_variable_file + + def get_variable_names(text): """ Get the list of variable names in the supplied text. diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index e078e71ce0..48dc0a2ac2 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -247,11 +247,12 @@ WLSDPLY-01720=to_boolean() method called with non-boolean value {0} so returning # wlsdeploy/util/variables.py WLSDPLY-01730=Failed to load variables file {0}: {1} -WLSDPLY-01731=Variables updated after encryption +WLSDPLY-01731=Variables updated WLSDPLY-01732=Variable {0} is not found in properties file WLSDPLY-01733=Variable file {0} cannot be read: {1} WLSDPLY-01734=No value in variable file {0} WLSDPLY-01735=Variable substitution for {0} is deprecated, use @@PROP:{1}@@ +WLSDPLY-01736=Default variable file name {0} # wlsdeploy/util/weblogic_helper.py WLSDPLY-01740=Encryption failed: Unable to locate SerializedSystemIni @@ -414,10 +415,10 @@ WLSDPLY-06005=Unable to clear the existing archive file. Correct the problem bef WLSDPLY-06006=Disconnect from the administration server failed: {0} WLSDPLY-06007=Closing the domain failed: {0} WLSDPLY-06008=Unable to create temporary file for writing the model: {0} -WLSDPLY-06009=Unable to add model file {0} to archive as {1}: {2} + WLSDPLY-06010={0} failed to clear the existing archive file at {1} of binaries: {2} WLSDPLY-06011={0} failed to discover domain {1} at {2} : {3} -WLSDPLY-06012={0} failed to persist the model to the archive file {1}: {2} + WLSDPLY-06013={0} failed to create the archive file at {1}: {2} WLSDPLY-06014=Filters applied to the model WLSDPLY-06015=Unable to run validation against the discovered model : {0} @@ -1163,6 +1164,12 @@ WLSDPLY-19532=No model variable injector file {0} WLSDPLY-19533=Will inject variable replacement strings into model using keyword directions found in model \ variable injector file {0} +# wlsdeploy/tool/variable_inject.py +WLSDPLY-19600=Use model variable injector file {0} from command line arguments +WLSDPLY-19601=Use variable injector keywords file {0} from command line arguments +WLSDPLY-19602=Use variable properties file {0} from command line arguments +WLSDPLY-19603=Archive file {0} was provided but does not contain a model file +WLSDPLY-19604=Update model with injected variables # Common tooling messages used by multiple tools WLSDPLY-20000={0} encountered an unexpected validation error: {1} @@ -1188,3 +1195,5 @@ WLSDPLY-20019=Filter entry in {0} has neither ID or path WLSDPLY-20020=Filter ID {0} is invalid WLSDPLY-20021=Filter path {0} does not exist WLSDPLY-20022=Error loading filter path {0} +WLSDPLY-20023={0} unable to add model file {1} to archive as {2}: {3} +WLSDPLY-20024={0} failed to persist the model to the archive file {1}: {2} \ No newline at end of file diff --git a/installer/src/main/bin/injectVariables.cmd b/installer/src/main/bin/injectVariables.cmd index 016c5efd30..f93fb59c28 100644 --- a/installer/src/main/bin/injectVariables.cmd +++ b/installer/src/main/bin/injectVariables.cmd @@ -52,7 +52,7 @@ SETLOCAL -SET WLSDEPLOY_PROGRAM_NAME=validateModel +SET WLSDEPLOY_PROGRAM_NAME=injectVariables SET SCRIPT_PATH=%~dp0 FOR %%i IN ("%SCRIPT_PATH%") DO SET SCRIPT_PATH=%%~fsi diff --git a/installer/src/main/bin/injectVariables.sh b/installer/src/main/bin/injectVariables.sh new file mode 100644 index 0000000000..c9a85de1ab --- /dev/null +++ b/installer/src/main/bin/injectVariables.sh @@ -0,0 +1,308 @@ +#!/bin/sh +# ***************************************************************************** +# injectVariables.sh +# +# Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 +# +# NAME +# injectVariables.sh - WLS Deploy tool to inject variables into the model +# +# DESCRIPTION +# This script will inject variable tokens into the model and persist the variables to the +# indicated variable file. This can be run against a model that has injected variables. Any +# injected variables will not be replaced. If the existing variable file was provided, the +# new injected variables will be appended to the file. +# +# +# This script uses the following command-line arguments directly, the rest +# of the arguments are passed down to the underlying python program: +# +# - -oracle_home The directory of the existing Oracle Home to use. +# This directory must exist and it is the caller^'s +# responsibility to verify that it does. This +# argument is required. +# +# - -domain_type The type of domain to create. This argument is +# is optional. If not specified, it defaults to WLS. +# +# - -wlst_path The path to the Oracle Home product directory under +# which to find the wlst.cmd script. This is only +# needed for pre-12.2.1 upper stack products like SOA. +# +# For example, for SOA 12.1.3, -wlst_path should be +# specified as $ORACLE_HOME/soa +# +# This script uses the following variables: +# +# JAVA_HOME - The location of the JDK to use. The caller must set +# this variable to a valid Java 7 (or later) JDK. +# +# WLSDEPLOY_HOME - The location of the WLS Deploy installation. +# If the caller sets this, the callers location will be +# honored provided it is an existing directory. +# Otherwise, the location will be calculated from the +# location of this script. +# +# WLSDEPLOY_PROPERTIES - Extra system properties to pass to WLST. The caller +# can use this environment variable to add additional +# system properties to the WLST environment. +# + +usage() { + echo "" + echo "Usage: $1 [-help]" + echo " -oracle_home " + ecgi " -model_file ^ | -archive_file ^" + echo " [-variable_injector_file ^]" + echo " [-variable_keywords_file ^]" + echo " [-variable_properties_file ^]"" + echo " [-domain_type ]" + echo " [-wlst_path ]" + echo "" + echo " where:" + echo " oracle-home - the existing Oracle Home directory for the domain" + echo "" + echo " model-file - the location of the model file in which variables will be injected." + echo " If not specified, the tool will look for the model" + echo " in the archive file. Either the model_file or the archive_file argument must be provided." + echo "" + echo " archive-file - the path to the archive file that contains a model in which the variables" + echo " will be injected. If the model-file argument is used, this argument will be" + echo " ignored. The archive file must contain a valid model." + echo "" + echo " variable-injector-file - the location of the variable injector file which contains the variable" + echo " injector keywords for this model injection run. If this argument is not provided," + echo " the model_variable_injector.json file must exist in the lib directory in the" + echo " WLSDEPLOY_HOME location." + echo "" + echo " variable-keywords-file - this argument overrides the INSTALLED version of the allowed variable keywords" + echo " for the variable injector. This argument is for advanced usage only. The installed" + echo " keywords file is located in the lib directory of WLSDEPLOY_HOME location." + echo "" + echo " variable-file - the location of the property file in which to store any variable names injected" + echo " into the model. This argument overrides the value in the model injector file." + echo " If the variable file is not listed in the model injector file, and this command" + echo " line argument is not used, the variable properties will be located and named" + echo " based on the model file or archive file name and location." + echo " If the variable file exists, new variable values will be appended to the file." + echo "" + echo " domain-type - the type of domain (e.g., WLS, JRF)." + echo " Used to locate wlst.cmd if wlst-path not specified" + echo "" + echo " wlst-path - the Oracle Home subdirectory of the wlst.cmd" + echo " script to use (e.g., ^/soa)" + echo "" +} + +umask 27 + +WLSDEPLOY_PROGRAM_NAME="injectVariables"; export WLSDEPLOY_PROGRAM_NAME + +if [ "${WLSDEPLOY_HOME}" = "" ]; then + BASEDIR="$( cd "$( dirname $0 )" && pwd )" + WLSDEPLOY_HOME=`cd "${BASEDIR}/.." ; pwd` + export WLSDEPLOY_HOME +elif [ ! -d ${WLSDEPLOY_HOME} ]; then + echo "Specified WLSDEPLOY_HOME of ${WLSDEPLOY_HOME} does not exist" >&2 + exit 2 +fi + +# +# Make sure that the JAVA_HOME environment variable is set to point to a +# JDK 7 or higher JVM (and that it isn't OpenJDK). +# +if [ "${JAVA_HOME}" = "" ]; then + echo "Please set the JAVA_HOME environment variable to point to a Java 7 installation" >&2 + exit 2 +elif [ ! -d "${JAVA_HOME}" ]; then + echo "Your JAVA_HOME environment variable to points to a non-existent directory: ${JAVA_HOME}" >&2 + exit 2 +fi + +if [ -x "${JAVA_HOME}/bin/java" ]; then + JAVA_EXE=${JAVA_HOME}/bin/java +else + echo "Java executable at ${JAVA_HOME}/bin/java either does not exist or is not executable" >&2 + exit 2 +fi + +JVM_OUTPUT=`${JAVA_EXE} -version 2>&1` +case "${JVM_OUTPUT}" in + *OpenJDK*) + echo "JAVA_HOME ${JAVA_HOME} contains OpenJDK, which is not supported" >&2 + exit 2 + ;; +esac + +JVM_FULL_VERSION=`${JAVA_EXE} -fullversion 2>&1 | awk -F "\"" '{ print $2 }'` +JVM_VERSION=`echo ${JVM_FULL_VERSION} | awk -F "." '{ print $2 }'` + +if [ ${JVM_VERSION} -lt 7 ]; then + echo "You are using an unsupported JDK version ${JVM_FULL_VERSION}" >&2 + exit 2 +else + echo "JDK version is ${JVM_FULL_VERSION}" +fi + +# +# Check to see if no args were given and print the usage message +# +if [[ $# = 0 ]]; then + usage `basename $0` + exit 0 +fi + +SCRIPT_ARGS="$*" + +# +# Find the args required to determine the WLST script to run +# + +while [[ $# > 1 ]]; do + key="$1" + case $key in + -help) + usage `basename $0` + exit 0 + ;; + -oracle_home) + ORACLE_HOME="$2" + shift + ;; + -domain_type) + DOMAIN_TYPE="$2" + shift + ;; + -wlst_path) + WLST_PATH_DIR="$2" + shift + ;; + *) + # unknown option + ;; + esac + shift # past arg or value +done + +# +# Check for values of required arguments for this script to continue. +# The underlying WLST script has other required arguments. +# +if [ "${ORACLE_HOME}" = "" ]; then + echo "Required argument ORACLE_HOME not provided" >&2 + usage `basename $0` + exit 99 +elif [ ! -d ${ORACLE_HOME} ]; then + echo "The specified ORACLE_HOME does not exist: ${ORACLE_HOME}" >&2 + exit 98 +fi + +# +# If the WLST_PATH_DIR is specified, validate that it contains the wlst.cmd script +# +if [ "${WLST_PATH_DIR}" != "" ]; then + if [ ! -d ${WLST_PATH_DIR} ]; then + echo "WLST_PATH_DIR specified does not exist: ${WLST_PATH_DIR}" >&2 + exit 98 + fi + WLST=${WLST_PATH_DIR}/common/bin/wlst.sh + if [ ! -x "${WLST}" ]; then + echo "WLST executable ${WLST} not found under specified WLST_PATH_DIR: ${WLST_PATH_DIR}" >&2 + exit 98 + fi + CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export CLASSPATH + WLST_EXT_CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export WLST_EXT_CLASSPATH +else + # + # Find the location for wlst.sh + # + WLST="" + USE_JRF_WLST=FALSE + if [ "${DOMAIN_TYPE}" = "WLS" ]; then + USE_JRF_WLST=FALSE + elif [ "${DOMAIN_TYPE}" = "RestrictedJRF" ]; then + USE_JRF_WLST=TRUE + elif [ "${DOMAIN_TYPE}" = "JRF" ]; then + USE_JRF_WLST=TRUE + else + echo "Domain type ${DOMAIN_TYPE} not recognized by shell script...assuming JRF is required" + fi + + if [ "${USE_JRF_WLST}" = "TRUE" ]; then + if [ -x ${ORACLE_HOME}/oracle_common/common/bin/wlst.sh ]; then + WLST=${ORACLE_HOME}/oracle_common/common/bin/wlst.sh + CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export CLASSPATH + WLST_EXT_CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export WLST_EXT_CLASSPATH + fi + else + if [ -x ${ORACLE_HOME}/wlserver_10.3/common/bin/wlst.sh ]; then + WLST=${ORACLE_HOME}/wlserver_10.3/common/bin/wlst.sh + CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export CLASSPATH + elif [ -x ${ORACLE_HOME}/wlserver_12.1/common/bin/wlst.sh ]; then + WLST=${ORACLE_HOME}/wlserver_12.1/common/bin/wlst.sh + CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export CLASSPATH + elif [ -x ${ORACLE_HOME}/wlserver/common/bin/wlst.sh -a -f ${ORACLE_HOME}/wlserver/.product.properties ]; then + WLST=${ORACLE_HOME}/wlserver/common/bin/wlst.sh + CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export CLASSPATH + else + WLST=${ORACLE_HOME}/oracle_common/common/bin/wlst.sh + WLST_EXT_CLASSPATH=${WLSDEPLOY_HOME}/lib/weblogic-deploy-core.jar; export WLST_EXT_CLASSPATH + fi + fi + + if [ "${WLST}" = "" ]; then + echo "Unable to determine WLS version in ${ORACLE_HOME} to determine WLST shell script to call" >&2 + exit 98 + fi +fi + +LOG_CONFIG_CLASS=oracle.weblogic.deploy.logging.WLSDeployLoggingConfig +WLST_PROPERTIES=-Dcom.oracle.cie.script.throwException=true +WLST_PROPERTIES="-Djava.util.logging.config.class=${LOG_CONFIG_CLASS} ${WLST_PROPERTIES} ${WLSDEPLOY_PROPERTIES}" +export WLST_PROPERTIES + +if [ "${WLSDEPLOY_LOG_PROPERTIES}" = "" ]; then + WLSDEPLOY_LOG_PROPERTIES=${WLSDEPLOY_HOME}/etc/logging.properties; export WLSDEPLOY_LOG_PROPERTIES +fi + +if [ "${WLSDEPLOY_LOG_DIRECTORY}" = "" ]; then + WLSDEPLOY_LOG_DIRECTORY=${WLSDEPLOY_HOME}/logs; export WLSDEPLOY_LOG_DIRECTORY +fi + +echo "JAVA_HOME = ${JAVA_HOME}" +echo "WLST_EXT_CLASSPATH = ${WLST_EXT_CLASSPATH}" +echo "CLASSPATH = ${CLASSPATH}" +echo "WLST_PROPERTIES = ${WLST_PROPERTIES}" + +PY_SCRIPTS_PATH=${WLSDEPLOY_HOME}/lib/python +echo "${WLST} ${PY_SCRIPTS_PATH}/variable_inject.py ${SCRIPT_ARGS}" + +"${WLST}" "${PY_SCRIPTS_PATH}/variable_inject.py" ${SCRIPT_ARGS} + +RETURN_CODE=$? +if [ ${RETURN_CODE} -eq 100 ]; then + usage `basename $0` + RETURN_CODE=0 +elif [ ${RETURN_CODE} -eq 99 ]; then + usage `basename $0` + echo "" + echo "injectVariables.sh failed due to the usage error shown above" >&2 +elif [ ${RETURN_CODE} -eq 98 ]; then + echo "" + echo "injectVariables.sh failed due to a parameter validation error" >&2 +elif [ ${RETURN_CODE} -eq 2 ]; then + echo "" + echo "injectVariables.sh failed (exit code = ${RETURN_CODE})" >&2 +elif [ ${RETURN_CODE} -eq 1 ]; then + echo "" + echo "injectVariables.sh completed but with some issues (exit code = ${RETURN_CODE})" >&2 +elif [ ${RETURN_CODE} -eq 0 ]; then + echo "" + echo "injectVariables.sh completed successfully (exit code = ${RETURN_CODE})" +else + # Unexpected return code so just print the message and exit... + echo "" + echo "injectVariables.sh failed (exit code = ${RETURN_CODE})" >&2 +fi +exit ${RETURN_CODE} diff --git a/installer/src/main/lib/injectors/host.json b/installer/src/main/lib/injectors/host.json index 0761849661..0a11d2cf46 100644 --- a/installer/src/main/lib/injectors/host.json +++ b/installer/src/main/lib/injectors/host.json @@ -9,13 +9,7 @@ "CoherenceClusterSystemResource.CoherenceResource.CoherenceClusterParams.UnicastListenAddress": {}, "CoherenceClusterSystemResource.CoherenceResource.CoherenceFederationParams.RemoteParticipantHost": {}, "CoherenceClusterSystemResource.FederationRemoteParticipantHost": {}, - "JDBCSystemResource.JdbcResource.JDBCOracleParams.OnsNodeList": { - "regexp": [ - { - "pattern": "[\w\\\\]" - } - ] - }, + "JDBCSystemResource.JdbcResource.JDBCOracleParams.OnsNodeList": {}, "JMSSystemResource.JmsResource.Template.Multicast.MulticastAddress": {}, "JMSSystemResource.JmsResource.Topic.Multicast.MulticastAddress": {}, "JMSSystemResource.JmsResource.UniformDistributedTopic.Multicast.MulticastAddress": {}, diff --git a/installer/src/main/samples/custom_injector.json b/installer/src/main/samples/custom_injector.json index acc8349073..bdbe60aac8 100644 --- a/installer/src/main/samples/custom_injector.json +++ b/installer/src/main/samples/custom_injector.json @@ -1,18 +1,3 @@ -//{ -// "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": -// { -// "regexp": [ -// { -// "pattern": "(?<=PORT=)[\w.-]+(?=\))", -// "suffix": "Port" -// }, -// { -// "pattern": "(?<=HOST=)[\w.-]+(?=\))", -// "suffix": "Host" -// } -// ] -// } -//} { "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": { From 6f9f06dfd086f86efdead2a3e1d8bc7685d5fadd Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 24 May 2018 15:54:21 -0500 Subject: [PATCH 20/52] add capability to append or update an existing variable file with injector --- core/src/main/python/discover.py | 24 ++--- core/src/main/python/encrypt.py | 2 +- core/src/main/python/variable_inject.py | 4 +- .../wlsdeploy/tool/util/variable_injector.py | 92 +++++++++++++------ .../main/python/wlsdeploy/util/variables.py | 7 +- .../deploy/messages/wlsdeploy_rb.properties | 6 +- .../src/test/python/variable_injector_test.py | 8 +- .../resources/variable_injector_keyword.json | 2 +- 8 files changed, 94 insertions(+), 51 deletions(-) diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index 500cc6bff4..ccdb804dc1 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -22,7 +22,6 @@ from oracle.weblogic.deploy.util import WebLogicDeployToolingVersion from oracle.weblogic.deploy.util import WLSDeployArchive from oracle.weblogic.deploy.util import WLSDeployArchiveIOException -from oracle.weblogic.deploy.validate import ValidateException sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0]))) @@ -37,7 +36,6 @@ from wlsdeploy.tool.discover.multi_tenant_discoverer import MultiTenantDiscoverer from wlsdeploy.tool.discover.resources_discoverer import ResourcesDiscoverer from wlsdeploy.tool.discover.topology_discoverer import TopologyDiscoverer -from wlsdeploy.tool.validate.validator import Validator from wlsdeploy.tool.util import filter_helper from wlsdeploy.util import wlst_helper from wlsdeploy.util import model_translator @@ -369,18 +367,20 @@ def __check_and_customize_model(model, model_context): if filter_helper.apply_filters(model.get_model(), "discover"): __logger.info('WLSDPLY-06014', _class_name=_class_name, method_name=_method_name) - inserted, variable_model, variable_file_name = VariableInjector(model.get_model(), model_context, WebLogicHelper( - __logger).get_actual_weblogic_version()).inject_variables_keyword_file() + inserted, variable_model, variable_file_name = VariableInjector(_program_name, model.get_model(), model_context, + WebLogicHelper( + __logger).get_actual_weblogic_version()).\ + inject_variables_keyword_file() if inserted: model = Model(variable_model) - try: - validator = Validator(model_context, wlst_mode=__wlst_mode) - - # no variables are generated by the discover tool - validator.validate_in_tool_mode(model.get_model(), variables_file_name=variable_file_name, - archive_file_name=model_context.get_archive_file_name()) - except ValidateException, ex: - __logger.warning('WLSDPLY-06015', ex.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) + # try: + # validator = Validator(model_context, wlst_mode=__wlst_mode) + # + # # no variables are generated by the discover tool + # validator.validate_in_tool_mode(model.get_model(), variables_file_name=variable_file_name, + # archive_file_name=model_context.get_archive_file_name()) + # except ValidateException, ex: + # __logger.warning('WLSDPLY-06015', ex.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) return model diff --git a/core/src/main/python/encrypt.py b/core/src/main/python/encrypt.py index 03cc3240c4..4f306f7365 100644 --- a/core/src/main/python/encrypt.py +++ b/core/src/main/python/encrypt.py @@ -190,7 +190,7 @@ def __encrypt_model_and_variables(model_context): if variable_change_count > 0: try: - variable_helper.write_variables(variables, variable_file) + variable_helper.write_variables(_program_name, variables, variable_file) __logger.info('WLSDPLY-04209', _program_name, variable_change_count, variable_file, class_name=_class_name, method_name=_method_name) except VariableException, ve: diff --git a/core/src/main/python/variable_inject.py b/core/src/main/python/variable_inject.py index b689643b34..4edeaab9ac 100644 --- a/core/src/main/python/variable_inject.py +++ b/core/src/main/python/variable_inject.py @@ -177,8 +177,8 @@ def __inject(model, model_context): :param model_context: the model context :return: True if variables were inserted into model: The updated model """ - __kwargs[variable_injector.VARIABLE_FILE_APPEND_ARG] = True - inserted, variable_model, variable_file_name = VariableInjector(model, model_context, + __kwargs[variable_injector.VARIABLE_FILE_APPEND_ARG] = variable_injector.VARIABLE_FILE_UPDATE + inserted, variable_model, variable_file_name = VariableInjector(_program_name, model, model_context, WebLogicHelper( __logger).get_actual_weblogic_version()). \ inject_variables_keyword_file(**__kwargs) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index b7547f9780..b10ccb292e 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -10,7 +10,6 @@ import oracle.weblogic.deploy.aliases.AliasException as AliasException import oracle.weblogic.deploy.json.JsonException as JsonException -import oracle.weblogic.deploy.util.PyOrderedDict as PyOrderedDict import oracle.weblogic.deploy.util.VariableException as VariableException import wlsdeploy.util.model as model_sections @@ -33,6 +32,9 @@ VARIABLE_FILE_NAME_ARG = 'variable_file_name' VARIABLE_FILE_NAME = 'variables.json' VARIABLE_FILE_APPEND_ARG = 'append_to_variables' +VARIABLE_FILE_APPEND = 'append' +VARIABLE_FILE_UPDATE = 'update' +VARIABLE_FILE_APPEND_VALS = [VARIABLE_FILE_APPEND, VARIABLE_FILE_UPDATE] # custom keyword in model injector file CUSTOM_KEYWORD = 'CUSTOM' KEYWORD_FILES = 'file-list' @@ -59,7 +61,15 @@ class VariableInjector(object): - def __init__(self, model, model_context=None, version=None): + def __init__(self, program_name, model, model_context=None, version=None): + """ + Construct an instance of the injector with the model and information used by the injector. + :param program_name: name of the calling tool + :param model: to be updated with variables + :param model_context: context with command line information + :param version: of model if model context is not provided + """ + self.__program_name = program_name self.__original = copy.deepcopy(model) self.__model = model self.__model_context = model_context @@ -86,7 +96,7 @@ def inject_variables_keyword_file(self, **kwargs): _logger.entering(class_name=_class_name, method_name=_method_name) variable_injector_location_file = _get_variable_injector_file_name(**kwargs) - variables_injector_dictionary = _load_variables_file(variable_injector_location_file) + variables_injector_dictionary = _load_variable_injector_file(variable_injector_location_file) variable_keywords_location_file = _get_variable_keywords_file_name(**kwargs) keywords_dictionary = _load_keywords_file(variable_keywords_location_file) @@ -101,12 +111,14 @@ def inject_variables_keyword_file(self, **kwargs): else: _logger.info('WLSDPLY-19533', variable_injector_location_file, class_name=_class_name, method_name=_method_name) - variable_file_location = self._replace_tokens(variable_file_location) + append, stage_dictionary = _load_variable_file(variable_file_location, **kwargs) injector_file_list = _create_injector_file_list(variables_injector_dictionary, keywords_dictionary, _get_keyword_files_location(**kwargs)) variables_file_dictionary = self.inject_variables_keyword_dictionary(injector_file_list) - variables_inserted = _write_variables_file(variables_file_dictionary, variable_file_location, - _get_append_to_variable_file(**kwargs)) + if variables_file_dictionary: + stage_dictionary.update(variables_file_dictionary) + variables_inserted = self._write_variables_file(stage_dictionary, variable_file_location, + append) if variables_inserted: _logger.info('WLSDPLY-19518', variable_file_location, class_name=_class_name, method_name=_method_name) @@ -124,20 +136,20 @@ def inject_variables_keyword_file(self, **kwargs): def inject_variables_keyword_dictionary(self, injector_file_list): """ Takes a variable keyword dictionary and returns a variables for file in a dictionary - :param injector_file_list: - :return: + :param injector_file_list: list of injector files for processing variable injection + :return: variables_dictionary containing the variable properties to persist to the variable file """ _method_name = 'inject_variables_keyword_dictionary' _logger.entering(injector_file_list, class_name=_class_name, method_name=_method_name) - variables_dictionary = PyOrderedDict() + variable_dictionary = dict() for filename in injector_file_list: injector_dictionary = _load_injector_file(self._replace_tokens(filename)) entries = self.inject_variables(injector_dictionary) if entries: _logger.finer('WLSDPLY-19513', filename, class_name=_class_name, method_name=_method_name) - variables_dictionary.update(entries) - _logger.exiting(class_name=_class_name, method_name=_method_name, result=variables_dictionary) - return variables_dictionary + variable_dictionary.update(entries) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_dictionary) + return variable_dictionary def inject_variables(self, injector_dictionary): """ @@ -422,7 +434,7 @@ def _log_mbean_not_found(self, mbean, replacement, location): code = ValidationCodes.INVALID try: code, __ = self.__aliases.is_valid_model_folder_name(location, mbean) - except AliasException, ae: + except AliasException: pass if code == ValidationCodes.INVALID: _logger.warning('WLSDPLY-19515', mbean, replacement, location.get_folder_path(), @@ -444,8 +456,46 @@ def _get_variable_file_name(self, variables_injector_dictionary, **kwargs): _logger.finer('WLSDPLY-19521', variable_file_location, class_name=_class_name, method_name=_method_name) else: variable_file_location = variables.get_default_variable_file_name(self.__model_context) + if variable_file_location: + variable_file_location = self._replace_tokens(variable_file_location) return variable_file_location + def _write_variables_file(self, variables_dictionary, variables_file_name, append): + _method_name = '_write_variables_file' + _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) + + written = False + if variables_dictionary: + try: + variables.write_variables(self.__program_name, variables_dictionary, variables_file_name, append) + written = True + except VariableException, ve: + _logger.warning('WLSDPLY-19507', variables_file_name, ve.getLocalizedMessage(), class_name=_class_name, + method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) + return written + + +def _load_variable_file(variable_file_location, **kwargs): + _method_name = '_load_variable_file' + append = False + variable_dictionary = dict() + if VARIABLE_FILE_APPEND_ARG in kwargs and kwargs[VARIABLE_FILE_APPEND_ARG] in VARIABLE_FILE_APPEND_VALS: + _logger.fine('append argument found {0}', kwargs[VARIABLE_FILE_APPEND_ARG] ) + if kwargs[VARIABLE_FILE_APPEND_ARG] == VARIABLE_FILE_APPEND: + _logger.fine('WLSDPLY-19536', variable_file_location, class_name=_class_name, method_name=_method_name) + append = True + elif kwargs[VARIABLE_FILE_APPEND_ARG] == VARIABLE_FILE_UPDATE and os.path.isfile(variable_file_location): + _logger.fine('WLSDPLY-19534', variable_file_location, class_name=_class_name, method_name=_method_name) + try: + variable_dictionary = variables.load_variables(variable_file_location) + except VariableException, ve: + _logger.warning('WLSDPLY-19537', variable_file_location, ve.getLocalizedMessage(), + class_name=_class_name, method_name=_method_name) + else: + _logger.fine('WLSDPLY-19535', variable_file_location, class_name=_class_name, method_name=_method_name) + return append, variable_dictionary + def _get_variable_injector_file_name(**kwargs): variable_injector_file_name = VARIABLE_INJECTOR_FILE_NAME @@ -467,7 +517,7 @@ def _get_variable_keywords_file_name(**kwargs): return os.path.join(_wlsdeploy_location, DEFAULT_FILE_LOCATION, variable_keywords_file_name) -def _load_variables_file(variable_injector_location): +def _load_variable_injector_file(variable_injector_location): _method_name = '_load_variables_file' _logger.entering(variable_injector_location, class_name=_class_name, method_name=_method_name) variables_dictionary = None @@ -555,20 +605,6 @@ def _get_append_to_variable_file(**kwargs): return False -def _write_variables_file(variables_dictionary, variables_file_name, append): - _method_name = '_write_variables_file' - _logger.entering(variables_dictionary, variables_file_name, class_name=_class_name, method_name=_method_name) - - written = False - if variables_dictionary: - try: - variables.write_variables(variables_dictionary, variables_file_name, append) - written = True - except VariableException, ve: - _logger.warning('WLSDPLY-19507', variables_file_name, ve.getLocalizedMessage(), class_name=_class_name, - method_name=_method_name) - _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) - return written def _format_variable_value(value): diff --git a/core/src/main/python/wlsdeploy/util/variables.py b/core/src/main/python/wlsdeploy/util/variables.py index 9a7117718e..826b6e783d 100644 --- a/core/src/main/python/wlsdeploy/util/variables.py +++ b/core/src/main/python/wlsdeploy/util/variables.py @@ -58,22 +58,23 @@ def load_variables(file_path): return variable_map -def write_variables(variable_map, file_path, append=False): +def write_variables(program_name, variable_map, file_path, append=False): """ Write the dictionary of variables to the specified file. + :param program_name: name of tool that invoked the method which will be written to the variable properties file :param variable_map: the dictionary of variables :param file_path: the file to which to write the properties :param append: defaults to False. Append properties to the end of file :raises VariableException if an error occurs while storing the variables in the file """ _method_name = 'write_variables' - _logger.entering(file_path, append, class_name=_class_name, method_name=_method_name) + _logger.entering(program_name, file_path, append, class_name=_class_name, method_name=_method_name) props = Properties() for key in variable_map: value = variable_map[key] props.setProperty(key, value) - comment = exception_helper.get_message('WLSDPLY-01731') + comment = exception_helper.get_message('WLSDPLY-01731', program_name) output_stream = None try: output_stream = FileOutputStream(File(file_path), Boolean(append)) diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 48dc0a2ac2..a5ddb3e8a5 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -247,7 +247,7 @@ WLSDPLY-01720=to_boolean() method called with non-boolean value {0} so returning # wlsdeploy/util/variables.py WLSDPLY-01730=Failed to load variables file {0}: {1} -WLSDPLY-01731=Variables updated +WLSDPLY-01731=Variables updated by {0} WLSDPLY-01732=Variable {0} is not found in properties file WLSDPLY-01733=Variable file {0} cannot be read: {1} WLSDPLY-01734=No value in variable file {0} @@ -1163,6 +1163,10 @@ WLSDPLY-19531=Invalid location {0} for variable injection into attribute {1} : { WLSDPLY-19532=No model variable injector file {0} WLSDPLY-19533=Will inject variable replacement strings into model using keyword directions found in model \ variable injector file {0} +WLSDPLY-19534=Update existing variable file {0} with injected variables +WLSDPLY-19535=Create new variable file {0} for injected variables +WLSDPLY-19536=Append existing variable file {0} with injected variables +WLSDPLY-19537=Unable to load variables from existing file {0} and will overwrite the file : {1} # wlsdeploy/tool/variable_inject.py WLSDPLY-19600=Use model variable injector file {0} from command line arguments diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index ccf4fe38d4..153393c456 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -2,6 +2,7 @@ Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. The Universal Permissive License (UPL), Version 1.0 """ +import os import unittest import wlsdeploy.util.variables as variables @@ -12,7 +13,7 @@ class VariableFileHelperTest(unittest.TestCase): _resources_dir = '../../test-classes' - _variable_file = _resources_dir + '/variables.properties' + _variable_file = _resources_dir + '/variable.injector.test.properties' _model_file = _resources_dir + '/variable_insertion.yaml' _variable_injector_keyword = 'variable_injector_keyword.json' _variable_injector_custom = 'variable_injector_custom.json' @@ -21,7 +22,7 @@ class VariableFileHelperTest(unittest.TestCase): def setUp(self): self.name = VariableFileHelperTest self._model = FileToPython(self._model_file).parse() - self._helper = VariableInjector(self._model, None, '12.2.1.3') + self._helper = VariableInjector(self.name, self._model, None, '12.2.1.3') def testSingleVariableReplacement(self): replacement_dict = dict() @@ -223,9 +224,10 @@ def testWithVariableHelperKeywords(self): variable_injector_path_name=self._resources_dir, variable_injector_file_name=self._variable_injector_keyword, variable_keywords_path_name=self._resources_dir, variable_keywords_file_name=self._keywords_file) - self.assertEqual(True, inserted) self.assertEqual(self._variable_file, variable_file_name) + self.assertEqual(True, inserted) actual = variables.load_variables(self._variable_file) + print actual self._compare_to_expected_dictionary(expected, actual) def _compare_to_expected_dictionary(self, expected, actual): diff --git a/core/src/test/resources/variable_injector_keyword.json b/core/src/test/resources/variable_injector_keyword.json index 51e9bc34c3..a71915be92 100644 --- a/core/src/test/resources/variable_injector_keyword.json +++ b/core/src/test/resources/variable_injector_keyword.json @@ -1,5 +1,5 @@ { - "variable_file_name": "../../test-classes/variables.properties", + "variable_file_name": "../../test-classes/variable.injector.test.properties", "PORT": {}, "URL": {}, "CREDENTIALS": {} From aa19eb6ba898dd0777eab92c01f6138de8620244 Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 24 May 2018 18:28:19 -0500 Subject: [PATCH 21/52] Add managedserver keyword --- .../wlsdeploy/tool/util/variable_injector.py | 61 ++++++++++++++----- .../tool/util/variable_injector_functions.py | 51 ++++++++++++++++ .../deploy/messages/wlsdeploy_rb.properties | 3 + .../src/test/python/variable_injector_test.py | 19 ++++++ .../src/main/samples/custom_injector.json | 3 +- 5 files changed, 120 insertions(+), 17 deletions(-) create mode 100644 core/src/main/python/wlsdeploy/tool/util/variable_injector_functions.py diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index b10ccb292e..c800b6d9ad 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -12,6 +12,7 @@ import oracle.weblogic.deploy.json.JsonException as JsonException import oracle.weblogic.deploy.util.VariableException as VariableException +import wlsdeploy.tool.util.variable_injector_functions as variable_injector_functions import wlsdeploy.util.model as model_sections import wlsdeploy.util.variables as variables from wlsdeploy.aliases.aliases import Aliases @@ -22,6 +23,8 @@ from wlsdeploy.logging.platform_logger import PlatformLogger WEBLOGIC_DEPLOY_HOME_TOKEN = '@@WLSDEPLOY@@' + +# KWARGS to modify how files are found and loaded on the inject_variables_keyword_file(..) function VARIABLE_INJECTOR_FILE_NAME = 'model_variable_injector.json' VARIABLE_KEYWORDS_FILE_NAME = 'variable_keywords.json' VARIABLE_INJECTOR_PATH_NAME_ARG = 'variable_injector_path_name' @@ -35,6 +38,8 @@ VARIABLE_FILE_APPEND = 'append' VARIABLE_FILE_UPDATE = 'update' VARIABLE_FILE_APPEND_VALS = [VARIABLE_FILE_APPEND, VARIABLE_FILE_UPDATE] + + # custom keyword in model injector file CUSTOM_KEYWORD = 'CUSTOM' KEYWORD_FILES = 'file-list' @@ -49,6 +54,17 @@ VARIABLE_SEP = '.' SUFFIX_SEP = '--' + +MANAGED_SERVERS = 'MANAGED_SERVERS' +ADMIN_SERVER = 'ADMIN_SERVER' + +# This could be changed into a loaded file so that others can add their own bits of code to +# selectively identify which names in an mbean list should be injected with properties +USER_KEYWORD_DICT = { + MANAGED_SERVERS: 'managed_server_list', + ADMIN_SERVER: 'admin_server_list' +} +# global variables for functions in VariableInjector _find_special_names_pattern = re.compile('[\[).+\]]') _fake_name_marker = 'fakename' _fake_name_replacement = re.compile('.' + _fake_name_marker) @@ -179,7 +195,7 @@ def __inject_variable(self, location, injector, injector_values): def _traverse_variables(model_section, mbean_list): if mbean_list: mbean = mbean_list.pop(0) - mbean, mbean_name_list = _find_special_name(mbean) + mbean, mbean_name_list = self._find_special_name(mbean) _logger.finer('WLSDPLY-19523', mbean, location.get_folder_path(), class_name=_class_name, method_name=_method_name) if mbean in model_section: @@ -221,7 +237,7 @@ def _traverse_variables(model_section, mbean_list): section = self.__model if start_mbean_list: # Find out in what section is the mbean top folder so can move to that section in the model - top_mbean, __ = _find_special_name(start_mbean_list[0]) + top_mbean, __ = self._find_special_name(start_mbean_list[0]) for entry in self.__section_keys: if entry in self.__model and top_mbean in self.__model[entry]: section = self.__model[entry] @@ -442,7 +458,7 @@ def _log_mbean_not_found(self, mbean, replacement, location): else: _logger.finer('WLSDPLY-19516', mbean, replacement, location.get_folder_path(), class_name=_class_name, method_name=_method_name) - + def _get_variable_file_name(self, variables_injector_dictionary, **kwargs): _method_name = '_get_variable_file_name' if VARIABLE_FILE_NAME_ARG in kwargs: @@ -475,13 +491,38 @@ def _write_variables_file(self, variables_dictionary, variables_file_name, appen _logger.exiting(class_name=_class_name, method_name=_method_name, result=written) return written + def _find_special_name(self, mbean): + mbean_name = mbean + mbean_name_list = [] + name_list = _find_special_names_pattern.split(mbean) + if name_list and len(name_list) > 1: + mbean_name = name_list[0] + mbean_name_list = name_list[1].split(',') + if mbean_name_list: + new_list = [] + for entry in mbean_name_list: + if entry in USER_KEYWORD_DICT: + _logger.fine('WLSDPLY-19538', entry, mbean) + try: + method = getattr(variable_injector_functions, USER_KEYWORD_DICT[entry]) + append_list = method(self.__model) + new_list.extend(append_list) + except AttributeError, e: + _logger.warning('WLSDPLY-19539', entry, USER_KEYWORD_DICT[entry], e) + new_list = mbean_name_list + break + else: + new_list.append(entry) + mbean_name_list = new_list + return mbean_name, mbean_name_list + def _load_variable_file(variable_file_location, **kwargs): _method_name = '_load_variable_file' append = False variable_dictionary = dict() if VARIABLE_FILE_APPEND_ARG in kwargs and kwargs[VARIABLE_FILE_APPEND_ARG] in VARIABLE_FILE_APPEND_VALS: - _logger.fine('append argument found {0}', kwargs[VARIABLE_FILE_APPEND_ARG] ) + _logger.fine('append argument found {0}', kwargs[VARIABLE_FILE_APPEND_ARG]) if kwargs[VARIABLE_FILE_APPEND_ARG] == VARIABLE_FILE_APPEND: _logger.fine('WLSDPLY-19536', variable_file_location, class_name=_class_name, method_name=_method_name) append = True @@ -605,8 +646,6 @@ def _get_append_to_variable_file(**kwargs): return False - - def _format_variable_value(value): if type(value) == bool: if value: @@ -667,16 +706,6 @@ def _split_injector(injector_path): return ml, attr -def _find_special_name(mbean): - mbean_name = mbean - mbean_name_list = [] - name_list = _find_special_names_pattern.split(mbean) - if name_list and len(name_list) > 1: - mbean_name = name_list[0] - mbean_name_list = name_list[1].split(',') - return mbean_name, mbean_name_list - - def __temporary_fix(injector_dictionary): # this is very dangerous - for now, if you want to escape a backslash, need to do 4 backslash. _method_name = '__temporary_fix' diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector_functions.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector_functions.py new file mode 100644 index 0000000000..3ceb150e6d --- /dev/null +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector_functions.py @@ -0,0 +1,51 @@ +""" +Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. +The Universal Permissive License (UPL), Version 1.0 +""" +from wlsdeploy.logging.platform_logger import PlatformLogger + +import wlsdeploy.aliases.model_constants as model_constants +import wlsdeploy.util.model as model_sections + +_class_name = 'variable_injector' +_logger = PlatformLogger('wlsdeploy.tool.util') + + +def managed_server_list(model): + """ + Return a managed server name list from the provided model. + :param model: to process for managed server list + :return: list of managed server names or empty list if no managed servers + """ + _method_name = 'managed_server_list' + _logger.entering(class_name=_class_name, method_name=_method_name) + ms_name_list = [] + topology_constant = model_sections.get_model_topology_key() + if topology_constant in model: + topology = model[topology_constant] + if model_constants.SERVER in topology: + ms_name_list = topology[model_constants.SERVER].keys() + if model_constants.ADMIN_SERVER_NAME in topology: + admin_server = topology[model_constants.ADMIN_SERVER_NAME] + if admin_server in ms_name_list: + ms_name_list.remove(admin_server) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=ms_name_list) + return ms_name_list + + +def admin_server_list(model): + """ + Return the domain admin server in list format + :param model: to process for admin server list + :return: admin server in list format + """ + _method_name = 'admin_server_list' + _logger.entering(class_name=_class_name, method_name=_method_name) + as_name_list = [] + topology_constant = model_sections.get_model_topology_key() + if topology_constant in model: + topology = model[topology_constant] + if topology and model_constants.ADMIN_SERVER_NAME in topology: + as_name_list.append(topology[model_constants.ADMIN_SERVER_NAME]) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=as_name_list) + return as_name_list diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index a5ddb3e8a5..799e28506f 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1167,6 +1167,9 @@ WLSDPLY-19534=Update existing variable file {0} with injected variables WLSDPLY-19535=Create new variable file {0} for injected variables WLSDPLY-19536=Append existing variable file {0} with injected variables WLSDPLY-19537=Unable to load variables from existing file {0} and will overwrite the file : {1} +WLSDPLY-19538=Found name keyword {0} for mbean {1} +WLSDPLY-19539=Unable to locate and call mbean name keyword {0} method {1} : {2}. Will inject variables using \ + full mbean name list # wlsdeploy/tool/variable_inject.py WLSDPLY-19600=Use model variable injector file {0} from command line arguments diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index 153393c456..15b64a7696 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -210,6 +210,25 @@ def testWithListMBeanName(self): actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) + def testWithManagedServerKeyword(self): + expected = dict() + expected['Server.m1.SSL.Enabled'] = 'True' + expected['Server.m2.SSL.Enabled'] = 'True' + replacement_dict = dict() + replacement_dict['Server[MANAGED_SERVERS].SSL.Enabled'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testWithMultiKeyword(self): + expected = dict() + expected['Server.AdminServer.SSL.Enabled'] = 'True' + expected['Server.m1.SSL.Enabled'] = 'True' + expected['Server.m2.SSL.Enabled'] = 'True' + replacement_dict = dict() + replacement_dict['Server[MANAGED_SERVERS,ADMIN_SERVER].SSL.Enabled'] = dict() + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + def testWithVariableHelperKeywords(self): expected = dict() expected['JMSSystemResource.MyJmsModule.JmsResource.ForeignServer.MyForeignServer.ConnectionURL'] \ diff --git a/installer/src/main/samples/custom_injector.json b/installer/src/main/samples/custom_injector.json index bdbe60aac8..1c218284d8 100644 --- a/installer/src/main/samples/custom_injector.json +++ b/installer/src/main/samples/custom_injector.json @@ -11,5 +11,6 @@ "suffix": "Host" } ] - } + }, + "Server[MANAGED_SERVERS].SSL.Enabled": {} } \ No newline at end of file From 6978b67a8759d06c2cf347e88cd93d94f489ef20 Mon Sep 17 00:00:00 2001 From: crountre Date: Fri, 25 May 2018 11:37:00 -0500 Subject: [PATCH 22/52] Add force attribute into model keyword --- .../wlsdeploy/tool/util/variable_injector.py | 14 +++++++++++++- .../deploy/messages/wlsdeploy_rb.properties | 2 ++ core/src/test/python/variable_injector_test.py | 11 +++++++++++ installer/src/main/samples/custom_injector.json | 7 ++++++- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index c800b6d9ad..6a15608537 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -6,6 +6,7 @@ import os import re +import java.lang.Boolean as Boolean import java.lang.IllegalArgumentException as IllegalArgumentException import oracle.weblogic.deploy.aliases.AliasException as AliasException @@ -225,6 +226,7 @@ def _traverse_variables(model_section, mbean_list): self._log_mbean_not_found(mbean, injector, location) return False else: + self._check_insert_attribute_model(location, model_section, attribute, injector_values) if attribute in model_section: returned_dict = self._variable_info(model_section, attribute, location, injector_values) if returned_dict: @@ -516,6 +518,16 @@ def _find_special_name(self, mbean): mbean_name_list = new_list return mbean_name, mbean_name_list + def _check_insert_attribute_model(self, location, model_section, attribute, injector_values): + _method_name = '_check_insert_attribute_model' + if attribute not in model_section and (FORCE in injector_values and Boolean(injector_values[FORCE])): + value = self.__aliases.get_model_attribute_default_value(location, attribute) + # This is the best I can do - need a get_default_value_model function(location, attribute) + __, wlst_value = self.__aliases.get_wlst_attribute_name_and_value(location, attribute, value) + _logger.fine('WLSDPLY-19540', attribute, location.get_folder_path(), wlst_value, + class_name=_class_name, method_name=_method_name) + model_section[attribute] = wlst_value + def _load_variable_file(variable_file_location, **kwargs): _method_name = '_load_variable_file' @@ -559,7 +571,7 @@ def _get_variable_keywords_file_name(**kwargs): def _load_variable_injector_file(variable_injector_location): - _method_name = '_load_variables_file' + _method_name = '_load_variable_injector_file' _logger.entering(variable_injector_location, class_name=_class_name, method_name=_method_name) variables_dictionary = None if os.path.isfile(variable_injector_location): diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 799e28506f..96106612b3 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1170,6 +1170,8 @@ WLSDPLY-19537=Unable to load variables from existing file {0} and will overwrite WLSDPLY-19538=Found name keyword {0} for mbean {1} WLSDPLY-19539=Unable to locate and call mbean name keyword {0} method {1} : {2}. Will inject variables using \ full mbean name list +WLSDPLY-19540=Attribute {0} not in the model at location {1} but the force attribute is present in the injector \ + keyword so the attribute will be added to the model with the default value {2} # wlsdeploy/tool/variable_inject.py WLSDPLY-19600=Use model variable injector file {0} from command line arguments diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index 15b64a7696..139055af6b 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -249,6 +249,17 @@ def testWithVariableHelperKeywords(self): print actual self._compare_to_expected_dictionary(expected, actual) + def testForceAttribute(self): + expected = dict() + expected['Server.AdminServer.SSL.HostnameVerificationIgnored'] = 'false' + expected['Server.m1.SSL.HostnameVerificationIgnored'] = 'false' + expected['Server.m2.SSL.HostnameVerificationIgnored'] = 'false' + replacement_dict = dict() + replacement_dict['Server.SSL.HostnameVerificationIgnored'] = dict() + replacement_dict['Server.SSL.HostnameVerificationIgnored'][variable_injector.FORCE] = True + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + def _compare_to_expected_dictionary(self, expected, actual): self.assertEqual(len(expected), len(actual), 'Not the same number of entries : expected=' + str(len(expected)) + ', actual=' + str( diff --git a/installer/src/main/samples/custom_injector.json b/installer/src/main/samples/custom_injector.json index 1c218284d8..e1632ff89d 100644 --- a/installer/src/main/samples/custom_injector.json +++ b/installer/src/main/samples/custom_injector.json @@ -1,4 +1,6 @@ { + "UnixMachine.NodeManager.ListenAddress": {}, + "UnixMachine.NodeManager.ListenPort": {}, "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": { "regexp": [ @@ -12,5 +14,8 @@ } ] }, - "Server[MANAGED_SERVERS].SSL.Enabled": {} + "Server[MANAGED_SERVERS].SSL.Enabled": {}, + "Server[ADMIN_SERVER].SSL.HostnameVerificationIgnored": { + "force": true + } } \ No newline at end of file From 6d79613d2def250242a677f48a6c2e74e5fb5a7b Mon Sep 17 00:00:00 2001 From: crountre Date: Fri, 25 May 2018 18:11:28 -0500 Subject: [PATCH 23/52] add variable value to injector --- .../wlsdeploy/tool/util/variable_injector.py | 57 ++++++++++++++----- .../deploy/messages/wlsdeploy_rb.properties | 2 + .../src/test/python/variable_injector_test.py | 44 ++++++++++++++ 3 files changed, 89 insertions(+), 14 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 6a15608537..a1e5aa3a49 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -52,6 +52,7 @@ REGEXP_SUFFIX = 'suffix' REGEXP_PATTERN = 'pattern' FORCE = 'force' +VARIABLE_VALUE = 'variable_value' VARIABLE_SEP = '.' SUFFIX_SEP = '--' @@ -280,11 +281,11 @@ def __format_variable_name_segment(self, location, attribute, suffix): def _variable_info(self, model, attribute, location, injector_values): # add code here to put in model if force in injector values if REGEXP in injector_values: - return self._process_regexp(model, attribute, location, injector_values[REGEXP]) + return self._process_regexp(model, attribute, location, injector_values) else: - return self._process_attribute(model, attribute, location) + return self._process_attribute(model, attribute, location, injector_values) - def _process_attribute(self, model, attribute, location): + def _process_attribute(self, model, attribute, location, injector_values): _method_name = '_process_attribute' _logger.entering(attribute, location.get_folder_path(), class_name=_class_name, method_name=_method_name) @@ -302,20 +303,22 @@ def _process_attribute(self, model, attribute, location): _logger.finer('WLSDPLY-19526', attribute_value, attribute, str(location), class_name=_class_name, method_name=_method_name) if variable_value: - variable_dict[variable_name] = variable_value + variable_dict[variable_name] = self._check_replace_variable_value(location, attribute, variable_value, + injector_values) _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_dict - def _process_regexp(self, model, attribute, location, regexp_list): + def _process_regexp(self, model, attribute, location, injector_values): if isinstance(model[attribute], dict): - return self._process_patterns_dictionary(attribute, model[attribute], location, regexp_list) + return self._process_patterns_dictionary(attribute, model[attribute], location, injector_values) elif type(model[attribute]) == list: - return self._process_patterns_list(attribute, model[attribute], location, regexp_list) + return self._process_patterns_list(attribute, model[attribute], location, injector_values) else: - return self._process_patterns_string(model, attribute, location, regexp_list) + return self._process_patterns_string(model, attribute, location, injector_values) - def _process_patterns_string(self, model, attribute, location, regexp_list): + def _process_patterns_string(self, model, attribute, location, injector_values): variable_dict = dict() + regexp_list = injector_values[REGEXP] for dictionary in regexp_list: pattern = None suffix = None @@ -325,7 +328,8 @@ def _process_patterns_string(self, model, attribute, location, regexp_list): suffix = dictionary[REGEXP_SUFFIX] variable_name, variable_value = self._process_pattern_string(model, attribute, location, pattern, suffix) if variable_value: - variable_dict[variable_name] = variable_value + variable_dict[variable_name] = self._check_replace_variable_value(location, attribute, variable_value, + injector_values) return variable_dict def _process_pattern_string(self, model, attribute, location, pattern, suffix): @@ -350,6 +354,7 @@ def _process_pattern_string(self, model, attribute, location, pattern, suffix): _logger.finer('WLSDPLY-19524', pattern, attribute, model[attribute], location.get_folder_path, class_name=_class_name, method_name=_method_name) + _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value @@ -362,8 +367,9 @@ def _find_segment_in_string(self, attribute, attribute_value, location, pattern, _format_as_property(variable_name)) return attribute_value, variable_name, variable_value - def _process_patterns_list(self, attribute, attribute_value, location, regexp_list): + def _process_patterns_list(self, attribute, attribute_value, location, injector_values): variable_dict = dict() + regexp_list = injector_values[REGEXP] for dictionary in regexp_list: pattern = None suffix = None @@ -374,7 +380,8 @@ def _process_patterns_list(self, attribute, attribute_value, location, regexp_li variable_name, variable_value = self._process_pattern_list(attribute, attribute_value, location, pattern, suffix) if variable_value: - variable_dict[variable_name] = variable_value + variable_dict[variable_name] = self._check_replace_variable_value(location, attribute, variable_value, + injector_values) return variable_dict def _process_pattern_list(self, attribute_name, attribute_list, location, pattern, suffix): @@ -399,8 +406,9 @@ def _process_pattern_list(self, attribute_name, attribute_list, location, patter _logger.exiting(class_name=_class_name, method_name=_method_name, result=variable_value) return variable_name, variable_value - def _process_patterns_dictionary(self, attribute, attribute_dict, location, regexp_list): + def _process_patterns_dictionary(self, attribute, attribute_dict, location, injector_values): variable_dict = dict() + regexp_list = injector_values[REGEXP] for dictionary in regexp_list: pattern = None suffix = None @@ -411,7 +419,8 @@ def _process_patterns_dictionary(self, attribute, attribute_dict, location, rege variable_name, variable_value = self._process_pattern_dictionary(attribute, attribute_dict, location, pattern, suffix) if variable_value: - variable_dict[variable_name] = variable_value + variable_dict[variable_name] = self._check_replace_variable_value(location, attribute, variable_value, + injector_values) return variable_dict def _process_pattern_dictionary(self, attribute_name, attribute_dict, location, regexp, suffix): @@ -497,9 +506,12 @@ def _find_special_name(self, mbean): mbean_name = mbean mbean_name_list = [] name_list = _find_special_names_pattern.split(mbean) + print 'mbean before split ', mbean + print 'after pattern split ', name_list if name_list and len(name_list) > 1: mbean_name = name_list[0] mbean_name_list = name_list[1].split(',') + print 'after second split ', mbean_name_list if mbean_name_list: new_list = [] for entry in mbean_name_list: @@ -516,6 +528,7 @@ def _find_special_name(self, mbean): else: new_list.append(entry) mbean_name_list = new_list + print 'mbean ', mbean_name, ' mbean_name_list ', mbean_name_list return mbean_name, mbean_name_list def _check_insert_attribute_model(self, location, model_section, attribute, injector_values): @@ -528,6 +541,21 @@ def _check_insert_attribute_model(self, location, model_section, attribute, inje class_name=_class_name, method_name=_method_name) model_section[attribute] = wlst_value + def _check_replace_variable_value(self, location, attribute, variable_value, injector_values): + _method_name = '_format_variable_value' + if VARIABLE_VALUE in injector_values: + value = injector_values[VARIABLE_VALUE] + # might add code to call a method to populate the replacement value + try: + self.__aliases.get_wlst_attribute_name_and_value(location, attribute, value) + variable_value = value + _logger.fine('WLSDPLY-19542', value, variable_value, attribute, location.get_folder_path(), + class_name=_class_name, method_name=_method_name) + except AliasException, ae: + _logger.warning('WLSDPLY-19541', value, attribute, location, ae.getLocalizedMessage(), + class_name=_class_name, method_name=_method_name) + return variable_value + def _load_variable_file(variable_file_location, **kwargs): _method_name = '_load_variable_file' @@ -705,6 +733,7 @@ def _format_as_property(prop_name): return '@@PROP:%s@@' % prop_name + def _split_injector(injector_path): """ Split the injector path into an mbean list and an attribute name from the injector path string diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 96106612b3..2697317381 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1172,6 +1172,8 @@ WLSDPLY-19539=Unable to locate and call mbean name keyword {0} method {1} : {2}. full mbean name list WLSDPLY-19540=Attribute {0} not in the model at location {1} but the force attribute is present in the injector \ keyword so the attribute will be added to the model with the default value {2} +WLSDPLY-19541=Replacement variable value {0} cannot be formatted for the attribute {1} at location {2} : {3} +WLSDPLY-19542=Variable value has been set to {0} and replaces the model value {1} for attribute {2} at location {3} # wlsdeploy/tool/variable_inject.py WLSDPLY-19600=Use model variable injector file {0} from command line arguments diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index 139055af6b..aeb6537b2d 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -260,6 +260,50 @@ def testForceAttribute(self): actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) + def testReplaceVariableValueAttribute(self): + expected = dict() + expected[ + 'JMSSystemResource.MyJmsModule.JmsResource.ForeignServer.MyForeignServer.JNDIProperty' + '.java.naming.security.principal.Value'] = 'k8s' + replacement_dict = dict() + replacement_dict['JMSSystemResource.JmsResource.ForeignServer.' + 'JNDIProperty[java.naming.security.principal].Value'] = dict() + replacement_dict['JMSSystemResource.JmsResource.ForeignServer.' + 'JNDIProperty[java.naming.security.principal].Value'][ + variable_injector.VARIABLE_VALUE] = 'k8s' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testReplaceVariableValueSegmentInString(self): + expected = dict() + expected['JDBCSystemResource.Database2.JdbcResource.JDBCDriverParams.URL--Host'] = \ + 'den00chv' + replacement_dict = dict() + replacement_dict['JDBCSystemResource[Database2].JdbcResource.JDBCDriverParams.URL'] = dict() + list_entry = dict() + list_entry[variable_injector.REGEXP_PATTERN] = '(?<=HOST=)[\w.-]+(?=\))' + list_entry[variable_injector.REGEXP_SUFFIX] = 'Host' + replacement_dict['JDBCSystemResource[Database2].JdbcResource.JDBCDriverParams.URL'][variable_injector.REGEXP] = [ + list_entry] + replacement_dict['JDBCSystemResource[Database2].JdbcResource.JDBCDriverParams.URL'][ + variable_injector.VARIABLE_VALUE] = 'den00chv' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + + def testReplaceVariableValueSegmentInDictionary(self): + expected = dict() + expected['MailSession.MailSession-0.Properties--SmtpHost'] = 'localhost' + expected['MailSession.MyMailSession.Properties--SmtpHost'] = 'localhost' + replacement_dict = dict() + replacement_dict['MailSession.Properties'] = dict() + list_entry = dict() + list_entry[variable_injector.REGEXP_PATTERN] = 'mail.smtp.host' + list_entry[variable_injector.REGEXP_SUFFIX] = 'SmtpHost' + replacement_dict['MailSession.Properties'][variable_injector.REGEXP] = [list_entry] + replacement_dict['MailSession.Properties'][variable_injector.VARIABLE_VALUE] = 'localhost' + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + def _compare_to_expected_dictionary(self, expected, actual): self.assertEqual(len(expected), len(actual), 'Not the same number of entries : expected=' + str(len(expected)) + ', actual=' + str( From 6385f68597179fe1128b85a8687457c8ce7da480 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 11:01:33 -0500 Subject: [PATCH 24/52] fix for pattern issue --- .../wlsdeploy/tool/util/variable_injector.py | 25 ++++++++++++------- .../deploy/messages/wlsdeploy_rb.properties | 1 + 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index a1e5aa3a49..c2fd813c3a 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -67,10 +67,11 @@ ADMIN_SERVER: 'admin_server_list' } # global variables for functions in VariableInjector -_find_special_names_pattern = re.compile('[\[).+\]]') +_find_special_names_pattern = re.compile('[\[\]]') _fake_name_marker = 'fakename' _fake_name_replacement = re.compile('.' + _fake_name_marker) _white_space_replacement = re.compile('\s') +_split_around_special_names = re.compile('([\w]+\[[\w\.,]+\])|\.') _wlsdeploy_location = os.environ.get('WLSDEPLOY_HOME') _class_name = 'variable_injector' @@ -506,12 +507,9 @@ def _find_special_name(self, mbean): mbean_name = mbean mbean_name_list = [] name_list = _find_special_names_pattern.split(mbean) - print 'mbean before split ', mbean - print 'after pattern split ', name_list if name_list and len(name_list) > 1: mbean_name = name_list[0] mbean_name_list = name_list[1].split(',') - print 'after second split ', mbean_name_list if mbean_name_list: new_list = [] for entry in mbean_name_list: @@ -528,7 +526,6 @@ def _find_special_name(self, mbean): else: new_list.append(entry) mbean_name_list = new_list - print 'mbean ', mbean_name, ' mbean_name_list ', mbean_name_list return mbean_name, mbean_name_list def _check_insert_attribute_model(self, location, model_section, attribute, injector_values): @@ -562,7 +559,6 @@ def _load_variable_file(variable_file_location, **kwargs): append = False variable_dictionary = dict() if VARIABLE_FILE_APPEND_ARG in kwargs and kwargs[VARIABLE_FILE_APPEND_ARG] in VARIABLE_FILE_APPEND_VALS: - _logger.fine('append argument found {0}', kwargs[VARIABLE_FILE_APPEND_ARG]) if kwargs[VARIABLE_FILE_APPEND_ARG] == VARIABLE_FILE_APPEND: _logger.fine('WLSDPLY-19536', variable_file_location, class_name=_class_name, method_name=_method_name) append = True @@ -733,18 +729,29 @@ def _format_as_property(prop_name): return '@@PROP:%s@@' % prop_name - def _split_injector(injector_path): """ Split the injector path into an mbean list and an attribute name from the injector path string :param injector_path: :return: attribute name:mbean list of mbean folder nodes """ + _method_name = '_split_injector' attr = None - ml = injector_path.split('.') + ml = _split_around_special_names.split(injector_path) + mbean_list = [] if len(ml) > 0: attr = ml.pop() - return ml, attr + for mbean_item in ml: + if mbean_item: + start = 0 + end = len(mbean_item) + if mbean_item.startswith('\.'): + start += 1 + if mbean_item.endswith('\.'): + end -= 1 + mbean_list.append(mbean_item[start:end]) + _logger.finer('WLSDPLY-19543', mbean_list, attr, class_name=_class_name, method_name=_method_name) + return mbean_list, attr def __temporary_fix(injector_dictionary): diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 2697317381..f5363c718c 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -1174,6 +1174,7 @@ WLSDPLY-19540=Attribute {0} not in the model at location {1} but the force attri keyword so the attribute will be added to the model with the default value {2} WLSDPLY-19541=Replacement variable value {0} cannot be formatted for the attribute {1} at location {2} : {3} WLSDPLY-19542=Variable value has been set to {0} and replaces the model value {1} for attribute {2} at location {3} +WLSDPLY-19543=Split injector value into mbean list {0} and attribute {1} # wlsdeploy/tool/variable_inject.py WLSDPLY-19600=Use model variable injector file {0} from command line arguments From f39b50fe9349eb48590334c7575c574b13dff196 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 14:46:08 -0500 Subject: [PATCH 25/52] Add variable injector to the readme file --- README.md | 214 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/README.md b/README.md index 7615d50b49..57a473f800 100644 --- a/README.md +++ b/README.md @@ -845,6 +845,220 @@ When creating the archive, the tool will try to gather all binaries, scripts, an 1. Any binaries referenced from the `ORACLE_HOME` will not be gathered, as they are assumed to exist in any target domain to which model-driven operations will be applied. Doing this is key to allowing the model to be WebLogic Server version independent. 2. In its current form, the discover domain Tool will only gather binaries and scripts that are accessible from the local machine. Warnings will be generated for any binaries or scripts that cannot be found but the configuration for those binaries will still be collected, where possible. It is the user's responsibility to add those missing files to the archive in the appropriate locations and edit the the model, as needed, to point to those files inside the archive using the relative path inside the archive (for example, `wlsdeploy/applications/myapp.ear`). + ----------------------------------------------- + Variable Injector Tool + ----------------------------------------------- + +An optional feature for the Discover Domain is the Variable Injector tool. The variable injector will replace a selected model attribute value with a property marker containing a unique variable name. This variable name and the replaced value are inserted into a variable property file. + +To enable the Variable Injector during the Discover Domain, you must place a json file named model_variable_injector.json into the /lib directory. This file must be manually created and contain one of the pre-defined keywords and/or a CUSTOM designated file. + +A variable name substitution is only performed once for an attribute during the variable injector run. The first substitution is not replaced by and subsequent matches. + +The supported keywords are as follows: + +- CREDENTIALS + All MBean credentials attribute values (user and password) are injected with a variable property string and the string and value placed in the variable property file + +- HOST + All MBean Host attribute values in the model are injected with a variable property string and the string and value placed in the variable property file + +- PORT + All MBean Port attribute values in the model are injected with a variable property string and the string and value placed in the variable property file + +- TARGET + All MBean Target attribute values in the model are injected with a variable property string and the string and value placed in the variable property file + +- TOPOLOGY + Special MBean attributes found in the topology section of the model are injected with a variable property string and the string value placed in the variable + property file. This includes server, machine and node manager ports, credentials and listen addresses, and cluster messaging modes, addresses and ports. + +- URL + All MBean URL attribute values in the model are injected with a variable property string and the string and value placed in the variable property file + +These special keywords are defined in a keyword file that is installed into the /lib directory. Each keyword is associated with an injector json file. +Each injector json file is installed into the /lib/injectors directory. An injector file contains the directives for the Variable Injector tool to +inject variables into specific model attributes. + +Here is an example of a model_variable_injector.json file using the PORT keyword. + +```json +{ + "PORT": {}, +} +``` + +Below is a model snippet that shows injected variables for the port values in a topology section. + +```$yaml + +topology: + Name: soa_domain + AdminServerName: AdminServer + Cluster: + soa_cluster: + osb_cluster: + Server: + AdminServer: + ListenAddress: myadmin.example.com + ListenPort: @@PROP:Server.AdminServer.ListenPort@@ + Machine: machine1 + SSL: + Enabled: true + ListenPort: @@PROP:Server.SSL.AdminServer.ListenPort@@ + soa_server1: + ListenAddress: managed1.example.com + ListenPort: @@PROP:Server.soa_server1.ListenPort@@ + Cluster: soa_cluster + Machine: machine2 + SSL: + Enabled: true + ListenPort: @@PROP:Server.SSL.soa_server1.ListenPort@@ + soa_server2: + ListenAddress: managed2.example.com + ListenPort: @@PROP:Server.so_server2.ListenPort@@ + Cluster: soa_cluster + Machine: machine3 + SSL: + Enabled: true + ListenPort: @@PROP:Server.SSL.soa_server2.ListenPort@@ + +And the resulting variable property file: + +```txt +Server.AdminServer.ListenPort=7001 +Server.AdminServer.SSL.ListenPort=7002 +Server.soa_server1.ListenPort=8001 +Server.soa_server1.SSL.ListenPort=8002 +Server.soa_server2.ListenPort=8001 +Server.soa_server2.SSL.ListenPort=8002 +``` + +To designate the name and location of the variable properties file, include the variable_file_name directive in the model_variable_injector.json file. + +```json +{ + "variable_file_name": "/home/savedomain/variables.properties", +} +``` + +If this directive is not included in the json file, the model or archive file name and location is used to create the variable properties file name and location. If the model_file command line +argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name +and location will be used to create the variable properties file name and location. + +As with the archive and model file, each run of the discover domain tool will overwrite the contents of an existing variable property file with the values from the current run. + +# Custom Variable Injector + +You may customize the injector attribute replacement by using the CUSTOM keyword in the model_variable_injector.json. The CUSTOM keyword identifies a list of one or more custom injector json files. +These injector json files will be processed first, in list order, and the resulting variable properties will not be replaced when processing additional keywords in the model variable injector json file. + +An entry in the injector json file is a key that specifies the unique MBean attribute that will be injected, and an optional set of directives. + +The injector is a period separated MBean hierarchy and attribute name as they are defined in the model. Do not enter the name of the model section into the injector key. + +For example, an injector key for the Server SSL Listen Port is Server.SSL.ListenPort. The Server.SSL identifies the hierarchy in the model to the attribute. + +```json +{ + "Server.SSL.ListenPort": {}, +} +``` + +An example of that hierarchy in a model (notice that the MBean name of AdminServer is NOT included): +```yaml +topology: + Server: + AdminServer: + ListenAddress: myadmin.example.com + ListenPort: 7001 + Machine: machine1 + SSL: + Enabled: true + ListenPort: 7002 +``` + +The Variable Injector tool will inject a unique variable name for every server in the model that has the Server/SSL/ListenPort attribute. + +# Custom special directives + +The following directives can refine the variable injection. + +- force: + For true, if the MBean hierarchy exists in the model, but the attribute does not, then the attribute will be added and persisted to the discovered model. The value stored in the + model is the weblogic default value. + +- variable_value: + Replace the model value with the specified value in the variable properties. This may be used in conjunction with the force directive, replacing the default value with the indicated value. + +-regexp: + A list of regexp patterns that will be applied to either the string values or map values of an attribute in the model. If the pattern matches, then the matching part of the + string or dictionary will be injected with a unique variable name. + -pattern: + the regular expression pattern to apply to the string value or map values of an attribute + -suffix: + The suffix name to append to each resulting variable name in the variable properties file + +The regexp list is useful when only a segment of a string value needs to be parameterized (making for quick manipulation of the variable properties). If more than one +pattern is specified in the regexp directive list, then a suffix MUST be added in order to generate a unique variable name. + +The following is an example of how to effectively use the regexp directive list to search for a segment in a string value. In this example, we want to search for +the host and port in each Oracle JDBC URL that uses the special Oracle URL notation, and create an entry for the host and port in the variable properties file. + +In the model, we expect to find a URL like the following: + +```$yaml + JDBCSystemResource: + Database1: + JdbcResource: + JDBCDriverParams: + URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=slc05til.us.oracle.com)(PORT=1521)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' +``` + +We create a directive in our custom injector json file: + +```json + "JDBCSystemResource.JdbcResource.JDBCDriverParams.URL": + { + "regexp": [ + { + "pattern": "(?<=PORT=)[\\w.-]+(?=\\))", + "suffix": "Port" + }, + { + "pattern": "(?<=HOST=)[\\w.-]+(?=\\))", + "suffix": "Host" + } + ] + }, +``` + +During the Discover Domain tool run, the pattern is applied to the URL string, and the resulting entries added to the variable properties: + +- URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host@@:)(PORT=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port@@)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' +- + JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host=slc05til.us.oracle.com + JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port=1521 + +## Selecting specific MBean names for variable injection + +The final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to parameterize an attribute for a specific server, or for all the managed servers. +To define a list of one or more names of an MBean, add a user directive to the injector keyword value. The user directive is added to the end of an MBean between brackets. For instance, to select only the admin server named AdminServer, add the +user directive to the Server MBean - Server[AdminServer]. To select servers soa_server1 and soa_server2 from the model, create the injector key Server[soa_server1,soa_server2] + +The injector tool recognizes two KEYWORDS for a user list, MANAGED_SERVERS (all the managed servers in the model) and ADMIN_SERVER (The admin server in the model). + +A custom injector to parameterize the admin server SSL listen port is: + +```json +{ + "Server[ADMIN_SERVER].SSL.ListenPort": {}, +} +``` + +A sample of a model_variable_injector.json file and a custom injector json file are installed in the /samples directory. + + ## Downloading and Installing the Software The Oracle WebLogic Server Deploy Tooling project repository is located at [`https://github.com/oracle/weblogic-deploy-tooling`](https://github.com/oracle/weblogic-deploy-tooling). Binary distributions of the `weblogic-deploy.zip` installer can be downloaded from the [GitHub Releases page](https://github.com/oracle/weblogic-deploy-tooling/releases). To install the software, simply unzip the `weblogic-deploy.zip` installer on a machine that has the desired versions of WebLogic Server installed. After being unzipped, the software is ready to use, just set the `JAVA_HOME` environment variable to point to a Java 7 or higher JDK and the shell scripts are ready to run. From 1a5ddc0c6c404087cd4dc9725d12f9d440a2dd24 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 14:48:14 -0500 Subject: [PATCH 26/52] Add variable injector to the readme file --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 57a473f800..149a8f9778 100644 --- a/README.md +++ b/README.md @@ -845,9 +845,7 @@ When creating the archive, the tool will try to gather all binaries, scripts, an 1. Any binaries referenced from the `ORACLE_HOME` will not be gathered, as they are assumed to exist in any target domain to which model-driven operations will be applied. Doing this is key to allowing the model to be WebLogic Server version independent. 2. In its current form, the discover domain Tool will only gather binaries and scripts that are accessible from the local machine. Warnings will be generated for any binaries or scripts that cannot be found but the configuration for those binaries will still be collected, where possible. It is the user's responsibility to add those missing files to the archive in the appropriate locations and edit the the model, as needed, to point to those files inside the archive using the relative path inside the archive (for example, `wlsdeploy/applications/myapp.ear`). - ----------------------------------------------- - Variable Injector Tool - ----------------------------------------------- +## The Variable Injector Tool An optional feature for the Discover Domain is the Variable Injector tool. The variable injector will replace a selected model attribute value with a property marker containing a unique variable name. This variable name and the replaced value are inserted into a variable property file. From 2f88e6857c2bfa4f4533aae87fab825060aec004 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 15:02:34 -0500 Subject: [PATCH 27/52] Add variable injector to the readme file --- README.md | 57 ++++++++++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 149a8f9778..3c249df4ea 100644 --- a/README.md +++ b/README.md @@ -849,9 +849,9 @@ When creating the archive, the tool will try to gather all binaries, scripts, an An optional feature for the Discover Domain is the Variable Injector tool. The variable injector will replace a selected model attribute value with a property marker containing a unique variable name. This variable name and the replaced value are inserted into a variable property file. -To enable the Variable Injector during the Discover Domain, you must place a json file named model_variable_injector.json into the /lib directory. This file must be manually created and contain one of the pre-defined keywords and/or a CUSTOM designated file. +To enable the Variable Injector during the Discover Domain, you must place a json file named model_variable_injector.json into the WLSDEPLOY/lib directory. This file must be manually created and contain one of the pre-defined keywords and/or a CUSTOM designated file. -A variable name substitution is only performed once for an attribute during the variable injector run. The first substitution is not replaced by and subsequent matches. +A variable name substitution is only performed once for an attribute during the variable injector run. The first substitution is not replaced by any subsequent matches. The supported keywords are as follows: @@ -874,9 +874,9 @@ The supported keywords are as follows: - URL All MBean URL attribute values in the model are injected with a variable property string and the string and value placed in the variable property file -These special keywords are defined in a keyword file that is installed into the /lib directory. Each keyword is associated with an injector json file. -Each injector json file is installed into the /lib/injectors directory. An injector file contains the directives for the Variable Injector tool to -inject variables into specific model attributes. +These special keywords are defined in a keyword file that is installed into the WLSDEPLOY/lib directory. Each keyword is associated with an injector json file. +Each injector json file is installed into the WLSDEPLOY/lib/injectors directory. An injector file contains the directives for the Variable Injector tool to +inject variable names into specified model attributes. Here is an example of a model_variable_injector.json file using the PORT keyword. @@ -886,7 +886,7 @@ Here is an example of a model_variable_injector.json file using the PORT keyword } ``` -Below is a model snippet that shows injected variables for the port values in a topology section. +Below is a model snippet that shows injected variables in the port attributes. ```$yaml @@ -914,7 +914,7 @@ topology: ListenPort: @@PROP:Server.SSL.soa_server1.ListenPort@@ soa_server2: ListenAddress: managed2.example.com - ListenPort: @@PROP:Server.so_server2.ListenPort@@ + ListenPort: @@PROP:Server.soa_server2.ListenPort@@ Cluster: soa_cluster Machine: machine3 SSL: @@ -923,14 +923,12 @@ topology: And the resulting variable property file: -```txt Server.AdminServer.ListenPort=7001 Server.AdminServer.SSL.ListenPort=7002 Server.soa_server1.ListenPort=8001 Server.soa_server1.SSL.ListenPort=8002 Server.soa_server2.ListenPort=8001 Server.soa_server2.SSL.ListenPort=8002 -``` To designate the name and location of the variable properties file, include the variable_file_name directive in the model_variable_injector.json file. @@ -942,18 +940,17 @@ To designate the name and location of the variable properties file, include the If this directive is not included in the json file, the model or archive file name and location is used to create the variable properties file name and location. If the model_file command line argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name -and location will be used to create the variable properties file name and location. +and location will be used. As with the archive and model file, each run of the discover domain tool will overwrite the contents of an existing variable property file with the values from the current run. -# Custom Variable Injector +### Custom Variable Injector You may customize the injector attribute replacement by using the CUSTOM keyword in the model_variable_injector.json. The CUSTOM keyword identifies a list of one or more custom injector json files. -These injector json files will be processed first, in list order, and the resulting variable properties will not be replaced when processing additional keywords in the model variable injector json file. - -An entry in the injector json file is a key that specifies the unique MBean attribute that will be injected, and an optional set of directives. +These injector json files will be processed by the Variable Injector tool first, in list order. Any resulting variable replacement will not be overlaid when processing additional keywords. -The injector is a period separated MBean hierarchy and attribute name as they are defined in the model. Do not enter the name of the model section into the injector key. +An entry in the injector json file is a key that specifies the unique MBean attribute to parameterize, and an optional set of directives. The injector is a period separated MBean hierarchy and attribute name as they are defined in the model. +Do not enter the name of the model section into the injector key. For example, an injector key for the Server SSL Listen Port is Server.SSL.ListenPort. The Server.SSL identifies the hierarchy in the model to the attribute. @@ -976,9 +973,9 @@ topology: ListenPort: 7002 ``` -The Variable Injector tool will inject a unique variable name for every server in the model that has the Server/SSL/ListenPort attribute. +For this example, the Variable Injector tool will inject a unique variable name for every server in the model that has the Server/SSL/ListenPort attribute. -# Custom special directives +#### Custom special directives The following directives can refine the variable injection. @@ -986,16 +983,16 @@ The following directives can refine the variable injection. For true, if the MBean hierarchy exists in the model, but the attribute does not, then the attribute will be added and persisted to the discovered model. The value stored in the model is the weblogic default value. -- variable_value: +- variable_value: Replace the model value with the specified value in the variable properties. This may be used in conjunction with the force directive, replacing the default value with the indicated value. --regexp: +- regexp: A list of regexp patterns that will be applied to either the string values or map values of an attribute in the model. If the pattern matches, then the matching part of the string or dictionary will be injected with a unique variable name. - -pattern: - the regular expression pattern to apply to the string value or map values of an attribute - -suffix: - The suffix name to append to each resulting variable name in the variable properties file +-- pattern: + the regular expression pattern to apply to the string value or map values of an attribute +-- suffix: + The suffix name to append to each resulting variable name in the variable properties file The regexp list is useful when only a segment of a string value needs to be parameterized (making for quick manipulation of the variable properties). If more than one pattern is specified in the regexp directive list, then a suffix MUST be added in order to generate a unique variable name. @@ -1033,14 +1030,14 @@ We create a directive in our custom injector json file: During the Discover Domain tool run, the pattern is applied to the URL string, and the resulting entries added to the variable properties: -- URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host@@:)(PORT=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port@@)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' -- - JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host=slc05til.us.oracle.com - JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port=1521 +URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host@@:)(PORT=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port@@)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' + +JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host=slc05til.us.oracle.com +JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port=1521 -## Selecting specific MBean names for variable injection +### Selecting specific MBean names for variable injection -The final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to parameterize an attribute for a specific server, or for all the managed servers. +The final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to parameterize an attribute just for a specific set of servers. To define a list of one or more names of an MBean, add a user directive to the injector keyword value. The user directive is added to the end of an MBean between brackets. For instance, to select only the admin server named AdminServer, add the user directive to the Server MBean - Server[AdminServer]. To select servers soa_server1 and soa_server2 from the model, create the injector key Server[soa_server1,soa_server2] @@ -1054,7 +1051,7 @@ A custom injector to parameterize the admin server SSL listen port is: } ``` -A sample of a model_variable_injector.json file and a custom injector json file are installed in the /samples directory. +A sample of a model_variable_injector.json file and a custom injector json file are installed in the WLSDEPLOY/samples directory. ## Downloading and Installing the Software From 1d82478a2173b97433ed7f7fabf48286c84046b8 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 15:05:11 -0500 Subject: [PATCH 28/52] Add variable injector to the readme file --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 3c249df4ea..41f61c7023 100644 --- a/README.md +++ b/README.md @@ -923,12 +923,14 @@ topology: And the resulting variable property file: +``` Server.AdminServer.ListenPort=7001 Server.AdminServer.SSL.ListenPort=7002 Server.soa_server1.ListenPort=8001 Server.soa_server1.SSL.ListenPort=8002 Server.soa_server2.ListenPort=8001 Server.soa_server2.SSL.ListenPort=8002 +``` To designate the name and location of the variable properties file, include the variable_file_name directive in the model_variable_injector.json file. @@ -1030,10 +1032,12 @@ We create a directive in our custom injector json file: During the Discover Domain tool run, the pattern is applied to the URL string, and the resulting entries added to the variable properties: +``` URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host@@:)(PORT=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port@@)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host=slc05til.us.oracle.com JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port=1521 +``` ### Selecting specific MBean names for variable injection From 7fbe287040b3a1eaf0835767001f809fffe502c5 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 15:08:08 -0500 Subject: [PATCH 29/52] Add variable injector to the readme file --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 41f61c7023..a60aeac54e 100644 --- a/README.md +++ b/README.md @@ -1039,11 +1039,11 @@ JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host=slc05til.us JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port=1521 ``` -### Selecting specific MBean names for variable injection +#### Selecting specific MBean names for variable injection The final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to parameterize an attribute just for a specific set of servers. To define a list of one or more names of an MBean, add a user directive to the injector keyword value. The user directive is added to the end of an MBean between brackets. For instance, to select only the admin server named AdminServer, add the -user directive to the Server MBean - Server[AdminServer]. To select servers soa_server1 and soa_server2 from the model, create the injector key Server[soa_server1,soa_server2] +user directive to the Server MBean - Server[AdminServer]. To select servers soa_server1 and soa_server2 from the model, the injector key is Server[soa_server1,soa_server2] The injector tool recognizes two KEYWORDS for a user list, MANAGED_SERVERS (all the managed servers in the model) and ADMIN_SERVER (The admin server in the model). @@ -1055,6 +1055,8 @@ A custom injector to parameterize the admin server SSL listen port is: } ``` +## Samples + A sample of a model_variable_injector.json file and a custom injector json file are installed in the WLSDEPLOY/samples directory. From 63abedd44952fc2fded2d794d25408503f927889 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 15:26:11 -0500 Subject: [PATCH 30/52] Add target injector file --- installer/src/main/lib/injectors/target.json | 40 ++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 installer/src/main/lib/injectors/target.json diff --git a/installer/src/main/lib/injectors/target.json b/installer/src/main/lib/injectors/target.json new file mode 100644 index 0000000000..8791804b16 --- /dev/null +++ b/installer/src/main/lib/injectors/target.json @@ -0,0 +1,40 @@ +{ + "AppDeployment.Target": {}, + "CoherenceClusterSystemResource.Target": {}, + "CoherenceClusterSystemResource.CoherenceCacheConfig.Target": {}, + "FileStore.Target": {}, + "ForeignJNDIProvider.Target": {}, + "JDBCStore.Target": {}, + "JDBCSystemResource.Target": {}, + "JMSServer.Target": {}, + "JMSServer.SubDeployment.Target": {}, + "Library.Target": {}, + "Library.SubDeployment.Target": {}, + "MailSession.Target": {}, + "MessagingBridge.Target": {}, + "PathService.Target": {}, + "ResourceGroup.Target": {}, + "SAFAgent.Target": {}, + "SelfTuning.Capacity.Target": {}, + "SelfTuning.ContextRequestClass.Target": {}, + "SelfTuning.ContextRequestClass.ContextCase.Target": {}, + "SelfTuning.FairShareRequestClass.Target": {}, + "SelfTuning.MaxThreadsConstraint.Target": {}, + "SelfTuning.MinThreadsConstraint.Target": {}, + "SelfTuning.ResponseTimeRequestClass.Target": {}, + "SelfTuning.WorkManager.Target": {}, + "Server.Cluster": {}, + "Server.DataSource.Target": {}, + "Server.TransactionLogJDBCStore.Target": {}, + "Server.WebServer.Target": {}, + "ServerTemplate.Cluster": {}, + "ServerTemplate.DataSource.Target": {}, + "ServerTemplate.TransactionLogJDBCStore.Target": {}, + "ServerTemplate.WebServer.Target": {}, + "ShutdownClass.Target": {}, + "StartupClass.Target": {}, + "VirtualHost.Target": {}, + "VirtualTarget.Target": {}, + "VirtualTarget.WebServer.Target": {}, + "WLDFSystemResource.Target": {} +} \ No newline at end of file From 8a4cead13e126ea681b4627d8464113a6570877f Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 15:55:38 -0500 Subject: [PATCH 31/52] remove unused python file --- core/src/main/python/discover.py | 31 ++++---- .../wlsdeploy/util/model_search_helper.py | 76 ------------------- installer/src/main/lib/variable_keywords.json | 1 + 3 files changed, 17 insertions(+), 91 deletions(-) delete mode 100644 core/src/main/python/wlsdeploy/util/model_search_helper.py diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index 4adea2dabe..33da0ac1aa 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -12,16 +12,16 @@ from java.lang import IllegalArgumentException from java.lang import IllegalStateException from java.lang import String - from oracle.weblogic.deploy.aliases import AliasException from oracle.weblogic.deploy.discover import DiscoverException from oracle.weblogic.deploy.util import CLAException from oracle.weblogic.deploy.util import FileUtils from oracle.weblogic.deploy.util import PyWLSTException from oracle.weblogic.deploy.util import TranslateException -from oracle.weblogic.deploy.util import WebLogicDeployToolingVersion from oracle.weblogic.deploy.util import WLSDeployArchive from oracle.weblogic.deploy.util import WLSDeployArchiveIOException +from oracle.weblogic.deploy.util import WebLogicDeployToolingVersion +from oracle.weblogic.deploy.validate import ValidateException sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0]))) @@ -29,7 +29,6 @@ from wlsdeploy.exception import exception_helper from wlsdeploy.logging.platform_logger import PlatformLogger from wlsdeploy.util import getcreds -from wlsdeploy.tool.util.variable_injector import VariableInjector from wlsdeploy.tool.discover import discoverer from wlsdeploy.tool.discover.deployments_discoverer import DeploymentsDiscoverer from wlsdeploy.tool.discover.domain_info_discoverer import DomainInfoDiscoverer @@ -37,6 +36,8 @@ from wlsdeploy.tool.discover.resources_discoverer import ResourcesDiscoverer from wlsdeploy.tool.discover.topology_discoverer import TopologyDiscoverer from wlsdeploy.tool.util import filter_helper +from wlsdeploy.tool.util.variable_injector import VariableInjector +from wlsdeploy.tool.validate.validator import Validator from wlsdeploy.util import wlst_helper from wlsdeploy.util import model_translator from wlsdeploy.util.cla_utils import CommandLineArgUtil @@ -373,25 +374,25 @@ def __check_and_customize_model(model, model_context): inject_variables_keyword_file() if inserted: model = Model(variable_model) - # try: - # validator = Validator(model_context, wlst_mode=__wlst_mode) - # - # # no variables are generated by the discover tool - # validator.validate_in_tool_mode(model.get_model(), variables_file_name=variable_file_name, - # archive_file_name=model_context.get_archive_file_name()) - # except ValidateException, ex: - # __logger.warning('WLSDPLY-06015', ex.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) + try: + validator = Validator(model_context, wlst_mode=__wlst_mode) + + # no variables are generated by the discover tool + validator.validate_in_tool_mode(model.get_model(), variables_file_name=variable_file_name, + archive_file_name=model_context.get_archive_file_name()) + except ValidateException, ex: + __logger.warning('WLSDPLY-06015', ex.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) return model -def __log_and_exit(exit_code, _class_name, _method_name): +def __log_and_exit(exit_code, class_name, _method_name): """ Helper method to log the exiting message and call sys.exit() :param exit_code: the exit code to use - :param _class_name: the class name to pass to the logger + :param class_name: the class name to pass to the logger :param _method_name: the method name to pass to the logger """ - __logger.exiting(result=exit_code, class_name=_class_name, method_name=_method_name) + __logger.exiting(result=exit_code, class_name=class_name, method_name=_method_name) sys.exit(exit_code) @@ -444,7 +445,7 @@ def main(args): except TranslateException, ex: __logger.severe('WLSDPLY-20024', _program_name, model_context.get_archive_file_name(), ex.getLocalizedMessage(), - error=ex, class_name=_class_name, method_name=_method_name) + error=ex, class_name=_class_name, method_name=_method_name) __log_and_exit(CommandLineArgUtil.PROG_ERROR_EXIT_CODE, _class_name, _method_name) __close_archive(model_context) diff --git a/core/src/main/python/wlsdeploy/util/model_search_helper.py b/core/src/main/python/wlsdeploy/util/model_search_helper.py deleted file mode 100644 index 1dc0936b42..0000000000 --- a/core/src/main/python/wlsdeploy/util/model_search_helper.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. -The Universal Permissive License (UPL), Version 1.0 -""" -from wlsdeploy.logging.platform_logger import PlatformLogger -import wlsdeploy.util.model as model_sections - -_class_name = 'ModelSearchHelper' -_logger = PlatformLogger('wlsdeploy.util') - - -class ModelSearchHelper(object): - """ - Class functions to facilitate searching the model. - """ - - def __init__(self, model): - """ - Encapsulate the model on which to perform searches. - :param model: to encapsulate - """ - self.__model = model - - def locate_attribute_value(self, section, attribute_path): - """ - Locate and return the dictionary entry for the attribute and value using the provided search pattern. -
:[....].[.${search pattern}] - where section is a section of the model, and the mbean is the path of mbeans that contains the attribute and - search pattern is a key=value pair within the attribute's value. - The model is traversed down to the - :param section: - :param attribute_path: - :return: - """ - _method_name = 'locate_attribute_value' - _logger.entering(attribute_path, class_name=_class_name, method_name=_method_name) - if section in model_sections.get_model_top_level_keys(): - model_section = self.__model[section] - mbean_list, attribute_name = _split_attribute_path(attribute_path) - for entry in mbean_list: - if entry in model_section: - model_section = model_section[entry] - else: - _logger.warning('WLSDPLY-19406', entry, attribute_path, section) - break - - - - _logger.exiting(class_name=_class_name, method_name=_method_name) - - -def _split_section(attribute_path): - """ - Split the section from the attribute path. - :param attribute_path: - :return: - """ - split_list = attribute_path.split(':', 1) - section = None - attribute = None - segment = None - if len(split_list) == 2 and split_list[0] in model_sections.get_model_top_level_keys(): - section = split_list[0] - attribute_split_list = split_list[1].split('${') - attribute = attribute_split_list[0] - if len(attribute_split_list) == 2: - segment = attribute_split_list[1][:len(attribute_split_list[1])-1] - return section, attribute, segment - - -def _split_attribute_path(attribute_path): - mbean_list = attribute_path.split('.') - attribute = None - if len(mbean_list) > 0: - attribute = mbean_list.pop() - return mbean_list, attribute diff --git a/installer/src/main/lib/variable_keywords.json b/installer/src/main/lib/variable_keywords.json index 49386112f5..9317b9b1bc 100644 --- a/installer/src/main/lib/variable_keywords.json +++ b/installer/src/main/lib/variable_keywords.json @@ -3,5 +3,6 @@ "PORT": "port.json", "HOST": "host.json", "URL": "url.json", + "TARGET": "target.json", "TOPOLOGY": "topology.json" } \ No newline at end of file From 4b6357f897f4be44b99ba86ea6043ae5cb54aeca Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:13:47 -0500 Subject: [PATCH 32/52] Update variable_injector_keyword.json --- core/src/test/resources/variable_injector_keyword.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/resources/variable_injector_keyword.json b/core/src/test/resources/variable_injector_keyword.json index a71915be92..ff408fb4d4 100644 --- a/core/src/test/resources/variable_injector_keyword.json +++ b/core/src/test/resources/variable_injector_keyword.json @@ -3,4 +3,4 @@ "PORT": {}, "URL": {}, "CREDENTIALS": {} -} \ No newline at end of file +} From 38ac79f138ceac7bd67868e836e092b93553e0e6 Mon Sep 17 00:00:00 2001 From: crountre Date: Tue, 29 May 2018 16:19:39 -0500 Subject: [PATCH 33/52] remove unused test file --- core/src/test/python/variable_injector_test.py | 1 - core/src/test/resources/variable_injector_custom.json | 4 ---- 2 files changed, 5 deletions(-) delete mode 100644 core/src/test/resources/variable_injector_custom.json diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index aeb6537b2d..4daa8dcabb 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -16,7 +16,6 @@ class VariableFileHelperTest(unittest.TestCase): _variable_file = _resources_dir + '/variable.injector.test.properties' _model_file = _resources_dir + '/variable_insertion.yaml' _variable_injector_keyword = 'variable_injector_keyword.json' - _variable_injector_custom = 'variable_injector_custom.json' _keywords_file = 'keywords.json' def setUp(self): diff --git a/core/src/test/resources/variable_injector_custom.json b/core/src/test/resources/variable_injector_custom.json deleted file mode 100644 index da12fa1944..0000000000 --- a/core/src/test/resources/variable_injector_custom.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "CUSTOM": { "file": "C:\Users\crountre\temp\custom_variables.properties" }, - "PORT": {} -} \ No newline at end of file From 66ef0cd8e48ded66ea3056d76b33d1704c8fb2f6 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:23:18 -0500 Subject: [PATCH 34/52] Add newline at end --- installer/src/main/lib/injectors/target.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/src/main/lib/injectors/target.json b/installer/src/main/lib/injectors/target.json index 8791804b16..68683cfd5a 100644 --- a/installer/src/main/lib/injectors/target.json +++ b/installer/src/main/lib/injectors/target.json @@ -37,4 +37,4 @@ "VirtualTarget.Target": {}, "VirtualTarget.WebServer.Target": {}, "WLDFSystemResource.Target": {} -} \ No newline at end of file +} From 3015943cc38c1540d5c5beef06863d9663ea0eb3 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:23:47 -0500 Subject: [PATCH 35/52] add newline to end of file --- installer/src/main/lib/variable_keywords.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/src/main/lib/variable_keywords.json b/installer/src/main/lib/variable_keywords.json index 9317b9b1bc..b340bcf2f4 100644 --- a/installer/src/main/lib/variable_keywords.json +++ b/installer/src/main/lib/variable_keywords.json @@ -5,4 +5,4 @@ "URL": "url.json", "TARGET": "target.json", "TOPOLOGY": "topology.json" -} \ No newline at end of file +} From 686a88fd048345eb4f5f15fd878f913ef6295230 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:24:15 -0500 Subject: [PATCH 36/52] add newline to eof --- installer/src/main/samples/custom_injector.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/src/main/samples/custom_injector.json b/installer/src/main/samples/custom_injector.json index e1632ff89d..75b0a04846 100644 --- a/installer/src/main/samples/custom_injector.json +++ b/installer/src/main/samples/custom_injector.json @@ -18,4 +18,4 @@ "Server[ADMIN_SERVER].SSL.HostnameVerificationIgnored": { "force": true } -} \ No newline at end of file +} From 3a3124f5b41f5b0eaa3ac314c740f2d465190cb4 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:24:38 -0500 Subject: [PATCH 37/52] add newline to eof --- installer/src/main/lib/injectors/url.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/src/main/lib/injectors/url.json b/installer/src/main/lib/injectors/url.json index 05e256a200..efd6a7e847 100644 --- a/installer/src/main/lib/injectors/url.json +++ b/installer/src/main/lib/injectors/url.json @@ -21,4 +21,4 @@ "ServerTemplate.SingleSignOnServices.OrganizationUrl": {}, "ServerTemplate.SingleSignOnServices.PublishedSiteUrl": {}, "WLDFSystemResource.WLDFResource.WatchNotification.RestNotification.EndpointUrl": {} -} \ No newline at end of file +} From 3cbdae5c5e26bd80e846b24ce375185a2b38759c Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:25:04 -0500 Subject: [PATCH 38/52] add newline to eof --- installer/src/main/lib/injectors/topology.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/src/main/lib/injectors/topology.json b/installer/src/main/lib/injectors/topology.json index 06e9fb5e98..183ae2bb65 100644 --- a/installer/src/main/lib/injectors/topology.json +++ b/installer/src/main/lib/injectors/topology.json @@ -28,4 +28,4 @@ "UnixMachine.NodeManager.NodeManagerHome": {}, "UnixMachine.NodeManager.PasswordEncrypted": {}, "UnixMachine.NodeManager.UserName": {} -} \ No newline at end of file +} From d8f4e529c179af3453bf78fe3efebde95a5a301a Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:25:33 -0500 Subject: [PATCH 39/52] add newline to eof --- installer/src/main/lib/injectors/host.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/src/main/lib/injectors/host.json b/installer/src/main/lib/injectors/host.json index 0a11d2cf46..bbe86c2a32 100644 --- a/installer/src/main/lib/injectors/host.json +++ b/installer/src/main/lib/injectors/host.json @@ -53,4 +53,4 @@ "VirtualHost.VirtualHostName": {}, "VirtualTarget.WebServer.FrontendHost": {}, "VirtualTarget.HostName": {} -} \ No newline at end of file +} From 872bc0592c305840e389a472ec6aacce1c247894 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:26:18 -0500 Subject: [PATCH 40/52] add newline to eof --- core/src/test/resources/keywords.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/resources/keywords.json b/core/src/test/resources/keywords.json index f4d86c0ceb..75de751503 100644 --- a/core/src/test/resources/keywords.json +++ b/core/src/test/resources/keywords.json @@ -3,4 +3,4 @@ "HOST": "host.json", "URL": "url.json", "CREDENTIALS": "credentials.json" -} \ No newline at end of file +} From 625af5fa2db8d10a2a366c8f084ce42ac0ab8dc5 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:26:44 -0500 Subject: [PATCH 41/52] add newline to eof --- core/src/test/resources/custom.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/resources/custom.json b/core/src/test/resources/custom.json index 742836bce9..493b0fba8c 100644 --- a/core/src/test/resources/custom.json +++ b/core/src/test/resources/custom.json @@ -1,4 +1,4 @@ { "Notes": {}, "Application.SecurityDDModel": {} -} \ No newline at end of file +} From 11120399909b9ebf61412f03df03dc2d2392cdd8 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:28:31 -0500 Subject: [PATCH 42/52] undo changes not used in message properties --- .../weblogic/deploy/messages/wlsdeploy_rb.properties | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index f5363c718c..e3d589a6f5 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -265,7 +265,7 @@ WLSDPLY-01750=The WebLogic Deploy Tooling {0} version is {1} WLSDPLY-01760=Failed to access key in map: {0} ############################################################################### -# Encrypt Messages (04000 - 04499) # +# Encrypt Messages (04000 - 04999) # ############################################################################### # oracle.weblogic.deploy.encrypt.EncryptionUtils @@ -304,13 +304,6 @@ WLSDPLY-04210={0} encrypted {1} model attributes and wrote them to file {2} WLSDPLY-04211={0} failed to write encrypted model to file {1}: {2} WLSDPLY-04212={0} failed to encrypt the password: {1} -############################################################################### -# Variable Injector Messages (04500 - 04999) # -############################################################################### -# wlsdeploy/tool/variable_inject.py -WLSDPLY-04500=User passphrase was empty or null - - ############################################################################### # Validate Messages (05000 - 05999) # ############################################################################### @@ -1208,4 +1201,4 @@ WLSDPLY-20020=Filter ID {0} is invalid WLSDPLY-20021=Filter path {0} does not exist WLSDPLY-20022=Error loading filter path {0} WLSDPLY-20023={0} unable to add model file {1} to archive as {2}: {3} -WLSDPLY-20024={0} failed to persist the model to the archive file {1}: {2} \ No newline at end of file +WLSDPLY-20024={0} failed to persist the model to the archive file {1}: {2} From 8db533313daf3eb6ddfbfd347308ab261c2d0901 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Tue, 29 May 2018 16:32:21 -0500 Subject: [PATCH 43/52] Fix conflict not caught by git during merge of master to branch --- .../oracle/weblogic/deploy/messages/wlsdeploy_rb.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index e3d589a6f5..6d4b375577 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -408,10 +408,10 @@ WLSDPLY-06005=Unable to clear the existing archive file. Correct the problem bef WLSDPLY-06006=Disconnect from the administration server failed: {0} WLSDPLY-06007=Closing the domain failed: {0} WLSDPLY-06008=Unable to create temporary file for writing the model: {0} - +WLSDPLY-06009=Unable to add model file {0} to archive as {1}: {2} WLSDPLY-06010={0} failed to clear the existing archive file at {1} of binaries: {2} WLSDPLY-06011={0} failed to discover domain {1} at {2} : {3} - +WLSDPLY-06012={0} failed to persist the model to the archive file {1}: {2} WLSDPLY-06013={0} failed to create the archive file at {1}: {2} WLSDPLY-06014=Filters applied to the model WLSDPLY-06015=Unable to run validation against the discovered model : {0} From 1e29db9c0ecd4ff5fb8a3a056a4b27be7c5e6a64 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Wed, 30 May 2018 12:50:09 -0500 Subject: [PATCH 44/52] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a60aeac54e..f6aee0a9be 100644 --- a/README.md +++ b/README.md @@ -851,7 +851,9 @@ An optional feature for the Discover Domain is the Variable Injector tool. The v To enable the Variable Injector during the Discover Domain, you must place a json file named model_variable_injector.json into the WLSDEPLOY/lib directory. This file must be manually created and contain one of the pre-defined keywords and/or a CUSTOM designated file. -A variable name substitution is only performed once for an attribute during the variable injector run. The first substitution is not replaced by any subsequent matches. +A variable name substitution is only performed once for an attribute during the variable injector run. The first substitution is not replaced by any subsequent matches. + +The Discover Domain tool calls the variable injector after the model has been discovered and any filters processed. The subsequent model and variable file are then validated with the validator tool. The supported keywords are as follows: @@ -874,9 +876,7 @@ The supported keywords are as follows: - URL All MBean URL attribute values in the model are injected with a variable property string and the string and value placed in the variable property file -These special keywords are defined in a keyword file that is installed into the WLSDEPLOY/lib directory. Each keyword is associated with an injector json file. -Each injector json file is installed into the WLSDEPLOY/lib/injectors directory. An injector file contains the directives for the Variable Injector tool to -inject variable names into specified model attributes. +These special keywords are defined in a keyword file that is installed into the WLSDEPLOY/lib directory. Each keyword is associated with an injector json file. Each injector json file is installed into the WLSDEPLOY/lib/injectors directory. An injector file contains the directives for the Variable Injector tool to inject variable names into specified model attributes. Here is an example of a model_variable_injector.json file using the PORT keyword. From 7e17a3379c7f3e562cc73633c3f65f4ea6542a66 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Wed, 30 May 2018 17:35:25 -0500 Subject: [PATCH 45/52] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f6aee0a9be..767f1ccc9b 100644 --- a/README.md +++ b/README.md @@ -847,11 +847,11 @@ When creating the archive, the tool will try to gather all binaries, scripts, an ## The Variable Injector Tool -An optional feature for the Discover Domain is the Variable Injector tool. The variable injector will replace a selected model attribute value with a property marker containing a unique variable name. This variable name and the replaced value are inserted into a variable property file. +The variable injector tool is used to parameterize a model with variables. These variables are assigned values by you in the external variable properties file. This facilitates the ease of using the same domain model to create new domains in different environments. You can run the variable injector tool as an option in the Discover Domain tool, or you can update a model with variables with the stand-alone command line interface. -To enable the Variable Injector during the Discover Domain, you must place a json file named model_variable_injector.json into the WLSDEPLOY/lib directory. This file must be manually created and contain one of the pre-defined keywords and/or a CUSTOM designated file. +To enable the Variable Injector during the Discover Domain, you must place a json file named model_variable_injector.json into the \/lib directory. This file must be manually created and contain one of the Variable Injector pre-defined keywords and/or a CUSTOM designated file. A keyword points to, and the custom file is, an injector directive file. These injector directives control how the variables are injected into the model and variable properties file. -A variable name substitution is only performed once for an attribute during the variable injector run. The first substitution is not replaced by any subsequent matches. +As the directives are processed by the Variable Injector tool, a model attribute is assessed for a match for parameterization, and a property token and a unique variable name is injected into the attribute in the model. The variable name, and the model value, are placed into the external variable properties file. A variable name substitution on a model attribute is only performed once during the variable injector run. The first substitution is not replaced by any subsequent matches. The Discover Domain tool calls the variable injector after the model has been discovered and any filters processed. The subsequent model and variable file are then validated with the validator tool. @@ -876,7 +876,7 @@ The supported keywords are as follows: - URL All MBean URL attribute values in the model are injected with a variable property string and the string and value placed in the variable property file -These special keywords are defined in a keyword file that is installed into the WLSDEPLOY/lib directory. Each keyword is associated with an injector json file. Each injector json file is installed into the WLSDEPLOY/lib/injectors directory. An injector file contains the directives for the Variable Injector tool to inject variable names into specified model attributes. +These special keywords are defined in a keyword file that is installed into the \/lib directory. Each keyword is associated with an injector json file. Each injector json file is installed into the \/lib/injectors directory. An injector file contains the directives for the Variable Injector tool to inject variable names into specified model attributes. Here is an example of a model_variable_injector.json file using the PORT keyword. From e6b3f27c7f37b1cdc0386f9dfc679971d9ed4364 Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 31 May 2018 08:14:38 -0500 Subject: [PATCH 46/52] Improve readme --- README.md | 61 +++++++++++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 767f1ccc9b..18887392aa 100644 --- a/README.md +++ b/README.md @@ -847,13 +847,13 @@ When creating the archive, the tool will try to gather all binaries, scripts, an ## The Variable Injector Tool -The variable injector tool is used to parameterize a model with variables. These variables are assigned values by you in the external variable properties file. This facilitates the ease of using the same domain model to create new domains in different environments. You can run the variable injector tool as an option in the Discover Domain tool, or you can update a model with variables with the stand-alone command line interface. +The variable injector tool is used to tokenize a model with variables. You assign a value for a variable in an external variable properties file. This facilitates using the same domain model to create new domains in different environments. You can run the variable injector tool as an option in the Discover Domain tool, or you can update a model with variables with the stand-alone command line interface. -To enable the Variable Injector during the Discover Domain, you must place a json file named model_variable_injector.json into the \/lib directory. This file must be manually created and contain one of the Variable Injector pre-defined keywords and/or a CUSTOM designated file. A keyword points to, and the custom file is, an injector directive file. These injector directives control how the variables are injected into the model and variable properties file. +To enable the Variable Injector during the Discover Domain, you place a json file named model_variable_injector.json into the \/lib directory. This file will tell the injector tool which model attributes to tokenize. You create it with one of the injector's pre-defined keywords and/or a CUSTOM file or files. A keyword points to, and the custom file is, an injector directive file. These injector directives control which variables are injected into the model and variable properties file. -As the directives are processed by the Variable Injector tool, a model attribute is assessed for a match for parameterization, and a property token and a unique variable name is injected into the attribute in the model. The variable name, and the model value, are placed into the external variable properties file. A variable name substitution on a model attribute is only performed once during the variable injector run. The first substitution is not replaced by any subsequent matches. +As these directives are applied to the model, each model attribute is assessed for a match. If the attribute matches a directive, then a property token and a unique variable name is injected into the attribute. The variable name, and the model value, are placed into the external variable properties file. A variable name substitution on a model attribute is only performed once during the variable injector run. The first substitution is not replaced by any subsequent matches. -The Discover Domain tool calls the variable injector after the model has been discovered and any filters processed. The subsequent model and variable file are then validated with the validator tool. +The Discover Domain tool calls the variable injector after the model has been discovered and any filters processed. After variables have been injected by the tool, the model and variable file are validated with the validator tool. The supported keywords are as follows: @@ -932,29 +932,17 @@ Server.soa_server2.ListenPort=8001 Server.soa_server2.SSL.ListenPort=8002 ``` -To designate the name and location of the variable properties file, include the variable_file_name directive in the model_variable_injector.json file. - -```json -{ - "variable_file_name": "/home/savedomain/variables.properties", -} -``` - -If this directive is not included in the json file, the model or archive file name and location is used to create the variable properties file name and location. If the model_file command line -argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name +To specify the name and location of the variable properties file, you can use the argument `-variable_properties_file` on the discover domain command line. If you use this command line argument, the model variable injector file must exist in the \/lib directory. +If the model variable injector file exists in the directory, but the command line argument is not used, the variable properties file will be named as follows: If the model_file command line argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name and location will be used. As with the archive and model file, each run of the discover domain tool will overwrite the contents of an existing variable property file with the values from the current run. ### Custom Variable Injector -You may customize the injector attribute replacement by using the CUSTOM keyword in the model_variable_injector.json. The CUSTOM keyword identifies a list of one or more custom injector json files. -These injector json files will be processed by the Variable Injector tool first, in list order. Any resulting variable replacement will not be overlaid when processing additional keywords. +You may designate custom injector directives by using the CUSTOM keyword in the model_variable_injector.json. On the CUSTOM keyword, you identify a list of one or more custom injector directive json files. An entry in the injector json file is a key that specifies the unique MBean attribute to tokenize, and an optional set of directives. The injector is a period separated MBean hierarchy and attribute name as they are defined in the model. Do not enter the name of the model section into the injector key. -An entry in the injector json file is a key that specifies the unique MBean attribute to parameterize, and an optional set of directives. The injector is a period separated MBean hierarchy and attribute name as they are defined in the model. -Do not enter the name of the model section into the injector key. - -For example, an injector key for the Server SSL Listen Port is Server.SSL.ListenPort. The Server.SSL identifies the hierarchy in the model to the attribute. +For example, an injector key for the Server SSL Listen Port is as below. ```json { @@ -962,7 +950,8 @@ For example, an injector key for the Server SSL Listen Port is Server.SSL.Listen } ``` -An example of that hierarchy in a model (notice that the MBean name of AdminServer is NOT included): +Note the hierarchy of MBeans in the model for the ListenPort attribute and note that the MBean name of AdminServer is NOT included in the directive: + ```yaml topology: Server: @@ -975,14 +964,14 @@ topology: ListenPort: 7002 ``` -For this example, the Variable Injector tool will inject a unique variable name for every server in the model that has the Server/SSL/ListenPort attribute. +These custom injector json files will be processed by the Variable Injector tool before keywords, each file processed in list order. A property injected into an attribute will not be replaced by any subsequent matches. #### Custom special directives The following directives can refine the variable injection. -- force: - For true, if the MBean hierarchy exists in the model, but the attribute does not, then the attribute will be added and persisted to the discovered model. The value stored in the +- force: + If the MBean hierarchy exists in the model, but the attribute does not, then the attribute will be added and persisted to the discovered model. The value stored in the model is the weblogic default value. - variable_value: @@ -990,14 +979,13 @@ The following directives can refine the variable injection. - regexp: A list of regexp patterns that will be applied to either the string values or map values of an attribute in the model. If the pattern matches, then the matching part of the - string or dictionary will be injected with a unique variable name. + string or dictionary will be injected with a property token and a unique variable name . -- pattern: the regular expression pattern to apply to the string value or map values of an attribute -- suffix: - The suffix name to append to each resulting variable name in the variable properties file + The suffix name to append to each resulting variable name in order to create a unique variable name -The regexp list is useful when only a segment of a string value needs to be parameterized (making for quick manipulation of the variable properties). If more than one -pattern is specified in the regexp directive list, then a suffix MUST be added in order to generate a unique variable name. +The regexp list is useful when only a segment of a string value or map needs to be injected (giving you a clean list of property values in the variable properties file). You can inject more than one token into a string or map with multiple patterns. However, when you have more than one pattern, you must provide a suffix for each. This allows the tool to generate a unique variable name for each token in the string or map. The following is an example of how to effectively use the regexp directive list to search for a segment in a string value. In this example, we want to search for the host and port in each Oracle JDBC URL that uses the special Oracle URL notation, and create an entry for the host and port in the variable properties file. @@ -1030,24 +1018,31 @@ We create a directive in our custom injector json file: }, ``` -During the Discover Domain tool run, the pattern is applied to the URL string, and the resulting entries added to the variable properties: +During the Discover Domain tool run, the pattern is applied to the URL string and tokens injected into the string: ``` URL: 'jdbc:oracle:thin:@(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(HOST=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host@@:)(PORT=@@PROP:JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port@@)))(CONNECT_DATA=(SERVICE_NAME=orcl.us.oracle.com)))' +``` +And The variables put in the properties file: + +``` JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Host=slc05til.us.oracle.com JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port=1521 ``` #### Selecting specific MBean names for variable injection -The final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to parameterize an attribute just for a specific set of servers. -To define a list of one or more names of an MBean, add a user directive to the injector keyword value. The user directive is added to the end of an MBean between brackets. For instance, to select only the admin server named AdminServer, add the -user directive to the Server MBean - Server[AdminServer]. To select servers soa_server1 and soa_server2 from the model, the injector key is Server[soa_server1,soa_server2] +The final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to tokenize an attribute just for a specific server. +To define a list of one or more names for a specific MBean in the injector directive hierarchy, format the list as follows: +``` +MBean[comma separated list of names] +`` +For instance, to select only the admin server named AdminServer for a Server directive, use the format Server[AdminServer]. To select servers soa_server1 and soa_server2 from the model, the injector key use the format Server[soa_server1,soa_server2] The injector tool recognizes two KEYWORDS for a user list, MANAGED_SERVERS (all the managed servers in the model) and ADMIN_SERVER (The admin server in the model). -A custom injector to parameterize the admin server SSL listen port is: +A custom injector for the admin server SSL listen port is: ```json { From 91076476fd3aa2cd4fc855025e521ac08f167666 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Thu, 31 May 2018 08:42:00 -0500 Subject: [PATCH 47/52] Update README.md --- README.md | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 18887392aa..2289e2adf7 100644 --- a/README.md +++ b/README.md @@ -849,11 +849,9 @@ When creating the archive, the tool will try to gather all binaries, scripts, an The variable injector tool is used to tokenize a model with variables. You assign a value for a variable in an external variable properties file. This facilitates using the same domain model to create new domains in different environments. You can run the variable injector tool as an option in the Discover Domain tool, or you can update a model with variables with the stand-alone command line interface. -To enable the Variable Injector during the Discover Domain, you place a json file named model_variable_injector.json into the \/lib directory. This file will tell the injector tool which model attributes to tokenize. You create it with one of the injector's pre-defined keywords and/or a CUSTOM file or files. A keyword points to, and the custom file is, an injector directive file. These injector directives control which variables are injected into the model and variable properties file. +To enable the Variable Injector during the Discover Domain, you place a json file named model_variable_injector.json into the \/lib directory. You create this file, and it contains pre-defined keywords and/or a CUSTOM list of files. A keyword points to, and the custom file is, an injector directive file. The tool applies the directives to the attributes in a model, and if the directive matches an attribute, then a property token with a unique variable name is injected into the attribute. The variable name and model value are placed into the external variable properties file. Variable injection on an attribute is only performed once. The property token is not replaced by any subsequent matches. -As these directives are applied to the model, each model attribute is assessed for a match. If the attribute matches a directive, then a property token and a unique variable name is injected into the attribute. The variable name, and the model value, are placed into the external variable properties file. A variable name substitution on a model attribute is only performed once during the variable injector run. The first substitution is not replaced by any subsequent matches. - -The Discover Domain tool calls the variable injector after the model has been discovered and any filters processed. After variables have been injected by the tool, the model and variable file are validated with the validator tool. +The Discover Domain tool calls the variable injector after the model has been discovered and all filters run. After variable injection, the model and variable file are validated with the validator tool. The supported keywords are as follows: @@ -876,7 +874,7 @@ The supported keywords are as follows: - URL All MBean URL attribute values in the model are injected with a variable property string and the string and value placed in the variable property file -These special keywords are defined in a keyword file that is installed into the \/lib directory. Each keyword is associated with an injector json file. Each injector json file is installed into the \/lib/injectors directory. An injector file contains the directives for the Variable Injector tool to inject variable names into specified model attributes. +Note that these keywords are defined in a file installed into the \/lib folder. Each keyword points to an injector json file that is located in the \/lib/injectors folder. An injector file contains directives specific to the keyword. You can look at these files to help your create your own CUSTOM injector file. Here is an example of a model_variable_injector.json file using the PORT keyword. @@ -932,17 +930,17 @@ Server.soa_server2.ListenPort=8001 Server.soa_server2.SSL.ListenPort=8002 ``` -To specify the name and location of the variable properties file, you can use the argument `-variable_properties_file` on the discover domain command line. If you use this command line argument, the model variable injector file must exist in the \/lib directory. -If the model variable injector file exists in the directory, but the command line argument is not used, the variable properties file will be named as follows: If the model_file command line argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name -and location will be used. +To specify the name and location of the variable properties file, you use the argument `-variable_properties_file` on the discover domain command line. If you use this command line argument, the model variable injector file must exist in the \/lib directory. If the model variable injector file exists in the directory, but the command line argument is not used, the variable properties file is created as follows: If the model_file command line argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name and location will be used. As with the archive and model file, each run of the discover domain tool will overwrite the contents of an existing variable property file with the values from the current run. ### Custom Variable Injector -You may designate custom injector directives by using the CUSTOM keyword in the model_variable_injector.json. On the CUSTOM keyword, you identify a list of one or more custom injector directive json files. An entry in the injector json file is a key that specifies the unique MBean attribute to tokenize, and an optional set of directives. The injector is a period separated MBean hierarchy and attribute name as they are defined in the model. Do not enter the name of the model section into the injector key. +You designate custom injector directives using the CUSTOM keyword in the model_variable_injector.json. The CUSTOM keyword requires a a list of one or more custom injector directive json files. + +An injector directive contains a key that identifies an attribute to be tokenized, and an optional set of directive properties. The key is a period separated MBean hierarchy and attribute name as they are defined in the model. Do not enter the name of the model section into the injector key. -For example, an injector key for the Server SSL Listen Port is as below. +For example, an injector key for the Server SSL Listen Port is as below. This directive contains no additional properties. ```json { @@ -966,9 +964,9 @@ topology: These custom injector json files will be processed by the Variable Injector tool before keywords, each file processed in list order. A property injected into an attribute will not be replaced by any subsequent matches. -#### Custom special directives +#### Custom directive properties -The following directives can refine the variable injection. +Include the following properties to refine the directive as specified. - force: If the MBean hierarchy exists in the model, but the attribute does not, then the attribute will be added and persisted to the discovered model. The value stored in the @@ -987,8 +985,7 @@ The following directives can refine the variable injection. The regexp list is useful when only a segment of a string value or map needs to be injected (giving you a clean list of property values in the variable properties file). You can inject more than one token into a string or map with multiple patterns. However, when you have more than one pattern, you must provide a suffix for each. This allows the tool to generate a unique variable name for each token in the string or map. -The following is an example of how to effectively use the regexp directive list to search for a segment in a string value. In this example, we want to search for -the host and port in each Oracle JDBC URL that uses the special Oracle URL notation, and create an entry for the host and port in the variable properties file. +The following is an example of how to effectively use the regexp directive list to search for a segment in a string value. In this example, we want to search for the host and port in each Oracle JDBC URL that uses the special Oracle URL notation, and create an entry for the host and port in the variable properties file. In the model, we expect to find a URL like the following: @@ -1033,8 +1030,7 @@ JDBCSystemResource.Database1.JdbcResource.JDBCDriverParams.URL--Port=1521 #### Selecting specific MBean names for variable injection -The final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to tokenize an attribute just for a specific server. -To define a list of one or more names for a specific MBean in the injector directive hierarchy, format the list as follows: +This final custom directive allows you to explicitly define which named entries for an MBean in the model you wish to inject properties. For instance, you might wish to tokenize an attribute just for a specific server. To define a list of one or more names for a specific MBean in the injector directive hierarchy, format the list as follows: ``` MBean[comma separated list of names] `` From e94e8ae3bc16e506443235ce498f2f3c45bdabee Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Thu, 31 May 2018 08:45:47 -0500 Subject: [PATCH 48/52] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2289e2adf7..43a84ad752 100644 --- a/README.md +++ b/README.md @@ -918,6 +918,7 @@ topology: SSL: Enabled: true ListenPort: @@PROP:Server.SSL.soa_server2.ListenPort@@ +``` And the resulting variable property file: From 41af69c2e9c4ce4d4bf17902db0588b149b0c319 Mon Sep 17 00:00:00 2001 From: Carolyn Rountree Date: Thu, 31 May 2018 08:53:24 -0500 Subject: [PATCH 49/52] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 43a84ad752..1d9df6999f 100644 --- a/README.md +++ b/README.md @@ -984,7 +984,7 @@ Include the following properties to refine the directive as specified. -- suffix: The suffix name to append to each resulting variable name in order to create a unique variable name -The regexp list is useful when only a segment of a string value or map needs to be injected (giving you a clean list of property values in the variable properties file). You can inject more than one token into a string or map with multiple patterns. However, when you have more than one pattern, you must provide a suffix for each. This allows the tool to generate a unique variable name for each token in the string or map. +The regexp list is useful when only a segment of a string value or map needs to be tokenized (giving you a clean list of property values in the variable properties file). You can inject more than one token into a string or map with multiple patterns. However, when you have more than one pattern, you must provide a suffix for each. This allows the tool to generate a unique variable name for each token in the string or map. The following is an example of how to effectively use the regexp directive list to search for a segment in a string value. In this example, we want to search for the host and port in each Oracle JDBC URL that uses the special Oracle URL notation, and create an entry for the host and port in the variable properties file. @@ -1035,7 +1035,7 @@ This final custom directive allows you to explicitly define which named entries ``` MBean[comma separated list of names] `` -For instance, to select only the admin server named AdminServer for a Server directive, use the format Server[AdminServer]. To select servers soa_server1 and soa_server2 from the model, the injector key use the format Server[soa_server1,soa_server2] +To select only the admin server named AdminServer for a Server directive, use the format Server[AdminServer]. To select servers soa_server1 and soa_server2, format the key as Server[soa_server1,soa_server2] The injector tool recognizes two KEYWORDS for a user list, MANAGED_SERVERS (all the managed servers in the model) and ADMIN_SERVER (The admin server in the model). From 058e8f21c52a8c7d540ac7c39ce258d6edafae4b Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 31 May 2018 11:11:59 -0500 Subject: [PATCH 50/52] changes for review suggestions --- core/src/main/python/discover.py | 25 ++++++++++ .../wlsdeploy/tool/util/variable_injector.py | 23 +++++---- .../deploy/messages/wlsdeploy_rb.properties | 9 ++-- .../src/test/python/variable_injector_test.py | 1 + .../resources/variable_injector_keyword.json | 1 - installer/src/main/bin/discoverDomain.cmd | 13 ++++- installer/src/main/bin/injectVariables.cmd | 49 +++++++++---------- .../main/samples/model_variable_injector.json | 1 - 8 files changed, 77 insertions(+), 45 deletions(-) diff --git a/core/src/main/python/discover.py b/core/src/main/python/discover.py index 33da0ac1aa..07580c5504 100644 --- a/core/src/main/python/discover.py +++ b/core/src/main/python/discover.py @@ -25,6 +25,7 @@ sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0]))) +import wlsdeploy.tool.util.variable_injector as variable_injector from wlsdeploy.aliases.wlst_modes import WlstModes from wlsdeploy.exception import exception_helper from wlsdeploy.logging.platform_logger import PlatformLogger @@ -60,6 +61,7 @@ # Used by shell script to locate WLST CommandLineArgUtil.MODEL_FILE_SWITCH, CommandLineArgUtil.DOMAIN_TYPE_SWITCH, + CommandLineArgUtil.VARIABLE_PROPERTIES_FILE_SWITCH, CommandLineArgUtil.ADMIN_URL_SWITCH, CommandLineArgUtil.ADMIN_USER_SWITCH, CommandLineArgUtil.ADMIN_PASS_SWITCH @@ -80,6 +82,7 @@ def __process_args(args): __verify_required_args_present(required_arg_map) __wlst_mode = __process_online_args(optional_arg_map) __process_archive_filename_arg(required_arg_map) + __process_variable_filename_arg(optional_arg_map) combined_arg_map = optional_arg_map.copy() combined_arg_map.update(required_arg_map) @@ -159,6 +162,28 @@ def __process_archive_filename_arg(required_arg_map): return +def __process_variable_filename_arg(optional_arg_map): + """ + If the variable filename argument is present, the required model variable injector json file must exist in + the WLSDEPLOY lib directory. + :param optional_arg_map: containing the variable file name + :raises: CLAException: if this argument is present but the model variable injector json does not exist + """ + _method_name = '__process_variable_filename_arg' + + if CommandLineArgUtil.VARIABLE_PROPERTIES_FILE_SWITCH in optional_arg_map: + variable_injector_file_name = variable_injector.get_default_variable_injector_file_name() + try: + FileUtils.validateExistingFile(variable_injector_file_name) + except IllegalArgumentException, ie: + ex = exception_helper.create_cla_exception('WLSDPLY-06021', optional_arg_map[ + CommandLineArgUtil.VARIABLE_PROPERTIES_FILE_SWITCH], variable_injector_file_name, + ie.getLocalizedMessage(), error=ie) + __logger.throwing(ex, class_name=_class_name, method_name=_method_name) + raise ex + return + + def __discover(model_context): """ Populate the model from the domain. diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index c2fd813c3a..60393b69a4 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -34,7 +34,7 @@ VARIABLE_KEYWORDS_FILE_NAME_ARG = 'variable_keywords_file_name' VARIABLE_INJECTOR_FILES_PATH_ARG = 'variable_injector_files_path_name' VARIABLE_FILE_NAME_ARG = 'variable_file_name' -VARIABLE_FILE_NAME = 'variables.json' +VARIABLE_FILE_NAME = 'variable.properties' VARIABLE_FILE_APPEND_ARG = 'append_to_variables' VARIABLE_FILE_APPEND = 'append' VARIABLE_FILE_UPDATE = 'update' @@ -123,7 +123,7 @@ def inject_variables_keyword_file(self, **kwargs): return_model = self.__original variable_file_location = None if variables_injector_dictionary and keywords_dictionary: - variable_file_location = self._get_variable_file_name(variables_injector_dictionary, **kwargs) + variable_file_location = self._get_variable_file_name(**kwargs) if not variable_file_location: _logger.warning('WLSDPLY-19520', variable_injector_location_file, class_name=_class_name, method_name=_method_name) @@ -471,17 +471,11 @@ def _log_mbean_not_found(self, mbean, replacement, location): _logger.finer('WLSDPLY-19516', mbean, replacement, location.get_folder_path(), class_name=_class_name, method_name=_method_name) - def _get_variable_file_name(self, variables_injector_dictionary, **kwargs): + def _get_variable_file_name(self, **kwargs): _method_name = '_get_variable_file_name' if VARIABLE_FILE_NAME_ARG in kwargs: variable_file_location = kwargs[VARIABLE_FILE_NAME_ARG] _logger.finer('WLSDPLY-19522', variable_file_location, class_name=_class_name, method_name=_method_name) - if VARIABLE_FILE_NAME_ARG in variables_injector_dictionary: - del variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] - elif VARIABLE_FILE_NAME_ARG in variables_injector_dictionary: - variable_file_location = variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] - del variables_injector_dictionary[VARIABLE_FILE_NAME_ARG] - _logger.finer('WLSDPLY-19521', variable_file_location, class_name=_class_name, method_name=_method_name) else: variable_file_location = variables.get_default_variable_file_name(self.__model_context) if variable_file_location: @@ -554,6 +548,15 @@ def _check_replace_variable_value(self, location, attribute, variable_value, inj return variable_value +def get_default_variable_injector_file_name(variable_injector_file_name=VARIABLE_INJECTOR_FILE_NAME): + """ + Return the default name and location of the model variable injector json file + :param variable_injector_file_name: if different than the default name + :return: file path in the wlsdeploy location and file name + """ + return os.path.join(_wlsdeploy_location, DEFAULT_FILE_LOCATION, variable_injector_file_name) + + def _load_variable_file(variable_file_location, **kwargs): _method_name = '_load_variable_file' append = False @@ -581,7 +584,7 @@ def _get_variable_injector_file_name(**kwargs): if VARIABLE_INJECTOR_PATH_NAME_ARG in kwargs: return os.path.join(kwargs[VARIABLE_INJECTOR_PATH_NAME_ARG], variable_injector_file_name) else: - return os.path.join(_wlsdeploy_location, DEFAULT_FILE_LOCATION, variable_injector_file_name) + return get_default_variable_injector_file_name(variable_injector_file_name) def _get_variable_keywords_file_name(**kwargs): diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index 6d4b375577..ec7b28f9a7 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -421,6 +421,8 @@ WLSDPLY-06018=Please enter the WebLogic administrator password WLSDPLY-06019=Failed to read the WebLogic administrator password input from the user: {0} WLSDPLY-06020=Discover of domain is not currently supported in online mode. Ignoring Admin credential arguments which \ are present for FUTURE TBD release. +WLSDPLY-06021=The variables file command line argument {0} was used but the model variables injector file \ + {1} does not exist : {2} # discoverer.py WLSDPLY-06100=Find attributes at location {0} @@ -1133,13 +1135,12 @@ WLSDPLY-19512=CUSTOM keyword was found in variables injector list but no files w keyword WLSDPLY-19513=Variables were located and inserted in the model using injector file {0} WLSDPLY-19514=Located mbean {0} in the model file -WLSDPLY-19515=Invalid mbean {0} found in replacement property {1} at location {2} -WLSDPLY-19516=MBean {0} not found in the model for replacement property {1} at location (2} -WLSDPLY-19517=Attribute {0} not found in the model for replacement property {1} at location {2} +WLSDPLY-19515=Invalid mbean {0} found in injector directive {1} at location {2} +WLSDPLY-19516=MBean {0} not found in the model for injector directive {1} at location {2} +WLSDPLY-19517=Attribute {0} not found in the model for injector directive {1} at location {2} WLSDPLY-19518=Variables were inserted into the model and written to the variables file {0} WLSDPLY-19519=No variables were inserted into the model during variable replacement WLSDPLY-19520=Variable injector file was found at location {0} but no variable properties file name was provided -WLSDPLY-19521=Variables property file name {0} extracted from model injector json file WLSDPLY-19522=Variables property file name {0} was passed as an argument to the variable injector WLSDPLY-19523=MBean {0} from injector path represents a named MBean folder at location {1} WLSDPLY-19524=Segment {0} for attribute {1} not found in model value {2} at location {3} diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index 4daa8dcabb..966d4a4312 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -239,6 +239,7 @@ def testWithVariableHelperKeywords(self): expected['Machine.machine1.NodeManager.PasswordEncrypted'] = '--FIX ME--' expected['Machine.machine1.NodeManager.UserName'] = 'admin' inserted, model, variable_file_name = self._helper.inject_variables_keyword_file( + variable_file_name=self._variable_file, variable_injector_path_name=self._resources_dir, variable_injector_file_name=self._variable_injector_keyword, variable_keywords_path_name=self._resources_dir, variable_keywords_file_name=self._keywords_file) diff --git a/core/src/test/resources/variable_injector_keyword.json b/core/src/test/resources/variable_injector_keyword.json index ff408fb4d4..2b8da6ac04 100644 --- a/core/src/test/resources/variable_injector_keyword.json +++ b/core/src/test/resources/variable_injector_keyword.json @@ -1,5 +1,4 @@ { - "variable_file_name": "../../test-classes/variable.injector.test.properties", "PORT": {}, "URL": {}, "CREDENTIALS": {} diff --git a/installer/src/main/bin/discoverDomain.cmd b/installer/src/main/bin/discoverDomain.cmd index 46a56727d2..f163d81570 100644 --- a/installer/src/main/bin/discoverDomain.cmd +++ b/installer/src/main/bin/discoverDomain.cmd @@ -115,6 +115,13 @@ IF %JVM_VERSION% LSS 7 ( ECHO JDK version is %JVM_FULL_VERSION%, setting JAVA_VENDOR to Sun... SET JAVA_VENDOR=Sun ) +@rem +@rem Check to see if no args were given and print the usage message +@rem +IF "%~1" == "" ( + SET RETURN_CODE=0 + GOTO usage +) @rem @rem Find the args required to determine the WLST script to run @@ -125,6 +132,10 @@ SET DOMAIN_TYPE= SET WLST_PATH_DIR= :arg_loop +IF "%1" == "-help" ( + SET RETURN_CODE=0 + GOTO usage +) IF "%1" == "-oracle_home" ( SET ORACLE_HOME=%2 SHIFT @@ -295,8 +306,6 @@ ECHO. ECHO discoverDomain.cmd failed ^(exit code = %RETURN_CODE%^) >&2 GOTO exit_script -:usage -ECHO. :usage ECHO. ECHO Usage: %~nx0 -oracle_home ^ diff --git a/installer/src/main/bin/injectVariables.cmd b/installer/src/main/bin/injectVariables.cmd index f93fb59c28..33b1fb7aa3 100644 --- a/installer/src/main/bin/injectVariables.cmd +++ b/installer/src/main/bin/injectVariables.cmd @@ -312,45 +312,40 @@ GOTO exit_script ECHO. ECHO Usage: %~nx0 [-help] ECHO -oracle_home ^ -ECHO -model_file ^ | -archive_file ^ +ECHO [-model_file ^] +ECHO [-archive_file ^] ECHO [-variable_injector_file ^] -ECHO [-variable_keywords_file ^] ECHO [-variable_properties_file ^] ECHO [-domain_type ^] ECHO [-wlst_path ^] ECHO. ECHO where: -ECHO oracle-home - the existing Oracle Home directory with the correct version for the model +ECHO oracle-home - the existing Oracle Home directory with the correct version for the model ECHO. -ECHO model-file - the location of the model file in which variables will be injected. -ECHO If not specified, the tool will look for the model -ECHO in the archive file. Either the model_file or the archive_file argument must be provided. +ECHO model-file - the location of the model file in which variables will be injected. +ECHO If not specified, the tool will look for the model +ECHO in the archive file. Either the model_file or the archive_file argument +ECHO must be provided. ECHO. -ECHO archive-file - the path to the archive file that contains a model in which the variables -ECHO will be injected. If the model-file argument is used, this argument will be -ECHO ignored. The archive file must contain a valid model. +ECHO archive-file - the path to the archive file that contains a model in which the variables +ECHO will be injected. If the model-file argument is used, this argument will be +ECHO ignored. The archive file must contain a valid model. ECHO. -ECHO variable-injector-file - the location of the variable injector file which contains the variable -ECHO injector keywords for this model injection run. If this argument is not provided, -ECHO the model_variable_injector.json file must exist in the lib directory in the -ECHO WLSDEPLOY_HOME location. +ECHO variable_properties_file - the location of the property file in which to store any variable names injected +ECHO into the model. If this command line argument is not specified, the variable +ECHO will be located and named based on the model file or archive file name and +ECHO location. If the file exists, the file will be updated with new variable values. ECHO. -ECHO variable-keywords-file - this argument overrides the INSTALLED version of the allowed variable keywords -ECHO for the variable injector. This argument is for advanced usage only. The installed -ECHO keywords file is located in the lib directory of WLSDEPLOY_HOME location. +ECHO variable-injector-file - the location of the variable injector file which contains the variable +ECHO injector keywords for this model injection run. If this argument is not provided, +ECHO the model_variable_injector.json file must exist in the lib directory in the +ECHO WLSDEPLOY_HOME location. ECHO. -ECHO variable-file - the location of the property file in which to store any variable names injected -ECHO into the model. This argument overrides the value in the model injector file. -ECHO If the variable file is not listed in the model injector file, and this command -ECHO line argument is not used, the variable properties will be located and named -ECHO based on the model file or archive file name and location. -ECHO If the variable file exists, new variable values will be appended to the file. +ECHO domain-type - the type of domain (e.g., WLS, JRF). +ECHO Used to locate wlst.cmd if wlst-path not specified ECHO. -ECHO domain-type - the type of domain (e.g., WLS, JRF). -ECHO Used to locate wlst.cmd if wlst-path not specified -ECHO. -ECHO wlst-path - the Oracle Home subdirectory of the wlst.cmd -ECHO script to use (e.g., ^\soa) +ECHO wlst-path - the Oracle Home subdirectory of the wlst.cmd +ECHO script to use (e.g., ^\soa) ECHO. :exit_script diff --git a/installer/src/main/samples/model_variable_injector.json b/installer/src/main/samples/model_variable_injector.json index d419deca0b..3b38c47e36 100644 --- a/installer/src/main/samples/model_variable_injector.json +++ b/installer/src/main/samples/model_variable_injector.json @@ -1,5 +1,4 @@ { - "variable_file_name": "@@WLSDEPLOY@@/samples/variables.properties", "CREDENTIALS": {}, "CUSTOM": { "file-list": [ "@@WLSDEPLOY@@/samples/custom_injector.json" ] From 48a7b047154acdedfb2897ff0df3b39e24a33b2a Mon Sep 17 00:00:00 2001 From: crountre Date: Thu, 7 Jun 2018 14:43:17 -0500 Subject: [PATCH 51/52] Add a few corrections and a unit test for aliases get default value --- .../wlsdeploy/tool/util/variable_injector.py | 19 +++++++------ .../src/test/python/variable_injector_test.py | 27 ++++++++++++------- .../test/resources/variable_insertion.yaml | 12 +++++++++ 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py index 60393b69a4..1031b1ce07 100644 --- a/core/src/main/python/wlsdeploy/tool/util/variable_injector.py +++ b/core/src/main/python/wlsdeploy/tool/util/variable_injector.py @@ -92,11 +92,15 @@ def __init__(self, program_name, model, model_context=None, version=None): self.__original = copy.deepcopy(model) self.__model = model self.__model_context = model_context + if self.__model_context: + self.__wlst_mode = self.__model_context.get_target_wlst_mode() + else: + self.__wlst_mode = WlstModes.OFFLINE self.__section_keys = model_sections.get_model_top_level_keys() self.__section_keys.remove(model_sections.get_model_domain_info_key()) if version: - self.__aliases = Aliases(model_context, WlstModes.OFFLINE, version, None) + self.__aliases = Aliases(model_context, self.__wlst_mode, version, None) else: self.__aliases = Aliases(model_context) @@ -526,26 +530,25 @@ def _check_insert_attribute_model(self, location, model_section, attribute, inje _method_name = '_check_insert_attribute_model' if attribute not in model_section and (FORCE in injector_values and Boolean(injector_values[FORCE])): value = self.__aliases.get_model_attribute_default_value(location, attribute) - # This is the best I can do - need a get_default_value_model function(location, attribute) - __, wlst_value = self.__aliases.get_wlst_attribute_name_and_value(location, attribute, value) - _logger.fine('WLSDPLY-19540', attribute, location.get_folder_path(), wlst_value, + _logger.fine('WLSDPLY-19540', attribute, location.get_folder_path(), value, class_name=_class_name, method_name=_method_name) - model_section[attribute] = wlst_value + model_section[attribute] = value def _check_replace_variable_value(self, location, attribute, variable_value, injector_values): _method_name = '_format_variable_value' + value = variable_value if VARIABLE_VALUE in injector_values: value = injector_values[VARIABLE_VALUE] # might add code to call a method to populate the replacement value try: - self.__aliases.get_wlst_attribute_name_and_value(location, attribute, value) - variable_value = value + wlst_attribute_name = self.__aliases.get_wlst_attribute_name(location, attribute) + __, value = self.__aliases.get_model_attribute_name_and_value(location, wlst_attribute_name, value) _logger.fine('WLSDPLY-19542', value, variable_value, attribute, location.get_folder_path(), class_name=_class_name, method_name=_method_name) except AliasException, ae: _logger.warning('WLSDPLY-19541', value, attribute, location, ae.getLocalizedMessage(), class_name=_class_name, method_name=_method_name) - return variable_value + return value def get_default_variable_injector_file_name(variable_injector_file_name=VARIABLE_INJECTOR_FILE_NAME): diff --git a/core/src/test/python/variable_injector_test.py b/core/src/test/python/variable_injector_test.py index 966d4a4312..2051ba3dfa 100644 --- a/core/src/test/python/variable_injector_test.py +++ b/core/src/test/python/variable_injector_test.py @@ -2,7 +2,6 @@ Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. The Universal Permissive License (UPL), Version 1.0 """ -import os import unittest import wlsdeploy.util.variables as variables @@ -156,10 +155,10 @@ def testWithSegmentInList(self): variable_injector.REGEXP] = [list_entry] actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) - list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ + wldf_list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ 'HarvestedType']['weblogic.management.runtime.ServerRuntimeMBean']['HarvestedAttribute'] found = False - for entry in list: + for entry in wldf_list: if entry == '@@PROP:WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.' \ 'weblogic.management.runtime.ServerRuntimeMBean.HarvestedAttribute@@': found = True @@ -179,11 +178,10 @@ def testWithSegmentInStringInList(self): variable_injector.REGEXP] = [list_entry] actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) - list = \ - self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester']['HarvestedType'][ - 'weblogic.management.runtime.ServerRuntimeMBean']['HarvestedInstance'] + wldf_list = self._model['resources']['WLDFSystemResource']['MyWldfModule']['WLDFResource']['Harvester'][ + 'HarvestedType']['weblogic.management.runtime.ServerRuntimeMBean']['HarvestedInstance'] found = False - for entry in list: + for entry in wldf_list: if entry == 'com.bea:Name=@@PROP:WLDFSystemResource.MyWldfModule.WLDFResource.Harvester.HarvestedType.' \ 'weblogic.management.runtime.ServerRuntimeMBean.HarvestedInstance--ManagedServer@@' \ ',Type=ServerRuntime': @@ -246,7 +244,6 @@ def testWithVariableHelperKeywords(self): self.assertEqual(self._variable_file, variable_file_name) self.assertEqual(True, inserted) actual = variables.load_variables(self._variable_file) - print actual self._compare_to_expected_dictionary(expected, actual) def testForceAttribute(self): @@ -260,6 +257,15 @@ def testForceAttribute(self): actual = self._helper.inject_variables(replacement_dict) self._compare_to_expected_dictionary(expected, actual) + def testForceAttributeWithTwoDefaults(self): + expected = dict() + expected['JMSSystemResource.MyJmsModule.JmsResource.Template.JmsTemplate.MaximumMessageSize'] = '0' + replacement_dict = dict() + replacement_dict['JMSSystemResource.JmsResource.Template.MaximumMessageSize'] = dict() + replacement_dict['JMSSystemResource.JmsResource.Template.MaximumMessageSize'][variable_injector.FORCE] = True + actual = self._helper.inject_variables(replacement_dict) + self._compare_to_expected_dictionary(expected, actual) + def testReplaceVariableValueAttribute(self): expected = dict() expected[ @@ -272,6 +278,7 @@ def testReplaceVariableValueAttribute(self): 'JNDIProperty[java.naming.security.principal].Value'][ variable_injector.VARIABLE_VALUE] = 'k8s' actual = self._helper.inject_variables(replacement_dict) + print actual self._compare_to_expected_dictionary(expected, actual) def testReplaceVariableValueSegmentInString(self): @@ -283,8 +290,8 @@ def testReplaceVariableValueSegmentInString(self): list_entry = dict() list_entry[variable_injector.REGEXP_PATTERN] = '(?<=HOST=)[\w.-]+(?=\))' list_entry[variable_injector.REGEXP_SUFFIX] = 'Host' - replacement_dict['JDBCSystemResource[Database2].JdbcResource.JDBCDriverParams.URL'][variable_injector.REGEXP] = [ - list_entry] + replacement_dict['JDBCSystemResource[Database2].JdbcResource.JDBCDriverParams.URL'][ + variable_injector.REGEXP] = [list_entry] replacement_dict['JDBCSystemResource[Database2].JdbcResource.JDBCDriverParams.URL'][ variable_injector.VARIABLE_VALUE] = 'den00chv' actual = self._helper.inject_variables(replacement_dict) diff --git a/core/src/test/resources/variable_insertion.yaml b/core/src/test/resources/variable_insertion.yaml index 378b602fdb..6fe7e99aca 100644 --- a/core/src/test/resources/variable_insertion.yaml +++ b/core/src/test/resources/variable_insertion.yaml @@ -35,6 +35,8 @@ topology: SSL: Enabled: true ListenPort: 9004 + DefaultFileStore: + InitialSize: 512 ListenPort: 9003 AdminServer: ListenAddress: 127.0.0.1 @@ -44,6 +46,13 @@ topology: ListenPort: 9002 ListenPort: 9001 resources: + SelfTuning: + MaxThreadsConstraint: + ThreeMax: + Count: 3 + WorkManager: + MaxThWM: + Target: mycluster MailSession: MyMailSession: Properties: @@ -97,6 +106,9 @@ resources: Key: bar Value: True JNDIPropertiesCredentialEncrypted: '--FIX ME--' + Template: + JmsTemplate: + InsertionPausedAtStartup: true JDBCSystemResource: Database1: DescriptorFileName: 'jdbc/Database1-3195-jdbc.xml' From 3b5bdb0c7eb92a6c3f35e38f131f51220c6638d6 Mon Sep 17 00:00:00 2001 From: Derek Sharpe Date: Fri, 8 Jun 2018 15:23:39 -0500 Subject: [PATCH 52/52] readme edits --- README.md | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 1d9df6999f..4e300bb671 100644 --- a/README.md +++ b/README.md @@ -847,40 +847,36 @@ When creating the archive, the tool will try to gather all binaries, scripts, an ## The Variable Injector Tool -The variable injector tool is used to tokenize a model with variables. You assign a value for a variable in an external variable properties file. This facilitates using the same domain model to create new domains in different environments. You can run the variable injector tool as an option in the Discover Domain tool, or you can update a model with variables with the stand-alone command line interface. +The variable injector tool is used to tokenize a model with variables. The values for these variables are assigned using an external property file. This facilitates using the same domain model to create new domains in different environments. The variable injector tool can be run as an option in the Discover Domain tool, or or from the stand-alone command line interface. -To enable the Variable Injector during the Discover Domain, you place a json file named model_variable_injector.json into the \/lib directory. You create this file, and it contains pre-defined keywords and/or a CUSTOM list of files. A keyword points to, and the custom file is, an injector directive file. The tool applies the directives to the attributes in a model, and if the directive matches an attribute, then a property token with a unique variable name is injected into the attribute. The variable name and model value are placed into the external variable properties file. Variable injection on an attribute is only performed once. The property token is not replaced by any subsequent matches. +To enable the Variable Injector during the Discover Domain, create a variable injector configuration by placing a json file named model_variable_injector.json into the \/lib directory using one or more of the pre-defined keywords and/or a CUSTOM list of files. A keyword points to an injector directive file. The tool applies the directives to the attributes in a model, and if the directive matches an attribute, then a property token with a unique variable name is injected into the model and replaces the attribute value. The variable name and model attribute value are placed into the external variable properties file. -The Discover Domain tool calls the variable injector after the model has been discovered and all filters run. After variable injection, the model and variable file are validated with the validator tool. +NOTE: Variable injection on an attribute is only performed once. The property token is not replaced by any subsequent matches. + +If variable injection is enabled, the Discover Domain Tool calls the variable injector after the model has been discovered and after all filters run, but before model validation. The supported keywords are as follows: -- CREDENTIALS - All MBean credentials attribute values (user and password) are injected with a variable property string and the string and value placed in the variable property file +- CREDENTIALS - All MBean credentials attribute values (user and password) are injected with a variable. -- HOST - All MBean Host attribute values in the model are injected with a variable property string and the string and value placed in the variable property file +- HOST - All MBean Host attribute values in the model are injected with a variable. -- PORT - All MBean Port attribute values in the model are injected with a variable property string and the string and value placed in the variable property file +- PORT - All MBean Port attribute values in the model are injected with a variable. -- TARGET - All MBean Target attribute values in the model are injected with a variable property string and the string and value placed in the variable property file +- TARGET - All MBean Target attribute values in the model are injected with a variable. -- TOPOLOGY - Special MBean attributes found in the topology section of the model are injected with a variable property string and the string value placed in the variable - property file. This includes server, machine and node manager ports, credentials and listen addresses, and cluster messaging modes, addresses and ports. +- TOPOLOGY - Common environmental MBean attributes found in the topology section of the model are injected with a variable. + This includes server, machine and node manager ports, credentials and listen addresses, and cluster messaging modes, addresses and ports. -- URL - All MBean URL attribute values in the model are injected with a variable property string and the string and value placed in the variable property file +- URL - All MBean URL attribute values in the model are injected with a variable. -Note that these keywords are defined in a file installed into the \/lib folder. Each keyword points to an injector json file that is located in the \/lib/injectors folder. An injector file contains directives specific to the keyword. You can look at these files to help your create your own CUSTOM injector file. +NOTE: The directives used by each pre-defined keyword are defined in an injector json file that is located in the \/lib/injectors folder. These files should not be changed, but could be used as . Here is an example of a model_variable_injector.json file using the PORT keyword. ```json { - "PORT": {}, + "PORT": {} } ``` @@ -931,25 +927,25 @@ Server.soa_server2.ListenPort=8001 Server.soa_server2.SSL.ListenPort=8002 ``` -To specify the name and location of the variable properties file, you use the argument `-variable_properties_file` on the discover domain command line. If you use this command line argument, the model variable injector file must exist in the \/lib directory. If the model variable injector file exists in the directory, but the command line argument is not used, the variable properties file is created as follows: If the model_file command line argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name and location will be used. +To specify the name and location of the variable properties file for the Discover Tool, use the argument `-variable_properties_file` on the command line. Usage of the variable_properties_file argument without the presence of the model variable injector file in the \/lib directory will cause an error condition and the tool will exit. If the model variable injector file exists in the directory, but the command line argument is not used, the variable properties file is created with the following defaults: If the model_file command line argument is used on the Discover Domain run, the properties file name and location will be the same as the model file, with the file extension `.properties`. If only the archive file argument is present, the archive file name and location will be used. As with the archive and model file, each run of the discover domain tool will overwrite the contents of an existing variable property file with the values from the current run. ### Custom Variable Injector -You designate custom injector directives using the CUSTOM keyword in the model_variable_injector.json. The CUSTOM keyword requires a a list of one or more custom injector directive json files. +To designate custom injector directives, use the CUSTOM keyword in the model_variable_injector.json. The CUSTOM keyword requires a a list of one or more custom injector directive json files. -An injector directive contains a key that identifies an attribute to be tokenized, and an optional set of directive properties. The key is a period separated MBean hierarchy and attribute name as they are defined in the model. Do not enter the name of the model section into the injector key. +An injector directive contains a key that identifies an attribute to be tokenized, and an optional set of directive properties. The key is a period separated MBean hierarchy and attribute name as they are defined in the model. Always exclude the name of the model section from the injector key. For example, an injector key for the Server SSL Listen Port is as below. This directive contains no additional properties. ```json { - "Server.SSL.ListenPort": {}, + "Server.SSL.ListenPort": {} } ``` -Note the hierarchy of MBeans in the model for the ListenPort attribute and note that the MBean name of AdminServer is NOT included in the directive: +NOTE: The hierarchy of MBeans in the model for the ListenPort attribute and note that the MBean name of AdminServer is NOT included in the directive: ```yaml topology: