Add app module deployment support (#1220)

* initial checkin

* fix password and user in jdbc standalone xml

* reset found flag

* add ons password token

* capture user in properties

* initial merge from discovery of application module and added logic for replacing token with clear text value from properties etc ...

* changes for url and sh script

* add function to handle encrypt only password fields in app module xml

* update logic to only encrypt password fields in jdbc app module

* add userid to sh script

* dont clear cache of other than credentials

* change level of warning message

* reverse revision

* remove extra character in comments

* remove unneeded argument

* replace with extra tokens

* requested fixes

* code smells

* encrypt password after all substitutions are done

* revert changes

* requested change

* add jms app module for create

* use helper method to be consistent

Co-authored-by: Carolyn Rountree <[email protected]>

Author: jshum2479

core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java

@@ -618,7 +618,7 @@ public static byte[] readInputStreamToByteArray(InputStream input) throws IOExce
     }
 
     public static File writeInputStreamToFile(InputStream input, String fileName) throws IOException {
-        File tmpdir = new File(System.getProperty("java.io.tmpdir"));
+        File tmpdir = getTmpDir();
         File file = new File(tmpdir, fileName);
         try (FileOutputStream fos = new FileOutputStream(file)) {
             byte[] byteArray = FileUtils.readInputStreamToByteArray(input);
@@ -627,6 +627,9 @@ public static File writeInputStreamToFile(InputStream input, String fileName) th
         return file;
     }
 
+    public static File getTmpDir() {
+        return new File(System.getProperty("java.io.tmpdir"));
+    }
 
     public static void extractZipFileContent(WLSDeployArchive archiveFile, String zipEntry, String extractPath)  {
         final String METHOD = "extractZipFileContent";

core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java

@@ -615,6 +615,15 @@ public String addApplication(String appPath) throws WLSDeployArchiveIOException
         return newName;
     }
 
+    public String replaceApplication(String appPath, String tempFile) throws WLSDeployArchiveIOException {
+        final String METHOD = "replaceApplication";
+        LOGGER.entering(CLASS, METHOD, appPath);
+        getZipFile().removeZipEntry(appPath);
+        String newName = addApplication(tempFile);
+        LOGGER.exiting(CLASS, METHOD, newName);
+        return newName;
+    }
+
     public String addApplicationFolder(String appName, String appPath)
             throws WLSDeployArchiveIOException {
         final String METHOD = "addApplicationFolder";

core/src/main/python/discover.py

@@ -219,21 +219,21 @@ def __process_domain_home(arg_map, wlst_mode):
         arg_map[CommandLineArgUtil.DOMAIN_HOME_SWITCH] = full_path
 
 
-def __discover(model_context, aliases, credential_injector, helper):
+def __discover(model_context, aliases, credential_injector, helper, extra_tokens):
     """
     Populate the model from the domain.
     :param model_context: the model context
     :param aliases: aliases instance for discover
     :param credential_injector: credential injector instance
     :param helper: wlst_helper instance
+    :param extra_tokens: dictionary to store non-credential tokens during credential search
     :return: the fully-populated model
     :raises DiscoverException: if an error occurred while discover the domain
     """
     _method_name = '__discover'
     model = Model()
     base_location = LocationContext()
     __connect_to_domain(model_context, helper)
-
     try:
         _add_domain_name(base_location, aliases, helper)
         DomainInfoDiscoverer(model_context, model.get_model_domain_info(), base_location, wlst_mode=__wlst_mode,
@@ -243,7 +243,8 @@ def __discover(model_context, aliases, credential_injector, helper):
         ResourcesDiscoverer(model_context, model.get_model_resources(), base_location, wlst_mode=__wlst_mode,
                             aliases=aliases, credential_injector=credential_injector).discover()
         DeploymentsDiscoverer(model_context, model.get_model_app_deployments(), base_location, wlst_mode=__wlst_mode,
-                              aliases=aliases, credential_injector=credential_injector).discover()
+                              aliases=aliases, credential_injector=credential_injector,
+                              extra_tokens=extra_tokens).discover()
         __discover_multi_tenant(model, model_context, base_location, aliases, credential_injector)
     except AliasException, ae:
         wls_version = WebLogicHelper(__logger).get_actual_weblogic_version()
@@ -448,14 +449,15 @@ def __persist_model(model, model_context):
     __logger.exiting(class_name=_class_name, method_name=_method_name)
 
 
-def __check_and_customize_model(model, model_context, aliases, credential_injector):
+def __check_and_customize_model(model, model_context, aliases, credential_injector, extra_tokens):
     """
     Customize the model dictionary before persisting. Validate the model after customization for informational
     purposes. Any validation errors will not stop the discovered model to be persisted.
     :param model: completely discovered model, before any tokenization
     :param model_context: configuration from command-line
     :param aliases: used for validation if model changes are made
     :param credential_injector: injector created to collect and tokenize credentials, possibly None
+    :param extra_tokens: dictionary to handle non-credential tokenized arguments
     """
     _method_name = '__check_and_customize_model'
     __logger.entering(class_name=_class_name, method_name=_method_name)
@@ -482,9 +484,12 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject
 
     # Apply the injectors specified in model_variable_injector.json, or in the target configuration.
     # Include the variable mappings that were collected in credential_cache.
+
     variable_injector = VariableInjector(_program_name, model.get_model(), model_context,
                                          WebLogicHelper(__logger).get_actual_weblogic_version(), credential_cache)
 
+    variable_injector.add_to_cache(dictionary=extra_tokens)
+
     inserted, variable_model, variable_file_name = variable_injector.inject_variables_keyword_file()
 
     if inserted:
@@ -579,10 +584,11 @@ def main(model_context):
         else:
             __logger.info('WLSDPLY-06024', class_name=_class_name, method_name=_method_name)
 
+        extra_tokens = {}
         try:
-            model = __discover(model_context, aliases, credential_injector, helper)
+            model = __discover(model_context, aliases, credential_injector, helper, extra_tokens)
 
-            model = __check_and_customize_model(model, model_context, aliases, credential_injector)
+            model = __check_and_customize_model(model, model_context, aliases, credential_injector, extra_tokens)
 
             __remote_report(model_context)
         except DiscoverException, ex:

core/src/main/python/wlsdeploy/aliases/model_constants.py

@@ -192,6 +192,7 @@
 MESSAGE_LOGGING_PARAMS = 'MessageLoggingParams'
 MESSAGING_BRIDGE = 'MessagingBridge'
 METHOD = 'Method'
+MODULE_TYPE = 'ModuleType'
 MULTICAST = 'Multicast'
 MULTICAST_ADDRESS = 'MulticastAddress'
 MULTICAST_PORT = 'MulticastPort'
@@ -287,6 +288,7 @@
 STORE = 'Store'
 SUB_DEPLOYMENT = 'SubDeployment'
 SUB_DEPLOYMENT_NAME = 'SubDeploymentName'
+SUB_MODULE_TARGETS = 'subModuleTargets'
 SYSTEM_COMPONENT = 'SystemComponent'
 SYSTEM_PASSWORD_VALIDATOR = 'SystemPasswordValidator'
 TARGET = 'Target'

core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py

@@ -28,6 +28,7 @@
 from wlsdeploy.aliases.model_constants import APPLICATION
 from wlsdeploy.aliases.model_constants import DEPLOYMENT_ORDER
 from wlsdeploy.aliases.model_constants import LIBRARY
+from wlsdeploy.aliases.model_constants import MODULE_TYPE
 from wlsdeploy.aliases.model_constants import PARTITION
 from wlsdeploy.aliases.model_constants import PLAN_DIR
 from wlsdeploy.aliases.model_constants import PLAN_PATH
@@ -38,6 +39,8 @@
 from wlsdeploy.aliases.model_constants import SECURITY_DD_MODEL
 from wlsdeploy.aliases.model_constants import SOURCE_PATH
 from wlsdeploy.aliases.model_constants import STAGE_MODE
+from wlsdeploy.aliases.model_constants import SUB_DEPLOYMENT
+from wlsdeploy.aliases.model_constants import SUB_MODULE_TARGETS
 from wlsdeploy.aliases.model_constants import TARGET
 from wlsdeploy.aliases.model_constants import TARGETS
 from wlsdeploy.aliases.wlst_modes import WlstModes
@@ -48,6 +51,9 @@
 from wlsdeploy.util import dictionary_utils
 from wlsdeploy.util import model_helper
 from wlsdeploy.util import string_utils
+from wlsdeploy.tool.util import appmodule_helper
+
+import wlsdeploy.util.variables as variables
 
 
 class ApplicationsDeployer(Deployer):
@@ -192,17 +198,23 @@ def __add_applications(self):
                     self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
                     raise ex
 
+            module_type = dictionary_utils.get_element(application, MODULE_TYPE)
 
             application_name = \
-                self.version_helper.get_application_versioned_name(app_source_path, application_name)
+                self.version_helper.get_application_versioned_name(app_source_path, application_name,
+                                                                   module_type=module_type)
 
             quoted_application_name = self.wlst_helper.get_quoted_name_for_wlst(application_name)
+
             application_location.add_name_token(application_token, quoted_application_name)
 
             self.wlst_helper.cd(root_path)
             deployer_utils.create_and_cd(application_location, existing_applications, self.aliases)
+
             self._set_attributes_and_add_subfolders(application_location, application)
+
             application_location.remove_name_token(application_token)
+            self.__substitute_appmodule_token(app_source_path, module_type)
 
             if app_source_path.startswith(WLSDeployArchive.ARCHIVE_STRUCT_APPS_TARGET_DIR):
                 plan_dir = dictionary_utils.get_element(application, PLAN_DIR)
@@ -211,6 +223,34 @@ def __add_applications(self):
 
         self.logger.exiting(class_name=self._class_name, method_name=_method_name)
 
+    def __substitute_appmodule_token(self, path, module_type):
+        # we need to substitute any token in the app module xml file
+        if  self.version_helper.is_module_type_app_module(module_type):
+            if os.path.isabs(path):
+                abspath = path
+            else:
+                abspath = self.model_context.get_domain_home() + os.sep + path
+
+            original_variables = {}
+            variable_file = self.model_context.get_variable_file()
+            if variable_file is not None and os.path.exists(variable_file):
+                original_variables = variables.load_variables(variable_file)
+
+            fh = open(abspath, 'r')
+            text = fh.read()
+            fh.close()
+            newtext = variables.substitute_value(text, original_variables, self.model_context)
+            # only jdbc and jms for now
+            if module_type == 'jdbc':
+                newtext = appmodule_helper.process_jdbc_appmodule_xml(newtext, self.model_context)
+            elif module_type == 'jms':
+                newtext = appmodule_helper.process_jms_appmodule_xml(newtext, self.model_context)
+
+            newfh = open(abspath, 'w')
+            newfh.write(newtext)
+            newfh.close()
+
+
     def __online_deploy_apps_and_libs(self, base_location):
         """
         Deploy shared libraries and applications in online mode.
@@ -937,7 +977,8 @@ def __deploy_model_libraries(self, model_libs, lib_location):
                     plan_file = dictionary_utils.get_element(lib_dict, PLAN_PATH)
                     targets = dictionary_utils.get_element(lib_dict, TARGET)
                     stage_mode = dictionary_utils.get_element(lib_dict, STAGE_MODE)
-                    options = _get_deploy_options(model_libs, lib_name, library_module='true')
+                    options, sub_module_targets = _get_deploy_options(model_libs, lib_name, library_module='true',
+                                                                      application_version_helper=self.version_helper)
                     for uses_path_tokens_attribute_name in uses_path_tokens_attribute_names:
                         if uses_path_tokens_attribute_name in lib_dict:
                             path = lib_dict[uses_path_tokens_attribute_name]
@@ -965,7 +1006,8 @@ def __deploy_model_applications(self, model_apps, app_location, deployed_applist
                     plan_file = dictionary_utils.get_element(app_dict, PLAN_PATH)
                     stage_mode = dictionary_utils.get_element(app_dict, STAGE_MODE)
                     targets = dictionary_utils.get_element(app_dict, TARGET)
-                    options = _get_deploy_options(model_apps, app_name, library_module='false')
+                    options, sub_module_targets  = _get_deploy_options(model_apps, app_name, library_module='false',
+                                                                       application_version_helper=self.version_helper)
 
                     # any attribute with 'uses_path_tokens' may be in the archive (such as SourcePath)
                     for uses_path_tokens_attribute_name in uses_path_tokens_attribute_names:
@@ -977,13 +1019,19 @@ def __deploy_model_applications(self, model_apps, app_location, deployed_applist
                     resource_group_template_name, resource_group_name, partition_name = \
                         self.__get_mt_names_from_location(location)
 
+                    module_type = dictionary_utils.get_element(app_dict, MODULE_TYPE)
+
                     new_app_name = self.__deploy_app_online(app_name, src_path, targets, plan=plan_file,
                                                             stage_mode=stage_mode, partition=partition_name,
                                                             resource_group=resource_group_name,
                                                             resource_group_template=resource_group_template_name,
+                                                            module_type=module_type,
+                                                            sub_module_targets=sub_module_targets,
                                                             options=options)
                     location.remove_name_token(token_name)
                     deployed_applist.append(new_app_name)
+                    self.__substitute_appmodule_token(path, module_type)
+
 
     def __get_mt_names_from_location(self, app_location):
         dummy_location = LocationContext()
@@ -1007,7 +1055,8 @@ def __get_mt_names_from_location(self, app_location):
         return resource_group_template_name, resource_group_name, partition_name
 
     def __deploy_app_online(self, application_name, source_path, targets, stage_mode=None, plan=None, partition=None,
-                            resource_group=None, resource_group_template=None, options=None):
+                            resource_group=None, resource_group_template=None, sub_module_targets=None,
+                            module_type = None, options=None):
         """
         Deploy an application or shared library in online mode.
         :param application_name: the name of the app or library from the model
@@ -1050,7 +1099,8 @@ def __deploy_app_online(self, application_name, source_path, targets, stage_mode
         if is_library:
             computed_name = self.version_helper.get_library_versioned_name(source_path, application_name)
         else:
-            computed_name = self.version_helper.get_application_versioned_name(source_path, application_name)
+            computed_name = self.version_helper.get_application_versioned_name(source_path, application_name,
+                                                                               module_type=module_type)
 
         application_name = computed_name
 
@@ -1079,6 +1129,9 @@ def __deploy_app_online(self, application_name, source_path, targets, stage_mode
                 kwargs[key] = value
         kwargs['timeout'] = self.model_context.get_model_config().get_deploy_timeout()
 
+        if self.version_helper.is_module_type_app_module(module_type) and sub_module_targets is not None:
+            kwargs[SUB_MODULE_TARGETS] = sub_module_targets
+
         self.logger.fine('WLSDPLY-09320', type_name, application_name, kwargs,
                          class_name=self._class_name, method_name=_method_name)
         self.wlst_helper.deploy_application(application_name, *args, **kwargs)
@@ -1093,7 +1146,6 @@ def __extract_source_path_from_archive(self, source_path, model_type, model_name
         :param model_name: the element name (my-app, etc.), used for logging
         """
         _method_name = '__extract_source_path_from_archive'
-
         # source path may be may be a single file (jar, war, etc.)
         if self.archive_helper.contains_file(source_path):
             self.archive_helper.extract_file(source_path)
@@ -1192,7 +1244,7 @@ def __start_all_apps(self, deployed_app_list, base_location):
             self.__start_app(app)
 
 
-def _get_deploy_options(model_apps, app_name, library_module):
+def _get_deploy_options(model_apps, app_name, library_module, application_version_helper):
     """
     Get the deploy command options.
     :param model_apps: the apps dictionary
@@ -1202,8 +1254,8 @@ def _get_deploy_options(model_apps, app_name, library_module):
     """
     deploy_options = OrderedDict()
     # not sure about altDD, altWlsDD
+    app = dictionary_utils.get_dictionary_element(model_apps, app_name)
     for option in [DEPLOYMENT_ORDER, SECURITY_DD_MODEL, PLAN_STAGING_MODE, STAGE_MODE]:
-        app = dictionary_utils.get_dictionary_element(model_apps, app_name)
         value = dictionary_utils.get_element(app, option)
 
         option_name = ''
@@ -1224,8 +1276,21 @@ def _get_deploy_options(model_apps, app_name, library_module):
 
     if len(deploy_options) == 0:
         deploy_options = None
-    return deploy_options
 
+    module_type = dictionary_utils.get_element(app, MODULE_TYPE)
+    sub_module_targets = ''
+    if application_version_helper.is_module_type_app_module(module_type):
+        sub_deployments = dictionary_utils.get_element(app, SUB_DEPLOYMENT)
+        if sub_deployments is not None:
+            for sub_deployment in sub_deployments:
+                if sub_module_targets != '':
+                    sub_module_targets += ','
+                name = sub_deployment
+                target = sub_deployments[name][TARGET]
+                sub_module_targets += '%s@%s' % (name, target)
+
+
+    return deploy_options, sub_module_targets
 
 def _find_deployorder_list(apps_dict, ordered_list, order):
     """

core/src/main/python/wlsdeploy/tool/deploy/applications_version_helper.py

@@ -82,7 +82,7 @@ def get_library_versioned_name(self, source_path, model_name, from_archive=False
         self.logger.exiting(class_name=self._class_name, method_name=_method_name, result=versioned_name)
         return versioned_name
 
-    def get_application_versioned_name(self, source_path, model_name, from_archive=False):
+    def get_application_versioned_name(self, source_path, model_name, from_archive=False, module_type=None):
         """
         Get the proper name of the deployable application that WLST requires in the target domain.
         This method is needed for the case where the application is explicitly versioned in its ear/war manifest.
@@ -100,6 +100,8 @@ def get_application_versioned_name(self, source_path, model_name, from_archive=F
 
         # if no manifest version is found, leave the original name unchanged
         versioned_name = model_name
+        if self.is_module_type_app_module(module_type):
+            return model_name
 
         try:
             manifest = self.__get_manifest(source_path, from_archive)
@@ -123,6 +125,12 @@ def get_application_versioned_name(self, source_path, model_name, from_archive=F
         self.logger.exiting(class_name=self._class_name, method_name=_method_name, result=versioned_name)
         return versioned_name
 
+    def is_module_type_app_module(self, module_type):
+        if module_type in [ 'jms', 'jdbc', 'wldf']:
+            return True
+        else:
+            return False
+
     def __get_manifest(self, source_path, from_archive):
         """
         Returns the manifest object for the specified path.