Update Verrazzano CRD ingress traits with application information (#1380)

* Update Verrazzano CRD ingress traits with application information

* Update unit test to verify ingress trait rule merging for Verrazzano

Author: rakillen

core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2020, 2022, Oracle and/or its affiliates.
+Copyright (c) 2020, 2023, Oracle and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 Methods for creating Kubernetes resource configuration files for Verrazzano.
@@ -8,12 +8,18 @@
 
 from java.io import File
 
+from wlsdeploy.aliases import alias_utils
 from wlsdeploy.aliases.location_context import LocationContext
 from wlsdeploy.aliases.model_constants import APPLICATION
 from wlsdeploy.aliases.model_constants import CLUSTER
+from wlsdeploy.aliases.model_constants import DYNAMIC_SERVERS
 from wlsdeploy.aliases.model_constants import JDBC_DRIVER_PARAMS
 from wlsdeploy.aliases.model_constants import JDBC_RESOURCE
 from wlsdeploy.aliases.model_constants import JDBC_SYSTEM_RESOURCE
+from wlsdeploy.aliases.model_constants import LISTEN_PORT
+from wlsdeploy.aliases.model_constants import SERVER
+from wlsdeploy.aliases.model_constants import SERVER_TEMPLATE
+from wlsdeploy.aliases.model_constants import TARGET
 from wlsdeploy.aliases.model_constants import URL
 from wlsdeploy.logging.platform_logger import PlatformLogger
 from wlsdeploy.tool.util import k8s_helper
@@ -52,14 +58,21 @@
 HAS_APPLICATIONS = 'hasApplications'
 HAS_CLUSTERS = 'hasClusters'
 HAS_DATASOURCES = 'hasDatasources'
+HAS_HOST_APPLICATIONS = 'hasHostApplications'
 HAS_MODEL = 'hasModel'
+HOST_APPLICATION_APPLICATIONS = 'applications'
+HOST_APPLICATION_HOST = 'host'
+HOST_APPLICATION_PORT = 'port'
+HOST_APPLICATIONS = 'hostApplications'
 NAMESPACE = 'namespace'
 REPLICAS = 'replicas'
 RUNTIME_ENCRYPTION_SECRET = "runtimeEncryptionSecret"
 SET_CLUSTER_REPLICAS = "setClusterReplicas"
 USE_PERSISTENT_VOLUME = "usePersistentVolume"
 WEBLOGIC_CREDENTIALS_SECRET = 'webLogicCredentialsSecret'
 
+DEFAULT_LISTEN_PORT = 7001
+
 
 def create_additional_output(model, model_context, aliases, credential_injector, exception_type,
                              domain_home_override=None):
@@ -255,6 +268,54 @@ def _build_template_hash(model, model_context, aliases, credential_injector, dom
     template_hash[APPLICATIONS] = apps
     template_hash[HAS_APPLICATIONS] = len(apps) != 0
 
+    # host applications - applications organized by host, for Verrazzano IngressTrait
+
+    app_map = {}
+    applications = dictionary_utils.get_dictionary_element(model.get_model_app_deployments(), APPLICATION)
+    for app_name in applications:
+        app_hash = dict()
+        app_hash[APPLICATION_NAME] = app_name
+        # this text is matched in crd_file_updater, be careful if changing
+        app_hash[APPLICATION_PREFIX] = '(path for ' + app_name + ')'
+
+        app_folder = dictionary_utils.get_dictionary_element(applications, app_name)
+        targets_value = dictionary_utils.get_dictionary_element(app_folder, TARGET)
+        targets = alias_utils.create_list(targets_value, 'WLSDPLY-01682')
+        for target in targets:
+            if target not in app_map:
+                app_map[target] = []
+            app_map[target].append(app_hash)
+
+    host_apps = []
+    target_keys = app_map.keys()
+    target_keys.sort()
+    for target_key in target_keys:
+        listen_port = DEFAULT_LISTEN_PORT
+        target_cluster = _find_cluster(model, target_key)
+        if target_cluster is not None:
+            full_host_name = k8s_helper.get_dns_name(domain_uid + '-cluster-' + target_key)
+            dynamic_servers = dictionary_utils.get_dictionary_element(target_cluster, DYNAMIC_SERVERS)
+            template_name = dictionary_utils.get_element(dynamic_servers, SERVER_TEMPLATE)
+            if template_name:
+                server_template = _find_server_template(model, template_name)
+                if server_template:
+                    listen_port = server_template[LISTEN_PORT] or listen_port
+        else:
+            full_host_name = k8s_helper.get_dns_name(domain_uid + '-' + target_key)
+            target_server = _find_server(model, target_key)
+            if target_server is not None:
+                listen_port = target_server[LISTEN_PORT] or listen_port
+
+        host_app = {
+            HOST_APPLICATION_HOST: full_host_name,
+            HOST_APPLICATION_PORT: str_helper.to_string(listen_port),
+            HOST_APPLICATION_APPLICATIONS: app_map[target_key]
+        }
+        host_apps.append(host_app)
+
+    template_hash[HOST_APPLICATIONS] = host_apps
+    template_hash[HAS_HOST_APPLICATIONS] = len(host_apps) != 0
+
     # additional secrets - exclude admin
 
     additional_secrets = []
@@ -280,3 +341,27 @@ def _build_template_hash(model, model_context, aliases, credential_injector, dom
     template_hash[HAS_ADDITIONAL_SECRETS] = len(additional_secrets) != 0
 
     return template_hash
+
+
+def _find_cluster(model, name):
+    cluster_map = dictionary_utils.get_dictionary_element(model.get_model_topology(), CLUSTER)
+    for cluster_name in cluster_map:
+        if name == cluster_name:
+            return cluster_map[cluster_name]
+    return None
+
+
+def _find_server(model, name):
+    server_map = dictionary_utils.get_dictionary_element(model.get_model_topology(), SERVER)
+    for server_name in server_map:
+        if name == server_name:
+            return server_map[server_name]
+    return None
+
+
+def _find_server_template(model, name):
+    template_map = dictionary_utils.get_dictionary_element(model.get_model_topology(), SERVER_TEMPLATE)
+    for template_name in template_map:
+        if name == template_name:
+            return template_map[template_name]
+    return None

core/src/main/python/wlsdeploy/tool/util/targets/crd_file_updater.py

@@ -4,6 +4,8 @@
 
 Methods to update an output file with information from the kubernetes section of the model.
 """
+import re
+
 from oracle.weblogic.deploy.util import PyOrderedDict
 from oracle.weblogic.deploy.util import PyRealBoolean
 from oracle.weblogic.deploy.yaml import YamlException
@@ -42,6 +44,19 @@
 VERRAZZANO_APPLICATION_KIND = 'ApplicationConfiguration'
 WORKLOAD = 'workload'
 
+# specific to Verrazzano application document
+COMPONENTS = "components"
+DESTINATION = "destination"
+INGRESS_TRAIT = "IngressTrait"
+PATH = "path"
+PATH_TYPE = "pathType"
+PATHS = "paths"
+RULES = "rules"
+TRAIT = "trait"
+TRAITS = "traits"
+
+PATH_SAMPLE_PATTERN = '^\\(path for.*\\)$'
+
 
 def update_from_model(crd_file, model, crd_helper):
     """
@@ -112,7 +127,7 @@ def _update_documents(crd_documents, model_content, crd_helper, output_file_path
                 found = True
 
             elif kind == VERRAZZANO_APPLICATION_KIND:
-                _update_crd(crd_document, model_content, 'application', crd_helper, output_file_path)
+                _update_crd_application(crd_document, model_content, crd_helper, output_file_path)
                 found = True
 
     if not found:
@@ -158,6 +173,20 @@ def _update_crd_cluster(crd_dictionary, model_dictionary, crd_helper, output_fil
             _update_dictionary(crd_dictionary, model_cluster, schema, None, cluster_crd_folder, output_file_path)
 
 
+def _update_crd_application(crd_dictionary, model_dictionary, crd_helper, output_file_path):
+    """
+    Update the CRD application dictionary from the model.
+    :param crd_dictionary: the CRD dictionary to be updated
+    :param model_dictionary: the model content to use for update
+    :param crd_helper: used to get CRD folder information
+    :param output_file_path: used for logging
+    """
+    _method_name = '_update_crd_application'
+
+    _update_crd(crd_dictionary, model_dictionary, 'application', crd_helper, output_file_path)
+    _add_application_comments(crd_dictionary)
+
+
 def _update_crd_component(crd_dictionary, model_dictionary, crd_helper, output_file_path):
     """
     Update the CRD component dictionary from the model.
@@ -418,6 +447,69 @@ def _add_weblogic_workload_comments(vz_dictionary):
         _add_cluster_spec_comments(cluster_spec)
 
 
+def _add_application_comments(vz_dictionary):
+    """
+    Add relevant comments to the Verrazzano application CRD dictionary for additional information.
+    :param vz_dictionary: the Verrazzano dictionary
+    """
+    spec_folder = dictionary_utils.get_dictionary_element(vz_dictionary, SPEC)
+    components = dictionary_utils.get_dictionary_element(spec_folder, COMPONENTS)
+    for component in components:
+        traits = _get_list_element(component, TRAITS)
+        for trait in traits:
+            trait_dictionary = dictionary_utils.get_dictionary_element(trait, TRAIT)
+            trait_kind = dictionary_utils.get_element(trait_dictionary, KIND)
+            if trait_kind == INGRESS_TRAIT:
+                _add_ingress_trait_comments(trait_dictionary)
+
+
+def _add_ingress_trait_comments(trait_dictionary):
+    """
+    Add relevant comments to the IngressTrait CRD dictionary for additional information.
+    Convert sample rule paths to comments if none were added from WDT model.
+    Remove sample rule paths if any paths were added from WDT model.
+    :param trait_dictionary: the IngressTrait dictionary
+    """
+    trait_spec = dictionary_utils.get_dictionary_element(trait_dictionary, SPEC)
+    rules = _get_list_element(trait_spec, RULES)
+    for rule in rules:
+        sample_paths = []
+        has_defined_paths = False
+        paths = _get_list_element(rule, PATHS)
+        for path in paths:
+            path_path = dictionary_utils.get_dictionary_element(path, PATH)
+            is_sample_path = re.search(PATH_SAMPLE_PATTERN, str_helper.to_string(path_path))
+            if is_sample_path:
+                sample_paths.append(path)
+            else:
+                has_defined_paths = True
+
+        if has_defined_paths:
+            for sample_path in sample_paths:
+                paths.remove(sample_path)
+        else:
+            rule.addComment(DESTINATION, PATHS + ':')
+            for sample_path in sample_paths:
+                rule.addComment(DESTINATION, '  - ' + PATH + ': ' + str_helper.to_string(sample_path[PATH]))
+                rule.addComment(DESTINATION, '    ' + PATH_TYPE + ': ' + str_helper.to_string(sample_path[PATH_TYPE]))
+            del rule[PATHS]
+
+
+def _get_list_element(dictionary, element_name):
+    """
+    Retrieve the value for the provided element name from the dictionary.
+    Return empty list if name is not in the dictionary.
+    :param dictionary: to find the element name
+    :param element_name: for which to retrieve the value
+    :return: value from the dictionary or empty list
+    """
+    if element_name in dictionary:
+        result = dictionary[element_name]
+    else:
+        result = []
+    return result
+
+
 def _get_or_create_dictionary(dictionary, key):
     if key not in dictionary:
         dictionary[key] = PyOrderedDict()

core/src/main/python/wlsdeploy/tool/util/targets/model_crd_helper.py

@@ -105,6 +105,7 @@ def get_product_helper(product_key, product_version, exception_type=ExceptionTyp
         application_folder = ModelCrdFolder("application", application_schema, False)
         application_folder.add_object_list_key('spec/components', 'componentName')
         application_folder.add_object_list_key('spec/components/traits', 'trait/kind')
+        application_folder.add_object_list_key('spec/components/traits/trait/spec/rules', 'destination/host')
         helper.add_crd_folder(application_folder)
 
         weblogic_schema_name = VZ_1_WEBLOGIC_SCHEMA_NAME

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -365,6 +365,7 @@ WLSDPLY-01678=Expected a list value for {0} in the target output file {1}, skipp
 WLSDPLY-01679=Add any credential secrets that are required to pull the image
 WLSDPLY-01680=Set a specific replica count for this cluster
 WLSDPLY-01681=Unable to create results file "{0}": {1}
+WLSDPLY-01682=Unable to convert target value of type {0} to a list
 
 # wlsdeploy/util/enum.py
 WLSDPLY-01700=The value {0} is not a valid value of the Enum type {1}

core/src/main/targetconfigs/templates/vz-application-v1.yaml

@@ -23,14 +23,16 @@ spec:
             kind: IngressTrait
             spec:
               rules:
-{{#hasApplications}}
-                - paths:
-{{/hasApplications}}
+{{#hostApplications}}
+                - destination:
+                    host: {{{host}}}
+                    port: {{{port}}}
+                  paths:
 {{#applications}}
-                    # application {{{applicationName}}}
                     - path: "{{{applicationPrefix}}}"
                       pathType: Prefix
 {{/applications}}
+{{/hostApplications}}
     - componentName: {{{domainPrefix}}}-configmap
 ---
 apiVersion: core.oam.dev/v1alpha2

core/src/test/python/wlsdeploy/tool/extract/extract_test.py

@@ -105,6 +105,19 @@ def testVerrazzanoModel(self):
         trait_list = self._traverse(component_list[0], 'traits')
         self._match_values("Application trait count", len(trait_list), 3)
 
+        ingress_trait = self._traverse(trait_list[1], 'trait')
+        rule_list = self._traverse(ingress_trait, 'spec', 'rules')
+        self._match_values("Ingress trait rule count", len(rule_list), 3)
+
+        # m1 has paths added from the verrazzano section
+        m1_rule = rule_list[0]
+        m1_path_list = self._traverse(m1_rule, 'paths')
+        self._match_values("Server 1 rule path count", len(m1_path_list), 2)
+
+        # m2 has no rules, only sample comments
+        m2_rule = rule_list[1]
+        self._match_values("Server 2 has no paths", 'paths' in m2_rule, False)
+
         configmap_resource = documents[2]
 
         # one entry was added to config map

core/src/test/resources/extract/model-3.yaml

@@ -5,6 +5,20 @@
 # This will test that fields from the verrazzano section of the model
 # are transferred to the resulting domain resource file
 
+topology:
+  Cluster:
+    mycluster:
+      DynamicServers:
+        ServerTemplate: template1
+  Server:
+    m1:
+      # no ListenPort, use default
+    m2:
+      ListenPort: 9005
+  ServerTemplate:
+    template1:
+      ListenPort: 9008
+
 resources:
   # template will create a configmap entry for this JDBC resource
   JDBCSystemResource:
@@ -13,6 +27,19 @@ resources:
         JDBCDriverParams:
           URL: 'jdbc:oracle:thin:@dbhost:1521/pdborcl'
 
+appDeployments:
+  # these apps will go into application IngressTrait rules
+  Application:
+    oneApp:
+      SourcePath: wlsdeploy/apps/oneApp.ear
+      Target: mycluster,m1
+    twoApp :
+      SourcePath: wlsdeploy/apps/twoApp.ear
+      Target: m2,mycluster
+    threeApp :
+      SourcePath: wlsdeploy/apps/threeApp.ear
+      Target: m2,m1
+
 verrazzano:
     application:
         spec:
@@ -24,6 +51,18 @@ verrazzano:
                             kind: MetricsTrait
                             spec:
                                 scraper: verrazzano-system/my-model-scraper
+                      - trait:  # should merge with template trait
+                            kind: IngressTrait
+                            spec:
+                                rules:
+                                  # assign specific paths to this destination
+                                  - destination:
+                                      host: base-domain-m1
+                                    paths:
+                                      - path: '/simple-ear-path'
+                                        pathType: Prefix
+                                      - path: '/simple-ear3-path'
+                                        pathType: Prefix
                       - trait:  # should add to template traits
                             apiVersion: oam.verrazzano.io/v1alpha1
                             kind: LoggingTrait