Added support for logging in JSON format to a file.

README.md

@@ -96,7 +96,7 @@ The `weblogic-logging-exporter.jar` will be available under the `target` directo
 
 ## Installation
 
-This section outlines the steps that are required to add the Weblogic Logging Exporter to Weblogic Server.
+This section outlines the steps that are required to add the WebLogic Logging Exporter to WebLogic Server.
 
 1. Download or build the WebLogic Logging Exporter as described above.
 
@@ -119,20 +119,16 @@ This section outlines the steps that are required to add the Weblogic Logging Ex
     </startup-class>
     ```
 
-1. Add `weblogic-logging-exporter.jar` and `snakeyaml-1.27.jar` to your classpath.
+1. Add `weblogic-logging-exporter.jar` to your classpath.
 
-   This project requires `snakeyaml` to parse the YAML configuration file.  If you built the project locally,
-   you can find this JAR file in your local maven repository at `~/.m2/repository/org/yaml/snakeyaml/1.27/snakeyaml-1.27.jar`.
-   Otherwise, you can download it from [Maven Central](https://search.maven.org/artifact/org.yaml/snakeyaml/1.27/bundle).
+   Place the file(s) in a suitable location, e.g. your domain directory.
 
-   Place this file in a suitable location, e.g. your domain directory.
-
-   Update the server classpath to include these two files.  This can be done by adding a statement to the end of your
+   Update the server classpath to include these file(s).  This can be done by adding a statement to the end of your
    `setDomainEnv.sh` script in your domain's `bin` directory as follows (this example assumes your domain
    directory is `/u01/base_domain`):
 
    ```
-   export CLASSPATH="/u01/base_domain/weblogic-logging-exporter.jar:/u01/base_domain/snakeyaml-1.27.jar:$CLASSPATH"
+   export CLASSPATH="/u01/base_domain/weblogic-logging-exporter.jar:$CLASSPATH"
    ```
 
 1. Create a configuration file for the WebLogic Logging Exporter.
@@ -163,7 +159,10 @@ This section outlines the steps that are required to add the Weblogic Logging Ex
    If you prefer to place the configuration file in a different location, you can set the environment variable
    `WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE` to point to the location of the file.
 
-1. Restart the servers to activate the changes.  After restarting the servers, they will load the WebLogic
+   If you want to write the JSON logs to a file instead of sending it elasticsearch directly use the following configuration
+   [file](samples/WebLogicFileLoggingExporter.yaml) and adjust it to your needs. Make sure to rename it to WebLogicLoggingExporter.yaml.
+
+6. Restart the servers to activate the changes.  After restarting the servers, they will load the WebLogic
    Logging Exporter and start sending their logs to the specified Elasticsearch instance.  You can then
    access them in Kibana as shown in the example below. You will need to create an index first and then go to
    the visualization page.

pom.xml

@@ -85,8 +85,9 @@
                 </configuration>
             </plugin>
             <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-failsafe-plugin</artifactId>
-                <version>2.22.1</version>
+                <version>2.22.2</version>
             </plugin>
             <!-- end -->
             <plugin>
@@ -109,6 +110,24 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.3.0</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
@@ -190,13 +209,24 @@
             <artifactId>snakeyaml</artifactId>
             <version>1.27</version>
         </dependency>
+        <dependency>
+            <groupId>co.elastic.logging</groupId>
+            <artifactId>jul-ecs-formatter</artifactId>
+            <version>1.2.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-jdk14</artifactId>
+            <version>1.7.32</version>
+        </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr-complete</artifactId>
             <version>3.5.2</version>
+            <!-- we need it in unit tests but the weblogic runtime provides it at run time -->
+            <scope>test</scope>
         </dependency>
     </dependencies>
-
     <reporting>
         <plugins>
             <plugin>

samples/WebLogicFileLoggingExporter.yaml

@@ -0,0 +1,15 @@
+# turn off the elasticsearch output
+weblogicLoggingExporterEnabled: false
+
+# configure json logging output
+writeToFileEnabled: true
+# %g is necessary for rollback of log files.
+outputFile: '/var/log/oracle/json-logging%g.log'
+# Max open files
+maxRollbackFiles: 3
+# Max file size in bytes, this is 50 MB
+maxFileSize: 52428800
+# Should the log file be appended to when the new logger is created
+appendToFile: true
+# Optional configuration for specifying which levels get logged to the json log file.
+# fileLoggingLogLevel: INFO

src/main/java/weblogic/logging/exporter/Startup.java

@@ -4,18 +4,21 @@
 package weblogic.logging.exporter;
 
 import java.io.File;
+import java.util.logging.FileHandler;
+import java.util.logging.Level;
 import java.util.logging.Logger;
+
 import weblogic.logging.LoggingHelper;
+import weblogic.logging.ServerLoggingHandler;
 import weblogic.logging.exporter.config.Config;
 
 public class Startup {
 
   private static final String DEFAULT_CONFIG_FILE = "config/WebLogicLoggingExporter.yaml";
 
   public static void main(String[] argv) {
-    System.out.println("======================= Weblogic Logging Exporter Startup class called");
+    System.out.println("======================= WebLogic Logging Exporter Startup class called");
     try {
-      // Logger logger = LoggingHelper.getDomainLogger();
       Logger logger = LoggingHelper.getServerLogger();
 
       /*
@@ -26,27 +29,41 @@ public static void main(String[] argv) {
       */
 
       String fileName =
-          System.getProperty(
-              "WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE",
-              System.getenv("WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE"));
+              System.getProperty(
+                      "WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE",
+                      System.getenv("WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE"));
       System.out.println(
-          "JavaProperty/EnvVariable WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE:" + fileName);
+              "JavaProperty/EnvVariable WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE:" + fileName);
       if (fileName == null || fileName.isEmpty()) {
         System.out.println(
-            "Env variable WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE is not set. Defaulting to:"
-                + DEFAULT_CONFIG_FILE);
+                "Env variable WEBLOGIC_LOGGING_EXPORTER_CONFIG_FILE is not set. Defaulting to:"
+                        + DEFAULT_CONFIG_FILE);
         fileName = DEFAULT_CONFIG_FILE;
       }
       File file = new File(fileName);
       System.out.println("Reading configuration from file name: " + file.getAbsolutePath());
       Config config = Config.loadConfig(file);
       System.out.println(config);
+
+      //  Elastic log handler is enabled or the file log handler
       if (config.getEnabled()) {
         logger.addHandler(new LogExportHandler(config));
+      } else if (config.isFileLoggingEnabled()) {
+        // Because of this bridge log messages in the applications themselves are being forwarded to the server logger.
+        // so that logging in ear/war artifacts are also visible to the server logger and appear in the JSON log file.
+        Logger.getLogger("").addHandler(new ServerLoggingHandler());
+
+        // Register a file handler using the provided config
+        FileHandler fh = new FileHandler(config.getOutputFile(), config.getMaxFileSize(), config.getGetMaxRollbackFiles(), config.getAppendToFile());
+        fh.setLevel(Level.parse(config.getFileLoggingLogLevel()));
+        fh.setFormatter(new WebLogicLogFormatter(config.getDomainUID()));
+        logger.addHandler(fh);
       } else {
-        System.out.println("WebLogic Logging Exporter is disabled");
+        System.out.println("WebLogic Elasticsearch Logging Exporter is disabled");
       }
-    } catch (Exception e) {
+      // also catch errors so that WebLogic does not crash when a required library was not placed in the classpath correctly.
+    } catch (Error | Exception e) {
+      System.out.println("======================= Something went wrong, the WebLogic Logging Exporter is not activated");
       e.printStackTrace();
     }
   }

src/main/java/weblogic/logging/exporter/WebLogicLogFormatter.java

@@ -0,0 +1,55 @@
+package weblogic.logging.exporter;
+
+import co.elastic.logging.jul.EcsFormatter;
+import org.slf4j.MDC;
+import weblogic.logging.WLLogRecord;
+
+import java.util.logging.LogRecord;
+
+public class WebLogicLogFormatter extends EcsFormatter {
+    public static final String FIELDS_MESSAGE_ID = "fields.messageID";
+    public static final String FIELDS_SERVER_NAME = "fields.serverName";
+    public static final String FIELDS_USER_ID = "fields.userId";
+    public static final String FIELDS_SUB_SYSTEM = "fields.subSystem";
+    public static final String FIELDS_MACHINE_NAME = "fields.machineName";
+    public static final String FIELDS_TRANSACTION_ID = "fields.transactionId";
+    public static final String FIELDS_DIAGNOSTIC_CONTEXT_ID = "fields.diagnosticContextId";
+    public static final String FIELDS_SEQUENCE_NUMBER = "fields.sequenceNumber";
+    public static final String FIELDS_DOMAIN_UID = "fields.domainUID";
+
+    private final String domainUID;
+
+    public WebLogicLogFormatter(String domainUID) {
+        this.domainUID = domainUID;
+    }
+
+    @Override
+    public String format(final LogRecord record) {
+        WLLogRecord wlLogRecord = (WLLogRecord) record;
+
+        MDC.put(FIELDS_MESSAGE_ID, wlLogRecord.getId());
+        MDC.put(FIELDS_SERVER_NAME, wlLogRecord.getServerName());
+        MDC.put(FIELDS_USER_ID, wlLogRecord.getUserId());
+        MDC.put(FIELDS_SUB_SYSTEM, wlLogRecord.getSubsystem());
+        MDC.put(FIELDS_MACHINE_NAME, wlLogRecord.getMachineName());
+        MDC.put(FIELDS_TRANSACTION_ID, wlLogRecord.getTransactionId());
+        MDC.put(FIELDS_DIAGNOSTIC_CONTEXT_ID, wlLogRecord.getDiagnosticContextId());
+        MDC.put(FIELDS_SEQUENCE_NUMBER, String.valueOf(wlLogRecord.getSequenceNumber()));
+        MDC.put(FIELDS_DOMAIN_UID, domainUID);
+
+        String result = super.format(wlLogRecord);
+
+        // Can't clear the whole MDC HashMap as there might be other records in there.
+        MDC.remove(FIELDS_MESSAGE_ID);
+        MDC.remove(FIELDS_SERVER_NAME);
+        MDC.remove(FIELDS_USER_ID);
+        MDC.remove(FIELDS_SUB_SYSTEM);
+        MDC.remove(FIELDS_MACHINE_NAME);
+        MDC.remove(FIELDS_TRANSACTION_ID);
+        MDC.remove(FIELDS_DIAGNOSTIC_CONTEXT_ID);
+        MDC.remove(FIELDS_SEQUENCE_NUMBER);
+        MDC.remove(FIELDS_DOMAIN_UID);
+
+        return result;
+    }
+}

src/main/java/weblogic/logging/exporter/config/Config.java

@@ -31,6 +31,13 @@ public class Config {
   private static final String INDEX_NAME = "weblogicLoggingIndexName";
   private static final String DOMAIN_UID = "domainUID";
 
+  private static final String WRITE_TO_FILE_ENABLED = "writeToFileEnabled";
+  private static final String OUTPUT_FILE = "outputFile";
+  private static final String MAX_ROLLBACK_FILES = "maxRollbackFiles";
+  private static final String MAX_FILE_SIZE = "maxFileSize";
+  private static final String APPEND_TO_FILE = "appendToFile";
+  private static final String FILE_LOGGING_LOG_LEVEL = "fileLoggingLogLevel";
+
   private String host = DEFAULT_HOST;
   private int port = DEFAULT_PORT;
   private String indexName = DEFAULT_INDEX_NAME;
@@ -40,6 +47,14 @@ public class Config {
   private final List<FilterConfig> filterConfigs = new ArrayList<>();
   private String domainUID = DEFAULT_DOMAIN_UID;
 
+  private boolean fileLoggingEnabled;
+  private String outputFile;
+  private Integer getMaxRollbackFiles;
+  private Integer maxFileSize;
+  private boolean appendToFile;
+
+  private String fileLoggingLogLevel = "INFO";
+
   private Config() {}
 
   private Config(Map<String, Object> yaml) {
@@ -76,6 +91,26 @@ private Config(Map<String, Object> yaml) {
       System.out.println("Index name is converted to all lower case : " + indexName);
     }
     if (yaml.containsKey(FILTERS)) appendFilters(yaml.get(FILTERS));
+
+    // File output
+    if (yaml.containsKey(WRITE_TO_FILE_ENABLED)) {
+      fileLoggingEnabled = MapUtils.getBooleanValue(yaml, WRITE_TO_FILE_ENABLED);
+    }
+    if (yaml.containsKey(OUTPUT_FILE)) {
+      outputFile = MapUtils.getStringValue(yaml, OUTPUT_FILE);
+    }
+    if (yaml.containsKey(MAX_ROLLBACK_FILES)) {
+      getMaxRollbackFiles = MapUtils.getIntegerValue(yaml, MAX_ROLLBACK_FILES);
+    }
+    if (yaml.containsKey(MAX_FILE_SIZE)) {
+      maxFileSize = MapUtils.getIntegerValue(yaml, MAX_FILE_SIZE);
+    }
+    if (yaml.containsKey(APPEND_TO_FILE)) {
+      appendToFile = MapUtils.getBooleanValue(yaml, APPEND_TO_FILE);
+    }
+    if (yaml.containsKey(FILE_LOGGING_LOG_LEVEL)) {
+      fileLoggingLogLevel = MapUtils.getStringValue(yaml, FILE_LOGGING_LOG_LEVEL);
+    }
   }
 
   public static Config loadConfig(File file) {
@@ -143,29 +178,22 @@ private static Config loadConfig(Map<String, Object> yamlConfig) {
 
   @Override
   public String toString() {
-    return "Config{"
-        + "weblogicLoggingIndexName='"
-        + indexName
-        + '\''
-        + ", publishHost='"
-        + host
-        + '\''
-        + ", publishPort="
-        + port
-        + ", weblogicLoggingExporterSeverity='"
-        + severity
-        + '\''
-        + ", weblogicLoggingExporterBulkSize='"
-        + bulkSize
-        + '\''
-        + ", enabled="
-        + enabled
-        + ", weblogicLoggingExporterFilters="
-        + filterConfigs
-        + ", domainUID='"
-        + domainUID
-        + '\''
-        + '}';
+    return "Config{" +
+            "host='" + host + '\'' +
+            ", port=" + port +
+            ", indexName='" + indexName + '\'' +
+            ", bulkSize=" + bulkSize +
+            ", enabled=" + enabled +
+            ", severity='" + severity + '\'' +
+            ", filterConfigs=" + filterConfigs +
+            ", domainUID='" + domainUID + '\'' +
+            ", fileLoggingEnabled=" + fileLoggingEnabled +
+            ", outputFile='" + outputFile + '\'' +
+            ", getMaxRollbackFiles=" + getMaxRollbackFiles +
+            ", maxFileSize=" + maxFileSize +
+            ", appendToFile=" + appendToFile +
+            ", fileLoggingLogLevel='" + fileLoggingLogLevel + '\'' +
+            '}';
   }
 
   public String getHost() {
@@ -199,4 +227,28 @@ public int getBulkSize() {
   public String getDomainUID() {
     return domainUID;
   }
+
+  public boolean isFileLoggingEnabled() {
+    return fileLoggingEnabled;
+  }
+
+  public String getOutputFile() {
+    return outputFile;
+  }
+
+  public Integer getGetMaxRollbackFiles() {
+    return getMaxRollbackFiles;
+  }
+
+  public Integer getMaxFileSize() {
+    return maxFileSize;
+  }
+
+  public boolean getAppendToFile() {
+    return appendToFile;
+  }
+
+  public String getFileLoggingLogLevel() {
+    return fileLoggingLogLevel;
+  }
 }