From 04d22367292029f058917bb521b103f46502a420 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 29 Sep 2023 11:16:14 +0000 Subject: [PATCH 1/2] add patch --- .../org/apache/hadoop/conf/Configuration.java | 45 ++++++++++++++----- .../apache/hadoop/conf/TestConfiguration.java | 28 ++++++++++++ 2 files changed, 62 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index c30ce0db84e22..a6398ea38f8ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -51,7 +51,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.ListIterator; import java.util.Map; @@ -318,7 +317,7 @@ private static boolean getRestrictParserDefault(Object resource) { /** * Configuration objects */ - private static final WeakHashMap REGISTRY = + private static WeakHashMap REGISTRY = new WeakHashMap(); /** @@ -826,9 +825,7 @@ public Configuration() { public Configuration(boolean loadDefaults) { this.loadDefaults = loadDefaults; - synchronized(Configuration.class) { - REGISTRY.put(this, null); - } + registerConfiguration(this); } /** @@ -862,18 +859,41 @@ public Configuration(Configuration other) { this.propertyTagsMap.putAll(other.propertyTagsMap); } - synchronized(Configuration.class) { - REGISTRY.put(this, null); - } + registerConfiguration(this); + this.classLoader = other.classLoader; this.loadDefaults = other.loadDefaults; setQuietMode(other.getQuietMode()); } + /** + * Registers the configuration to enable reloading. + */ + private static void registerConfiguration(Configuration conf) { + WeakHashMap registry = REGISTRY; + if (registry != null) { + synchronized (Configuration.class) { + registry.put(conf, null); + } + } + } + + public static synchronized void setConfigurationRegistryEnabled(boolean enable) { + if (enable) { + REGISTRY = new WeakHashMap(); + } else { + REGISTRY = null; + } + } + /** * Reload existing configuration instances. */ public static synchronized void reloadExistingConfigurations() { + if (REGISTRY == null) { + LOG.debug("Configuration registry is disabled!"); + return; + } if (LOG.isDebugEnabled()) { LOG.debug("Reloading " + REGISTRY.keySet().size() + " existing configurations"); @@ -889,11 +909,14 @@ public static synchronized void reloadExistingConfigurations() { * @param name file name. File should be present in the classpath. */ public static synchronized void addDefaultResource(String name) { + WeakHashMap registry = REGISTRY; if(!defaultResources.contains(name)) { defaultResources.add(name); - for(Configuration conf : REGISTRY.keySet()) { - if(conf.loadDefaults) { - conf.reloadConfiguration(); + if (registry != null) { + for (Configuration conf : registry.keySet()) { + if (conf.loadDefaults) { + conf.reloadConfiguration(); + } } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 64beb7b484296..14d9fff7f3e85 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -45,6 +45,9 @@ import java.util.Properties; import java.util.Random; import java.util.Set; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import static java.util.concurrent.TimeUnit.*; @@ -64,6 +67,7 @@ import static org.junit.Assert.*; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.concurrent.AtomicInitializer; import org.apache.hadoop.conf.Configuration.IntegerRanges; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; @@ -2553,4 +2557,28 @@ public void testResourceRace() { // Thread 1 config.get("secondParse"); } + + @Test + public void testReloadingCanBeDisabled() { + try { + + AtomicInteger reloadCount = new AtomicInteger(0); + Configuration config = new Configuration() { + @Override + public synchronized void reloadConfiguration() { + reloadCount.incrementAndGet(); + } + }; + assertThat(reloadCount.get(), is(0)); + Configuration.reloadExistingConfigurations(); + assertThat(reloadCount.get(), is(1)); + Configuration.setConfigurationRegistryEnabled(false); + assertThat(reloadCount.get(), is(1)); + Configuration.reloadExistingConfigurations(); + assertThat(reloadCount.get(), is(1)); + + } finally { + Configuration.setConfigurationRegistryEnabled(true); + } + } } From 3e75579702138208b25cce075672c6319670145b Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 29 Sep 2023 11:21:37 +0000 Subject: [PATCH 2/2] address comment --- .../src/main/java/org/apache/hadoop/conf/Configuration.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 61db4956b1dbf..fb8e44f542b46 100755 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -883,7 +883,9 @@ private static void registerConfiguration(Configuration conf) { public static synchronized void setConfigurationRegistryEnabled(boolean enable) { if (enable) { - REGISTRY = new WeakHashMap(); + if (REGISTRY == null) { + REGISTRY = new WeakHashMap(); + } } else { REGISTRY = null; }