From 2ab048fa15ab00dfe5071d88b80f5a91f7ccf063 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 28 Jan 2025 23:19:41 +0800 Subject: [PATCH 1/5] HADOOP-19405. hadoop-aws tests have stopped running. --- hadoop-tools/hadoop-aws/pom.xml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index e3c1b023e772f..1d04107ff5b48 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -581,25 +581,5 @@ bcpkix-jdk18on test - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - org.junit.jupiter - junit-jupiter-params - test - - - org.junit.platform - junit-platform-launcher - test - From c06571803c15f06a8cbdd83ef60eb0e75c086266 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Wed, 29 Jan 2025 00:46:44 +0800 Subject: [PATCH 2/5] HADOOP-19405. hadoop-aws tests have stopped running. --- hadoop-project/pom.xml | 2 +- hadoop-tools/hadoop-aws/pom.xml | 25 +++++++++++++++++++++++++ hadoop-tools/hadoop-azure/pom.xml | 5 +++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b51881d58faaa..c8230d659a599 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -186,7 +186,7 @@ -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError ${extraJavaTestArgs} - 3.0.0-M1 + 3.0.0-M4 ${maven-surefire-plugin.version} ${maven-surefire-plugin.version} diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 1d04107ff5b48..a2929dffa2ffb 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -581,5 +581,30 @@ bcpkix-jdk18on test + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.platform + junit-platform-launcher + test + + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 29fb5019af8cf..ea3065a796fb7 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -372,6 +372,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + From 695e3144e7c79ed4813ffe86c02f8d167f744d05 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Thu, 30 Jan 2025 23:26:55 +0800 Subject: [PATCH 3/5] HADOOP-19405. hadoop-aws tests have stopped running. --- .../hadoop-mapreduce-client-app/pom.xml | 5 +++++ .../hadoop-mapreduce-client-core/pom.xml | 5 +++++ .../hadoop-mapreduce-client-hs/pom.xml | 5 +++++ .../hadoop-mapreduce-client-jobclient/pom.xml | 5 +++++ .../hadoop-mapreduce-client-nativetask/pom.xml | 5 +++++ .../hadoop-mapreduce-client-shuffle/pom.xml | 5 +++++ hadoop-tools/hadoop-archive-logs/pom.xml | 5 +++++ hadoop-tools/hadoop-archives/pom.xml | 5 +++++ hadoop-tools/hadoop-azure-datalake/pom.xml | 6 +++++- hadoop-tools/hadoop-datajoin/pom.xml | 5 +++++ hadoop-tools/hadoop-distcp/pom.xml | 5 +++++ .../hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml | 5 +++++ .../hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml | 5 +++++ .../hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml | 5 +++++ hadoop-tools/hadoop-extras/pom.xml | 5 +++++ hadoop-tools/hadoop-federation-balance/pom.xml | 5 +++++ hadoop-tools/hadoop-fs2img/pom.xml | 5 +++++ hadoop-tools/hadoop-gridmix/pom.xml | 5 +++++ hadoop-tools/hadoop-resourceestimator/pom.xml | 5 +++++ hadoop-tools/hadoop-rumen/pom.xml | 5 +++++ hadoop-tools/hadoop-sls/pom.xml | 5 +++++ hadoop-tools/hadoop-streaming/pom.xml | 5 +++++ .../hadoop-yarn-applications-distributedshell/pom.xml | 5 +++++ .../hadoop-yarn-services/hadoop-yarn-services-core/pom.xml | 6 +++++- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 5 +++++ .../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 5 +++++ .../hadoop-yarn-server-globalpolicygenerator/pom.xml | 5 +++++ .../hadoop-yarn-server-nodemanager/pom.xml | 5 +++++ .../hadoop-yarn-server-resourcemanager/pom.xml | 5 +++++ .../pom.xml | 5 +++++ .../hadoop-yarn-server-timelineservice-hbase-tests/pom.xml | 5 +++++ 31 files changed, 155 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index 4bd8ca970e525..b54e96d213a01 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -168,6 +168,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index ef424e3e992ee..81457e6845ec3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -92,6 +92,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index d8c9eaf19bb20..57f954ecd3841 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -115,6 +115,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index 8305867ba0827..b5deed6231e32 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -148,6 +148,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml index 5ffa4f52b3080..f162b7037ee59 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -80,6 +80,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml index fee97311ee589..b1c1b51c81fc9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml @@ -66,6 +66,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml index 7fe6e80cd4ac7..755f8229f3066 100644 --- a/hadoop-tools/hadoop-archive-logs/pom.xml +++ b/hadoop-tools/hadoop-archive-logs/pom.xml @@ -162,6 +162,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml index ab70389865c87..18c270cae9dd2 100644 --- a/hadoop-tools/hadoop-archives/pom.xml +++ b/hadoop-tools/hadoop-archives/pom.xml @@ -123,6 +123,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index f772cce3733a3..b578c548a1a9f 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -196,6 +196,10 @@ junit-platform-launcher test - + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml index 344c22282c74b..1c29a62d64227 100644 --- a/hadoop-tools/hadoop-datajoin/pom.xml +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -129,6 +129,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 37ccc0c2b45d0..b5a0c0d53b6a5 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -140,6 +140,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml index 8f571338b1d8f..e8c186c92192e 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/pom.xml @@ -69,6 +69,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml index 307e4bb0d6b50..2cc2cddf0b3e8 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/pom.xml @@ -122,6 +122,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml index f23e9b5437738..4be890ad333a0 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-workload/pom.xml @@ -69,6 +69,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml index 9da8f885917d6..86a192ce9ab0c 100644 --- a/hadoop-tools/hadoop-extras/pom.xml +++ b/hadoop-tools/hadoop-extras/pom.xml @@ -154,6 +154,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-federation-balance/pom.xml b/hadoop-tools/hadoop-federation-balance/pom.xml index 98c82c01ccc13..1211dd06d30ca 100644 --- a/hadoop-tools/hadoop-federation-balance/pom.xml +++ b/hadoop-tools/hadoop-federation-balance/pom.xml @@ -144,6 +144,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-fs2img/pom.xml b/hadoop-tools/hadoop-fs2img/pom.xml index 4a68b328b0e38..b91be930b06c8 100644 --- a/hadoop-tools/hadoop-fs2img/pom.xml +++ b/hadoop-tools/hadoop-fs2img/pom.xml @@ -97,6 +97,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index 862984fadabed..404c6b9f1496f 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -144,6 +144,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-resourceestimator/pom.xml b/hadoop-tools/hadoop-resourceestimator/pom.xml index 3c7d9cd036447..a7319d7cddb56 100644 --- a/hadoop-tools/hadoop-resourceestimator/pom.xml +++ b/hadoop-tools/hadoop-resourceestimator/pom.xml @@ -98,6 +98,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index 133c328bf1c9a..a2758acd560ea 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -123,6 +123,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml index 5c1a05364164d..142af5ff68ade 100644 --- a/hadoop-tools/hadoop-sls/pom.xml +++ b/hadoop-tools/hadoop-sls/pom.xml @@ -98,6 +98,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index 5f7ab7692f3ca..c5c793ea799f4 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -150,6 +150,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index d38934828adb6..493f89d1350dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -182,6 +182,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 6abd8f45ca874..317964557f883 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -289,7 +289,11 @@ junit-platform-launcher test - + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 377a0fbf459b7..c88256e7da2b0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -191,6 +191,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index bd5869fa3a2c6..c409403c84f08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -195,6 +195,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml index 3d0d24320de60..9e624835fda61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml @@ -149,6 +149,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index cfb1e95bd6011..df87a7b18efff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -104,6 +104,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + org.slf4j slf4j-api diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index f307a0b509890..365ae8cd30501 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -356,6 +356,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml index ffdd4ed0b6062..2df7c08d104cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml @@ -183,6 +183,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index 025bc3f139ed6..a2b68722ca1ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -131,6 +131,11 @@ junit-platform-launcher test + + org.junit.vintage + junit-vintage-engine + test + From b0c6e5b0f879dea538060b208bc2157ef2341557 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 1 Feb 2025 12:31:06 +0800 Subject: [PATCH 4/5] HADOOP-19405. Fix Junit Tests. --- .../v2/app/webapp/TestAMWebServices.java | 10 +- .../v2/hs/webapp/TestHsWebServices.java | 13 ++- hadoop-project/pom.xml | 2 +- .../timelineservice/ApplicationEntity.java | 7 +- .../yarn/client/api/impl/TestAMRMClient.java | 105 ++++++++++++------ .../TestLocalityMulticastAMRMProxyPolicy.java | 2 +- .../GlobalPolicyGenerator.java | 3 +- .../webapp/GPGWebApp.java | 2 + .../webapp/RMWebAppFilter.java | 1 + 9 files changed, 97 insertions(+), 48 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java index 2cc59ed664266..681487fceee54 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java @@ -28,13 +28,13 @@ import java.util.Set; import javax.ws.rs.core.MediaType; +import javax.ws.rs.NotAcceptableException; import javax.ws.rs.NotFoundException; -import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; import javax.ws.rs.core.Response; +import static javax.ws.rs.core.Response.Status.NOT_ACCEPTABLE; import static javax.ws.rs.core.Response.Status.NOT_FOUND; -import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -231,15 +231,15 @@ public void testInvalidUri2() throws JSONException, Exception { @Test public void testInvalidAccept() throws JSONException, Exception { - WebTarget r = target(); + WebTarget r = targetWithJsonObject(); String responseStr = ""; try { responseStr = r.path("ws").path("v1").path("mapreduce") .request(MediaType.TEXT_PLAIN).get(String.class); fail("should have thrown exception on invalid uri"); - } catch (ServiceUnavailableException sue) { + } catch (NotAcceptableException sue) { Response response = sue.getResponse(); - assertResponseStatusCode(SERVICE_UNAVAILABLE, response.getStatusInfo()); + assertResponseStatusCode(NOT_ACCEPTABLE, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java index c34df08b7ff7c..b320868242e29 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java @@ -20,13 +20,14 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import java.io.StringReader; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.NotAcceptableException; import javax.ws.rs.NotFoundException; -import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; @@ -232,12 +233,12 @@ public void testInvalidAccept() throws JSONException, Exception { WebTarget r = target(); String responseStr = ""; try { - Response response = - r.path("ws").path("v1").path("history").request(MediaType.TEXT_PLAIN).get(); - throw new ServiceUnavailableException(response); - } catch (ServiceUnavailableException ue) { + responseStr = r.path("ws").path("v1").path("history"). + request(MediaType.TEXT_PLAIN).get(String.class); + fail("should have thrown exception on invalid uri"); + } catch (NotAcceptableException ue) { Response response = ue.getResponse(); - assertResponseStatusCode(Response.Status.SERVICE_UNAVAILABLE, + assertResponseStatusCode(Response.Status.NOT_ACCEPTABLE, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c8230d659a599..a4a9b94c863d0 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -185,7 +185,7 @@ --add-opens=java.base/sun.security.x509=ALL-UNNAMED - -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError ${extraJavaTestArgs} + -Xmx2048m -Xss2m -XX:+HeapDumpOnOutOfMemoryError ${extraJavaTestArgs} 3.0.0-M4 ${maven-surefire-plugin.version} ${maven-surefire-plugin.version} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 20226aa8e9c68..58f65f34e1f1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -43,7 +43,12 @@ public ApplicationEntity(TimelineEntity entity) { } public String getQueue() { - return getInfo().get(QUEUE_INFO_KEY).toString(); + if (getInfo() != null) { + if (getInfo().containsKey(QUEUE_INFO_KEY)) { + return getInfo().get(QUEUE_INFO_KEY).toString(); + } + } + return ""; } public void setQueue(String queue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index fff27bd008471..7c9dbaeaf368f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -92,7 +92,7 @@ * Test application master client class to resource manager. */ @RunWith(value = Parameterized.class) -public class TestAMRMClient extends BaseAMRMClientTest{ +public class TestAMRMClient extends BaseAMRMClientTest { private final static int DEFAULT_ITERATION = 3; @@ -123,6 +123,10 @@ public void testAMRMClientNoMatchingRequests() List> matches = amClient.getMatchingRequests(priority, node, testCapability1); assertEquals("Expected no matching requests.", matches.size(), 0); + amClient.close(); + if (amClient != null && amClient.getServiceState() == STATE.STARTED) { + amClient.stop(); + } } @Test (timeout=60000) @@ -257,6 +261,7 @@ public void testAMRMClientMatchingFit() throws YarnException, IOException { null, null); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -390,6 +395,7 @@ public void testAMRMClientMatchingFitExecType() null, null); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -444,6 +450,7 @@ public void testAMRMClientMatchingFitInferredRack() null); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -560,18 +567,20 @@ public void testAMRMClientMatchStorage() throws YarnException, IOException { } } - assertEquals(2, allocatedContainerCount); + assertTrue(allocatedContainerCount <= 2); AllocateResponse allocResponse = amClient.allocate(0.1f); assertEquals(0, amClient.release.size()); assertEquals(0, amClient.ask.size()); - assertEquals(0, allocResponse.getAllocatedContainers().size()); + assertTrue(allocResponse.getAllocatedContainers().size() >= 0); + // 0 requests left. everything got cleaned up - assertTrue(remoteRequestsTable.isEmpty()); + // assertTrue(remoteRequestsTable.isEmpty()); amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -637,7 +646,7 @@ public void testAllocationWithBlacklist() throws YarnException, IOException { amClient.addContainerRequest(storedContainer2); allocatedContainerCount = getAllocatedContainersNumber(amClient, DEFAULT_ITERATION); - assertEquals(2, allocatedContainerCount); + assertTrue(allocatedContainerCount <= 2); // Test in case exception in allocate(), blacklist is kept assertTrue(amClient.blacklistAdditions.isEmpty()); @@ -657,6 +666,7 @@ public void testAllocationWithBlacklist() throws YarnException, IOException { assertEquals(1, amClient.blacklistAdditions.size()); } } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -709,6 +719,7 @@ public void testAMRMClientWithBlacklist() throws YarnException, IOException { assertEquals(2, amClient.blacklistAdditions.size()); assertEquals(1, amClient.blacklistRemovals.size()); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -737,13 +748,13 @@ private int getAllocatedContainersNumber( return allocatedContainerCount; } - @Test (timeout=60000) - public void testAMRMClient() throws YarnException, IOException { + @Test (timeout=90000) + public void testAMRMClient() throws YarnException, IOException, InterruptedException, TimeoutException { initAMRMClientAndTest(false); } @Test (timeout=60000) - public void testAMRMClientAllocReqId() throws YarnException, IOException { + public void testAMRMClientAllocReqId() throws Exception { initAMRMClientAndTest(true); } @@ -759,7 +770,7 @@ public void testAMRMClientWithSaslEncryption() throws Exception { } protected void initAMRMClientAndTest(boolean useAllocReqId) - throws YarnException, IOException { + throws YarnException, IOException, InterruptedException, TimeoutException { AMRMClient amClient = null; try { // start am rm client @@ -777,15 +788,16 @@ protected void initAMRMClientAndTest(boolean useAllocReqId) amClient.registerApplicationMaster("Host", 10000, ""); if (useAllocReqId) { - testAllocRequestId((AMRMClientImpl)amClient); + testAllocRequestId((AMRMClientImpl) amClient); } else { testAllocation((AMRMClientImpl) amClient); } amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, - null, null); + null, null); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -793,7 +805,7 @@ protected void initAMRMClientAndTest(boolean useAllocReqId) } @Test(timeout=30000) - public void testAskWithNodeLabels() { + public void testAskWithNodeLabels() throws IOException { AMRMClientImpl client = new AMRMClientImpl(); @@ -838,6 +850,11 @@ public void testAskWithNodeLabels() { Assert.assertNull(req.getNodeLabelExpression()); } } + + client.close(); + if (client != null && client.getServiceState() == STATE.STARTED) { + client.stop(); + } } private void verifyAddRequestFailed(AMRMClient client, @@ -851,7 +868,7 @@ private void verifyAddRequestFailed(AMRMClient client, } @Test(timeout=30000) - public void testAskWithInvalidNodeLabels() { + public void testAskWithInvalidNodeLabels() throws IOException { AMRMClientImpl client = new AMRMClientImpl(); @@ -859,6 +876,11 @@ public void testAskWithInvalidNodeLabels() { verifyAddRequestFailed(client, new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x && y")); + + client.close(); + if (client != null && client.getServiceState() == STATE.STARTED) { + client.stop(); + } } @Test(timeout=60000) @@ -897,6 +919,7 @@ public void testAMRMClientWithContainerResourceChange() amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -911,13 +934,21 @@ private List allocateAndStartContainers( amClient.addContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); } - // send allocation requests - amClient.allocate(0.1f); - // let NM heartbeat to RM and trigger allocations - triggerSchedulingWithNMHeartBeat(); - // get allocations - AllocateResponse allocResponse = amClient.allocate(0.1f); - List containers = allocResponse.getAllocatedContainers(); + int iterationsLeft = 15; + int allocatedContainerCount = 0; + List containers = new ArrayList<>(); + while (allocatedContainerCount <= num && iterationsLeft-- > 0) { + AllocateResponse allocResponse = amClient.allocate(0.1f); + allocatedContainerCount += allocResponse.getAllocatedContainers().size(); + for (Container container : allocResponse.getAllocatedContainers()) { + containers.add(container); + } + if (iterationsLeft > 0) { + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); + } + } + assertEquals(num, containers.size()); // build container launch context @@ -1108,6 +1139,10 @@ public void testAMRMContainerPromotionAndDemotionWithAutoUpdate() waitForNMContextUpdate(updatedContainer, ExecutionType.OPPORTUNISTIC); amClient.close(); + + if (amClient != null && amClient.getServiceState() == STATE.STARTED) { + amClient.stop(); + } } private AllocateResponse waitForAllocation(AMRMClient amrmClient, @@ -1522,7 +1557,7 @@ private void testAllocation(final AMRMClientImpl amClient) // RM should allocate container within 2 calls to allocate() int allocatedContainerCount = 0; - int iterationsLeft = 3; + int iterationsLeft = 15; Set releases = new TreeSet(); amClient.getNMTokenCache().clearCache(); @@ -1561,8 +1596,8 @@ private void testAllocation(final AMRMClientImpl amClient) assertTrue(receivedNMTokens.size() > 0 && receivedNMTokens.size() <= nodeCount); - assertEquals(allocatedContainerCount, containersRequestedAny); - assertEquals(2, releases.size()); + assertTrue(allocatedContainerCount <= containersRequestedAny); + assertTrue(releases.size() <= 2); assertEquals(0, amClient.ask.size()); // need to tell the AMRMClient that we dont need these resources anymore @@ -1605,14 +1640,14 @@ public AllocateResponse answer(InvocationOnMock invocation) amClient.rmClient = realRM; } - assertEquals(2, amClient.release.size()); + assertTrue(amClient.release.size() <= 2); assertEquals(3, amClient.ask.size()); snoopRequest = amClient.ask.iterator().next(); // verify that the remove request made in between makeRequest and allocate // has not been lost assertEquals(0, snoopRequest.getNumContainers()); - waitForContainerCompletion(3, amClient, releases); + // waitForContainerCompletion(15, amClient, releases); } private void waitForContainerCompletion(int numIterations, @@ -1623,7 +1658,7 @@ private void waitForContainerCompletion(int numIterations, // inform RM of rejection AllocateResponse allocResponse = amClient.allocate(0.1f); // RM did not send new containers because AM does not need any - assertEquals(0, allocResponse.getAllocatedContainers().size()); + // assertEquals(0, allocResponse.getAllocatedContainers().size()); if(allocResponse.getCompletedContainersStatuses().size() > 0) { for(ContainerStatus cStatus :allocResponse .getCompletedContainersStatuses()) { @@ -1634,7 +1669,7 @@ private void waitForContainerCompletion(int numIterations, } } } - if(numIterations > 0) { + if (numIterations > 0) { // let NM heartbeat to RM and trigger allocations triggerSchedulingWithNMHeartBeat(); } @@ -1645,7 +1680,7 @@ private void waitForContainerCompletion(int numIterations, private void testAllocRequestId( final AMRMClientImpl amClient) throws YarnException, - IOException { + IOException, InterruptedException, TimeoutException { // setup container request assertEquals(0, amClient.ask.size()); @@ -1699,18 +1734,19 @@ private void testAllocRequestId( } } - assertEquals(containersRequestedAny, allocatedContainers.size()); + assertTrue(allocatedContainers.size() <= containersRequestedAny); Set expAllocIds = new HashSet<>( Arrays.asList(Long.valueOf(0), Long.valueOf(1), Long.valueOf(2))); Set actAllocIds = new HashSet<>(); for (Container ac : allocatedContainers) { actAllocIds.add(Long.valueOf(ac.getAllocationRequestId())); } - assertEquals(expAllocIds, actAllocIds); - assertEquals(3, amClient.release.size()); + + assertTrue(expAllocIds.containsAll(actAllocIds)); + assertTrue(releases.size() <= 3); assertEquals(0, amClient.ask.size()); - waitForContainerCompletion(3, amClient, releases); + // waitForContainerCompletion(15, amClient, releases); } private void assertNumContainers(AMRMClientImpl amClient, @@ -1749,7 +1785,7 @@ public Boolean get() { }; @Test - public void testWaitFor() throws InterruptedException { + public void testWaitFor() throws InterruptedException, IOException { AMRMClientImpl amClient = null; CountDownSupplier countDownChecker = new CountDownSupplier(); @@ -1763,6 +1799,7 @@ public void testWaitFor() throws InterruptedException { amClient.waitFor(countDownChecker, 1000); assertEquals(3, countDownChecker.counter); } finally { + amClient.close(); if (amClient != null) { amClient.stop(); } @@ -1896,6 +1933,7 @@ public ApplicationMasterProtocol run() { null, null); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -2004,6 +2042,7 @@ public void testGetMatchingFitWithProfiles() throws Exception { matches = amClient.getMatchingRequests(priority, node, testCapability4); assertEquals(1, matches.size()); } finally { + amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java index d7102e03e738f..4172c6afb94f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java @@ -197,7 +197,7 @@ public void testSplitBasedOnHeadroom() throws Exception { checkTotalContainerAllocation(response, 100); } - @Test(timeout = 5000) + @Test(timeout = 8000) public void testStressPolicy() throws Exception { // Tests how the headroom info are used to split based on the capacity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index ed44f3171d8ed..3854123b2693a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -300,7 +300,8 @@ public void startWepApp() { LOG.info("Instantiating GPGWebApp at {}.", webAppAddress); GPGWebApp gpgWebApp = new GPGWebApp(this); webApp = WebApps.$for("gpg", GPGContext.class, this.gpgContext, - "ws").at(webAppAddress).withResourceConfig(gpgWebApp.resourceConfig()).start(gpgWebApp); + "gpg-ws").at(webAppAddress). + withResourceConfig(gpgWebApp.resourceConfig()).start(gpgWebApp); } @SuppressWarnings("resource") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java index 1dd808c9ec699..52a523debfbc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; @@ -59,6 +60,7 @@ private class JerseyBinder extends AbstractBinder { @Override protected void configure() { bind(gpg).to(GlobalPolicyGenerator.class).named("gpg"); + bind(gpg.getConfig()).to(Configuration.class).named("conf"); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index f47ca4eb94793..31cdd092c0ab2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -250,6 +250,7 @@ private String ahsRedirectPath(String uri, RMWebApp rmWebApp) { private boolean shouldRedirect(RMWebApp rmWebApp, String uri) { return !uri.equals("/" + rmWebApp.wsName() + "/v1/cluster/info") + && !uri.equals("/ws/v1/cluster/info") && !uri.equals("/" + rmWebApp.name() + "/cluster") && !uri.startsWith(ProxyUriUtils.PROXY_BASE) && !NON_REDIRECTED_URIS.contains(uri); From 629debfe72dd8ad0c5471273a2ab97a893ed5a2f Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 2 Feb 2025 10:39:36 +0800 Subject: [PATCH 5/5] =?UTF-8?q?HADOOP-19405.=20Revert=20Mapreduce=E3=80=81?= =?UTF-8?q?Yarn=20Commit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hadoop-mapreduce-client-app/pom.xml | 5 - .../v2/app/webapp/TestAMWebServices.java | 10 +- .../hadoop-mapreduce-client-core/pom.xml | 5 - .../hadoop-mapreduce-client-hs/pom.xml | 5 - .../v2/hs/webapp/TestHsWebServices.java | 13 +-- .../hadoop-mapreduce-client-jobclient/pom.xml | 5 - .../pom.xml | 5 - .../hadoop-mapreduce-client-shuffle/pom.xml | 5 - .../timelineservice/ApplicationEntity.java | 7 +- .../pom.xml | 5 - .../hadoop-yarn-services-core/pom.xml | 6 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 5 - .../yarn/client/api/impl/TestAMRMClient.java | 105 ++++++------------ .../hadoop-yarn-server-common/pom.xml | 5 - .../TestLocalityMulticastAMRMProxyPolicy.java | 2 +- .../pom.xml | 5 - .../GlobalPolicyGenerator.java | 3 +- .../webapp/GPGWebApp.java | 2 - .../hadoop-yarn-server-nodemanager/pom.xml | 5 - .../pom.xml | 5 - .../webapp/RMWebAppFilter.java | 1 - .../pom.xml | 5 - .../pom.xml | 5 - 23 files changed, 48 insertions(+), 171 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index b54e96d213a01..4bd8ca970e525 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -168,11 +168,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java index 681487fceee54..2cc59ed664266 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServices.java @@ -28,13 +28,13 @@ import java.util.Set; import javax.ws.rs.core.MediaType; -import javax.ws.rs.NotAcceptableException; import javax.ws.rs.NotFoundException; +import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; import javax.ws.rs.core.Response; -import static javax.ws.rs.core.Response.Status.NOT_ACCEPTABLE; import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static javax.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -231,15 +231,15 @@ public void testInvalidUri2() throws JSONException, Exception { @Test public void testInvalidAccept() throws JSONException, Exception { - WebTarget r = targetWithJsonObject(); + WebTarget r = target(); String responseStr = ""; try { responseStr = r.path("ws").path("v1").path("mapreduce") .request(MediaType.TEXT_PLAIN).get(String.class); fail("should have thrown exception on invalid uri"); - } catch (NotAcceptableException sue) { + } catch (ServiceUnavailableException sue) { Response response = sue.getResponse(); - assertResponseStatusCode(NOT_ACCEPTABLE, response.getStatusInfo()); + assertResponseStatusCode(SERVICE_UNAVAILABLE, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index 81457e6845ec3..ef424e3e992ee 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -92,11 +92,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index 57f954ecd3841..d8c9eaf19bb20 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -115,11 +115,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java index b320868242e29..c34df08b7ff7c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServices.java @@ -20,14 +20,13 @@ import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import java.io.StringReader; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.NotAcceptableException; import javax.ws.rs.NotFoundException; +import javax.ws.rs.ServiceUnavailableException; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; import javax.ws.rs.core.MediaType; @@ -233,12 +232,12 @@ public void testInvalidAccept() throws JSONException, Exception { WebTarget r = target(); String responseStr = ""; try { - responseStr = r.path("ws").path("v1").path("history"). - request(MediaType.TEXT_PLAIN).get(String.class); - fail("should have thrown exception on invalid uri"); - } catch (NotAcceptableException ue) { + Response response = + r.path("ws").path("v1").path("history").request(MediaType.TEXT_PLAIN).get(); + throw new ServiceUnavailableException(response); + } catch (ServiceUnavailableException ue) { Response response = ue.getResponse(); - assertResponseStatusCode(Response.Status.NOT_ACCEPTABLE, + assertResponseStatusCode(Response.Status.SERVICE_UNAVAILABLE, response.getStatusInfo()); WebServicesTestUtils.checkStringMatch( "error string exists and shouldn't", "", responseStr); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index b5deed6231e32..8305867ba0827 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -148,11 +148,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml index f162b7037ee59..5ffa4f52b3080 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml @@ -80,11 +80,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml index b1c1b51c81fc9..fee97311ee589 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml @@ -66,11 +66,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java index 58f65f34e1f1f..20226aa8e9c68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java @@ -43,12 +43,7 @@ public ApplicationEntity(TimelineEntity entity) { } public String getQueue() { - if (getInfo() != null) { - if (getInfo().containsKey(QUEUE_INFO_KEY)) { - return getInfo().get(QUEUE_INFO_KEY).toString(); - } - } - return ""; + return getInfo().get(QUEUE_INFO_KEY).toString(); } public void setQueue(String queue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index 493f89d1350dc..d38934828adb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -182,11 +182,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 317964557f883..6abd8f45ca874 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -289,11 +289,7 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index c88256e7da2b0..377a0fbf459b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -191,11 +191,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 7c9dbaeaf368f..fff27bd008471 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -92,7 +92,7 @@ * Test application master client class to resource manager. */ @RunWith(value = Parameterized.class) -public class TestAMRMClient extends BaseAMRMClientTest { +public class TestAMRMClient extends BaseAMRMClientTest{ private final static int DEFAULT_ITERATION = 3; @@ -123,10 +123,6 @@ public void testAMRMClientNoMatchingRequests() List> matches = amClient.getMatchingRequests(priority, node, testCapability1); assertEquals("Expected no matching requests.", matches.size(), 0); - amClient.close(); - if (amClient != null && amClient.getServiceState() == STATE.STARTED) { - amClient.stop(); - } } @Test (timeout=60000) @@ -261,7 +257,6 @@ public void testAMRMClientMatchingFit() throws YarnException, IOException { null, null); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -395,7 +390,6 @@ public void testAMRMClientMatchingFitExecType() null, null); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -450,7 +444,6 @@ public void testAMRMClientMatchingFitInferredRack() null); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -567,20 +560,18 @@ public void testAMRMClientMatchStorage() throws YarnException, IOException { } } - assertTrue(allocatedContainerCount <= 2); + assertEquals(2, allocatedContainerCount); AllocateResponse allocResponse = amClient.allocate(0.1f); assertEquals(0, amClient.release.size()); assertEquals(0, amClient.ask.size()); - assertTrue(allocResponse.getAllocatedContainers().size() >= 0); - + assertEquals(0, allocResponse.getAllocatedContainers().size()); // 0 requests left. everything got cleaned up - // assertTrue(remoteRequestsTable.isEmpty()); + assertTrue(remoteRequestsTable.isEmpty()); amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -646,7 +637,7 @@ public void testAllocationWithBlacklist() throws YarnException, IOException { amClient.addContainerRequest(storedContainer2); allocatedContainerCount = getAllocatedContainersNumber(amClient, DEFAULT_ITERATION); - assertTrue(allocatedContainerCount <= 2); + assertEquals(2, allocatedContainerCount); // Test in case exception in allocate(), blacklist is kept assertTrue(amClient.blacklistAdditions.isEmpty()); @@ -666,7 +657,6 @@ public void testAllocationWithBlacklist() throws YarnException, IOException { assertEquals(1, amClient.blacklistAdditions.size()); } } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -719,7 +709,6 @@ public void testAMRMClientWithBlacklist() throws YarnException, IOException { assertEquals(2, amClient.blacklistAdditions.size()); assertEquals(1, amClient.blacklistRemovals.size()); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -748,13 +737,13 @@ private int getAllocatedContainersNumber( return allocatedContainerCount; } - @Test (timeout=90000) - public void testAMRMClient() throws YarnException, IOException, InterruptedException, TimeoutException { + @Test (timeout=60000) + public void testAMRMClient() throws YarnException, IOException { initAMRMClientAndTest(false); } @Test (timeout=60000) - public void testAMRMClientAllocReqId() throws Exception { + public void testAMRMClientAllocReqId() throws YarnException, IOException { initAMRMClientAndTest(true); } @@ -770,7 +759,7 @@ public void testAMRMClientWithSaslEncryption() throws Exception { } protected void initAMRMClientAndTest(boolean useAllocReqId) - throws YarnException, IOException, InterruptedException, TimeoutException { + throws YarnException, IOException { AMRMClient amClient = null; try { // start am rm client @@ -788,16 +777,15 @@ protected void initAMRMClientAndTest(boolean useAllocReqId) amClient.registerApplicationMaster("Host", 10000, ""); if (useAllocReqId) { - testAllocRequestId((AMRMClientImpl) amClient); + testAllocRequestId((AMRMClientImpl)amClient); } else { testAllocation((AMRMClientImpl) amClient); } amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, - null, null); + null, null); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -805,7 +793,7 @@ protected void initAMRMClientAndTest(boolean useAllocReqId) } @Test(timeout=30000) - public void testAskWithNodeLabels() throws IOException { + public void testAskWithNodeLabels() { AMRMClientImpl client = new AMRMClientImpl(); @@ -850,11 +838,6 @@ public void testAskWithNodeLabels() throws IOException { Assert.assertNull(req.getNodeLabelExpression()); } } - - client.close(); - if (client != null && client.getServiceState() == STATE.STARTED) { - client.stop(); - } } private void verifyAddRequestFailed(AMRMClient client, @@ -868,7 +851,7 @@ private void verifyAddRequestFailed(AMRMClient client, } @Test(timeout=30000) - public void testAskWithInvalidNodeLabels() throws IOException { + public void testAskWithInvalidNodeLabels() { AMRMClientImpl client = new AMRMClientImpl(); @@ -876,11 +859,6 @@ public void testAskWithInvalidNodeLabels() throws IOException { verifyAddRequestFailed(client, new ContainerRequest(Resource.newInstance(1024, 1), null, null, Priority.UNDEFINED, true, "x && y")); - - client.close(); - if (client != null && client.getServiceState() == STATE.STARTED) { - client.stop(); - } } @Test(timeout=60000) @@ -919,7 +897,6 @@ public void testAMRMClientWithContainerResourceChange() amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -934,21 +911,13 @@ private List allocateAndStartContainers( amClient.addContainerRequest( new ContainerRequest(capability, nodes, racks, priority)); } - int iterationsLeft = 15; - int allocatedContainerCount = 0; - List containers = new ArrayList<>(); - while (allocatedContainerCount <= num && iterationsLeft-- > 0) { - AllocateResponse allocResponse = amClient.allocate(0.1f); - allocatedContainerCount += allocResponse.getAllocatedContainers().size(); - for (Container container : allocResponse.getAllocatedContainers()) { - containers.add(container); - } - if (iterationsLeft > 0) { - // let NM heartbeat to RM and trigger allocations - triggerSchedulingWithNMHeartBeat(); - } - } - + // send allocation requests + amClient.allocate(0.1f); + // let NM heartbeat to RM and trigger allocations + triggerSchedulingWithNMHeartBeat(); + // get allocations + AllocateResponse allocResponse = amClient.allocate(0.1f); + List containers = allocResponse.getAllocatedContainers(); assertEquals(num, containers.size()); // build container launch context @@ -1139,10 +1108,6 @@ public void testAMRMContainerPromotionAndDemotionWithAutoUpdate() waitForNMContextUpdate(updatedContainer, ExecutionType.OPPORTUNISTIC); amClient.close(); - - if (amClient != null && amClient.getServiceState() == STATE.STARTED) { - amClient.stop(); - } } private AllocateResponse waitForAllocation(AMRMClient amrmClient, @@ -1557,7 +1522,7 @@ private void testAllocation(final AMRMClientImpl amClient) // RM should allocate container within 2 calls to allocate() int allocatedContainerCount = 0; - int iterationsLeft = 15; + int iterationsLeft = 3; Set releases = new TreeSet(); amClient.getNMTokenCache().clearCache(); @@ -1596,8 +1561,8 @@ private void testAllocation(final AMRMClientImpl amClient) assertTrue(receivedNMTokens.size() > 0 && receivedNMTokens.size() <= nodeCount); - assertTrue(allocatedContainerCount <= containersRequestedAny); - assertTrue(releases.size() <= 2); + assertEquals(allocatedContainerCount, containersRequestedAny); + assertEquals(2, releases.size()); assertEquals(0, amClient.ask.size()); // need to tell the AMRMClient that we dont need these resources anymore @@ -1640,14 +1605,14 @@ public AllocateResponse answer(InvocationOnMock invocation) amClient.rmClient = realRM; } - assertTrue(amClient.release.size() <= 2); + assertEquals(2, amClient.release.size()); assertEquals(3, amClient.ask.size()); snoopRequest = amClient.ask.iterator().next(); // verify that the remove request made in between makeRequest and allocate // has not been lost assertEquals(0, snoopRequest.getNumContainers()); - // waitForContainerCompletion(15, amClient, releases); + waitForContainerCompletion(3, amClient, releases); } private void waitForContainerCompletion(int numIterations, @@ -1658,7 +1623,7 @@ private void waitForContainerCompletion(int numIterations, // inform RM of rejection AllocateResponse allocResponse = amClient.allocate(0.1f); // RM did not send new containers because AM does not need any - // assertEquals(0, allocResponse.getAllocatedContainers().size()); + assertEquals(0, allocResponse.getAllocatedContainers().size()); if(allocResponse.getCompletedContainersStatuses().size() > 0) { for(ContainerStatus cStatus :allocResponse .getCompletedContainersStatuses()) { @@ -1669,7 +1634,7 @@ private void waitForContainerCompletion(int numIterations, } } } - if (numIterations > 0) { + if(numIterations > 0) { // let NM heartbeat to RM and trigger allocations triggerSchedulingWithNMHeartBeat(); } @@ -1680,7 +1645,7 @@ private void waitForContainerCompletion(int numIterations, private void testAllocRequestId( final AMRMClientImpl amClient) throws YarnException, - IOException, InterruptedException, TimeoutException { + IOException { // setup container request assertEquals(0, amClient.ask.size()); @@ -1734,19 +1699,18 @@ private void testAllocRequestId( } } - assertTrue(allocatedContainers.size() <= containersRequestedAny); + assertEquals(containersRequestedAny, allocatedContainers.size()); Set expAllocIds = new HashSet<>( Arrays.asList(Long.valueOf(0), Long.valueOf(1), Long.valueOf(2))); Set actAllocIds = new HashSet<>(); for (Container ac : allocatedContainers) { actAllocIds.add(Long.valueOf(ac.getAllocationRequestId())); } - - assertTrue(expAllocIds.containsAll(actAllocIds)); - assertTrue(releases.size() <= 3); + assertEquals(expAllocIds, actAllocIds); + assertEquals(3, amClient.release.size()); assertEquals(0, amClient.ask.size()); - // waitForContainerCompletion(15, amClient, releases); + waitForContainerCompletion(3, amClient, releases); } private void assertNumContainers(AMRMClientImpl amClient, @@ -1785,7 +1749,7 @@ public Boolean get() { }; @Test - public void testWaitFor() throws InterruptedException, IOException { + public void testWaitFor() throws InterruptedException { AMRMClientImpl amClient = null; CountDownSupplier countDownChecker = new CountDownSupplier(); @@ -1799,7 +1763,6 @@ public void testWaitFor() throws InterruptedException, IOException { amClient.waitFor(countDownChecker, 1000); assertEquals(3, countDownChecker.counter); } finally { - amClient.close(); if (amClient != null) { amClient.stop(); } @@ -1933,7 +1896,6 @@ public ApplicationMasterProtocol run() { null, null); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } @@ -2042,7 +2004,6 @@ public void testGetMatchingFitWithProfiles() throws Exception { matches = amClient.getMatchingRequests(priority, node, testCapability4); assertEquals(1, matches.size()); } finally { - amClient.close(); if (amClient != null && amClient.getServiceState() == STATE.STARTED) { amClient.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index c409403c84f08..bd5869fa3a2c6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -195,11 +195,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java index 4172c6afb94f5..d7102e03e738f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java @@ -197,7 +197,7 @@ public void testSplitBasedOnHeadroom() throws Exception { checkTotalContainerAllocation(response, 100); } - @Test(timeout = 8000) + @Test(timeout = 5000) public void testStressPolicy() throws Exception { // Tests how the headroom info are used to split based on the capacity diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml index 9e624835fda61..3d0d24320de60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml @@ -149,11 +149,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java index 3854123b2693a..ed44f3171d8ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/GlobalPolicyGenerator.java @@ -300,8 +300,7 @@ public void startWepApp() { LOG.info("Instantiating GPGWebApp at {}.", webAppAddress); GPGWebApp gpgWebApp = new GPGWebApp(this); webApp = WebApps.$for("gpg", GPGContext.class, this.gpgContext, - "gpg-ws").at(webAppAddress). - withResourceConfig(gpgWebApp.resourceConfig()).start(gpgWebApp); + "ws").at(webAppAddress).withResourceConfig(gpgWebApp.resourceConfig()).start(gpgWebApp); } @SuppressWarnings("resource") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java index 52a523debfbc5..1dd808c9ec699 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator; import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; @@ -60,7 +59,6 @@ private class JerseyBinder extends AbstractBinder { @Override protected void configure() { bind(gpg).to(GlobalPolicyGenerator.class).named("gpg"); - bind(gpg.getConfig()).to(Configuration.class).named("conf"); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index df87a7b18efff..cfb1e95bd6011 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -104,11 +104,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - org.slf4j slf4j-api diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 365ae8cd30501..f307a0b509890 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -356,11 +356,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java index 31cdd092c0ab2..f47ca4eb94793 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java @@ -250,7 +250,6 @@ private String ahsRedirectPath(String uri, RMWebApp rmWebApp) { private boolean shouldRedirect(RMWebApp rmWebApp, String uri) { return !uri.equals("/" + rmWebApp.wsName() + "/v1/cluster/info") - && !uri.equals("/ws/v1/cluster/info") && !uri.equals("/" + rmWebApp.name() + "/cluster") && !uri.startsWith(ProxyUriUtils.PROXY_BASE) && !NON_REDIRECTED_URIS.contains(uri); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml index 2df7c08d104cb..ffdd4ed0b6062 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/pom.xml @@ -183,11 +183,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index a2b68722ca1ac..025bc3f139ed6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -131,11 +131,6 @@ junit-platform-launcher test - - org.junit.vintage - junit-vintage-engine - test -