Skip to content

YARN-11753. Ensure NM is marked unhealthy if the ProcessBuilder reports an issue with the container-executor #7290

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 20, 2025
Merged

YARN-11753. Ensure NM is marked unhealthy if the ProcessBuilder reports an issue with the container-executor #7290

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e8c3e84
YARN-11753. Ensure NM is marked unhealthy if the ProcessBuilder repor…
brumi1024 Jan 15, 2025
716725e
Add null check.
brumi1024 Jan 16, 2025
e3bccc6
Fix checkstyle.
brumi1024 Jan 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions ...nager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,10 +466,12 @@ public void startLocalizer(LocalizerStartContext ctx)
Throwable cause = e.getCause() != null ? e.getCause() : e;
if (cause instanceof IOException) {
IOException io = (IOException) cause;
if (io.getMessage().contains("No such file or directory")) {
String containerExecutorPath = getContainerExecutorExecutablePath(conf);
if (io.getMessage() != null && io.getMessage().contains("Cannot run program \"" +
containerExecutorPath + "\"")) {
throw new ConfigurationException("Application " + appId + " initialization failed" +
"(exitCode=" + exitCode + "). Container executor not found at "
+ getContainerExecutorExecutablePath(conf), e);
+ containerExecutorPath, e);
}
}

Expand Down
62 changes: 34 additions & 28 deletions ...t/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -628,15 +628,17 @@ protected PrivilegedOperationExecutor getPrivilegedOperationExecutor() {
when(context.getEnvironment()).thenReturn(env);
Path workDir = new Path("/tmp");

LocalizerStartContext lsc = new LocalizerStartContext.Builder()
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
.setNmAddr(address)
.setUser(appSubmitter)
.setAppId(appId.toString())
.setLocId("12345")
.setDirsHandler(dirService)
.build();

try {
lce.startLocalizer(new LocalizerStartContext.Builder()
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
.setNmAddr(address)
.setUser(appSubmitter)
.setAppId(appId.toString())
.setLocId("12345")
.setDirsHandler(dirService)
.build());
lce.startLocalizer(lsc);
Assert.fail("startLocalizer should have thrown an exception");
} catch (IOException e) {
assertTrue("Unexpected exception " + e,
Expand All @@ -648,50 +650,54 @@ protected PrivilegedOperationExecutor getPrivilegedOperationExecutor() {
LinuxContainerExecutor.ExitCode.INVALID_CONFIG_FILE.getExitCode(),
};

for (int i = 0; i < exitCodesToThrow.length; i++) {
int exitCode = exitCodesToThrow[i];
for (int exitCode : exitCodesToThrow) {
doThrow(new PrivilegedOperationException("invalid config", exitCode, null, null))
.when(spyPrivilegedExecutor).executePrivilegedOperation(
any(), any(PrivilegedOperation.class),
any(), any(), anyBoolean(), anyBoolean());

try {
lce.startLocalizer(new LocalizerStartContext.Builder()
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
.setNmAddr(address)
.setUser(appSubmitter)
.setAppId(appId.toString())
.setLocId("12345")
.setDirsHandler(dirService)
.build());
lce.startLocalizer(lsc);
Assert.fail("startLocalizer should have thrown a ConfigurationException");
} catch (ConfigurationException e) {
assertTrue("Unexpected exception " + e,
e.getMessage().contains("exitCode=" + exitCode));
}
}

// Assert that we do catch an IOException thrown by the ProcessBuilder.start
// method as a misconfiguration
String containerExecutorPath = lce.getContainerExecutorExecutablePath(conf);
doThrow(new PrivilegedOperationException("IO error",
new IOException("No such file or directory")))
new IOException("Cannot run program \""+ containerExecutorPath + "\"")))
.when(spyPrivilegedExecutor).executePrivilegedOperation(
any(), any(PrivilegedOperation.class),
any(), any(), anyBoolean(), anyBoolean());

try {
lce.startLocalizer(new LocalizerStartContext.Builder()
.setNmPrivateContainerTokens(nmPrivateCTokensPath)
.setNmAddr(address)
.setUser(appSubmitter)
.setAppId(appId.toString())
.setLocId("12345")
.setDirsHandler(dirService)
.build());
Assert.fail("startLocalizer should have thrown a ConfigurationException");
lce.startLocalizer(lsc);
Assert.fail("startLocalizer should have thrown an ConfigurationException");
} catch (ConfigurationException e) {
assertTrue("Unexpected exception " + e,
e.getMessage().contains("Container executor not found"));
}

// Assert that we do not catch every IOException as a misconfiguration
doThrow(new PrivilegedOperationException("IO error",
new IOException("No such file or directory")))
.when(spyPrivilegedExecutor).executePrivilegedOperation(
any(), any(PrivilegedOperation.class),
any(), any(), anyBoolean(), anyBoolean());

try {
lce.startLocalizer(lsc);
Assert.fail("startLocalizer should have thrown an IOException");
} catch (ConfigurationException e) {
Assert.fail("startLocalizer should not have thrown a ConfigurationException");
} catch (IOException e) {
assertTrue("Unexpected exception " + e,
e.getMessage().contains("exitCode"));
}

doThrow(new PrivilegedOperationException("interrupted"))
.when(spyPrivilegedExecutor).executePrivilegedOperation(
Expand Down
Loading