YARN-11687. Update CGroupsResourceCalculator to track usages using cgroupv2

- unify common code between v1 and v2

Author: K0K0V0K

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/AbstractCGroupsResourceCalculator.java

@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
+
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.util.SysInfoLinux;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+
+/**
+ * Common code base for the CGroupsResourceCalculator implementations.
+ */
+public abstract class AbstractCGroupsResourceCalculator extends ResourceCalculatorProcessTree {
+  protected final String pid;
+  protected Clock clock;
+
+  @VisibleForTesting
+  long jiffyLengthMs = SysInfoLinux.JIFFY_LENGTH_IN_MILLIS;
+  @VisibleForTesting
+  CpuTimeTracker cpuTimeTracker;
+  @VisibleForTesting
+  CGroupsHandler cGroupsHandler;
+
+  /**
+   * Create resource calculator for the container that has the specified pid.
+   * @param pid A pid from the cgroup or null for all containers
+   */
+  public AbstractCGroupsResourceCalculator(String pid) {
+    super(pid);
+    this.pid = pid;
+  }
+
+  @Override
+  public String getProcessTreeDump() {
+    // We do not have a process tree in cgroups return just the pid for tracking
+    return pid;
+  }
+
+  @Override
+  public boolean checkPidPgrpidForMatch() {
+    // We do not have a process tree in cgroups returning default ok
+    return true;
+  }
+
+  @Override
+  public float getCpuUsagePercent() {
+    return cpuTimeTracker.getCpuTrackerUsagePercent();
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsResourceCalculator.java

@@ -58,7 +58,7 @@
  * If the real virtual memory is required please use the legacy procfs based
  * resource calculator or CombinedResourceCalculator.
  */
-public class CGroupsResourceCalculator extends ResourceCalculatorProcessTree {
+public class CGroupsResourceCalculator extends AbstractCGroupsResourceCalculator {
   enum Result {
     Continue,
     Exit
@@ -76,9 +76,7 @@ enum Result {
   private static final Pattern CGROUP_FILE_FORMAT = Pattern.compile(
       "^(\\d+):([^:]+):/(.*)$");
   private final String procfsDir;
-  private CGroupsHandler cGroupsHandler;
 
-  private String pid;
   private File cpuStat;
   private File memStat;
   private File memswStat;
@@ -87,10 +85,6 @@ enum Result {
   private long processPhysicalMemory;
   private long processVirtualMemory;
 
-  private final long jiffyLengthMs;
-  private final CpuTimeTracker cpuTimeTracker;
-  private Clock clock;
-
   /**
    * Create resource calculator for all Yarn containers.
    */
@@ -122,10 +116,9 @@ public CGroupsResourceCalculator(String pid) {
                             CGroupsHandler cGroupsHandler,
                             Clock clock,
                             long jiffyLengthMs0) {
-    super(pid);
+    super(pid != null && pid.equals("0") ? "1" : pid);
     this.procfsDir = procfsDir;
     this.cGroupsHandler = cGroupsHandler;
-    this.pid = pid != null && pid.equals("0") ? "1" : pid;
     this.jiffyLengthMs = jiffyLengthMs0;
     this.cpuTimeTracker =
         new CpuTimeTracker(this.jiffyLengthMs);
@@ -143,12 +136,6 @@ public void initialize() throws YarnException {
     setCGroupFilePaths();
   }
 
-  @Override
-  public float getCpuUsagePercent() {
-    LOG.debug("Process {} jiffies:{}", pid, processTotalJiffies);
-    return cpuTimeTracker.getCpuTrackerUsagePercent();
-  }
-
   @Override
   public long getCumulativeCpuTime() {
     if (jiffyLengthMs < 0) {
@@ -191,18 +178,6 @@ public void updateProcessTree() {
     }
   }
 
-  @Override
-  public String getProcessTreeDump() {
-    // We do not have a process tree in cgroups return just the pid for tracking
-    return pid;
-  }
-
-  @Override
-  public boolean checkPidPgrpidForMatch() {
-    // We do not have a process tree in cgroups returning default ok
-    return true;
-  }
-
   /**
    * Checks if the CGroupsResourceCalculator is available on this system.
    * This assumes that Linux container executor is already initialized.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsV2ResourceCalculator.java

@@ -18,20 +18,6 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.classification.VisibleForTesting;
-import org.apache.hadoop.util.CpuTimeTracker;
-import org.apache.hadoop.util.SysInfoLinux;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
-import org.apache.hadoop.yarn.util.SystemClock;
-
 import java.io.IOException;
 import java.math.BigInteger;
 import java.nio.charset.StandardCharsets;
@@ -44,34 +30,34 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.classification.VisibleForTesting;
+import org.apache.hadoop.util.CpuTimeTracker;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
 /**
  * A CGroupV2 file-system based Resource calculator without the process tree features.
  *
  * The feature only works if cluster runs in pure V2 version, because when we read the
  * /proc/{pid}/cgroup file currently we can not handle multiple lines.
  */
-public class CGroupsV2ResourceCalculator extends ResourceCalculatorProcessTree {
+public class CGroupsV2ResourceCalculator extends AbstractCGroupsResourceCalculator {
   private static final Logger LOG = LoggerFactory.getLogger(CGroupsV2ResourceCalculator.class);
   private final Map<String, String> stats = new ConcurrentHashMap<>();
-  private final Clock clock = SystemClock.getInstance();
-  private final String pid;
 
-  @VisibleForTesting
-  long jiffyLengthMs = SysInfoLinux.JIFFY_LENGTH_IN_MILLIS;
   @VisibleForTesting
   String root = "/";
-  @VisibleForTesting
-  CpuTimeTracker cpuTimeTracker;
-  @VisibleForTesting
-  CGroupsHandler cGroupsHandler;
 
   /**
    * Create resource calculator for the container that has the specified pid.
    * @param pid A pid from the cgroup or null for all containers
    */
   public CGroupsV2ResourceCalculator(String pid) {
     super(pid);
-    this.pid = pid;
   }
 
   @Override
@@ -80,11 +66,6 @@ public void initialize() throws YarnException {
     cGroupsHandler = ResourceHandlerModule.getCGroupsHandler();
   }
 
-  @Override
-  public float getCpuUsagePercent() {
-    return cpuTimeTracker.getCpuTrackerUsagePercent();
-  }
-
   @Override
   public long getCumulativeCpuTime() {
     // https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files
@@ -109,18 +90,6 @@ public long getVirtualMemorySize(int olderThanAge) {
         : getStat("memory.stat#vmalloc");
   }
 
-  @Override
-  public String getProcessTreeDump() {
-    // We do not have a process tree in cgroups return just the pid for tracking
-    return pid;
-  }
-
-  @Override
-  public boolean checkPidPgrpidForMatch() {
-    // We do not have a process tree in cgroups returning default ok
-    return true;
-  }
-
   @Override
   public void updateProcessTree() {
     try (Stream<Path> cGroupFiles = Files.list(getCGroupPath())){