Merge branch 'apache:trunk' into YARN-11764

Author: slfan1989

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/TestName.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.test;
+
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+/**
+ * This is a custom JUnit5 `RegisterExtension`
+ * we created to obtain the methond name of the executing function.
+ */
+public class TestName implements BeforeEachCallback {
+
+  private volatile String name;
+
+  @Override
+  public void beforeEach(ExtensionContext extensionContext) throws Exception {
+    name = extensionContext.getTestMethod().get().getName();
+  }
+
+  public String getMethodName() {
+    return this.name;
+  }
+}

hadoop-project/pom.xml

@@ -1589,7 +1589,7 @@
       <dependency>
         <groupId>com.aliyun.oss</groupId>
         <artifactId>aliyun-sdk-oss</artifactId>
-        <version>3.13.2</version>
+        <version>3.18.1</version>
         <exclusions>
           <exclusion>
             <groupId>org.apache.httpcomponents</groupId>

hadoop-tools/hadoop-aliyun/pom.xml

@@ -165,5 +165,31 @@
       <scope>test</scope>
       <type>jar</type>
     </dependency>
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.vintage</groupId>
+      <artifactId>junit-vintage-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java

@@ -73,6 +73,7 @@
 import java.util.ListIterator;
 import java.util.NoSuchElementException;
 import java.util.stream.Collectors;
+import com.aliyun.oss.common.comm.SignVersion;
 
 import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
 
@@ -113,6 +114,16 @@ public void initialize(URI uri, Configuration conf, String user,
         conf.get(USER_AGENT_PREFIX, USER_AGENT_PREFIX_DEFAULT) + ", Hadoop/"
             + VersionInfo.getVersion());
 
+    String region = conf.get(REGION_KEY, "");
+    String signatureVersion = conf.get(SIGNATURE_VERSION_KEY, SIGNATURE_VERSION_DEFAULT);
+    if ("V4".equalsIgnoreCase(signatureVersion)) {
+      clientConf.setSignatureVersion(SignVersion.V4);
+      if (StringUtils.isEmpty(region)) {
+        LOG.error("Signature version is V4 ,but region is empty.");
+        throw new IOException("SignVersion is V4 but region is empty");
+      }
+    }
+
     String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
     int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
     if (StringUtils.isNotEmpty(proxyHost)) {
@@ -171,6 +182,11 @@ public void initialize(URI uri, Configuration conf, String user,
       statistics.incrementWriteOps(1);
     }
 
+    if (StringUtils.isNotEmpty(region)) {
+      ossClient.setRegion(region);
+      LOG.debug("ossClient setRegion {}", region);
+    }
+
     maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
     int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION);
     if (listVersion < 1 || listVersion > 2) {

hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java

@@ -211,4 +211,19 @@ private Constants() {
   public static final String LIST_VERSION = "fs.oss.list.version";
 
   public static final int DEFAULT_LIST_VERSION = 2;
+
+  /**
+   * OSS signature version.
+   */
+  public static final String SIGNATURE_VERSION_KEY = "fs.oss.signatureversion";
+
+  /**
+   * OSS signature version DEFAULT {@value}.
+   */
+  public static final String SIGNATURE_VERSION_DEFAULT = "V1";
+
+  /**
+   * OSS region {@value}.
+   */
+  public static final String REGION_KEY = "fs.oss.region";
 }

hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java

@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.REGION_KEY;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SIGNATURE_VERSION_KEY;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.junit.Assert.*;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Tests Aliyun OSS system.
+ */
+public class ITAliyunOSSSignatureV4 {
+  private static final Logger LOG = LoggerFactory.getLogger(ITAliyunOSSSignatureV4.class);
+  private Configuration conf;
+  private URI testURI;
+  private Path testFile = new Path("ITAliyunOSSSignatureV4/atestr");
+
+  @Before
+  public void setUp() throws Exception {
+    conf = new Configuration();
+    String bucketUri = conf.get("test.fs.oss.name");
+    LOG.debug("bucketUri={}", bucketUri);
+    testURI = URI.create(bucketUri);
+  }
+
+  @Test
+  public void testV4() throws IOException {
+    conf.set(SIGNATURE_VERSION_KEY, "V4");
+    conf.set(REGION_KEY, "cn-hongkong");
+    AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+    fs.initialize(testURI, conf);
+    assumeNotNull(fs);
+
+    createFile(fs, testFile, true, dataset(256, 0, 255));
+    FileStatus status = fs.getFileStatus(testFile);
+    fs.delete(testFile);
+    fs.close();
+  }
+
+  @Test
+  public void testDefaultSignatureVersion() throws IOException {
+    AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+    fs.initialize(testURI, conf);
+    assumeNotNull(fs);
+
+    Path testFile2 = new Path("/test/atestr");
+    createFile(fs, testFile2, true, dataset(256, 0, 255));
+    FileStatus status = fs.getFileStatus(testFile2);
+    fs.delete(testFile2);
+    fs.close();
+  }
+
+  @Test
+  public void testV4WithoutRegion() throws IOException {
+    conf.set(SIGNATURE_VERSION_KEY, "V4");
+    AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
+    IOException expectedException = null;
+    try {
+      fs.initialize(testURI, conf);
+    } catch (IOException e) {
+      LOG.warn("use V4 , but do not set region, get exception={}", e);
+      expectedException = e;
+      assertEquals("use V4 , but do not set region", e.getMessage(),
+              "SignVersion is V4 but region is empty");
+    }
+    assertNotNull(expectedException);
+  }
+}

hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties

@@ -21,3 +21,6 @@ log4j.threshold=ALL
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# Log all oss classes
+log4j.logger.org.apache.hadoop.fs.aliyun.oss=DEBUG

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java

@@ -65,6 +65,8 @@ public enum AzureServiceErrorCode {
   COPY_BLOB_ABORTED("CopyBlobAborted", HttpURLConnection.HTTP_INTERNAL_ERROR, null),
   BLOB_OPERATION_NOT_SUPPORTED("BlobOperationNotSupported", HttpURLConnection.HTTP_CONFLICT, null),
   INVALID_APPEND_OPERATION("InvalidAppendOperation", HttpURLConnection.HTTP_CONFLICT, null),
+  UNAUTHORIZED_BLOB_OVERWRITE("UnauthorizedBlobOverwrite", HttpURLConnection.HTTP_FORBIDDEN,
+          "This request is not authorized to perform blob overwrites."),
   UNKNOWN(null, -1, null);
 
   private final String errorCode;

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsDfsClient.java

@@ -42,6 +42,7 @@
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
 import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore;
 import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
@@ -132,6 +133,8 @@
 import static org.apache.hadoop.fs.azurebfs.constants.HttpQueryParams.QUERY_PARAM_RETAIN_UNCOMMITTED_DATA;
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.RENAME_DESTINATION_PARENT_PATH_NOT_FOUND;
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.SOURCE_PATH_NOT_FOUND;
+import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.UNAUTHORIZED_BLOB_OVERWRITE;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_FILE_ALREADY_EXISTS;
 
 /**
  * AbfsClient interacting with the DFS Endpoint.
@@ -702,6 +705,14 @@ public AbfsClientRenameResult renamePath(
         throw e;
       }
 
+      // ref: HADOOP-19393. Write permission checks can occur before validating
+      // rename operation's validity. If there is an existing destination path, it may be rejected
+      // with an authorization error. Catching and throwing FileAlreadyExistsException instead.
+      if (op.getResult().getStorageErrorCode()
+          .equals(UNAUTHORIZED_BLOB_OVERWRITE.getErrorCode())){
+        throw new FileAlreadyExistsException(ERR_FILE_ALREADY_EXISTS);
+      }
+
       // ref: HADOOP-18242. Rename failure occurring due to a rare case of
       // tracking metadata being in incomplete state.
       if (op.getResult().getStorageErrorCode()

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java

@@ -28,6 +28,7 @@
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public final class AbfsErrors {
+  public static final String ERR_FILE_ALREADY_EXISTS = "File already exists.";
   public static final String ERR_WRITE_WITHOUT_LEASE = "Attempted to write to file without lease";
   public static final String ERR_LEASE_EXPIRED = "A lease ID was specified, but the lease for the resource has expired.";
   public static final String ERR_LEASE_EXPIRED_BLOB = "A lease ID was specified, but the lease for the blob has expired.";