Skip to content

Commit 4e922b4

Browse files
author
HarshitGupta
committed
Minor Changes
1 parent 1f56e2a commit 4e922b4

File tree

5 files changed

+45
-4
lines changed

5 files changed

+45
-4
lines changed

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1087,6 +1087,7 @@ protected RequestFactory createRequestFactory() {
10871087
.withRequestPreparer(getAuditManager()::requestCreated)
10881088
.withContentEncoding(contentEncoding)
10891089
.withStorageClass(storageClass)
1090+
.withMultipartEnabled(isMultipartEnabled)
10901091
.build();
10911092
}
10921093

@@ -1839,7 +1840,7 @@ private FSDataOutputStream innerCreateFile(
18391840

18401841
if(!checkDiskBuffer(getConf())){
18411842
throw new IOException("Unable to create OutputStream with the given"
1842-
+ "multipart upload and buffer configuration.");
1843+
+ " multipart upload and buffer configuration.");
18431844
}
18441845

18451846
final S3ABlockOutputStream.BlockOutputStreamBuilder builder =
@@ -5424,4 +5425,8 @@ public RequestFactory getRequestFactory() {
54245425
public boolean isCSEEnabled() {
54255426
return isCSEEnabled;
54265427
}
5428+
5429+
public boolean isMultipartEnabled() {
5430+
return isMultipartEnabled;
5431+
}
54275432
}

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import javax.annotation.Nullable;
2222
import java.io.File;
23+
import java.io.IOException;
2324
import java.io.InputStream;
2425
import java.util.List;
2526
import java.util.Optional;
@@ -199,7 +200,7 @@ AbortMultipartUploadRequest newAbortMultipartUploadRequest(
199200
*/
200201
InitiateMultipartUploadRequest newMultipartUploadRequest(
201202
String destKey,
202-
@Nullable PutObjectOptions options);
203+
@Nullable PutObjectOptions options) throws IOException;
203204

204205
/**
205206
* Complete a multipart upload.

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,10 @@ protected AbstractS3ACommitter(
217217
LOG.debug("{} instantiated for job \"{}\" ID {} with destination {}",
218218
role, jobName(context), jobIdString(context), outputPath);
219219
S3AFileSystem fs = getDestS3AFS();
220+
if (!fs.isMultipartEnabled()) {
221+
throw new PathCommitException(outputPath, "Multipart uploads are disabled for the FileSystem,"
222+
+ " the committer can't proceed.");
223+
}
220224
// set this thread's context with the job ID.
221225
// audit spans created in this thread will pick
222226
// up this value., including the commit operations instance

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ public class RequestFactoryImpl implements RequestFactory {
124124
*/
125125
private final StorageClass storageClass;
126126

127+
/**
128+
* Is Multipart Enabled
129+
*/
130+
private final boolean isMultipartEnabled;
131+
127132
/**
128133
* Constructor.
129134
* @param builder builder with all the configuration.
@@ -137,6 +142,7 @@ protected RequestFactoryImpl(
137142
this.requestPreparer = builder.requestPreparer;
138143
this.contentEncoding = builder.contentEncoding;
139144
this.storageClass = builder.storageClass;
145+
this.isMultipartEnabled = builder.isMultipartEnabled;
140146
}
141147

142148
/**
@@ -460,7 +466,10 @@ public AbortMultipartUploadRequest newAbortMultipartUploadRequest(
460466
@Override
461467
public InitiateMultipartUploadRequest newMultipartUploadRequest(
462468
final String destKey,
463-
@Nullable final PutObjectOptions options) {
469+
@Nullable final PutObjectOptions options) throws IOException {
470+
if(!isMultipartEnabled){
471+
throw new IOException("Multipart uploads are disabled on the given filesystem.");
472+
}
464473
final ObjectMetadata objectMetadata = newObjectMetadata(-1);
465474
maybeSetMetadata(options, objectMetadata);
466475
final InitiateMultipartUploadRequest initiateMPURequest =
@@ -682,6 +691,11 @@ public static final class RequestFactoryBuilder {
682691
*/
683692
private PrepareRequest requestPreparer;
684693

694+
/**
695+
* Is Multipart Enabled on the path.
696+
*/
697+
private boolean isMultipartEnabled = true;
698+
685699
private RequestFactoryBuilder() {
686700
}
687701

@@ -767,6 +781,18 @@ public RequestFactoryBuilder withRequestPreparer(
767781
this.requestPreparer = value;
768782
return this;
769783
}
784+
785+
/**
786+
* Multipart enabled
787+
*
788+
* @param value new value
789+
* @return the builder
790+
*/
791+
public RequestFactoryBuilder withMultipartEnabled(
792+
final boolean value) {
793+
this.isMultipartEnabled = value;
794+
return this;
795+
}
770796
}
771797

772798
/**

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import java.io.ByteArrayInputStream;
2222
import java.io.File;
23+
import java.io.IOException;
2324
import java.util.ArrayList;
2425
import java.util.concurrent.atomic.AtomicLong;
2526

@@ -173,7 +174,11 @@ private void createFactoryObjects(RequestFactory factory) {
173174
a(factory.newListObjectsV1Request(path, "/", 1));
174175
a(factory.newListNextBatchOfObjectsRequest(new ObjectListing()));
175176
a(factory.newListObjectsV2Request(path, "/", 1));
176-
a(factory.newMultipartUploadRequest(path, null));
177+
try {
178+
a(factory.newMultipartUploadRequest(path, null));
179+
} catch (IOException e) {
180+
throw new RuntimeException(e);
181+
}
177182
File srcfile = new File("/tmp/a");
178183
a(factory.newPutObjectRequest(path,
179184
factory.newObjectMetadata(-1), null, srcfile));

0 commit comments

Comments
 (0)