Skip to content

Commit 0d21404

Browse files
committed
fixed the problem. see #1. Just took this PR code and added a new endpiont so the difference could be seen.
1 parent 7e3d61e commit 0d21404

File tree

4 files changed

+44
-5
lines changed

4 files changed

+44
-5
lines changed

MsAzureBlobstorageLargeFileProblemApp.run.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<module name="downloader" />
44
<option name="SHORTEN_COMMAND_LINE" value="ARGS_FILE" />
55
<option name="SPRING_BOOT_MAIN_CLASS" value="com.azureproblem.blob.application.MsAzureBlobstorageLargeFileProblemApp" />
6-
<option name="VM_PARAMETERS" value="-Xmx5G -XX:+CrashOnOutOfMemoryError" />
6+
<option name="VM_PARAMETERS" value="-Xmx500M -XX:+CrashOnOutOfMemoryError" />
77
<extension name="coverage">
88
<pattern>
99
<option name="PATTERN" value="com.azureproblem.blob.application.*" />

MsAzureBlobstorageLargeFileProblemFileServerApp.run.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<module name="fileserver" />
44
<option name="SHORTEN_COMMAND_LINE" value="ARGS_FILE" />
55
<option name="SPRING_BOOT_MAIN_CLASS" value="com.azureproblem.bug.application.MsAzureBlobstorageLargeFileProblemFileServerApp" />
6-
<option name="VM_PARAMETERS" value="-Xmx5G -XX:+CrashOnOutOfMemoryError" />
6+
<option name="VM_PARAMETERS" value="-Xmx20G -XX:+CrashOnOutOfMemoryError" />
77
<extension name="coverage">
88
<pattern>
99
<option name="PATTERN" value="com.azureproblem.bug.application.*" />

README.md

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,18 @@ please checkout the log of the app. You see that it only this msg and the app ha
4545

4646
It is strange that the app can write (stream) any file to the file system. But we can not stream
4747
something bigger than 250MB to the blobstorage, although we use a Output stream the same as in
48-
the File case.
48+
the File case.
4949

5050
## Generate Testdata on the client side and store it to the BlobStorage
5151

52-
This calls generate test data on the client side and writes it directly to the Blob Storage. Not
52+
This calls generate test data on the client side and writes it directly to the Blob Storage. Not
5353
Data is transmitted via REST.
54-
`localhost:8544/trigger-generation-of-random-data-to-blobstore/260`
54+
`localhost:8544/trigger-generation-of-random-data-to-blobstore/260`
55+
56+
# Solution
57+
58+
The Problem could be solved by using `BlobAsyncClient`. In the code I added a working version see
59+
`trigger-download-to-blob-working-ok`.
60+
61+
See also https://github.com/git9999999/azure-blob-large-file-upload-problem/pull/1 Thansk a lot
62+
to alzimmermsft.

downloader/src/main/java/com/azureproblem/blob/controller/AzureBlobBugDownloaderController.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
import static org.apache.commons.lang3.ObjectUtils.isEmpty;
44

5+
import com.azure.storage.blob.BlobContainerAsyncClient;
56
import com.azure.storage.blob.BlobContainerClient;
7+
import com.azure.storage.blob.BlobServiceAsyncClient;
68
import com.azure.storage.blob.BlobServiceClient;
79
import com.azure.storage.blob.BlobServiceClientBuilder;
810
import com.azure.storage.blob.models.ParallelTransferOptions;
@@ -11,6 +13,7 @@
1113
import java.io.FileOutputStream;
1214
import java.io.IOException;
1315
import java.io.OutputStream;
16+
import java.nio.ByteBuffer;
1417
import java.nio.file.Files;
1518
import java.nio.file.Path;
1619
import java.security.SecureRandom;
@@ -48,11 +51,14 @@ public class AzureBlobBugDownloaderController {
4851
private final WebClient webClient;
4952

5053
private final ParallelTransferOptions parallelTransferOptions;
54+
private final BlobContainerAsyncClient blobContainerAsyncClient;
5155

5256

5357
public AzureBlobBugDownloaderController(BlobServiceClientBuilder blobServiceClientBuilder) {
5458
this.blobServiceClient = blobServiceClientBuilder.buildClient();
5559
this.containerClient = this.blobServiceClient.getBlobContainerClient(BLOB_CONTAINER_NAME);
60+
BlobServiceAsyncClient blobServiceAsyncClient = blobServiceClientBuilder.buildAsyncClient();
61+
this.blobContainerAsyncClient = blobServiceAsyncClient.getBlobContainerAsyncClient(BLOB_CONTAINER_NAME);
5662
log.debug("technicalAppIp '{}' technicalAppPort '{}'", "8545", "8545");
5763
this.webClient = msCommonWebClientBuilder().baseUrl(createUrl("client", "localhost", "8545")).build();
5864

@@ -156,6 +162,31 @@ public void triggerDownloadToBlob(@PathVariable int fileSizeInMb) {
156162
}
157163

158164

165+
@GetMapping(path = "/trigger-download-to-blob-working-ok/{fileSizeInMb}")
166+
public void triggerDownloadToBlobWorkingOk(@PathVariable int fileSizeInMb) {
167+
log.info("triggerDownloadToBlob");
168+
169+
var flux = this.webClient
170+
.get()
171+
.uri("/serve-file/" + fileSizeInMb)
172+
.accept(MediaType.APPLICATION_OCTET_STREAM)
173+
.exchangeToFlux(clientResponse -> clientResponse.body(BodyExtractors.toDataBuffers()));
174+
175+
var destination = "TestDownloadToAzureBlobStorage" + System.currentTimeMillis() + ".pdf";
176+
177+
var blobClientTarget = this.blobContainerAsyncClient.getBlobAsyncClient(destination);
178+
179+
blobClientTarget.upload(flux.map(dataBuffer -> {
180+
ByteBuffer buffer = ByteBuffer.allocate(dataBuffer.readableByteCount());
181+
dataBuffer.toByteBuffer(buffer);
182+
DataBufferUtils.release(dataBuffer);
183+
return buffer;
184+
}), this.parallelTransferOptions).block();
185+
186+
log.info("!!!!!!!!!!!!!!!!!!!!!!!!!!!! end download of {}", destination);
187+
}
188+
189+
159190
/**
160191
* https://stackoverflow.com/questions/76575117/can-not-transfer-more-than-250mb-with-databufferutils-write-to-azure-blob-storag/76580753#76580753
161192
*/

0 commit comments

Comments
 (0)