Skip to content

[Preview] SDK types - BlobClient and BlobContainerClient #813

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 23 commits into from
Mar 31, 2025
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
da00603
initial commit
ahmedmuhsin Feb 21, 2025
5ed671b
hook up model binding data and create one invocationChain factory per…
ahmedmuhsin Feb 28, 2025
c02b9cd
add parameter checking logic and an object cache
ahmedmuhsin Mar 6, 2025
6389f5c
move sdktype lib
ahmedmuhsin Mar 10, 2025
16793b6
remove the need to use middlewarecontext in sdktype
ahmedmuhsin Mar 11, 2025
814e655
move sdktypes lib to java-additions
ahmedmuhsin Mar 11, 2025
98d648c
update protobuf to a version that supports arm
ahmedmuhsin Mar 27, 2025
619b528
use sdkmetadata instead of sdktypes directly
ahmedmuhsin Mar 27, 2025
339dfd0
add shaded azure-identity in case user doesn't include it
ahmedmuhsin Mar 28, 2025
07c6a42
pipeline workaround
ahmedmuhsin Mar 28, 2025
755f9c9
add emulated tests for blobclient and blobcontainerclient
ahmedmuhsin Mar 28, 2025
cc13be7
remove azure-identity from the worker since it causes conflicts even …
ahmedmuhsin Mar 28, 2025
e3dc1c9
fix emulated tests
ahmedmuhsin Mar 28, 2025
477754a
put sdk types feature behind an app setting
ahmedmuhsin Mar 31, 2025
3650df0
fix build for linux testing
ahmedmuhsin Mar 31, 2025
f975d10
remove depends
ahmedmuhsin Mar 31, 2025
f53a3df
update flag name
ahmedmuhsin Mar 31, 2025
ca7e2cb
fix dependsOn
ahmedmuhsin Mar 31, 2025
11fbfdf
update build
ahmedmuhsin Mar 31, 2025
9221c84
update official build pipeline
ahmedmuhsin Mar 31, 2025
3b68fb5
change branch back to sdk-types
ahmedmuhsin Mar 31, 2025
c239cfe
disable 17 and 21 temporarily due to a spotbugs plugin issue
ahmedmuhsin Mar 31, 2025
98bb299
update repo and branch
ahmedmuhsin Mar 31, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ public static class Constants
public static string TriggerInputBindingBlobContainer = "test-triggerinput-java-new";
public static string InputBindingBlobContainer = "test-input-java-new";
public static string OutputBindingBlobContainer = "test-output-java-new";
public static string TriggerInputBindingBlobClientSdk = "test-triggerinput-blobclient";
public static string TriggerInputBindingBlobContainerClientSdk = "test-triggerinput-blobcontclient";

// Xunit Fixtures and Collections
public const string FunctionAppCollectionName = "FunctionAppCollection";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,17 @@ public async static Task ClearBlobContainers()
await ClearBlobContainer(Constants.TriggerInputBindingBlobContainer);
await ClearBlobContainer(Constants.InputBindingBlobContainer);
await ClearBlobContainer(Constants.OutputBindingBlobContainer);
await ClearBlobContainer(Constants.TriggerInputBindingBlobClientSdk);
await ClearBlobContainer(Constants.TriggerInputBindingBlobContainerClientSdk);
}

public async static Task CreateBlobContainers()
{
await CreateBlobContainer(Constants.TriggerInputBindingBlobContainer);
await CreateBlobContainer(Constants.InputBindingBlobContainer);
await CreateBlobContainer(Constants.OutputBindingBlobContainer);
await CreateBlobContainer(Constants.TriggerInputBindingBlobClientSdk);
await CreateBlobContainer(Constants.TriggerInputBindingBlobContainerClientSdk);
}

public async static Task UpdloadFileToContainer(string containerName, string expectedFileName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,5 +110,45 @@ public async Task BlobTriggerToBlob_Succeeds()

Assert.Equal("Hello World", result);
}

[Fact]
[Trait("Category", "SdkTypes")]
public async Task BlobTriggerToBlob_BlobClient_Succeeds()
{
string fileName = Guid.NewGuid().ToString();

//cleanup
await StorageHelpers.ClearBlobContainers();

//Setup
await StorageHelpers.CreateBlobContainers();

//Trigger
await StorageHelpers.UpdloadFileToContainer(Constants.TriggerInputBindingBlobClientSdk, fileName);

//Verify
string result = await StorageHelpers.DownloadFileFromContainer(Constants.OutputBindingBlobContainer, "testfile");
Assert.Equal("Hello World", result);
}

[Fact]
[Trait("Category", "SdkTypes")]
public async Task BlobTriggerToBlob_BlobContainerClient_Succeeds()
{
string fileName = Guid.NewGuid().ToString();

//cleanup
await StorageHelpers.ClearBlobContainers();

//Setup
await StorageHelpers.CreateBlobContainers();

//Trigger
await StorageHelpers.UpdloadFileToContainer(Constants.TriggerInputBindingBlobContainerClientSdk, fileName);

//Verify
string result = await StorageHelpers.DownloadFileFromContainer(Constants.OutputBindingBlobContainer, "testfile");
Assert.Equal("Hello World", result);
}
}
}
12 changes: 11 additions & 1 deletion emulatedtests/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,16 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.8</java.version>
<azure.functions.maven.plugin.version>1.18.0</azure.functions.maven.plugin.version>
<azure.functions.maven.plugin.version>1.37.1</azure.functions.maven.plugin.version>
<azure.functions.java.library.version>3.1.0</azure.functions.java.library.version>
<azure.functions.java.library.sql.version>2.1.0</azure.functions.java.library.sql.version>
<durabletask.azure.functions>1.0.0-beta.1</durabletask.azure.functions>
<functionAppName>azure-functions-java-emulatedtests</functionAppName>
<functionAppRegion>westus</functionAppRegion>
<functionResourceGroup>java-functions-group</functionResourceGroup>
<start-class>com.microsoft.azure.functions.endtoend.springcloud.Config</start-class>
<!-- Empty by default: no exclusion -->
<excludedClassPattern>**/___noop___.java</excludedClassPattern>
</properties>

<repositories>
Expand Down Expand Up @@ -112,6 +114,11 @@
<artifactId>spring-cloud-starter-function-web</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
<version>12.29.0</version>
</dependency>
</dependencies>

<dependencyManagement>
Expand All @@ -136,6 +143,9 @@
<source>${java.version}</source>
<target>${java.version}</target>
<encoding>${project.build.sourceEncoding}</encoding>
<excludes>
<exclude>${excludedClassPattern}</exclude>
</excludes>
</configuration>
</plugin>
<plugin>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package com.microsoft.azure.functions.endtoend;

import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.microsoft.azure.functions.ExecutionContext;
import com.microsoft.azure.functions.OutputBinding;
import com.microsoft.azure.functions.annotation.*;

import java.io.ByteArrayOutputStream;

/**
* Azure Functions with Azure Storage Blob.
*/
public class BlobTriggerSdkTypesTests {
/**
* This function will be invoked when a new or updated blob is detected at the specified path. The blob contents are provided as input to this function.
*/
@FunctionName("BlobTriggerUsingBlobClientToBlobTest")
@StorageAccount("AzureWebJobsStorage")
public void BlobTriggerToBlobTest_BlobClient(
@BlobTrigger(name = "triggerBlob", path = "test-triggerinput-blobclient/{name}", dataType = "binary") BlobClient triggerBlobClient,
@BindingName("name") String fileName,
@BlobOutput(name = "outputBlob", path = "test-output-java-new/testfile.txt", dataType = "binary") OutputBinding<byte[]> outputBlob,
final ExecutionContext context
) {
context.getLogger().info("BlobTriggerUsingBlobClient triggered for blob: " + fileName);

// Download the blob content
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
triggerBlobClient.downloadStream(outputStream);

// Set the downloaded content as output
outputBlob.setValue(outputStream.toByteArray());
context.getLogger().info("Uploaded blob " + fileName + " to container test-output-java-new/testfile.txt");
}

/**
* This function will be invoked when a new or updated blob is detected at the specified path. The blob contents are provided as input to this function.
*/
@FunctionName("BlobTriggerUsingBlobContainerClientToBlobTest")
@StorageAccount("AzureWebJobsStorage")
public void BlobTriggerToBlobTest_BlobContainerClient(
@BlobTrigger(name = "triggerBlob", path = "test-triggerinput-blobcontclient/{name}", dataType = "binary") BlobContainerClient triggerBlobContainerClient,
@BindingName("name") String fileName,
@BlobOutput(name = "outputBlob", path = "test-output-java-new/testfile.txt", dataType = "binary") OutputBinding<byte[]> outputBlob,
final ExecutionContext context
) {
context.getLogger().info("BlobTriggerUsingBlobContainerClient triggered for blob: " + fileName);

// Download the blob content
BlobClient triggerBlobClient = triggerBlobContainerClient.getBlobClient(fileName);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
triggerBlobClient.downloadStream(outputStream);

// Set the downloaded content as output
outputBlob.setValue(outputStream.toByteArray());
context.getLogger().info("Uploaded blob " + fileName + " to container test-output-java-new/testfile.txt");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@

import com.microsoft.azure.functions.annotation.*;

import java.util.List;
import java.util.Optional;

import com.microsoft.azure.functions.*;

/**
Expand Down Expand Up @@ -56,7 +53,7 @@ public void BlobTriggerStringTest(
context.getLogger().info("Java Blob trigger function BlobTriggerStringTest processed a blob.\n Name: " + fileName + "\n Content: " + triggerBlobText);
outputBlob.setValue(triggerBlobText);
}

public static class TestBlobData {
public String blobText;
}
Expand Down
7 changes: 7 additions & 0 deletions eng/ci/public-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ resources:
name: 1ESPipelineTemplates/1ESPipelineTemplates
ref: refs/tags/release

parameters:
- name: runSdkTypesTests
type: boolean
default: false

extends:
template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
parameters:
Expand Down Expand Up @@ -55,10 +60,12 @@ extends:
- template: /eng/ci/templates/jobs/run-emulated-tests-windows.yml@self
parameters:
poolName: 1es-pool-azfunc-public
runSdkTypesTests: ${{ parameters.runSdkTypesTests }}

- stage: TestLinux
dependsOn: Build
jobs:
- template: /eng/ci/templates/jobs/run-emulated-tests-linux.yml@self
parameters:
poolName: 1es-pool-azfunc-public
runSdkTypesTests: ${{ parameters.runSdkTypesTests }}
3 changes: 3 additions & 0 deletions eng/ci/templates/jobs/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ jobs:
- pwsh: |
java -version
displayName: 'Check default java version'
- pwsh: |
.\installLibsLocally.ps1
displayName: 'Install java-additions locally'
- pwsh: |
mvn clean package
displayName: 'Build java worker'
97 changes: 74 additions & 23 deletions eng/ci/templates/jobs/run-emulated-tests-linux.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
parameters:
poolName: ''
- name: poolName
type: string
default: ''
- name: runSdkTypesTests
type: boolean
default: false

jobs:
- job: "TestLinux"
Expand Down Expand Up @@ -57,18 +62,6 @@ jobs:
- pwsh: |
java -version
displayName: 'Check default java version'
- pwsh: |
if ("$(isTag)"){
$buildNumber="$(Build.SourceBranchName)"
Write-Host "Found git tag."
}
else {
$buildNumber="$(Build.BuildNumber)-v4"
Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
}
Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
.\package-pipeline.ps1 -buildNumber $buildNumber
displayName: 'Executing build script'
- task: UseDotNet@2
displayName: 'Install .NET 6'
inputs:
Expand All @@ -87,6 +80,30 @@ jobs:
jdkDestinationDirectory: "$(downloadPath)/externals"
cleanDestinationDirectory: true
displayName: 'Setup Java for Linux'
- bash: |
docker compose -f emulatedtests/utils/docker-compose.yml pull
docker compose -f emulatedtests/utils/docker-compose.yml up -d
displayName: 'Install Azurite and Start Emulators'
- pwsh: |
.\installLibsLocally.ps1
displayName: 'Install java-additions locally'
- pwsh: |
if ("$(isTag)"){
$buildNumber="$(Build.SourceBranchName)"
Write-Host "Found git tag."
}
else {
$buildNumber="$(Build.BuildNumber)-v4"
Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
}
Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
.\package-pipeline.ps1 -buildNumber $buildNumber
displayName: 'Executing build script'
- pwsh: |
cd ./emulatedtests
mvn clean package -DexcludedClassPattern="**/BlobTriggerSdkTypesTests.java" `-Dmaven`.javadoc`.skip=true `-Dmaven`.test`.skip `-Dorg`.slf4j`.simpleLogger`.log`.org`.apache`.maven`.cli`.transfer`.Slf4jMavenTransferListener=warn `-B
Copy-Item "confluent_cloud_cacert.pem" "./target/azure-functions/azure-functions-java-emulatedtests"
displayName: 'Package Java for E2E'
- pwsh: |
.\setup-tests-pipeline.ps1
displayName: 'Setup test environment -- Install the Core Tools'
Expand All @@ -96,23 +113,57 @@ jobs:
export PATH=$PATH:./Azure.Functions.Cli
func --version
displayName: 'Setup Core Tools - Linux'
- pwsh: |
cd ./emulatedtests
mvn clean package `-Dmaven`.javadoc`.skip=true `-Dmaven`.test`.skip `-Dorg`.slf4j`.simpleLogger`.log`.org`.apache`.maven`.cli`.transfer`.Slf4jMavenTransferListener=warn `-B
Copy-Item "confluent_cloud_cacert.pem" "./target/azure-functions/azure-functions-java-emulatedtests"
displayName: 'Package Java for E2E'
- bash: |
docker compose -f emulatedtests/utils/docker-compose.yml pull
docker compose -f emulatedtests/utils/docker-compose.yml up -d
displayName: 'Install Azurite and Start Emulators'
- task: DotNetCoreCLI@2
retryCountOnTaskFailure: 3
inputs:
command: 'test'
projects: |
emulatedtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E.csproj
arguments: '--filter "Category!=SdkTypes"'
env:
JAVA_HOME: $(JavaHome)
AzureWebJobsStorage: "UseDevelopmentStorage=true"
displayName: 'Build & Run tests'
continueOnError: false
continueOnError: false
# ------------------------------------------
# Conditionally run an additional set of steps for "SDK types" scenario
# ------------------------------------------
- ${{ if eq(parameters.runSdkTypesTests, true) }}:
- pwsh: |
if ("$(isTag)"){
$buildNumber="$(Build.SourceBranchName)"
Write-Host "Found git tag."
}
else {
$buildNumber="$(Build.BuildNumber)-v4"
Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
}
Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
.\package-pipeline.ps1 -buildNumber $buildNumber
displayName: 'Executing build script'
- pwsh: |
cd ./emulatedtests
mvn clean package `-Dmaven`.javadoc`.skip=true `-Dmaven`.test`.skip `-Dorg`.slf4j`.simpleLogger`.log`.org`.apache`.maven`.cli`.transfer`.Slf4jMavenTransferListener=warn `-B
Copy-Item "confluent_cloud_cacert.pem" "./target/azure-functions/azure-functions-java-emulatedtests"
displayName: 'Package Java for E2E'
- pwsh: |
.\setup-tests-pipeline.ps1
displayName: 'Setup test environment -- Install the Core Tools'
- bash: |
chmod +x ./Azure.Functions.Cli/func
chmod +x ./Azure.Functions.Cli/gozip
export PATH=$PATH:./Azure.Functions.Cli
func --version
displayName: 'Setup Core Tools - Linux'
- task: DotNetCoreCLI@2
retryCountOnTaskFailure: 3
inputs:
command: 'test'
projects: |
emulatedtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E.csproj
env:
JAVA_HOME: $(JavaHome)
AzureWebJobsStorage: "UseDevelopmentStorage=true"
ENABLE_SDK_TYPES: "true"
displayName: 'Build & Run tests'
continueOnError: false
Loading
Loading