From 8525a1bef7b4827bd6f732450fa45e33f5dd22ee Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 14:02:46 -0500 Subject: [PATCH 01/15] Error if invalid connection --- .../extensions/bindings/blob/blobClient.py | 17 +++++++++++-- .../bindings/blob/blobClientConverter.py | 1 - .../bindings/blob/containerClient.py | 1 - .../bindings/blob/storageStreamDownloader.py | 1 - .../tests/test_blobclient.py | 25 +++++++++++++++++++ 5 files changed, 40 insertions(+), 5 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index d571882..36dbe44 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -6,12 +6,12 @@ from typing import Union from azure.storage.blob import BlobClient as BlobClientSdk +from azure.storage.blob import BlobServiceClient from azurefunctions.extensions.base import Datum, SdkType class BlobClient(SdkType): def __init__(self, *, data: Union[bytes, Datum]) -> None: - # model_binding_data properties self._data = data self._version = None @@ -25,12 +25,25 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = os.getenv(content_json["Connection"]) + self._connection = content_json["Connection"] self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] + def validate_connection_string(self): + """ + Validates the connection string. If the connection string is + not an App Setting, an error will be thrown. + """ + if not os.getenv(self._connection): + raise ValueError( + f"Storage account connection string {self._connection} does not exist. " + f"Please make sure that it is a defined App Setting." + ) + self._connection = os.getenv(self._connection) + # Returns a BlobClient def get_sdk_type(self): + BlobClient.validate_connection_string(self) if self._data: return BlobClientSdk.from_connection_string( conn_str=self._connection, diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClientConverter.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClientConverter.py index f7fd4aa..f07193c 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClientConverter.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClientConverter.py @@ -16,7 +16,6 @@ class BlobClientConverter( binding="blob", trigger="blobTrigger", ): - @classmethod def check_input_type_annotation(cls, pytype: type) -> bool: return issubclass( diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index de6e6e9..6f1d77f 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -11,7 +11,6 @@ class ContainerClient(SdkType): def __init__(self, *, data: Union[bytes, Datum]) -> None: - # model_binding_data properties self._data = data self._version = "" diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index 2d26757..893a020 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -11,7 +11,6 @@ class StorageStreamDownloader(SdkType): def __init__(self, *, data: Union[bytes, Datum]) -> None: - # model_binding_data properties self._data = data or {} self._version = "" diff --git a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py index eed412f..893b20f 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py @@ -126,6 +126,31 @@ def test_input_populated(self): self.assertIsNotNone(sdk_result) self.assertIsInstance(sdk_result, BlobClientSdk) + def test_invalid_input_populated(self): + content = { + "Connection": "NotARealConnectionString", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + with self.assertRaises(ValueError) as e: + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: BlobClient = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=BlobClient + ) + self.assertEqual( + e.exception, + "Storage account connection string NotARealConnectionString does not exist. " + "Please make sure that it is a defined App Setting.", + ) + def test_input_invalid_pytype(self): content = { "Connection": "AzureWebJobsStorage", From 7dbe27be0c4a8d8859075ee1277e0c1874f075e8 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 14:09:57 -0500 Subject: [PATCH 02/15] Error for all types --- .../bindings/blob/containerClient.py | 15 ++++++++++- .../bindings/blob/storageStreamDownloader.py | 15 ++++++++++- .../tests/test_blobclient.py | 2 +- .../tests/test_containerclient.py | 25 +++++++++++++++++++ .../tests/test_ssd.py | 25 +++++++++++++++++++ 5 files changed, 79 insertions(+), 3 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index 6f1d77f..63aa995 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -24,12 +24,25 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = os.getenv(content_json["Connection"]) + self._connection = content_json["Connection"] self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] + def validate_connection_string(self): + """ + Validates the connection string. If the connection string is + not an App Setting, an error will be thrown. + """ + if not os.getenv(self._connection): + raise ValueError( + f"Storage account connection string {self._connection} does not exist. " + f"Please make sure that it is a defined App Setting." + ) + self._connection = os.getenv(self._connection) + # Returns a ContainerClient def get_sdk_type(self): + ContainerClient.validate_connection_string(self) if self._data: return ContainerClientSdk.from_connection_string( conn_str=self._connection, container_name=self._containerName diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index 893a020..57c3c78 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -24,12 +24,25 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = os.getenv(content_json["Connection"]) + self._connection = content_json["Connection"] self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] + def validate_connection_string(self): + """ + Validates the connection string. If the connection string is + not an App Setting, an error will be thrown. + """ + if not os.getenv(self._connection): + raise ValueError( + f"Storage account connection string {self._connection} does not exist. " + f"Please make sure that it is a defined App Setting." + ) + self._connection = os.getenv(self._connection) + # Returns a StorageStreamDownloader def get_sdk_type(self): + StorageStreamDownloader.validate_connection_string(self) if self._data: # Create BlobClient blob_client = BlobClientSdk.from_connection_string( diff --git a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py index 893b20f..d60b7ee 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py @@ -146,7 +146,7 @@ def test_invalid_input_populated(self): data=datum, trigger_metadata=None, pytype=BlobClient ) self.assertEqual( - e.exception, + e.exception.args[0], "Storage account connection string NotARealConnectionString does not exist. " "Please make sure that it is a defined App Setting.", ) diff --git a/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py b/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py index c207117..3ab8c3b 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py @@ -124,6 +124,31 @@ def test_input_populated(self): self.assertIsNotNone(sdk_result) self.assertIsInstance(sdk_result, ContainerClientSdk) + def test_invalid_input_populated(self): + content = { + "Connection": "NotARealConnectionString", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + with self.assertRaises(ValueError) as e: + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: ContainerClient = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=ContainerClient + ) + self.assertEqual( + e.exception.args[0], + "Storage account connection string NotARealConnectionString does not exist. " + "Please make sure that it is a defined App Setting.", + ) + def test_input_invalid_pytype(self): content = { "Connection": "AzureWebJobsStorage", diff --git a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py index 56ed88c..e848a0e 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py @@ -129,6 +129,31 @@ def test_input_populated(self): self.assertIsNotNone(sdk_result) self.assertIsInstance(sdk_result, SSDSdk) + def test_invalid_input_populated(self): + content = { + "Connection": "NotARealConnectionString", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + with self.assertRaises(ValueError) as e: + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: StorageStreamDownloader = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=StorageStreamDownloader + ) + self.assertEqual( + e.exception.args[0], + "Storage account connection string NotARealConnectionString does not exist. " + "Please make sure that it is a defined App Setting.", + ) + def test_input_invalid_pytype(self): content = { "Connection": "AzureWebJobsStorage", From 0bdd1eea81104e88db1dfc72aeef9e83ce587e29 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 15:49:24 -0500 Subject: [PATCH 03/15] Validate during type init --- .../extensions/bindings/blob/blobClient.py | 28 +++++++++---------- .../bindings/blob/containerClient.py | 27 +++++++++--------- .../bindings/blob/storageStreamDownloader.py | 27 +++++++++--------- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index 36dbe44..d541919 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -25,25 +25,12 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = content_json["Connection"] + self._connection = validate_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] - def validate_connection_string(self): - """ - Validates the connection string. If the connection string is - not an App Setting, an error will be thrown. - """ - if not os.getenv(self._connection): - raise ValueError( - f"Storage account connection string {self._connection} does not exist. " - f"Please make sure that it is a defined App Setting." - ) - self._connection = os.getenv(self._connection) - # Returns a BlobClient def get_sdk_type(self): - BlobClient.validate_connection_string(self) if self._data: return BlobClientSdk.from_connection_string( conn_str=self._connection, @@ -52,3 +39,16 @@ def get_sdk_type(self): ) else: return None + + +def validate_connection_string(connection_string: str) -> str: + """ + Validates the connection string. If the connection string is + not an App Setting, an error will be thrown. + """ + if not os.getenv(connection_string): + raise ValueError( + f"Storage account connection string {connection_string} does not exist. " + f"Please make sure that it is a defined App Setting." + ) + return os.getenv(connection_string) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index 63aa995..882b815 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -24,22 +24,10 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = content_json["Connection"] + self._connection = validate_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] - def validate_connection_string(self): - """ - Validates the connection string. If the connection string is - not an App Setting, an error will be thrown. - """ - if not os.getenv(self._connection): - raise ValueError( - f"Storage account connection string {self._connection} does not exist. " - f"Please make sure that it is a defined App Setting." - ) - self._connection = os.getenv(self._connection) - # Returns a ContainerClient def get_sdk_type(self): ContainerClient.validate_connection_string(self) @@ -49,3 +37,16 @@ def get_sdk_type(self): ) else: return None + + +def validate_connection_string(connection_string: str) -> str: + """ + Validates the connection string. If the connection string is + not an App Setting, an error will be thrown. + """ + if not os.getenv(connection_string): + raise ValueError( + f"Storage account connection string {connection_string} does not exist. " + f"Please make sure that it is a defined App Setting." + ) + return os.getenv(connection_string) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index 57c3c78..06e0520 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -24,22 +24,10 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = content_json["Connection"] + self._connection = validate_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] - def validate_connection_string(self): - """ - Validates the connection string. If the connection string is - not an App Setting, an error will be thrown. - """ - if not os.getenv(self._connection): - raise ValueError( - f"Storage account connection string {self._connection} does not exist. " - f"Please make sure that it is a defined App Setting." - ) - self._connection = os.getenv(self._connection) - # Returns a StorageStreamDownloader def get_sdk_type(self): StorageStreamDownloader.validate_connection_string(self) @@ -54,3 +42,16 @@ def get_sdk_type(self): return blob_client.download_blob() else: return None + + +def validate_connection_string(connection_string: str) -> str: + """ + Validates the connection string. If the connection string is + not an App Setting, an error will be thrown. + """ + if not os.getenv(connection_string): + raise ValueError( + f"Storage account connection string {connection_string} does not exist. " + f"Please make sure that it is a defined App Setting." + ) + return os.getenv(connection_string) From 2cc79762ebf88722b1655a74fea6ebd5f250d9a0 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 16:03:47 -0500 Subject: [PATCH 04/15] Missed refs --- .../azurefunctions/extensions/bindings/blob/containerClient.py | 1 - .../extensions/bindings/blob/storageStreamDownloader.py | 1 - 2 files changed, 2 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index 882b815..75a99ac 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -30,7 +30,6 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: # Returns a ContainerClient def get_sdk_type(self): - ContainerClient.validate_connection_string(self) if self._data: return ContainerClientSdk.from_connection_string( conn_str=self._connection, container_name=self._containerName diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index 06e0520..afcfb87 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -30,7 +30,6 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: # Returns a StorageStreamDownloader def get_sdk_type(self): - StorageStreamDownloader.validate_connection_string(self) if self._data: # Create BlobClient blob_client = BlobClientSdk.from_connection_string( From 63e32abd6f4b0bb536a5b44571d0435c761397d2 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 16:20:15 -0500 Subject: [PATCH 05/15] Removed testing line --- .../azurefunctions/extensions/bindings/blob/blobClient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index d541919..35257f7 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -6,7 +6,6 @@ from typing import Union from azure.storage.blob import BlobClient as BlobClientSdk -from azure.storage.blob import BlobServiceClient from azurefunctions.extensions.base import Datum, SdkType From 320ec6b96717a642b52d0562c6e1bbda543a74d3 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 17:17:03 -0500 Subject: [PATCH 06/15] Managed identity for Blob Client --- .github/workflows/ci_ut_ext_blob_workflow.yml | 2 + .../extensions/bindings/blob/blobClient.py | 60 ++++++++++++++++--- .../tests/test_blobclient.py | 54 +++++++++++++++++ 3 files changed, 107 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_ut_ext_blob_workflow.yml b/.github/workflows/ci_ut_ext_blob_workflow.yml index c8c6123..0bea425 100644 --- a/.github/workflows/ci_ut_ext_blob_workflow.yml +++ b/.github/workflows/ci_ut_ext_blob_workflow.yml @@ -38,6 +38,8 @@ jobs: working-directory: azurefunctions-extensions-bindings-blob env: AzureWebJobsStorage: ${{ secrets.AzureWebJobsStorage }} + input: ${{ secrets.input____serviceUri }} + trigger: ${{ secrets.trigger__blobServiceUri }} run: | python -m pytest -q --instafail --cov=. --cov-report xml --cov-branch --ignore tests/ tests diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index 35257f7..666a8c3 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -5,14 +5,16 @@ import os from typing import Union -from azure.storage.blob import BlobClient as BlobClientSdk +from azure.storage.blob import BlobServiceClient from azurefunctions.extensions.base import Datum, SdkType class BlobClient(SdkType): + def __init__(self, *, data: Union[bytes, Datum]) -> None: # model_binding_data properties self._data = data + self._using_managed_identity = False self._version = None self._source = None self._content_type = None @@ -24,30 +26,70 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) + self._using_managed_identity = using_managed_identity(content_json["Connection"]) self._connection = validate_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] - # Returns a BlobClient def get_sdk_type(self): + ''' + When using Managed Identity, the only way to create a BlobClient is + through a BlobServiceClient. There are two ways to create a + BlobServiceClient: + 1. Through the constructor: this is the only option when using Managed Identity + 2. Through from_connection_string: this is the only option when not using Managed Identity + + We track if Managed Identity is being used through a flag. + ''' if self._data: - return BlobClientSdk.from_connection_string( - conn_str=self._connection, - container_name=self._containerName, - blob_name=self._blobName, - ) + if self._using_managed_identity: + blob_service_client = BlobServiceClient(account_url=self._connection) + return blob_service_client.get_blob_client( + container=self._containerName, + blob=self._blobName, + ) + else: + blob_service_client = BlobServiceClient.from_connection_string(self._connection) + return blob_service_client.get_blob_client( + container=self._containerName, + blob=self._blobName, + ) else: return None +def using_managed_identity(connection_name: str) -> bool: + return ((os.getenv(connection_name + "__serviceUri") is not None) + or (os.getenv(connection_name + "__blobServiceUri") is not None)) + + def validate_connection_string(connection_string: str) -> str: """ Validates the connection string. If the connection string is not an App Setting, an error will be thrown. + + When using managed identity, the connection string variable name is formatted like so: + Input: __serviceUri + Trigger: __blobServiceUri + The variable received will be . Therefore, we need to append + the suffix to obtain the storage URI and create the client. + + If managed identity is being used, we set the using_managed_identity property to True. + + There are four cases: + 1. Not using managed identity: the environment variable exists as is + 2. Using managed identity for blob input: __serviceUri must be appended + 3. Using managed identity for blob trigger: __blobServiceUri must be appended + 4. None of these cases existed, so the connection variable is invalid. """ - if not os.getenv(connection_string): + if os.getenv(connection_string): + return os.getenv(connection_string) + elif os.getenv(connection_string + "__serviceUri"): + return os.getenv(connection_string + "__serviceUri") + elif os.getenv(connection_string + "__blobServiceUri"): + return os.getenv(connection_string + "__blobServiceUri") + else: raise ValueError( f"Storage account connection string {connection_string} does not exist. " f"Please make sure that it is a defined App Setting." ) - return os.getenv(connection_string) diff --git a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py index d60b7ee..585f63c 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py @@ -151,6 +151,60 @@ def test_invalid_input_populated(self): "Please make sure that it is a defined App Setting.", ) + def test_input_populated_managed_identity_input(self): + content = { + "Connection": "input", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: BlobClient = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=BlobClient + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, BlobClientSdk) + + sdk_result = BlobClient(data=datum.value).get_sdk_type() + + self.assertIsNotNone(sdk_result) + self.assertIsInstance(sdk_result, BlobClientSdk) + + def test_input_populated_managed_identity_trigger(self): + content = { + "Connection": "trigger", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: BlobClient = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=BlobClient + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, BlobClientSdk) + + sdk_result = BlobClient(data=datum.value).get_sdk_type() + + self.assertIsNotNone(sdk_result) + self.assertIsInstance(sdk_result, BlobClientSdk) + def test_input_invalid_pytype(self): content = { "Connection": "AzureWebJobsStorage", From 477779dcb5ac43a6abee62ec5415f7a180bfad5c Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 17:21:32 -0500 Subject: [PATCH 07/15] Secrets typo --- .github/workflows/ci_ut_ext_blob_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_ut_ext_blob_workflow.yml b/.github/workflows/ci_ut_ext_blob_workflow.yml index 0bea425..df2fb58 100644 --- a/.github/workflows/ci_ut_ext_blob_workflow.yml +++ b/.github/workflows/ci_ut_ext_blob_workflow.yml @@ -38,7 +38,7 @@ jobs: working-directory: azurefunctions-extensions-bindings-blob env: AzureWebJobsStorage: ${{ secrets.AzureWebJobsStorage }} - input: ${{ secrets.input____serviceUri }} + input: ${{ secrets.input__serviceUri }} trigger: ${{ secrets.trigger__blobServiceUri }} run: | python -m pytest -q --instafail --cov=. --cov-report xml --cov-branch --ignore tests/ tests From 26012d50f3329c82af33926847d542542a651ac7 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Fri, 10 May 2024 17:22:06 -0500 Subject: [PATCH 08/15] Secrets typo --- .../extensions/bindings/blob/blobClient.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index 666a8c3..310ee99 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -10,7 +10,6 @@ class BlobClient(SdkType): - def __init__(self, *, data: Union[bytes, Datum]) -> None: # model_binding_data properties self._data = data @@ -26,13 +25,15 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._using_managed_identity = using_managed_identity(content_json["Connection"]) + self._using_managed_identity = using_managed_identity( + content_json["Connection"] + ) self._connection = validate_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] def get_sdk_type(self): - ''' + """ When using Managed Identity, the only way to create a BlobClient is through a BlobServiceClient. There are two ways to create a BlobServiceClient: @@ -40,7 +41,7 @@ def get_sdk_type(self): 2. Through from_connection_string: this is the only option when not using Managed Identity We track if Managed Identity is being used through a flag. - ''' + """ if self._data: if self._using_managed_identity: blob_service_client = BlobServiceClient(account_url=self._connection) @@ -49,7 +50,9 @@ def get_sdk_type(self): blob=self._blobName, ) else: - blob_service_client = BlobServiceClient.from_connection_string(self._connection) + blob_service_client = BlobServiceClient.from_connection_string( + self._connection + ) return blob_service_client.get_blob_client( container=self._containerName, blob=self._blobName, @@ -59,8 +62,9 @@ def get_sdk_type(self): def using_managed_identity(connection_name: str) -> bool: - return ((os.getenv(connection_name + "__serviceUri") is not None) - or (os.getenv(connection_name + "__blobServiceUri") is not None)) + return (os.getenv(connection_name + "__serviceUri") is not None) or ( + os.getenv(connection_name + "__blobServiceUri") is not None + ) def validate_connection_string(connection_string: str) -> str: From b6bbcef59837feeefcf1b731e9bb5050a8b0dd28 Mon Sep 17 00:00:00 2001 From: hallvictoria Date: Mon, 13 May 2024 13:31:41 -0500 Subject: [PATCH 09/15] MI for CC and SSD --- .../extensions/bindings/blob/blobClient.py | 2 - .../bindings/blob/containerClient.py | 27 ++++++++-- .../bindings/blob/storageStreamDownloader.py | 37 +++++++++---- .../tests/test_containerclient.py | 54 +++++++++++++++++++ .../tests/test_ssd.py | 54 +++++++++++++++++++ 5 files changed, 158 insertions(+), 16 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index 310ee99..0707456 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -78,8 +78,6 @@ def validate_connection_string(connection_string: str) -> str: The variable received will be . Therefore, we need to append the suffix to obtain the storage URI and create the client. - If managed identity is being used, we set the using_managed_identity property to True. - There are four cases: 1. Not using managed identity: the environment variable exists as is 2. Using managed identity for blob input: __serviceUri must be appended diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index 75a99ac..80e1385 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -5,7 +5,7 @@ import os from typing import Union -from azure.storage.blob import ContainerClient as ContainerClientSdk +from azure.storage.blob import BlobServiceClient from azurefunctions.extensions.base import Datum, SdkType @@ -13,6 +13,7 @@ class ContainerClient(SdkType): def __init__(self, *, data: Union[bytes, Datum]) -> None: # model_binding_data properties self._data = data + self._using_managed_identity = False self._version = "" self._source = "" self._content_type = "" @@ -24,6 +25,9 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) + self._using_managed_identity = using_managed_identity( + content_json["Connection"] + ) self._connection = validate_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] @@ -31,13 +35,28 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: # Returns a ContainerClient def get_sdk_type(self): if self._data: - return ContainerClientSdk.from_connection_string( - conn_str=self._connection, container_name=self._containerName - ) + if self._using_managed_identity: + blob_service_client = BlobServiceClient(account_url=self._connection) + return blob_service_client.get_container_client( + container=self._containerName + ) + else: + blob_service_client = BlobServiceClient.from_connection_string( + self._connection + ) + return blob_service_client.get_container_client( + container=self._containerName + ) else: return None +def using_managed_identity(connection_name: str) -> bool: + return (os.getenv(connection_name + "__serviceUri") is not None) or ( + os.getenv(connection_name + "__blobServiceUri") is not None + ) + + def validate_connection_string(connection_string: str) -> str: """ Validates the connection string. If the connection string is diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index afcfb87..9203465 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -5,14 +5,15 @@ import os from typing import Union -from azure.storage.blob import BlobClient as BlobClientSdk +from azure.storage.blob import BlobServiceClient from azurefunctions.extensions.base import Datum, SdkType class StorageStreamDownloader(SdkType): def __init__(self, *, data: Union[bytes, Datum]) -> None: # model_binding_data properties - self._data = data or {} + self._data = data + self._using_managed_identity = False self._version = "" self._source = "" self._content_type = "" @@ -24,6 +25,9 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) + self._using_managed_identity = using_managed_identity( + content_json["Connection"] + ) self._connection = validate_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] @@ -31,18 +35,31 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: # Returns a StorageStreamDownloader def get_sdk_type(self): if self._data: - # Create BlobClient - blob_client = BlobClientSdk.from_connection_string( - conn_str=self._connection, - container_name=self._containerName, - blob_name=self._blobName, - ) - # download_blob() returns a StorageStreamDownloader object - return blob_client.download_blob() + if self._using_managed_identity: + blob_service_client = BlobServiceClient(account_url=self._connection) + # download_blob() returns a StorageStreamDownloader object + return blob_service_client.get_blob_client( + container=self._containerName, + blob=self._blobName, + ).download_blob() + else: + blob_service_client = BlobServiceClient.from_connection_string( + self._connection + ) + return blob_service_client.get_blob_client( + container=self._containerName, + blob=self._blobName, + ).download_blob() else: return None +def using_managed_identity(connection_name: str) -> bool: + return (os.getenv(connection_name + "__serviceUri") is not None) or ( + os.getenv(connection_name + "__blobServiceUri") is not None + ) + + def validate_connection_string(connection_string: str) -> str: """ Validates the connection string. If the connection string is diff --git a/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py b/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py index 3ab8c3b..b40860e 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py @@ -149,6 +149,60 @@ def test_invalid_input_populated(self): "Please make sure that it is a defined App Setting.", ) + def test_input_populated_managed_identity_input(self): + content = { + "Connection": "input", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: ContainerClient = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=ContainerClient + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, ContainerClientSdk) + + sdk_result = ContainerClient(data=datum.value).get_sdk_type() + + self.assertIsNotNone(sdk_result) + self.assertIsInstance(sdk_result, ContainerClientSdk) + + def test_input_populated_managed_identity_trigger(self): + content = { + "Connection": "trigger", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: ContainerClient = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=ContainerClient + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, ContainerClientSdk) + + sdk_result = ContainerClient(data=datum.value).get_sdk_type() + + self.assertIsNotNone(sdk_result) + self.assertIsInstance(sdk_result, ContainerClientSdk) + def test_input_invalid_pytype(self): content = { "Connection": "AzureWebJobsStorage", diff --git a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py index e848a0e..89dc60f 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py @@ -154,6 +154,60 @@ def test_invalid_input_populated(self): "Please make sure that it is a defined App Setting.", ) + def test_input_populated_managed_identity_input(self): + content = { + "Connection": "input", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: StorageStreamDownloader = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=StorageStreamDownloader + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, SSDSdk) + + sdk_result = StorageStreamDownloader(data=datum.value).get_sdk_type() + + self.assertIsNotNone(sdk_result) + self.assertIsInstance(sdk_result, SSDSdk) + + def test_input_populated_managed_identity_trigger(self): + content = { + "Connection": "trigger", + "ContainerName": "test-blob", + "BlobName": "text.txt", + } + + sample_mbd = MockMBD( + version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content=json.dumps(content), + ) + + datum: Datum = Datum(value=sample_mbd, type="model_binding_data") + result: StorageStreamDownloader = BlobClientConverter.decode( + data=datum, trigger_metadata=None, pytype=StorageStreamDownloader + ) + + self.assertIsNotNone(result) + self.assertIsInstance(result, SSDSdk) + + sdk_result = StorageStreamDownloader(data=datum.value).get_sdk_type() + + self.assertIsNotNone(sdk_result) + self.assertIsInstance(sdk_result, SSDSdk) + def test_input_invalid_pytype(self): content = { "Connection": "AzureWebJobsStorage", From 0f39dc16647b83c5998726780b093316993f227c Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 16 Aug 2024 16:04:04 -0500 Subject: [PATCH 10/15] lint --- .../azurefunctions/extensions/bindings/blob/utils.py | 3 ++- azurefunctions-extensions-bindings-blob/tests/test_ssd.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py index d0e2919..e237335 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py @@ -37,10 +37,11 @@ def get_connection_string(connection_string: str) -> str: f"Please make sure that it is a defined App Setting." ) + def using_managed_identity(connection_name: str) -> bool: """ To determine if managed identity is being used, we check if the provided - connection string has either of the two suffixes: + connection string has either of the two suffixes: __serviceUri or __blobServiceUri. """ return (os.getenv(connection_name + "__serviceUri") is not None) or ( diff --git a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py index aecce46..9871b5a 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py @@ -153,7 +153,7 @@ def test_invalid_input_populated(self): "Storage account connection string NotARealConnectionString does not exist. " "Please make sure that it is a defined App Setting.", ) - + def test_none_input_populated(self): content = { "Connection": None, From d321443eb3047116e6bcce39f85b08c91c917a29 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 16 Aug 2024 16:10:26 -0500 Subject: [PATCH 11/15] reorder checks --- .../azurefunctions/extensions/bindings/blob/blobClient.py | 2 +- .../azurefunctions/extensions/bindings/blob/containerClient.py | 2 +- .../extensions/bindings/blob/storageStreamDownloader.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index ca77e72..3fc8b5e 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -25,10 +25,10 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) + self._connection = get_connection_string(content_json["Connection"]) self._using_managed_identity = using_managed_identity( content_json["Connection"] ) - self._connection = get_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index 074f919..2c259a0 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -25,10 +25,10 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) + self._connection = get_connection_string(content_json["Connection"]) self._using_managed_identity = using_managed_identity( content_json["Connection"] ) - self._connection = get_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index 3f4efd5..4e989c0 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -25,10 +25,10 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) + self._connection = get_connection_string(content_json["Connection"]) self._using_managed_identity = using_managed_identity( content_json["Connection"] ) - self._connection = get_connection_string(content_json["Connection"]) self._containerName = content_json["ContainerName"] self._blobName = content_json["BlobName"] From e6085b8d74ddce14180661e383bc6abb5f8662c6 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Fri, 16 Aug 2024 16:23:04 -0500 Subject: [PATCH 12/15] remove duplicate test --- .../tests/test_blobclient.py | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py index eca0185..77a367a 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py @@ -175,31 +175,6 @@ def test_none_input_populated(self): "Storage account connection string cannot be none. Please provide a connection string.", ) - def test_invalid_input_populated(self): - content = { - "Connection": "NotARealConnectionString", - "ContainerName": "test-blob", - "BlobName": "text.txt", - } - - sample_mbd = MockMBD( - version="1.0", - source="AzureStorageBlobs", - content_type="application/json", - content=json.dumps(content), - ) - - with self.assertRaises(ValueError) as e: - datum: Datum = Datum(value=sample_mbd, type="model_binding_data") - result: BlobClient = BlobClientConverter.decode( - data=datum, trigger_metadata=None, pytype=BlobClient - ) - self.assertEqual( - e.exception.args[0], - "Storage account connection string NotARealConnectionString does not exist. " - "Please make sure that it is a defined App Setting.", - ) - def test_input_populated_managed_identity_input(self): content = { "Connection": "input", From 4d1cc4d391d70356aeacf65f2c83c5813ace46d9 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 20 Aug 2024 13:50:06 -0500 Subject: [PATCH 13/15] refactor env var check, client return --- .../extensions/bindings/blob/blobClient.py | 12 ++++-------- .../extensions/bindings/blob/containerClient.py | 9 +++------ .../bindings/blob/storageStreamDownloader.py | 14 +++++--------- .../extensions/bindings/blob/utils.py | 10 +++++----- 4 files changed, 17 insertions(+), 28 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index 3fc8b5e..48c13d2 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -45,17 +45,13 @@ def get_sdk_type(self): if self._data: if self._using_managed_identity: blob_service_client = BlobServiceClient(account_url=self._connection) - return blob_service_client.get_blob_client( - container=self._containerName, - blob=self._blobName, - ) else: blob_service_client = BlobServiceClient.from_connection_string( self._connection ) - return blob_service_client.get_blob_client( - container=self._containerName, - blob=self._blobName, - ) + return blob_service_client.get_blob_client( + container=self._containerName, + blob=self._blobName, + ) else: return None diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index 2c259a0..c92b76b 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -37,15 +37,12 @@ def get_sdk_type(self): if self._data: if self._using_managed_identity: blob_service_client = BlobServiceClient(account_url=self._connection) - return blob_service_client.get_container_client( - container=self._containerName - ) else: blob_service_client = BlobServiceClient.from_connection_string( self._connection ) - return blob_service_client.get_container_client( - container=self._containerName - ) + return blob_service_client.get_container_client( + container=self._containerName + ) else: return None diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index 4e989c0..a3ce8d1 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -37,18 +37,14 @@ def get_sdk_type(self): if self._data: if self._using_managed_identity: blob_service_client = BlobServiceClient(account_url=self._connection) - # download_blob() returns a StorageStreamDownloader object - return blob_service_client.get_blob_client( - container=self._containerName, - blob=self._blobName, - ).download_blob() else: blob_service_client = BlobServiceClient.from_connection_string( self._connection ) - return blob_service_client.get_blob_client( - container=self._containerName, - blob=self._blobName, - ).download_blob() + # download_blob() returns a StorageStreamDownloader object + return blob_service_client.get_blob_client( + container=self._containerName, + blob=self._blobName, + ).download_blob() else: return None diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py index e237335..a829fdf 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/utils.py @@ -20,16 +20,16 @@ def get_connection_string(connection_string: str) -> str: 3. Using managed identity for blob trigger: __blobServiceUri must be appended 4. None of these cases existed, so the connection variable is invalid. """ - if connection_string == None: + if connection_string is None: raise ValueError( - "Storage account connection string cannot be none. " + "Storage account connection string cannot be None. " "Please provide a connection string." ) - elif os.getenv(connection_string): + elif connection_string in os.environ: return os.getenv(connection_string) - elif os.getenv(connection_string + "__serviceUri"): + elif connection_string + "__serviceUri" in os.environ: return os.getenv(connection_string + "__serviceUri") - elif os.getenv(connection_string + "__blobServiceUri"): + elif connection_string + "__blobServiceUri" in os.environ: return os.getenv(connection_string + "__blobServiceUri") else: raise ValueError( From 4db63e21915c3256a4fb5383e2617cc1f4429dca Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 20 Aug 2024 14:18:03 -0500 Subject: [PATCH 14/15] fix tests --- .../tests/test_blobclient.py | 2 +- .../tests/test_containerclient.py | 2 +- azurefunctions-extensions-bindings-blob/tests/test_ssd.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py index 77a367a..8d087b6 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_blobclient.py @@ -172,7 +172,7 @@ def test_none_input_populated(self): ) self.assertEqual( e.exception.args[0], - "Storage account connection string cannot be none. Please provide a connection string.", + "Storage account connection string cannot be None. Please provide a connection string.", ) def test_input_populated_managed_identity_input(self): diff --git a/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py b/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py index 52e4615..b624743 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_containerclient.py @@ -170,7 +170,7 @@ def test_none_input_populated(self): ) self.assertEqual( e.exception.args[0], - "Storage account connection string cannot be none. Please provide a connection string.", + "Storage account connection string cannot be None. Please provide a connection string.", ) def test_input_populated_managed_identity_input(self): diff --git a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py index 9871b5a..17c86e2 100644 --- a/azurefunctions-extensions-bindings-blob/tests/test_ssd.py +++ b/azurefunctions-extensions-bindings-blob/tests/test_ssd.py @@ -175,7 +175,7 @@ def test_none_input_populated(self): ) self.assertEqual( e.exception.args[0], - "Storage account connection string cannot be none. Please provide a connection string.", + "Storage account connection string cannot be None. Please provide a connection string.", ) def test_input_populated_managed_identity_input(self): From aad623e6550e7b3a9e748d013a88ee14ce7a4cc5 Mon Sep 17 00:00:00 2001 From: Victoria Hall Date: Tue, 8 Oct 2024 13:54:10 -0500 Subject: [PATCH 15/15] refactor client creation, safe dict lookup --- .../extensions/bindings/blob/blobClient.py | 19 +++++++++---------- .../bindings/blob/containerClient.py | 19 +++++++++---------- .../bindings/blob/storageStreamDownloader.py | 19 +++++++++---------- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py index 48c13d2..491cf38 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/blobClient.py @@ -25,12 +25,12 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = get_connection_string(content_json["Connection"]) + self._connection = get_connection_string(content_json.get("Connection")) self._using_managed_identity = using_managed_identity( - content_json["Connection"] + content_json.get("Connection") ) - self._containerName = content_json["ContainerName"] - self._blobName = content_json["BlobName"] + self._containerName = content_json.get("ContainerName") + self._blobName = content_json.get("BlobName") def get_sdk_type(self): """ @@ -43,12 +43,11 @@ def get_sdk_type(self): We track if Managed Identity is being used through a flag. """ if self._data: - if self._using_managed_identity: - blob_service_client = BlobServiceClient(account_url=self._connection) - else: - blob_service_client = BlobServiceClient.from_connection_string( - self._connection - ) + blob_service_client = ( + BlobServiceClient(account_url=self._connection) + if self._using_managed_identity + else BlobServiceClient.from_connection_string(self._connection) + ) return blob_service_client.get_blob_client( container=self._containerName, blob=self._blobName, diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py index c92b76b..f261200 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/containerClient.py @@ -25,22 +25,21 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = get_connection_string(content_json["Connection"]) + self._connection = get_connection_string(content_json.get("Connection")) self._using_managed_identity = using_managed_identity( - content_json["Connection"] + content_json.get("Connection") ) - self._containerName = content_json["ContainerName"] - self._blobName = content_json["BlobName"] + self._containerName = content_json.get("ContainerName") + self._blobName = content_json.get("BlobName") # Returns a ContainerClient def get_sdk_type(self): if self._data: - if self._using_managed_identity: - blob_service_client = BlobServiceClient(account_url=self._connection) - else: - blob_service_client = BlobServiceClient.from_connection_string( - self._connection - ) + blob_service_client = ( + BlobServiceClient(account_url=self._connection) + if self._using_managed_identity + else BlobServiceClient.from_connection_string(self._connection) + ) return blob_service_client.get_container_client( container=self._containerName ) diff --git a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py index a3ce8d1..5a437eb 100644 --- a/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py +++ b/azurefunctions-extensions-bindings-blob/azurefunctions/extensions/bindings/blob/storageStreamDownloader.py @@ -25,22 +25,21 @@ def __init__(self, *, data: Union[bytes, Datum]) -> None: self._source = data.source self._content_type = data.content_type content_json = json.loads(data.content) - self._connection = get_connection_string(content_json["Connection"]) + self._connection = get_connection_string(content_json.get("Connection")) self._using_managed_identity = using_managed_identity( - content_json["Connection"] + content_json.get("Connection") ) - self._containerName = content_json["ContainerName"] - self._blobName = content_json["BlobName"] + self._containerName = content_json.get("ContainerName") + self._blobName = content_json.get("BlobName") # Returns a StorageStreamDownloader def get_sdk_type(self): if self._data: - if self._using_managed_identity: - blob_service_client = BlobServiceClient(account_url=self._connection) - else: - blob_service_client = BlobServiceClient.from_connection_string( - self._connection - ) + blob_service_client = ( + BlobServiceClient(account_url=self._connection) + if self._using_managed_identity + else BlobServiceClient.from_connection_string(self._connection) + ) # download_blob() returns a StorageStreamDownloader object return blob_service_client.get_blob_client( container=self._containerName,