diff --git a/client/datalake/scripts/cli.py b/client/datalake/scripts/cli.py index 3b1207d..648ed57 100644 --- a/client/datalake/scripts/cli.py +++ b/client/datalake/scripts/cli.py @@ -68,7 +68,7 @@ def wrapped(*args, **kwargs): 'environment variables, which can be overridden by ' 'command-line arguments.'), envvar='DATALAKE_CONFIG') -@click.option('-u', '--storage-url', +@click.option('-U', '--storage-url', help=('The URL to the top-level storage resource where ' 'datalake will archive all the files (e.g., ' 's3://my-datalake). DATALAKE_STORAGE_URL is the ' diff --git a/client/datalake/tests/conftest.py b/client/datalake/tests/conftest.py index 99aac43..494c03e 100644 --- a/client/datalake/tests/conftest.py +++ b/client/datalake/tests/conftest.py @@ -18,9 +18,17 @@ import os import six +try: + from moto import mock_aws +except ImportError: + # Ugh. This is necessary because the client tests rely on a later version + # of moto than the ingester and api tests. And that newer version changed + # the object mock_s3 -> mock_aws. + from moto import mock_s3 as mock_aws +except ImportError: + pass try: - from moto import mock_s3 import boto3 from six.moves.urllib.parse import urlparse import json @@ -142,7 +150,7 @@ def tear_down(): @pytest.fixture def s3_connection(aws_connector): - with mock_s3(): + with mock_aws(): yield boto3.resource('s3') diff --git a/client/pyproject.toml b/client/pyproject.toml index 6a147ef..73c63c0 100644 --- a/client/pyproject.toml +++ b/client/pyproject.toml @@ -37,7 +37,7 @@ dynamic = ["version"] test = [ 'pytest<8.0.0', 'pytest-cov>=2.5.1,<4', - 'moto[s3]>4,<5', + 'moto[s3]>4', 'twine<4.0.0', 'pip>=20.0.0,<22.0.0', 'wheel<0.38.0', diff --git a/client/test/test_archive.py b/client/test/test_archive.py index a04a314..99b455d 100644 --- a/client/test/test_archive.py +++ b/client/test/test_archive.py @@ -36,6 +36,19 @@ def test_push_file(archive, random_metadata, tmpfile, s3_object): def test_push_large_file( monkeypatch, archive, random_metadata, tmpfile, s3_object): monkeypatch.setenv('DATALAKE_CHUNK_SIZE_MB', '5') + from botocore.httpchecksum import StreamingChecksumBody + + # When testing with Moto, newer versions of boto3/botocore cause a + # FlexibleChecksumError because Moto doesn't supply the checksum headers + # that boto3 now expects. The standard config objects can fail to apply + # to the underlying S3 transfer manager. + # The most reliable way to disable this for tests is to directly patch + # the validation function in botocore to do nothing. + monkeypatch.setattr( + StreamingChecksumBody, + "_validate_checksum", + lambda _: None) + expected_content = ('big data' * 1024 * 1024).encode('utf-8') f = tmpfile(expected_content) url = archive.prepare_metadata_and_push(f, **random_metadata) diff --git a/client/test/test_cli.py b/client/test/test_cli.py index 4ce2365..cce3459 100644 --- a/client/test/test_cli.py +++ b/client/test/test_cli.py @@ -101,3 +101,12 @@ def test_push_with_translation_expression(cli_tester, tmpdir): cmd += '--what=job --start=now --end=now --where=hostname ' cmd += str(f) cli_tester(cmd) + + +@pytest.mark.xfail(reason="We don't want the CLI to raise warnings.", + strict=True) +def test_no_duplicate_parameters(cli_tester): + # Unfortunately there is no clean way to test if a warning is NOT emitted. + # So we have to test that a warning IS emitted and XFAIL. + with pytest.warns(UserWarning, match='Remove its duplicate as parameters'): + cli_tester('--help')