CLIENT: Dedupe the -u option

client/datalake/scripts/cli.py

@@ -68,7 +68,7 @@ def wrapped(*args, **kwargs):
                     'environment variables, which can be overridden by '
                     'command-line arguments.'),
               envvar='DATALAKE_CONFIG')
-@click.option('-u', '--storage-url',
+@click.option('-U', '--storage-url',
               help=('The URL to the top-level storage resource where '
                     'datalake will archive all the files (e.g., '
                     's3://my-datalake). DATALAKE_STORAGE_URL is the '

client/datalake/tests/conftest.py

@@ -18,9 +18,17 @@
 import os
 import six
 
+try:
+    from moto import mock_aws
+except ImportError:
+    # Ugh. This is necessary because the client tests rely on a later version
+    # of moto than the ingester and api tests. And that newer version changed
+    # the object mock_s3 -> mock_aws.
+    from moto import mock_s3 as mock_aws
+except ImportError:
+    pass
 
 try:
-    from moto import mock_s3
     import boto3
     from six.moves.urllib.parse import urlparse
     import json
@@ -142,7 +150,7 @@ def tear_down():
 
 @pytest.fixture
 def s3_connection(aws_connector):
-    with mock_s3():
+    with mock_aws():
         yield boto3.resource('s3')
 
 

client/pyproject.toml

@@ -37,7 +37,7 @@ dynamic = ["version"]
 test = [
     'pytest<8.0.0',
     'pytest-cov>=2.5.1,<4',
-    'moto[s3]>4,<5',
+    'moto[s3]>4',
     'twine<4.0.0',
     'pip>=20.0.0,<22.0.0',
     'wheel<0.38.0',

client/test/test_archive.py

@@ -36,6 +36,19 @@ def test_push_file(archive, random_metadata, tmpfile, s3_object):
 def test_push_large_file(
         monkeypatch, archive, random_metadata, tmpfile, s3_object):
     monkeypatch.setenv('DATALAKE_CHUNK_SIZE_MB', '5')
+    from botocore.httpchecksum import StreamingChecksumBody
+
+    # When testing with Moto, newer versions of boto3/botocore cause a
+    # FlexibleChecksumError because Moto doesn't supply the checksum headers
+    # that boto3 now expects. The standard config objects can fail to apply
+    # to the underlying S3 transfer manager.
+    # The most reliable way to disable this for tests is to directly patch
+    # the validation function in botocore to do nothing.
+    monkeypatch.setattr(
+        StreamingChecksumBody,
+        "_validate_checksum",
+        lambda _: None)
+
     expected_content = ('big data' * 1024 * 1024).encode('utf-8')
     f = tmpfile(expected_content)
     url = archive.prepare_metadata_and_push(f, **random_metadata)

client/test/test_cli.py

@@ -101,3 +101,12 @@ def test_push_with_translation_expression(cli_tester, tmpdir):
     cmd += '--what=job --start=now --end=now --where=hostname '
     cmd += str(f)
     cli_tester(cmd)
+
+
+@pytest.mark.xfail(reason="We don't want the CLI to raise warnings.",
+                   strict=True)
+def test_no_duplicate_parameters(cli_tester):
+    # Unfortunately there is no clean way to test if a warning is NOT emitted.
+    # So we have to test that a warning IS emitted and XFAIL.
+    with pytest.warns(UserWarning, match='Remove its duplicate as parameters'):
+        cli_tester('--help')