diff --git a/README.md b/README.md index 02d9b972..3dfbbf5e 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ You can download the [latest snapshot release][download] or pick it up from Mave com.amazonaws aws-dynamodb-encryption-java - 1.11.2 + 1.12.0 ``` @@ -164,4 +164,4 @@ For signing, the user specified signing key can be either symmetric or asymmetri [materialprovider]: src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/EncryptionMaterialsProvider.java [privatekey]: http://docs.oracle.com/javase/7/docs/api/java/security/PrivateKey.html [secretkey]: http://docs.oracle.com/javase/7/docs/api/javax/crypto/SecretKey.html -[download]: https://github.com/aws/aws-dynamodb-encryption-java/releases/tag/1.11.2 +[download]: https://github.com/aws/aws-dynamodb-encryption-java/releases/tag/1.12.0 diff --git a/pom.xml b/pom.xml index 375cfa95..0144f4d9 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ com.amazonaws aws-dynamodb-encryption-java - 1.11.2 + 1.12.0 jar aws-dynamodb-encryption-java @@ -43,7 +43,7 @@ com.amazonaws aws-java-sdk-bom - 1.11.315 + 1.11.380 pom import @@ -64,14 +64,14 @@ junit junit - 4.8.1 + 4.12 test org.bouncycastle bcprov-ext-jdk15on - 1.50 + 1.60 test @@ -114,8 +114,8 @@ maven-compiler-plugin 3.1 - 1.7 - 1.7 + 1.8 + 1.8 diff --git a/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptor.java b/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptor.java index 3ff8837c..76be51a9 100644 --- a/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptor.java +++ b/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptor.java @@ -31,6 +31,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import javax.crypto.Cipher; import javax.crypto.SecretKey; @@ -58,7 +60,16 @@ public class DynamoDBEncryptor { private static final String DEFAULT_DESCRIPTION_BASE = "amzn-ddb-map-"; // Same as the Mapper private static final Charset UTF8 = Charset.forName("UTF-8"); private static final String SYMMETRIC_ENCRYPTION_MODE = "/CBC/PKCS5Padding"; - + private static final ConcurrentHashMap BLOCK_SIZE_CACHE = new ConcurrentHashMap<>(); + private static final Function BLOCK_SIZE_CALCULATOR = (transformation) -> { + try { + final Cipher c = Cipher.getInstance(transformation); + return c.getBlockSize(); + } catch (final GeneralSecurityException ex) { + throw new IllegalArgumentException("Algorithm does not exist", ex); + } + }; + private static final int CURRENT_VERSION = 0; private String signatureFieldName = DEFAULT_SIGNATURE_FIELD; @@ -339,7 +350,7 @@ private void actualDecryption(Map itemAttributes, final String encryptionMode = encryptionKey != null ? encryptionKey.getAlgorithm() + materialDescription.get(symmetricEncryptionModeHeader) : null; Cipher cipher = null; - int ivSize = -1; + int blockSize = -1; for (Map.Entry entry: itemAttributes.entrySet()) { Set flags = attributeFlags.get(entry.getKey()); @@ -354,15 +365,13 @@ private void actualDecryption(Map itemAttributes, plainText = ByteBuffer.wrap(((DelegatedKey)encryptionKey).decrypt(toByteArray(cipherText), null, encryptionMode)); } else { if (cipher == null) { - cipher = Cipher.getInstance( - encryptionMode); - ivSize = cipher.getBlockSize(); + blockSize = getBlockSize(encryptionMode); + cipher = Cipher.getInstance(encryptionMode); } - byte[] iv = new byte[ivSize]; + byte[] iv = new byte[blockSize]; cipherText.get(iv); cipher.init(Cipher.DECRYPT_MODE, encryptionKey, new IvParameterSpec(iv), Utils.getRng()); - plainText = ByteBuffer.allocate( - cipher.getOutputSize(cipherText.remaining())); + plainText = ByteBuffer.allocate(cipher.getOutputSize(cipherText.remaining())); cipher.doFinal(cipherText, plainText); plainText.rewind(); } @@ -371,6 +380,10 @@ private void actualDecryption(Map itemAttributes, } } + protected static int getBlockSize(final String encryptionMode) { + return BLOCK_SIZE_CACHE.computeIfAbsent(encryptionMode, BLOCK_SIZE_CALCULATOR); + } + /** * This method has the side effect of replacing the plaintext * attribute-values of "itemAttributes" with ciphertext attribute-values @@ -388,7 +401,7 @@ private void actualEncryption(Map itemAttributes, encryptionMode = encryptionKey.getAlgorithm() + SYMMETRIC_ENCRYPTION_MODE; } Cipher cipher = null; - int ivSize = -1; + int blockSize = -1; for (Map.Entry entry: itemAttributes.entrySet()) { Set flags = attributeFlags.get(entry.getKey()); @@ -405,16 +418,22 @@ private void actualEncryption(Map itemAttributes, dk.encrypt(toByteArray(plainText), null, encryptionMode)); } else { if (cipher == null) { + blockSize = getBlockSize(encryptionMode); cipher = Cipher.getInstance(encryptionMode); - ivSize = cipher.getBlockSize(); } // Encryption format: // Note a unique iv is generated per attribute - byte[] iv = Utils.getRandom(ivSize); - cipher.init(Cipher.ENCRYPT_MODE, encryptionKey, new IvParameterSpec(iv), Utils.getRng()); - cipherText = ByteBuffer.allocate(ivSize + cipher.getOutputSize(plainText.remaining())); - cipherText.put(iv); + cipher.init(Cipher.ENCRYPT_MODE, encryptionKey, Utils.getRng()); + cipherText = ByteBuffer.allocate(blockSize + cipher.getOutputSize(plainText.remaining())); + cipherText.position(blockSize); cipher.doFinal(plainText, cipherText); + cipherText.flip(); + final byte[] iv = cipher.getIV(); + if (iv.length != blockSize) { + throw new IllegalStateException(String.format("Generated IV length (%d) not equal to block size (%d)", + iv.length, blockSize)); + } + cipherText.put(iv); cipherText.rewind(); } // Replace the plaintext attribute value with the encrypted content @@ -539,17 +558,22 @@ protected static Map unmarshallDescription(AttributeValue attrib attributeValue.getB().reset(); } } - + private static byte[] toByteArray(ByteBuffer buffer) { - if (buffer.hasArray()) { + buffer = buffer.duplicate(); + // We can only return the array directly if: + // 1. The ByteBuffer exposes an array + // 2. The ByteBuffer starts at the beginning of the array + // 3. The ByteBuffer uses the entire array + if (buffer.hasArray() && buffer.arrayOffset() == 0) { byte[] result = buffer.array(); - buffer.rewind(); - return result; - } else { - byte[] result = new byte[buffer.remaining()]; - buffer.get(result); - buffer.rewind(); - return result; + if (buffer.remaining() == result.length) { + return result; + } } + + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; } } diff --git a/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStore.java b/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStore.java index 28c04a30..98506ffd 100644 --- a/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStore.java +++ b/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStore.java @@ -142,6 +142,32 @@ public long getVersionFromMaterialDescription(final Map descript throw new IllegalArgumentException("No meta id found"); } } + + /** + * This API retrieves the intermediate keys from the source region and replicates it in the target region. + * @param materialName + * @param version + * @param targetMetaStore + */ + public void replicate(final String materialName, final long version, final MetaStore targetMetaStore) { + try { + final Map ddbKey = new HashMap(); + ddbKey.put(DEFAULT_HASH_KEY, new AttributeValue().withS(materialName)); + ddbKey.put(DEFAULT_RANGE_KEY, new AttributeValue().withN(Long.toString(version))); + final Map item = ddbGet(ddbKey); + if (item == null || item.isEmpty()) { + throw new IndexOutOfBoundsException("No material found: " + materialName + "#" + version); + } + + final Map plainText = getPlainText(item); + final Map encryptedText = targetMetaStore.getEncryptedText(plainText); + final PutItemRequest put = new PutItemRequest().withTableName(targetMetaStore.tableName).withItem(encryptedText) + .withExpected(doesNotExist); + targetMetaStore.ddb.putItem(put); + } catch (ConditionalCheckFailedException e) { + //Item already present. + } + } /** * Creates a DynamoDB Table with the correct properties to be used with a ProviderStore. */ @@ -187,36 +213,43 @@ private Map encryptKeys(final String name, final long ve plaintext .put(INTEGRITY_KEY_FIELD, new AttributeValue().withB(ByteBuffer.wrap(integrityKey.getEncoded()))); plaintext.put(INTEGRITY_ALGORITHM_FIELD, new AttributeValue().withS(integrityKey.getAlgorithm())); + return getEncryptedText(plaintext); + } + + private EncryptionMaterialsProvider decryptProvider(final Map item) { + final Map plaintext = getPlainText(item); + final String type = plaintext.get(MATERIAL_TYPE_VERSION).getS(); + final SecretKey encryptionKey; + final SecretKey integrityKey; + // This switch statement is to make future extensibility easier and more obvious + switch (type) { + case "0": // Only currently supported type + encryptionKey = new SecretKeySpec(plaintext.get(ENCRYPTION_KEY_FIELD).getB().array(), + plaintext.get(ENCRYPTION_ALGORITHM_FIELD).getS()); + integrityKey = new SecretKeySpec(plaintext.get(INTEGRITY_KEY_FIELD).getB().array(), plaintext + .get(INTEGRITY_ALGORITHM_FIELD).getS()); + break; + default: + throw new IllegalStateException("Unsupported material type: " + type); + } + return new WrappedMaterialsProvider(encryptionKey, encryptionKey, integrityKey, + buildDescription(plaintext)); + } + + private Map getPlainText(Map item) { try { - return encryptor.encryptAllFieldsExcept(plaintext, ddbCtx, DEFAULT_HASH_KEY, - DEFAULT_RANGE_KEY); + return encryptor.decryptAllFieldsExcept(item, + ddbCtx, DEFAULT_HASH_KEY, DEFAULT_RANGE_KEY); } catch (final GeneralSecurityException e) { throw new AmazonClientException(e); } } - private EncryptionMaterialsProvider decryptProvider(final Map item) { + private Map getEncryptedText(Map plaintext) { try { - final Map plaintext = encryptor.decryptAllFieldsExcept(item, - ddbCtx, DEFAULT_HASH_KEY, DEFAULT_RANGE_KEY); - - final String type = plaintext.get(MATERIAL_TYPE_VERSION).getS(); - final SecretKey encryptionKey; - final SecretKey integrityKey; - // This switch statement is to make future extensibility easier and more obvious - switch (type) { - case "0": // Only currently supported type - encryptionKey = new SecretKeySpec(plaintext.get(ENCRYPTION_KEY_FIELD).getB().array(), - plaintext.get(ENCRYPTION_ALGORITHM_FIELD).getS()); - integrityKey = new SecretKeySpec(plaintext.get(INTEGRITY_KEY_FIELD).getB().array(), plaintext - .get(INTEGRITY_ALGORITHM_FIELD).getS()); - break; - default: - throw new IllegalStateException("Unsupported material type: " + type); - } - return new WrappedMaterialsProvider(encryptionKey, encryptionKey, integrityKey, - buildDescription(plaintext)); + return encryptor.encryptAllFieldsExcept(plaintext, ddbCtx, DEFAULT_HASH_KEY, + DEFAULT_RANGE_KEY); } catch (final GeneralSecurityException e) { throw new AmazonClientException(e); } diff --git a/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/internal/LRUCache.java b/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/internal/LRUCache.java index 4ab39212..00c79eb7 100644 --- a/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/internal/LRUCache.java +++ b/src/main/java/com/amazonaws/services/dynamodbv2/datamodeling/internal/LRUCache.java @@ -14,14 +14,15 @@ */ package com.amazonaws.services.dynamodbv2.datamodeling.internal; -import java.util.AbstractMap.SimpleImmutableEntry; +import com.amazonaws.annotation.ThreadSafe; + +import java.util.ArrayList; import java.util.Collections; +import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - -import com.amazonaws.annotation.ThreadSafe; +import java.util.Map.Entry; /** * A bounded cache that has a LRU eviction policy when the cache is full. @@ -98,11 +99,16 @@ public void clear() { // The more complicated logic is to ensure that the listener is // actually called for all entries. if (listener != null) { - Set keys = new TreeSet(map.keySet()); - for (String key : keys) { - T val = map.get(key); - listener.onRemoval(new SimpleImmutableEntry(key, val)); - map.remove(key); + List> removedEntries = new ArrayList>(); + synchronized (map) { + Iterator> it = map.entrySet().iterator(); + while(it.hasNext()) { + removedEntries.add(it.next()); + it.remove(); + } + } + for (Entry entry : removedEntries) { + listener.onRemoval(entry); } } else { map.clear(); @@ -126,7 +132,7 @@ private LRUHashMap(final int maxSize, final RemovalListener listener) { } @Override - protected boolean removeEldestEntry(final Map.Entry eldest) { + protected boolean removeEldestEntry(final Entry eldest) { if (size() > maxSize) { if (listener != null) { listener.onRemoval(eldest); @@ -138,6 +144,6 @@ protected boolean removeEldestEntry(final Map.Entry eldest) { } public static interface RemovalListener { - public void onRemoval(Map.Entry entry); + public void onRemoval(Entry entry); } } diff --git a/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptorTest.java b/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptorTest.java index 7ebb2e70..4748b370 100644 --- a/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptorTest.java +++ b/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DynamoDBEncryptorTest.java @@ -14,12 +14,14 @@ */ package com.amazonaws.services.dynamodbv2.datamodeling.encryption; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.security.GeneralSecurityException; import java.security.InvalidAlgorithmParameterException; @@ -27,7 +29,6 @@ import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; -import java.security.SecureRandom; import java.security.Security; import java.security.SignatureException; import java.util.Collection; @@ -118,7 +119,7 @@ public void testSetMaterialDescriptionFieldName() { @Test public void fullEncryption() throws GeneralSecurityException { - Map encryptedAttributes = + Map encryptedAttributes = encryptor.encryptAllFieldsExcept(Collections.unmodifiableMap(attribs), context, "hashKey", "rangeKey", "version"); assertThat(encryptedAttributes, AttrMatcher.invert(attribs)); @@ -298,6 +299,40 @@ public void EcdsaSignedOnlyBadSignature() throws GeneralSecurityException { encryptor.decryptAllFieldsExcept(encryptedAttributes, context, attribs.keySet().toArray(new String[0])); } + @Test + public void toByteArray() throws ReflectiveOperationException { + final byte[] expected = new byte[] {0, 1, 2, 3, 4, 5}; + assertToByteArray("Wrap", expected, ByteBuffer.wrap(expected)); + assertToByteArray("Wrap-RO", expected, ByteBuffer.wrap(expected).asReadOnlyBuffer()); + + assertToByteArray("Wrap-Truncated-Sliced", expected, ByteBuffer.wrap(new byte[] {0, 1, 2, 3, 4, 5, 6}, 0, 6).slice()); + assertToByteArray("Wrap-Offset-Sliced", expected, ByteBuffer.wrap(new byte[] {6, 0, 1, 2, 3, 4, 5, 6}, 1, 6).slice()); + assertToByteArray("Wrap-Truncated", expected, ByteBuffer.wrap(new byte[] {0, 1, 2, 3, 4, 5, 6}, 0, 6)); + assertToByteArray("Wrap-Offset", expected, ByteBuffer.wrap(new byte[] {6, 0, 1, 2, 3, 4, 5, 6}, 1, 6)); + + ByteBuffer buff = ByteBuffer.allocate(expected.length + 10); + buff.put(expected); + buff.flip(); + assertToByteArray("Normal", expected, buff); + + buff = ByteBuffer.allocateDirect(expected.length + 10); + buff.put(expected); + buff.flip(); + assertToByteArray("Direct", expected, buff); + } + + private void assertToByteArray(final String msg, final byte[] expected, final ByteBuffer testValue) throws ReflectiveOperationException { + Method m = DynamoDBEncryptor.class.getDeclaredMethod("toByteArray", ByteBuffer.class); + m.setAccessible(true); + + int oldPosition = testValue.position(); + int oldLimit = testValue.limit(); + + assertArrayEquals(msg + ":Array", expected, (byte[]) m.invoke(null, testValue)); + assertEquals(msg + ":Position", oldPosition, testValue.position()); + assertEquals(msg + ":Limit", oldLimit, testValue.limit()); + } + private void assertAttrEquals(AttributeValue o1, AttributeValue o2) { Assert.assertEquals(o1.getB(), o2.getB()); assertSetsEqual(o1.getBS(), o2.getBS()); diff --git a/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStoreTests.java b/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStoreTests.java index 91f48337..f1960fc1 100644 --- a/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStoreTests.java +++ b/src/test/java/com/amazonaws/services/dynamodbv2/datamodeling/encryption/providers/store/MetaStoreTests.java @@ -36,29 +36,40 @@ import com.amazonaws.services.dynamodbv2.datamodeling.encryption.materials.EncryptionMaterials; import com.amazonaws.services.dynamodbv2.datamodeling.encryption.providers.EncryptionMaterialsProvider; import com.amazonaws.services.dynamodbv2.datamodeling.encryption.providers.SymmetricStaticProvider; -import com.amazonaws.services.dynamodbv2.datamodeling.encryption.providers.store.MetaStore; -import com.amazonaws.services.dynamodbv2.datamodeling.encryption.providers.store.ProviderStore; import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; public class MetaStoreTests { - private static final String TABLE_NAME = "keystoreTable"; + private static final String SOURCE_TABLE_NAME = "keystoreTable"; + private static final String DESTINATION_TABLE_NAME = "keystoreDestinationTable"; private static final String MATERIAL_NAME = "material"; private static final SecretKey AES_KEY = new SecretKeySpec(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, "AES"); + private static final SecretKey TARGET_AES_KEY = new SecretKeySpec(new byte[] { 0, + 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 }, "AES"); private static final SecretKey HMAC_KEY = new SecretKeySpec(new byte[] { 0, 1, 2, 3, 4, 5, 6, 7 }, "HmacSHA256"); + private static final SecretKey TARGET_HMAC_KEY = new SecretKeySpec(new byte[] { 0, + 2, 4, 6, 8, 10, 12, 14 }, "HmacSHA256"); private static final EncryptionMaterialsProvider BASE_PROVIDER = new SymmetricStaticProvider(AES_KEY, HMAC_KEY); + private static final EncryptionMaterialsProvider TARGET_BASE_PROVIDER = new SymmetricStaticProvider(TARGET_AES_KEY, TARGET_HMAC_KEY); private static final DynamoDBEncryptor ENCRYPTOR = DynamoDBEncryptor.getInstance(BASE_PROVIDER); + private static final DynamoDBEncryptor TARGET_ENCRYPTOR = DynamoDBEncryptor.getInstance(TARGET_BASE_PROVIDER); private AmazonDynamoDB client; - private ProviderStore store; + private AmazonDynamoDB targetClient; + private MetaStore store; + private MetaStore targetStore; private EncryptionContext ctx; @Before public void setup() { client = synchronize(DynamoDBEmbedded.create(), AmazonDynamoDB.class); - MetaStore.createTable(client, TABLE_NAME, new ProvisionedThroughput(1L, 1L)); - store = new MetaStore(client, TABLE_NAME, ENCRYPTOR); + targetClient = synchronize(DynamoDBEmbedded.create(), AmazonDynamoDB.class); + MetaStore.createTable(client, SOURCE_TABLE_NAME, new ProvisionedThroughput(1L, 1L)); + //Creating Targeted DynamoDB Object + MetaStore.createTable(targetClient, DESTINATION_TABLE_NAME, new ProvisionedThroughput(1L, 1L)); + store = new MetaStore(client, SOURCE_TABLE_NAME, ENCRYPTOR); + targetStore = new MetaStore(targetClient, DESTINATION_TABLE_NAME, TARGET_ENCRYPTOR); ctx = new EncryptionContext.Builder().build(); } @@ -172,6 +183,28 @@ public void getOrCreateCollision() { assertEquals(eMat.getSigningKey(), dMat.getVerificationKey()); } + @Test + public void replicateIntermediateKeysTest() { + assertEquals(-1, store.getMaxVersion(MATERIAL_NAME)); + + final EncryptionMaterialsProvider prov1 = store.getOrCreate(MATERIAL_NAME, 0); + assertEquals(0, store.getMaxVersion(MATERIAL_NAME)); + + store.replicate(MATERIAL_NAME, 0, targetStore); + assertEquals(0, targetStore.getMaxVersion(MATERIAL_NAME)); + + final EncryptionMaterials eMat = prov1.getEncryptionMaterials(ctx); + final DecryptionMaterials dMat = targetStore.getProvider(MATERIAL_NAME, 0).getDecryptionMaterials(ctx(eMat)); + + assertEquals(eMat.getEncryptionKey(), dMat.getDecryptionKey()); + assertEquals(eMat.getSigningKey(), dMat.getVerificationKey()); + } + + @Test(expected = IndexOutOfBoundsException.class) + public void replicateIntermediateKeysWhenMaterialNotFoundTest() { + store.replicate(MATERIAL_NAME, 0, targetStore); + } + @Test public void newProviderCollision() throws InterruptedException { final SlowNewProvider slowProv = new SlowNewProvider(); @@ -207,7 +240,7 @@ private static EncryptionContext ctx(final EncryptionMaterials mat) { private class SlowNewProvider extends Thread { public volatile EncryptionMaterialsProvider result; - public ProviderStore slowStore = new MetaStore(client, TABLE_NAME, ENCRYPTOR) { + public ProviderStore slowStore = new MetaStore(client, SOURCE_TABLE_NAME, ENCRYPTOR) { @Override public EncryptionMaterialsProvider newProvider(final String materialName) { final long nextId = getMaxVersion(materialName) + 1;