Skip to content

Commit 1b2ec92

Browse files
xerialfrsyuki
xerial
and
frsyuki
authored
Timestamp support (#565)
* Add add support for Timestamp type * Add timestamp tests * Fix timestamp pack overflow * Add timestamp value support * Support timestamp in unpackValue * Use AirSpec for integration with ScalaCheck * Fix ValueFactoryTest * Remove scalatest * Remove xerial-core dependency * Remove unnecessary dependencies * Variable.writeTo roundtrip tests * Use consistent code style Co-authored-by: Sadayuki Furuhashi <[email protected]>
1 parent 2864da3 commit 1b2ec92

31 files changed

+1934
-956
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ Here is a list of sbt commands for daily development:
7070
> ~test:compile # Compile both source and test codes
7171
> ~test # Run tests upon source code change
7272
> ~testOnly *MessagePackTest # Run tests in the specified class
73-
> ~testOnly *MessagePackTest -- -n prim # Run the test tagged as "prim"
73+
> ~testOnly *MessagePackTest -- (pattern) # Run tests matching the pattern
7474
> project msgpack-core # Focus on a specific project
7575
> package # Create a jar file in the target folder of each project
7676
> findbugs # Produce findbugs report in target/findbugs

‎build.sbt

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ Global / concurrentRestrictions := Seq(
55
Tags.limit(Tags.Test, 1)
66
)
77

8+
val AIRFRAME_VERSION = "20.4.1"
9+
810
// Use dynamic snapshot version strings for non tagged versions
911
ThisBuild / dynverSonatypeSnapshots := true
1012
// Use coursier friendly version separator
@@ -71,15 +73,19 @@ lazy val msgpackCore = Project(id = "msgpack-core", base = file("msgpack-core"))
7173
"org.msgpack.value",
7274
"org.msgpack.value.impl"
7375
),
76+
testFrameworks += new TestFramework("wvlet.airspec.Framework"),
7477
libraryDependencies ++= Seq(
7578
// msgpack-core should have no external dependencies
7679
junitInterface,
77-
"org.scalatest" %% "scalatest" % "3.2.8" % "test",
78-
"org.scalacheck" %% "scalacheck" % "1.15.4" % "test",
79-
"org.xerial" %% "xerial-core" % "3.6.0" % "test",
80-
"org.msgpack" % "msgpack" % "0.6.12" % "test",
81-
"commons-codec" % "commons-codec" % "1.12" % "test",
82-
"com.typesafe.akka" %% "akka-actor" % "2.5.23" % "test"
80+
"org.wvlet.airframe" %% "airframe-json" % AIRFRAME_VERSION % "test",
81+
"org.wvlet.airframe" %% "airspec" % AIRFRAME_VERSION % "test",
82+
// Add property testing support with forAll methods
83+
"org.scalacheck" %% "scalacheck" % "1.15.4" % "test",
84+
// For performance comparison with msgpack v6
85+
"org.msgpack" % "msgpack" % "0.6.12" % "test",
86+
// For integration test with Akka
87+
"com.typesafe.akka" %% "akka-actor" % "2.5.23" % "test",
88+
"org.scala-lang.modules" %% "scala-collection-compat" % "2.4.3" % "test"
8389
)
8490
)
8591

‎msgpack-core/src/main/java/org/msgpack/core/MessagePack.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ public static final boolean isFixedRaw(byte b)
165165
public static final byte MAP32 = (byte) 0xdf;
166166

167167
public static final byte NEGFIXINT_PREFIX = (byte) 0xe0;
168+
169+
public static final byte EXT_TIMESTAMP = (byte) -1;
168170
}
169171

170172
private MessagePack()

‎msgpack-core/src/main/java/org/msgpack/core/MessagePacker.java

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import java.nio.charset.CharsetEncoder;
3333
import java.nio.charset.CoderResult;
3434
import java.nio.charset.CodingErrorAction;
35+
import java.time.Instant;
3536

3637
import static org.msgpack.core.MessagePack.Code.ARRAY16;
3738
import static org.msgpack.core.MessagePack.Code.ARRAY32;
@@ -41,6 +42,7 @@
4142
import static org.msgpack.core.MessagePack.Code.EXT16;
4243
import static org.msgpack.core.MessagePack.Code.EXT32;
4344
import static org.msgpack.core.MessagePack.Code.EXT8;
45+
import static org.msgpack.core.MessagePack.Code.EXT_TIMESTAMP;
4446
import static org.msgpack.core.MessagePack.Code.FALSE;
4547
import static org.msgpack.core.MessagePack.Code.FIXARRAY_PREFIX;
4648
import static org.msgpack.core.MessagePack.Code.FIXEXT1;
@@ -798,6 +800,111 @@ else if (s.length() < (1 << 16)) {
798800
return this;
799801
}
800802

803+
/**
804+
* Writes a Timestamp value.
805+
*
806+
* <p>
807+
* This method writes a timestamp value using timestamp format family.
808+
*
809+
* @param instant the timestamp to be written
810+
* @return this packer
811+
* @throws IOException when underlying output throws IOException
812+
*/
813+
public MessagePacker packTimestamp(Instant instant)
814+
throws IOException
815+
{
816+
return packTimestamp(instant.getEpochSecond(), instant.getNano());
817+
}
818+
819+
/**
820+
* Writes a Timesamp value using a millisecond value (e.g., System.currentTimeMillis())
821+
* @param millis the millisecond value
822+
* @return this packer
823+
* @throws IOException when underlying output throws IOException
824+
*/
825+
public MessagePacker packTimestamp(long millis)
826+
throws IOException
827+
{
828+
return packTimestamp(Instant.ofEpochMilli(millis));
829+
}
830+
831+
private static final long NANOS_PER_SECOND = 1000000000L;
832+
833+
/**
834+
* Writes a Timestamp value.
835+
*
836+
* <p>
837+
* This method writes a timestamp value using timestamp format family.
838+
*
839+
* @param epochSecond the number of seconds from 1970-01-01T00:00:00Z
840+
* @param nanoAdjustment the nanosecond adjustment to the number of seconds, positive or negative
841+
* @return this
842+
* @throws IOException when underlying output throws IOException
843+
* @throws ArithmeticException when epochSecond plus nanoAdjustment in seconds exceeds the range of long
844+
*/
845+
public MessagePacker packTimestamp(long epochSecond, int nanoAdjustment)
846+
throws IOException, ArithmeticException
847+
{
848+
long sec = Math.addExact(epochSecond, Math.floorDiv(nanoAdjustment, NANOS_PER_SECOND));
849+
long nsec = Math.floorMod((long) nanoAdjustment, NANOS_PER_SECOND);
850+
851+
if (sec >>> 34 == 0) {
852+
// sec can be serialized in 34 bits.
853+
long data64 = (nsec << 34) | sec;
854+
if ((data64 & 0xffffffff00000000L) == 0L) {
855+
// sec can be serialized in 32 bits and nsec is 0.
856+
// use timestamp 32
857+
writeTimestamp32((int) sec);
858+
}
859+
else {
860+
// sec exceeded 32 bits or nsec is not 0.
861+
// use timestamp 64
862+
writeTimestamp64(data64);
863+
}
864+
}
865+
else {
866+
// use timestamp 96 format
867+
writeTimestamp96(sec, (int) nsec);
868+
}
869+
return this;
870+
}
871+
872+
private void writeTimestamp32(int sec)
873+
throws IOException
874+
{
875+
// timestamp 32 in fixext 4
876+
ensureCapacity(6);
877+
buffer.putByte(position++, FIXEXT4);
878+
buffer.putByte(position++, EXT_TIMESTAMP);
879+
buffer.putInt(position, sec);
880+
position += 4;
881+
}
882+
883+
private void writeTimestamp64(long data64)
884+
throws IOException
885+
{
886+
// timestamp 64 in fixext 8
887+
ensureCapacity(10);
888+
buffer.putByte(position++, FIXEXT8);
889+
buffer.putByte(position++, EXT_TIMESTAMP);
890+
buffer.putLong(position, data64);
891+
position += 8;
892+
}
893+
894+
private void writeTimestamp96(long sec, int nsec)
895+
throws IOException
896+
{
897+
// timestamp 96 in ext 8
898+
ensureCapacity(15);
899+
buffer.putByte(position++, EXT8);
900+
buffer.putByte(position++, (byte) 12); // length of nsec and sec
901+
buffer.putByte(position++, EXT_TIMESTAMP);
902+
buffer.putInt(position, nsec);
903+
position += 4;
904+
buffer.putLong(position, sec);
905+
position += 8;
906+
}
907+
801908
/**
802909
* Writes header of an Array value.
803910
* <p>

‎msgpack-core/src/main/java/org/msgpack/core/MessageUnpacker.java

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@
3232
import java.nio.charset.CharsetDecoder;
3333
import java.nio.charset.CoderResult;
3434
import java.nio.charset.CodingErrorAction;
35+
import java.time.Instant;
3536

37+
import static org.msgpack.core.MessagePack.Code.EXT_TIMESTAMP;
3638
import static org.msgpack.core.Preconditions.checkNotNull;
3739

3840
/**
@@ -595,6 +597,12 @@ private static MessagePackException unexpected(String expected, byte b)
595597
}
596598
}
597599

600+
private static MessagePackException unexpectedExtension(String expected, int expectedType, int actualType)
601+
{
602+
return new MessageTypeException(String.format("Expected extension type %s (%d), but got extension type %d",
603+
expected, expectedType, actualType));
604+
}
605+
598606
public ImmutableValue unpackValue()
599607
throws IOException
600608
{
@@ -643,7 +651,12 @@ public ImmutableValue unpackValue()
643651
}
644652
case EXTENSION: {
645653
ExtensionTypeHeader extHeader = unpackExtensionTypeHeader();
646-
return ValueFactory.newExtension(extHeader.getType(), readPayload(extHeader.getLength()));
654+
switch (extHeader.getType()) {
655+
case EXT_TIMESTAMP:
656+
return ValueFactory.newTimestamp(unpackTimestamp(extHeader));
657+
default:
658+
return ValueFactory.newExtension(extHeader.getType(), readPayload(extHeader.getLength()));
659+
}
647660
}
648661
default:
649662
throw new MessageNeverUsedFormatException("Unknown value type");
@@ -707,7 +720,13 @@ public Variable unpackValue(Variable var)
707720
}
708721
case EXTENSION: {
709722
ExtensionTypeHeader extHeader = unpackExtensionTypeHeader();
710-
var.setExtensionValue(extHeader.getType(), readPayload(extHeader.getLength()));
723+
switch (extHeader.getType()) {
724+
case EXT_TIMESTAMP:
725+
var.setTimestampValue(unpackTimestamp(extHeader));
726+
break;
727+
default:
728+
var.setExtensionValue(extHeader.getType(), readPayload(extHeader.getLength()));
729+
}
711730
return var;
712731
}
713732
default:
@@ -1257,6 +1276,45 @@ private String decodeStringFastPath(int length)
12571276
}
12581277
}
12591278

1279+
public Instant unpackTimestamp()
1280+
throws IOException
1281+
{
1282+
ExtensionTypeHeader ext = unpackExtensionTypeHeader();
1283+
return unpackTimestamp(ext);
1284+
}
1285+
1286+
/**
1287+
* Internal method that can be used only when the extension type header is already read.
1288+
*/
1289+
private Instant unpackTimestamp(ExtensionTypeHeader ext) throws IOException
1290+
{
1291+
if (ext.getType() != EXT_TIMESTAMP) {
1292+
throw unexpectedExtension("Timestamp", EXT_TIMESTAMP, ext.getType());
1293+
}
1294+
switch (ext.getLength()) {
1295+
case 4: {
1296+
// Need to convert Java's int (int32) to uint32
1297+
long u32 = readInt() & 0xffffffffL;
1298+
return Instant.ofEpochSecond(u32);
1299+
}
1300+
case 8: {
1301+
long data64 = readLong();
1302+
int nsec = (int) (data64 >>> 34);
1303+
long sec = data64 & 0x00000003ffffffffL;
1304+
return Instant.ofEpochSecond(sec, nsec);
1305+
}
1306+
case 12: {
1307+
// Need to convert Java's int (int32) to uint32
1308+
long nsecU32 = readInt() & 0xffffffffL;
1309+
long sec = readLong();
1310+
return Instant.ofEpochSecond(sec, nsecU32);
1311+
}
1312+
default:
1313+
throw new MessageFormatException(String.format("Timestamp extension type (%d) expects 4, 8, or 12 bytes of payload but got %d bytes",
1314+
EXT_TIMESTAMP, ext.getLength()));
1315+
}
1316+
}
1317+
12601318
/**
12611319
* Reads header of an array.
12621320
*

‎msgpack-core/src/main/java/org/msgpack/value/ImmutableTimestampValue.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
//
2+
// MessagePack for Java
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
//
16+
package org.msgpack.value;
17+
18+
/**
19+
* Immutable representation of MessagePack's Timestamp type.
20+
*
21+
* @see org.msgpack.value.TimestampValue
22+
*/
23+
public interface ImmutableTimestampValue
24+
extends TimestampValue, ImmutableValue
25+
{
26+
}

‎msgpack-core/src/main/java/org/msgpack/value/ImmutableValue.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,7 @@ public interface ImmutableValue
4747

4848
@Override
4949
public ImmutableStringValue asStringValue();
50+
51+
@Override
52+
public ImmutableTimestampValue asTimestampValue();
5053
}

‎msgpack-core/src/main/java/org/msgpack/value/TimestampValue.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
//
2+
// MessagePack for Java
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
//
16+
package org.msgpack.value;
17+
18+
import java.time.Instant;
19+
20+
/**
21+
* Value representation of MessagePack's Timestamp type.
22+
*/
23+
public interface TimestampValue
24+
extends ExtensionValue
25+
{
26+
long getEpochSecond();
27+
28+
int getNano();
29+
30+
long toEpochMillis();
31+
32+
Instant toInstant();
33+
}

‎msgpack-core/src/main/java/org/msgpack/value/Value.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package org.msgpack.value;
1717

1818
import org.msgpack.core.MessagePacker;
19+
import org.msgpack.core.MessageTypeCastException;
1920

2021
import java.io.IOException;
2122

@@ -180,6 +181,14 @@ public interface Value
180181
*/
181182
boolean isExtensionValue();
182183

184+
/**
185+
* Returns true if the type of this value is Timestamp.
186+
*
187+
* If this method returns true, {@code asTimestamp} never throws exceptions.
188+
* Note that you can't use <code>instanceof</code> or cast <code>((MapValue) thisValue)</code> to check type of a value because type of a mutable value is variable.
189+
*/
190+
boolean isTimestampValue();
191+
183192
/**
184193
* Returns the value as {@code NilValue}. Otherwise throws {@code MessageTypeCastException}.
185194
*
@@ -280,6 +289,15 @@ public interface Value
280289
*/
281290
ExtensionValue asExtensionValue();
282291

292+
/**
293+
* Returns the value as {@code TimestampValue}. Otherwise throws {@code MessageTypeCastException}.
294+
*
295+
* Note that you can't use <code>instanceof</code> or cast <code>((TimestampValue) thisValue)</code> to check type of a value because type of a mutable value is variable.
296+
*
297+
* @throws MessageTypeCastException If type of this value is not Map.
298+
*/
299+
TimestampValue asTimestampValue();
300+
283301
/**
284302
* Serializes the value using the specified {@code MessagePacker}
285303
*

0 commit comments

Comments
 (0)