Skip to content

Tensor datatostring #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 26 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
c57a2e7
Merge pull request #3 from tensorflow/master
JimClarke5 Oct 8, 2020
09fc07e
Merge pull request #4 from tensorflow/master
JimClarke5 Oct 27, 2020
a99dcb4
Merge pull request #5 from tensorflow/master
JimClarke5 Nov 17, 2020
ba294ea
Merge pull request #6 from tensorflow/master
JimClarke5 Nov 19, 2020
04f419a
Merge pull request #7 from tensorflow/master
JimClarke5 Dec 30, 2020
02e7ebf
Merge pull request #8 from tensorflow/master
JimClarke5 Jan 29, 2021
e0c9ed8
Merge pull request #9 from tensorflow/master
JimClarke5 Feb 1, 2021
5b0374b
Merge pull request #10 from tensorflow/master
JimClarke5 Feb 11, 2021
e038bbd
Merge pull request #11 from tensorflow/master
JimClarke5 Feb 23, 2021
def3051
Merge pull request #13 from tensorflow/master
JimClarke5 Mar 3, 2021
11748ae
Merge pull request #15 from tensorflow/master
JimClarke5 Mar 21, 2021
9e90751
Added Tensor.dataToString().
cowwoc Apr 3, 2021
b3a2ab7
Added missing import.
cowwoc Apr 3, 2021
e30d9b5
Documentation fix.
cowwoc Apr 3, 2021
8bf627a
Documentation fix.
cowwoc Apr 3, 2021
61e7cad
maxWidth was truncating text mid-element.
cowwoc Apr 3, 2021
ffdf879
* Added Operand.dataToString().
cowwoc Apr 4, 2021
da04cd4
Fixed test.
cowwoc Apr 4, 2021
5821e99
Leaving out comma after closing brackets.
cowwoc Apr 4, 2021
6f8dedc
Data type tests, wrap strings in quotes
rnett Apr 5, 2021
98d320d
Cleanup in response to PR comments.
cowwoc Apr 6, 2021
921e077
Merge pull request #1 from rnett/feature/tensor-datatostring
cowwoc Apr 6, 2021
a9412ea
Merge pull request #16 from tensorflow/master
JimClarke5 Apr 9, 2021
7e56987
Merge remote-tracking branch 'upstream/master' into feature/tensor-da…
cowwoc Apr 14, 2021
9c54fa4
Merge branch 'feature/tensor-datatostring' of https://github.com/coww…
JimClarke5 Apr 14, 2021
aa0631f
Changes based on PR comments.
JimClarke5 Apr 16, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -364,10 +364,10 @@ public final class Ops {

public final SignalOps signal;

public final TrainOps train;

public final QuantizationOps quantization;

public final TrainOps train;

private final Scope scope;

private Ops(Scope scope) {
Expand All @@ -390,8 +390,8 @@ private Ops(Scope scope) {
math = new MathOps(this);
audio = new AudioOps(this);
signal = new SignalOps(this);
train = new TrainOps(this);
quantization = new QuantizationOps(this);
train = new TrainOps(this);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

package org.tensorflow;

import org.tensorflow.TensorPrinter.Options;
import org.tensorflow.ndarray.Shape;
import org.tensorflow.ndarray.Shaped;
import org.tensorflow.op.Op;
Expand Down Expand Up @@ -56,7 +57,7 @@ public interface Operand<T extends TType> extends Op, Shaped {
/**
* Returns the tensor at this operand.
*
* <i>Only works when running in an eager execution</i>
* <p><i>Only works when running in an eager execution</i>
*
* @return the tensor
* @throws IllegalStateException if this is an operand of a graph
Expand All @@ -66,14 +67,34 @@ default T asTensor() {
}

/**
* Returns the tensor type of this operand
* Returns the String representation of the tensor elements at this operand.
*
* @return the String representation of the tensor elements
* @throws IllegalStateException if this is an operand of a graph
*/
default String print() {
return asTensor().print();
}

/**
* Returns the String representation of the tensor elements at this operand.
*
* @param options overrides the default configuration
* @return the String representation of the tensor elements
* @throws IllegalStateException if this is an operand of a graph
*/
default String print(Options options) {
return asTensor().print(options);
}

/** Returns the tensor type of this operand */
default Class<T> type() {
return asOutput().type();
}

/**
* Returns the (possibly partially known) shape of the tensor referred to by the {@link Output} of this operand.
* Returns the (possibly partially known) shape of the tensor referred to by the {@link Output} of
* this operand.
*/
@Override
default Shape shape() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,21 @@

package org.tensorflow;

import java.util.function.Consumer;
import org.tensorflow.ndarray.Shape;
import org.tensorflow.ndarray.Shaped;
import org.tensorflow.ndarray.buffer.ByteDataBuffer;
import org.tensorflow.proto.framework.DataType;
import org.tensorflow.types.family.TType;

import java.util.function.Consumer;

/**
* A statically typed multi-dimensional array.
*
* <p>There are two categories of tensors in TensorFlow Java: {@link TType typed tensors} and
* {@link RawTensor raw tensors}. The former maps the tensor native memory to an
* n-dimensional typed data space, allowing direct I/O operations from the JVM, while the latter
* is only a reference to a native tensor allowing basic operations and flat data access.</p>
* <p>There are two categories of tensors in TensorFlow Java: {@link TType typed tensors} and {@link
* RawTensor raw tensors}. The former maps the tensor native memory to an n-dimensional typed data
* space, allowing direct I/O operations from the JVM, while the latter is only a reference to a
* native tensor allowing basic operations and flat data access.
*
* <p><b>WARNING:</b> Resources consumed by the Tensor object <b>must</b> be explicitly freed by
* invoking the {@link #close()} method when the object is no longer needed. For example, using a
Expand All @@ -39,6 +40,7 @@
* doSomethingWith(t);
* }
* }</pre>
*
* <p>Instances of a Tensor are <b>not</b> thread-safe.
*/
public interface Tensor extends Shaped, AutoCloseable {
Expand All @@ -54,9 +56,9 @@ public interface Tensor extends Shaped, AutoCloseable {
* @param shape shape of the tensor
* @return an allocated but uninitialized tensor
* @throws IllegalArgumentException if elements of the given {@code type} are of variable length
* (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially
* {@link Shape#hasUnknownDimension() unknown}
* (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially {@link
* Shape#hasUnknownDimension() unknown}
* @throws IllegalStateException if tensor failed to be allocated
*/
static <T extends TType> T of(Class<T> type, Shape shape) {
Expand All @@ -67,27 +69,27 @@ static <T extends TType> T of(Class<T> type, Shape shape) {
* Allocates a tensor of a given datatype, shape and size.
*
* <p>This method is identical to {@link #of(Class, Shape)}, except that the final size of the
* tensor can be explicitly set instead of computing it from the datatype and shape, which could be
* larger than the actual space required to store the data but not smaller.
* tensor can be explicitly set instead of computing it from the datatype and shape, which could
* be larger than the actual space required to store the data but not smaller.
*
* @param <T> the tensor type
* @param type the tensor type class
* @param shape shape of the tensor
* @param size size in bytes of the tensor or -1 to compute the size from the shape
* @return an allocated but uninitialized tensor
* @see #of(Class, Shape)
* @throws IllegalArgumentException if {@code size} is smaller than the minimum space required to
* store the tensor data
* @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given
* {@code type} are of variable length (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially
* {@link Shape#hasUnknownDimension() unknown}
* store the tensor data
* @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given {@code
* type} are of variable length (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially {@link
* Shape#hasUnknownDimension() unknown}
* @throws IllegalStateException if tensor failed to be allocated
* @see #of(Class, Shape)
*/
static <T extends TType> T of(Class<T> type, Shape shape, long size) {
RawTensor tensor = RawTensor.allocate(type, shape, size);
try {
return (T)tensor.asTypedTensor();
return (T) tensor.asTypedTensor();
} catch (Exception e) {
tensor.close();
throw e;
Expand All @@ -114,12 +116,13 @@ static <T extends TType> T of(Class<T> type, Shape shape, long size) {
* @param <T> the tensor type
* @param type the tensor type class
* @param shape shape of the tensor
* @param dataInitializer method receiving accessor to the allocated tensor data for initialization
* @param dataInitializer method receiving accessor to the allocated tensor data for
* initialization
* @return an allocated and initialized tensor
* @throws IllegalArgumentException if elements of the given {@code type} are of variable length
* (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially
* {@link Shape#hasUnknownDimension() unknown}
* (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially {@link
* Shape#hasUnknownDimension() unknown}
* @throws IllegalStateException if tensor failed to be allocated
*/
static <T extends TType> T of(Class<T> type, Shape shape, Consumer<T> dataInitializer) {
Expand All @@ -129,28 +132,30 @@ static <T extends TType> T of(Class<T> type, Shape shape, Consumer<T> dataInitia
/**
* Allocates a tensor of a given datatype, shape and size.
*
* <p>This method is identical to {@link #of(Class, Shape, Consumer)}, except that the final
* size for the tensor can be explicitly set instead of being computed from the datatype and shape.
* <p>This method is identical to {@link #of(Class, Shape, Consumer)}, except that the final size
* for the tensor can be explicitly set instead of being computed from the datatype and shape.
*
* <p>This could be useful for tensor types that stores data but also metadata in the tensor memory,
* such as the lookup table in a tensor of strings.
* <p>This could be useful for tensor types that stores data but also metadata in the tensor
* memory, such as the lookup table in a tensor of strings.
*
* @param <T> the tensor type
* @param type the tensor type class
* @param shape shape of the tensor
* @param size size in bytes of the tensor or -1 to compute the size from the shape
* @param dataInitializer method receiving accessor to the allocated tensor data for initialization
* @param dataInitializer method receiving accessor to the allocated tensor data for
* initialization
* @return an allocated and initialized tensor
* @see #of(Class, Shape, long, Consumer)
* @throws IllegalArgumentException if {@code size} is smaller than the minimum space required to
* store the tensor data
* @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given
* {@code type} are of variable length (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially
* {@link Shape#hasUnknownDimension() unknown}
* store the tensor data
* @throws IllegalArgumentException if {@code size} is set to -1 but elements of the given {@code
* type} are of variable length (e.g. strings)
* @throws IllegalArgumentException if {@code shape} is totally or partially {@link
* Shape#hasUnknownDimension() unknown}
* @throws IllegalStateException if tensor failed to be allocated
* @see #of(Class, Shape, long, Consumer)
*/
static <T extends TType> T of(Class<T> type, Shape shape, long size, Consumer<T> dataInitializer) {
static <T extends TType> T of(
Class<T> type, Shape shape, long size, Consumer<T> dataInitializer) {
T tensor = of(type, shape, size);
try {
dataInitializer.accept(tensor);
Expand All @@ -172,34 +177,65 @@ static <T extends TType> T of(Class<T> type, Shape shape, long size, Consumer<T>
* @param shape the tensor shape.
* @param rawData a buffer containing the tensor raw data.
* @throws IllegalArgumentException if {@code rawData} is not large enough to contain the tensor
* data
* @throws IllegalArgumentException if {@code shape} is totally or partially
* {@link Shape#hasUnknownDimension() unknown}
* data
* @throws IllegalArgumentException if {@code shape} is totally or partially {@link
* Shape#hasUnknownDimension() unknown}
* @throws IllegalStateException if tensor failed to be allocated with the given parameters
*/
static <T extends TType> T of(Class<T> type, Shape shape, ByteDataBuffer rawData) {
return of(type, shape, rawData.size(), t -> rawData.copyTo(t.asRawTensor().data(), rawData.size()));
return of(
type, shape, rawData.size(), t -> rawData.copyTo(t.asRawTensor().data(), rawData.size()));
}

/** Returns the {@link DataType} of elements stored in the tensor. */
DataType dataType();

/** Returns the size, in bytes, of the tensor data. */
long numBytes();

/**
* Returns the {@link DataType} of elements stored in the tensor.
* Gets the String representation of the tensor elements
*
* @return the String representation of the tensor elements
*/
DataType dataType();
default String print() {
return new TensorPrinter(this).print();
}

/**
* Returns the size, in bytes, of the tensor data.
* Gets the String representation of the tensor elements
*
* @param printOptions the options for the {@link TensorPrinter} object.
* @return the String representation of the tensor elements
*/
long numBytes();
default String print(TensorPrinter.Options printOptions) {
return new TensorPrinter(this, printOptions).print();
}

/**
* Returns the shape of the tensor.
* Get a {@link TensorPrinter} for this tensor.
*
* @return the {@link TensorPrinter} for this tensor.
*/
@Override
Shape shape();
default TensorPrinter printer() {
return new TensorPrinter(this);
}

/**
* Returns a raw (untyped) representation of this tensor
* Get a {@link TensorPrinter} for this tensor.
*
* @param printOptions the options for the {@link TensorPrinter} object.
* @return the {@link TensorPrinter} for this tensor.
*/
default TensorPrinter printer(TensorPrinter.Options printOptions) {
return new TensorPrinter(this, printOptions);
}

/** Returns the shape of the tensor. */
@Override
Shape shape();

/** Returns a raw (untyped) representation of this tensor */
RawTensor asRawTensor();

/**
Expand All @@ -212,4 +248,22 @@ static <T extends TType> T of(Class<T> type, Shape shape, ByteDataBuffer rawData
*/
@Override
void close();

class ToStringOptions {

private Integer maxWidth;

private ToStringOptions() {}

/**
* Sets the maximum width of the output in characters.
*
* @param maxWidth the maximum width of the output in characters ({@code null} if unlimited).
* This limit may surpassed if the first or last element are too long.
*/
public ToStringOptions maxWidth(Integer maxWidth) {
this.maxWidth = maxWidth;
return this;
}
}
}
Loading