Skip to content

ARROW-335: Improve Type apis and toString() by encapsulating flatbuffers better #172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
julienledem wants to merge 3 commits into from
Closed

ARROW-335: Improve Type apis and toString() by encapsulating flatbuffers better #172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
41d5627
ARROW-335: Improve Type apis and toString() by encapsulating flatbuff…
julienledem Oct 12, 2016
262ae9f
review feedback
julienledem Oct 13, 2016
546aa02
fix rebase issues
julienledem Nov 8, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions java/vector/src/main/codegen/data/ArrowTypes.tdd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@
},
{
name: "Union",
fields: [{name: "mode", type: short}, {name: "typeIds", type: "int[]"}]
fields: [{name: "mode", type: short, valueType: UnionMode}, {name: "typeIds", type: "int[]"}]
},
{
name: "Int",
fields: [{name: "bitWidth", type: int}, {name: "isSigned", type: boolean}]
},
{
name: "FloatingPoint",
fields: [{name: precision, type: short}]
fields: [{name: precision, type: short, valueType: FloatingPointPrecision}]
},
{
name: "Utf8",
Expand Down Expand Up @@ -62,11 +62,11 @@
},
{
name: "Timestamp",
fields: [{name: "unit", type: short}]
fields: [{name: "unit", type: short, valueType: TimeUnit}]
},
{
name: "Interval",
fields: [{name: "unit", type: short}]
fields: [{name: "unit", type: short, valueType: IntervalUnit}]
}
]
}
172 changes: 38 additions & 134 deletions java/vector/src/main/codegen/templates/ArrowType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,30 +23,18 @@
package org.apache.arrow.vector.types.pojo;

import com.google.flatbuffers.FlatBufferBuilder;
import org.apache.arrow.flatbuf.Type;

import java.io.IOException;
import java.util.Objects;

import org.apache.arrow.flatbuf.Precision;
import org.apache.arrow.flatbuf.UnionMode;
import org.apache.arrow.flatbuf.TimeUnit;
import org.apache.arrow.flatbuf.IntervalUnit;
import org.apache.arrow.flatbuf.Type;

import org.apache.arrow.vector.types.*;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;

/**
* Arrow types
Expand All @@ -57,119 +45,31 @@
property = "name")
@JsonSubTypes({
<#list arrowTypes.types as type>
@JsonSubTypes.Type(value = ArrowType.${type.name}.class, name = "${type.name?remove_ending("_")?lower_case}"),
@JsonSubTypes.Type(value = ArrowType.${type.name?remove_ending("_")}.class, name = "${type.name?remove_ending("_")?lower_case}"),
</#list>
})
public abstract class ArrowType {

private static class FloatingPointPrecisionSerializer extends JsonSerializer<Short> {
@Override
public void serialize(Short precision,
JsonGenerator jsonGenerator,
SerializerProvider serializerProvider)
throws IOException, JsonProcessingException {
jsonGenerator.writeObject(Precision.name(precision));
}
}

private static class FloatingPointPrecisionDeserializer extends JsonDeserializer<Short> {
@Override
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
String name = p.getText();
switch(name) {
case "HALF":
return Precision.HALF;
case "SINGLE":
return Precision.SINGLE;
case "DOUBLE":
return Precision.DOUBLE;
default:
throw new IllegalArgumentException("unknown precision: " + name);
}
}
}

private static class UnionModeSerializer extends JsonSerializer<Short> {
@Override
public void serialize(Short mode,
JsonGenerator jsonGenerator,
SerializerProvider serializerProvider)
throws IOException, JsonProcessingException {
jsonGenerator.writeObject(UnionMode.name(mode));
}
}

private static class UnionModeDeserializer extends JsonDeserializer<Short> {
@Override
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
String name = p.getText();
switch(name) {
case "Sparse":
return UnionMode.Sparse;
case "Dense":
return UnionMode.Dense;
default:
throw new IllegalArgumentException("unknown union mode: " + name);
}
}
}

private static class TimestampUnitSerializer extends JsonSerializer<Short> {
@Override
public void serialize(Short unit,
JsonGenerator jsonGenerator,
SerializerProvider serializerProvider)
throws IOException, JsonProcessingException {
jsonGenerator.writeObject(TimeUnit.name(unit));
}
}

private static class TimestampUnitDeserializer extends JsonDeserializer<Short> {
@Override
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
String name = p.getText();
switch(name) {
case "SECOND":
return TimeUnit.SECOND;
case "MILLISECOND":
return TimeUnit.MILLISECOND;
case "MICROSECOND":
return TimeUnit.MICROSECOND;
case "NANOSECOND":
return TimeUnit.NANOSECOND;
default:
throw new IllegalArgumentException("unknown time unit: " + name);
}
}
}
public static enum ArrowTypeID {
<#list arrowTypes.types as type>
<#assign name = type.name>
${name?remove_ending("_")}(Type.${name}),
</#list>
NONE(Type.NONE);

private final byte flatbufType;

private static class IntervalUnitSerializer extends JsonSerializer<Short> {
@Override
public void serialize(Short unit,
JsonGenerator jsonGenerator,
SerializerProvider serializerProvider)
throws IOException, JsonProcessingException {
jsonGenerator.writeObject(IntervalUnit.name(unit));
public byte getFlatbufID() {
return this.flatbufType;
}
}

private static class IntervalUnitDeserializer extends JsonDeserializer<Short> {
@Override
public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
String name = p.getText();
switch(name) {
case "YEAR_MONTH":
return IntervalUnit.YEAR_MONTH;
case "DAY_TIME":
return IntervalUnit.DAY_TIME;
default:
throw new IllegalArgumentException("unknown interval unit: " + name);
}
private ArrowTypeID(byte flatbufType) {
this.flatbufType = flatbufType;
}
}

@JsonIgnore
public abstract byte getTypeType();
public abstract ArrowTypeID getTypeID();
public abstract int getType(FlatBufferBuilder builder);
public abstract <T> T accept(ArrowTypeVisitor<T> visitor);

Expand All @@ -183,28 +83,30 @@ public Short deserialize(JsonParser p, DeserializationContext ctxt) throws IOExc
*/
public static interface ArrowTypeVisitor<T> {
<#list arrowTypes.types as type>
T visit(${type.name} type);
T visit(${type.name?remove_ending("_")} type);
</#list>
}

<#list arrowTypes.types as type>
<#assign name = type.name>
<#assign name = type.name?remove_ending("_")>
<#assign fields = type.fields>
public static class ${name} extends ArrowType {
public static final byte TYPE_TYPE = Type.${name};
public static final ArrowTypeID TYPE_TYPE = ArrowTypeID.${name};
<#if type.fields?size == 0>
public static final ${name} INSTANCE = new ${name}();
</#if>

<#list fields as field>
${field.type} ${field.name};
<#assign fieldType = field.valueType!field.type>
${fieldType} ${field.name};
</#list>

<#if type.fields?size != 0>
@JsonCreator
public ${type.name}(
<#list type.fields as field>
<#if field.type == "short"> @JsonDeserialize(using = ${type.name}${field.name?cap_first}Deserializer.class) </#if>@JsonProperty("${field.name}") ${field.type} ${field.name}<#if field_has_next>, </#if>
<#assign fieldType = field.valueType!field.type>
@JsonProperty("${field.name}") ${fieldType} ${field.name}<#if field_has_next>, </#if>
</#list>
) {
<#list type.fields as field>
Expand All @@ -214,7 +116,7 @@ public static class ${name} extends ArrowType {
</#if>

@Override
public byte getTypeType() {
public ArrowTypeID getTypeID() {
return TYPE_TYPE;
}

Expand All @@ -235,27 +137,29 @@ public int getType(FlatBufferBuilder builder) {
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, ${field.name});
}
<#else>
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, this.${field.name});
org.apache.arrow.flatbuf.${type.name}.add${field.name?cap_first}(builder, this.${field.name}<#if field.valueType??>.getFlatbufID()</#if>);
</#if>
</#list>
return org.apache.arrow.flatbuf.${type.name}.end${type.name}(builder);
}

<#list fields as field>
<#if field.type == "short">
@JsonSerialize(using = ${type.name}${field.name?cap_first}Serializer.class)
</#if>
public ${field.type} get${field.name?cap_first}() {
<#assign fieldType = field.valueType!field.type>
public ${fieldType} get${field.name?cap_first}() {
return ${field.name};
}
</#list>

public String toString() {
return "${name}{"
return "${name}"
<#if fields?size != 0>
+ "("
<#list fields as field>
+ <#if field.type == "int[]">java.util.Arrays.toString(${field.name})<#else>${field.name}</#if><#if field_has_next> + ", " </#if>
+ <#if field.type == "int[]">java.util.Arrays.toString(${field.name})<#else>${field.name}</#if><#if field_has_next> + ", " </#if>
</#list>
+ "}";
+ ")"
</#if>
;
}

@Override
Expand All @@ -265,7 +169,7 @@ public int hashCode() {

@Override
public boolean equals(Object obj) {
if (!(obj instanceof ${type.name})) {
if (!(obj instanceof ${name})) {
return false;
}
<#if type.fields?size == 0>
Expand All @@ -287,7 +191,7 @@ public <T> T accept(ArrowTypeVisitor<T> visitor) {
public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.apache.arrow.flatbuf.Field field) {
switch(field.typeType()) {
<#list arrowTypes.types as type>
<#assign name = type.name>
<#assign name = type.name?remove_ending("_")>
<#assign nameLower = type.name?lower_case>
<#assign fields = type.fields>
case Type.${type.name}: {
Expand All @@ -302,7 +206,7 @@ public static org.apache.arrow.vector.types.pojo.ArrowType getTypeForField(org.a
${field.type} ${field.name} = ${nameLower}Type.${field.name}();
</#if>
</#list>
return new ${type.name}(<#list type.fields as field>${field.name}<#if field_has_next>, </#if></#list>);
return new ${name}(<#list type.fields as field><#if field.valueType??>${field.valueType}.fromFlatbufID(${field.name})<#else>${field.name}</#if><#if field_has_next>, </#if></#list>);
}
</#list>
default:
Expand Down
10 changes: 5 additions & 5 deletions java/vector/src/main/codegen/templates/NullableValueVectors.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,15 +99,15 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#elseif minor.class == "Time">
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Time(), null);
<#elseif minor.class == "Float4">
field = new Field(name, true, new FloatingPoint(Precision.SINGLE), null);
field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), null);
<#elseif minor.class == "Float8">
field = new Field(name, true, new FloatingPoint(Precision.DOUBLE), null);
field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null);
<#elseif minor.class == "TimeStamp">
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.flatbuf.TimeUnit.MILLISECOND), null);
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), null);
<#elseif minor.class == "IntervalDay">
field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.DAY_TIME), null);
field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), null);
<#elseif minor.class == "IntervalYear">
field = new Field(name, true, new Interval(org.apache.arrow.flatbuf.IntervalUnit.YEAR_MONTH), null);
field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.YEAR_MONTH), null);
<#elseif minor.class == "VarChar">
field = new Field(name, true, new Utf8(), null);
<#elseif minor.class == "VarBinary">
Expand Down
4 changes: 1 addition & 3 deletions java/vector/src/main/codegen/templates/UnionVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.List;

<@pp.dropOutputFile />
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />

Expand All @@ -35,7 +33,7 @@
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.schema.ArrowFieldNode;

import static org.apache.arrow.flatbuf.UnionMode.Sparse;
import static org.apache.arrow.vector.types.UnionMode.Sparse;



Expand Down
5 changes: 2 additions & 3 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.Collections;
import java.util.Iterator;

import org.apache.arrow.flatbuf.Type;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.BaseValueVector;
Expand Down Expand Up @@ -159,9 +158,9 @@ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorT
created = true;
}

if (vector.getField().getType().getTypeType() != minorType.getType().getTypeType()) {
if (vector.getField().getType().getTypeID() != minorType.getType().getTypeID()) {
final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
Type.name(minorType.getType().getTypeType()), Type.name(vector.getField().getType().getTypeType()));
minorType.getType().getTypeID(), vector.getField().getType().getTypeID());
throw new SchemaChangeRuntimeException(msg);
}

Expand Down
4 changes: 2 additions & 2 deletions java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import org.apache.arrow.vector.holders.ComplexHolder;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Struct_;
import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringHashMap;
Expand Down Expand Up @@ -290,7 +290,7 @@ public Field getField() {
for (ValueVector child : getChildren()) {
children.add(child.getField());
}
return new Field(name, false, Struct_.INSTANCE, children);
return new Field(name, false, Struct.INSTANCE, children);
}

@Override
Expand Down
Loading