Skip to content

ARROW-113: Make BaseValueVector#MAX_ALLOCATION_SIZE configurable #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
laurentgo wants to merge 1 commit into from
Closed

ARROW-113: Make BaseValueVector#MAX_ALLOCATION_SIZE configurable #65

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,24 @@
*/
package org.apache.arrow.vector;

import io.netty.buffer.ArrowBuf;

import java.util.Iterator;

import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.types.MaterializedField;
import org.apache.arrow.vector.util.TransferPair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;

import io.netty.buffer.ArrowBuf;

public abstract class BaseValueVector implements ValueVector {
private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);

public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
public static final String MAX_ALLOCATION_SIZE_PROPERTY = "arrow.vector.max_allocation_bytes";
public static final int MAX_ALLOCATION_SIZE = Integer.getInteger(MAX_ALLOCATION_SIZE_PROPERTY, Integer.MAX_VALUE);
public static final int INITIAL_VALUE_ALLOCATION = 4096;

protected final BufferAllocator allocator;
Expand Down Expand Up @@ -99,6 +100,7 @@ protected BaseMutator() { }
public void generateTestData(int values) {}

//TODO: consider making mutator stateless(if possible) on another issue.
@Override
public void reset() {}
}

Expand Down
36 changes: 30 additions & 6 deletions java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,12 @@

import java.nio.charset.Charset;

import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.RepeatedListVector;
import org.apache.arrow.vector.complex.RepeatedMapVector;
import org.apache.arrow.vector.types.MaterializedField;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.BasicTypeHelper;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.holders.BitHolder;
Copy link
Contributor

@jacques-n jacques-n Apr 20, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like your ide is switching the order of imports. Can you add checkstyle so we can avoid having jumping imports between different commits?

Copy link
Contributor Author

@laurentgo laurentgo Apr 20, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, I can add checkstyle plugin for import order. Any preference? http://checkstyle.sourceforge.net/config_imports.html#CustomImportOrder_Example (I'm using the Eclipse order, and I forgot to disable "organize imports on save action")

Copy link
Contributor Author

@laurentgo laurentgo Apr 26, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ping @jacques-n

import org.apache.arrow.vector.holders.IntHolder;
import org.apache.arrow.vector.holders.NullableFloat4Holder;
Expand All @@ -44,10 +40,16 @@
import org.apache.arrow.vector.holders.RepeatedVarBinaryHolder;
import org.apache.arrow.vector.holders.UInt4Holder;
import org.apache.arrow.vector.holders.VarCharHolder;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.types.MaterializedField;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.BasicTypeHelper;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExternalResource;


public class TestValueVector {
Expand All @@ -57,6 +59,28 @@ public class TestValueVector {

private BufferAllocator allocator;

// Rule to adjust MAX_ALLOCATION_SIZE and restore it back after the tests
@Rule
public final ExternalResource rule = new ExternalResource() {
private final String systemValue = System.getProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
private final String testValue = Long.toString(32*1024*1024);

@Override
protected void before() throws Throwable {
System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, testValue);
}

@Override
protected void after() {
if (systemValue != null) {
System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, systemValue);
}
else {
System.clearProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
}
}
};

@Before
public void init() {
allocator = new RootAllocator(Long.MAX_VALUE);
Expand Down