[Jython-checkins] jython: Add implementations of PyBuffer based on java.nio.ByteBuffer
jeff.allen
jython-checkins at python.org
Sat Aug 27 09:12:24 EDT 2016
https://hg.python.org/jython/rev/369b40954d96
changeset: 7942:369b40954d96
user: Jeff Allen <ja.py at farowl.co.uk>
date: Sat Jun 11 07:39:53 2016 +0100
summary:
Add implementations of PyBuffer based on java.nio.ByteBuffer
Implementations and a test using the recent refactoring of the array-
based buffer implementation. More refactoring and API change is pending,
see comments for a start.
files:
src/org/python/core/buffer/BaseArrayBuffer.java | 4 +-
src/org/python/core/buffer/BaseNIOBuffer.java | 345 ++++++++++
src/org/python/core/buffer/SimpleBuffer.java | 2 +
src/org/python/core/buffer/SimpleNIOBuffer.java | 206 +++++
src/org/python/core/buffer/Strided1DNIOBuffer.java | 219 ++++++
tests/java/org/python/core/PyBufferNIOTest.java | 316 +++++++++
tests/java/org/python/core/PyBufferTest.java | 17 +-
7 files changed, 1097 insertions(+), 12 deletions(-)
diff --git a/src/org/python/core/buffer/BaseArrayBuffer.java b/src/org/python/core/buffer/BaseArrayBuffer.java
--- a/src/org/python/core/buffer/BaseArrayBuffer.java
+++ b/src/org/python/core/buffer/BaseArrayBuffer.java
@@ -9,8 +9,7 @@
/**
* Base implementation of the Buffer API for when the storage implementation is <code>byte[]</code>.
* The description of {@link BaseBuffer} mostly applies. Methods provided or overridden here are
- * appropriate to 1-dimensional arrays backed by <code>byte[]</code>.
- *
+ * appropriate to 1-dimensional arrays, of any item size, backed by <code>byte[]</code>.
*/
public abstract class BaseArrayBuffer extends BaseBuffer implements PyBuffer {
@@ -275,5 +274,4 @@
}
}
}
-
}
diff --git a/src/org/python/core/buffer/BaseNIOBuffer.java b/src/org/python/core/buffer/BaseNIOBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/BaseNIOBuffer.java
@@ -0,0 +1,345 @@
+package org.python.core.buffer;
+
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
+import java.nio.ReadOnlyBufferException;
+
+import org.python.core.PyBUF;
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
+
+/**
+ * Base implementation of the Buffer API for when the storage implementation is
+ * <code>java.nio.ByteBuffer</code>. The description of {@link BaseBuffer} mostly applies. Methods
+ * provided or overridden here are appropriate to 1-dimensional arrays, of any item size, backed by
+ * a <code>ByteBuffer</code>.
+ */
+public abstract class BaseNIOBuffer extends BaseBuffer implements PyBuffer {
+
+ /**
+ * A {@link java.nio.ByteBuffer} (possibly a direct buffer) wrapping the storage that the
+ * exporter is sharing with the consumer. The data to be exposed may be only a subset of the
+ * bytes in the buffer, defined by the navigation information <code>index0</code>,
+ * <code>shape</code>, <code>strides</code>, etc., usually defined in the constructor.
+ * <p>
+ * Implementations must not adjust the position and limit of <code>storage</code> after
+ * construction. It will generally be a duplicate of (not a reference to) a ByteBuffer held by
+ * the client code. The capacity and backing store are fixed in construction, and the position
+ * will always be {@link #index0}. The limit is always higher than any valid data, and in the
+ * case of a contiguous buffer (with positive stride), is exactly just beyond the last item, so
+ * that a series of ByteBuffer.get operations will yield the data.
+ */
+ protected ByteBuffer storage;
+
+ /**
+ * Partially construct an instance of <code>BaseNIOBuffer</code> in support of a sub-class,
+ * specifying the 'feature flags', or at least a starting set to be adjusted later. These are
+ * the features of the buffer exported, not the flags that form the consumer's request. The
+ * buffer will be read-only and/or backed by a (heap) array according to the properties of the
+ * <code>ByteBuffer</code> passed in. {@link PyBUF#FORMAT} is implicitly added to the feature
+ * flags. To complete initialisation, the sub-class normally must assign: {@link #index0}) and
+ * the navigation arrays ({@link #shape}, {@link #strides}), and call
+ * {@link #checkRequestFlags(int)} passing the consumer's request flags.
+ *
+ * @param storage the <code>ByteBuffer</code> wrapping the exported object state. NOTE: this
+ * <code>PyBuffer</code> keeps a reference and may manipulate the position, mark and
+ * limit hereafter. Use {@link ByteBuffer#duplicate()} to give it an isolated copy.
+ * @param featureFlags bit pattern that specifies the actual features allowed/required
+ */
+ protected BaseNIOBuffer(ByteBuffer storage, int featureFlags) {
+ super(featureFlags & ~(WRITABLE | AS_ARRAY));
+ this.storage = storage;
+
+ // Deduce other feature flags from the client's ByteBuffer
+ if (!storage.isReadOnly()) {
+ addFeatureFlags(WRITABLE);
+ }
+ if (storage.hasArray()) {
+ addFeatureFlags(AS_ARRAY);
+ }
+ }
+
+ @Override
+ protected int getSize() {
+ return shape[0];
+ }
+
+ @Override
+ public int getLen() {
+ return shape[0] * getItemsize();
+ }
+
+ @Override
+ protected byte byteAtImpl(int byteIndex) throws IndexOutOfBoundsException {
+ return storage.get(byteIndex);
+ }
+
+ @Override
+ protected void storeAtImpl(byte value, int byteIndex) throws IndexOutOfBoundsException,
+ PyException {
+ // XXX consider catching ReadonlyBufferException instead of checking (and others: index?)
+ checkWritable();
+ storage.put(byteIndex, value);
+ }
+
+ @Override
+ protected int byteIndex(int... indices) throws IndexOutOfBoundsException {
+ // BaseBuffer implementation can be simplified since if indices.length!=1 we error.
+ checkDimension(indices.length); // throws if != 1
+ return byteIndex(indices[0]);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Specialised to one-dimensional, possibly strided buffer.
+ */
+ @Override
+ protected int calcGreatestIndex() {
+ int stride = strides[0];
+ if (stride == 1) {
+ return index0 + shape[0] - 1;
+ } else if (stride > 0) {
+ return index0 + (shape[0] - 1) * stride;
+ } else {
+ return index0 - 1;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Specialised to one-dimensional, possibly strided buffer.
+ */
+ @Override
+ protected int calcLeastIndex() {
+ int stride = strides[0];
+ if (stride < 0) {
+ return index0 + (shape[0] - 1) * stride;
+ } else {
+ return index0;
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseNIOBuffer</code> deals with the general
+ * one-dimensional case of arbitrary item size and stride.
+ */
+ @Override
+ public void copyTo(int srcIndex, byte[] dest, int destPos, int count)
+ throws IndexOutOfBoundsException {
+ // Wrap the destination, taking care to reflect the necessary range we shall write.
+ ByteBuffer destBuf = ByteBuffer.wrap(dest, destPos, count * getItemsize());
+ copyTo(srcIndex, destBuf, count);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseBuffer</code> deals with the general one-dimensional
+ * case of arbitrary item size and stride.
+ */
+ // XXX Should this become part of the PyBUffer interface?
+ public void copyTo(ByteBuffer dest) throws BufferOverflowException, ReadOnlyBufferException,
+ PyException {
+ // Note shape[0] is the number of items in the buffer
+ copyTo(0, dest, shape[0]);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseNIOBuffer</code> deals with the general
+ * one-dimensional case of arbitrary item size and stride.
+ */
+ // XXX Should this become part of the PyBuffer interface?
+ protected void copyTo(int srcIndex, ByteBuffer dest, int count) throws BufferOverflowException,
+ ReadOnlyBufferException, IndexOutOfBoundsException, PyException {
+
+ if (count > 0) {
+
+ ByteBuffer src = getNIOByteBuffer(srcIndex);
+
+ // Pick up attributes necessary to choose an efficient copy strategy
+ int itemsize = getItemsize();
+ int stride = getStrides()[0];
+
+ // Strategy depends on whether items are laid end-to-end contiguously or there are gaps
+ if (stride == itemsize) {
+ // stride == itemsize: straight copy of contiguous bytes
+ src.limit(src.position() + count * itemsize);
+ dest.put(src);
+
+ } else if (itemsize == 1) {
+ // Non-contiguous copy: single byte items
+ int pos = src.position();
+ for (int i = 0; i < count; i++) {
+ src.position(pos);
+ dest.put(src.get());
+ pos += stride;
+ }
+
+ } else {
+ // Non-contiguous copy: each time, copy itemsize bytes then skip
+ int pos = src.position();
+ for (int i = 0; i < count; i++) {
+ src.limit(pos + itemsize).position(pos);
+ dest.put(src);
+ pos += stride;
+ }
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseNIOBuffer</code> deals with the general
+ * one-dimensional case of arbitrary item size and stride.
+ */
+ @Override
+ public void copyFrom(byte[] src, int srcPos, int destIndex, int count)
+ throws IndexOutOfBoundsException, PyException {
+ // Wrap the source, taking care to reflect the range we shall read.
+ ByteBuffer srcBuf = ByteBuffer.wrap(src, srcPos, count * getItemsize());
+ copyFrom(srcBuf, destIndex, count);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseNIOBuffer</code> deals with the general
+ * one-dimensional case of arbitrary item size and stride.
+ */
+ // XXX Should this become part of the PyBUffer interface?
+ protected void copyFrom(ByteBuffer src, int dstIndex, int count)
+ throws IndexOutOfBoundsException, PyException {
+
+ checkWritable();
+
+ if (count > 0) {
+
+ ByteBuffer dst = getNIOByteBuffer(dstIndex);
+
+ // Pick up attributes necessary to choose an efficient copy strategy
+ int itemsize = getItemsize();
+ int stride = getStrides()[0];
+ int skip = stride - itemsize;
+
+ // Strategy depends on whether items are laid end-to-end or there are gaps
+ if (skip == 0) {
+ // Straight copy of contiguous bytes
+ dst.put(src);
+
+ } else if (itemsize == 1) {
+ // Non-contiguous copy: single byte items
+ int pos = dst.position();
+ for (int i = 0; i < count; i++) {
+ dst.position(pos);
+ dst.put(src.get());
+ // Next byte written will be here
+ pos += stride;
+ }
+
+ } else {
+ // Non-contiguous copy: each time, copy itemsize bytes at a time
+ int pos = dst.position();
+ for (int i = 0; i < count; i++) {
+ dst.position(pos);
+ // Delineate the next itemsize bytes in the src
+ src.limit(src.position() + itemsize);
+ dst.put(src);
+ // Next byte written will be here
+ pos += stride;
+ }
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The default implementation in <code>BaseNIOBuffer</code> deals with the general
+ * one-dimensional case.
+ */
+ @Override
+ public void copyFrom(PyBuffer src) throws IndexOutOfBoundsException, PyException {
+
+ int length = getLen();
+ int itemsize = getItemsize();
+
+ // Valid operation only if writable and same length and itemsize
+ checkWritable();
+ if (src.getLen() != length || src.getItemsize() != itemsize) {
+ throw differentStructure();
+ }
+
+ if (length > 0) {
+ // Pick up attributes necessary to choose an efficient copy strategy
+ int stride = getStrides()[0];
+ int skip = stride - itemsize;
+
+ ByteBuffer dst = getNIOByteBuffer();
+
+ // Strategy depends on whether destination items are laid end-to-end or there are gaps
+ if (skip == 0) {
+ // Straight copy to contiguous bytes
+ for (int i = 0; i < length; i++) {
+ dst.put(src.byteAt(i));
+ }
+
+ } else if (itemsize == 1) {
+ // Non-contiguous copy: single byte items
+ int pos = dst.position();
+ for (int i = 0; i < length; i++) {
+ dst.put(pos, src.byteAt(i));
+ pos += stride;
+ }
+
+ } else {
+ // Non-contiguous copy: each time, and itemsize > 1
+ int pos = dst.position();
+ int s = 0;
+ for (int i = 0; i < length; i++) {
+ for (int j = 0; j < itemsize; j++) {
+ dst.put(pos++, src.byteAt(s++));
+ }
+ pos += skip;
+ }
+ }
+ }
+
+ }
+
+ @Override
+ protected ByteBuffer getNIOByteBufferImpl() {
+ return storage.duplicate();
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public Pointer getBuf() {
+ checkHasArray();
+ return new Pointer(storage.array(), index0);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Specialised in <code>BaseArrayBuffer</code> to one dimension.
+ */
+ @Override
+ public boolean isContiguous(char order) {
+ if ("CFA".indexOf(order) < 0) {
+ return false;
+ } else {
+ if (getShape()[0] < 2) {
+ return true;
+ } else {
+ return getStrides()[0] == getItemsize();
+ }
+ }
+ }
+}
diff --git a/src/org/python/core/buffer/SimpleBuffer.java b/src/org/python/core/buffer/SimpleBuffer.java
--- a/src/org/python/core/buffer/SimpleBuffer.java
+++ b/src/org/python/core/buffer/SimpleBuffer.java
@@ -131,6 +131,8 @@
return index0 + index;
}
+ // XXX Consider moving to clauses in getBufferSlice(int, int, int, int)
+ // to avoid delegation loop where that delegates to this but in BaseBuffer the reverse.
@Override
public PyBuffer getBufferSlice(int flags, int start, int count) {
if (count > 0) {
diff --git a/src/org/python/core/buffer/SimpleNIOBuffer.java b/src/org/python/core/buffer/SimpleNIOBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/SimpleNIOBuffer.java
@@ -0,0 +1,206 @@
+package org.python.core.buffer;
+
+import java.nio.ByteBuffer;
+
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
+
+/**
+ * Buffer API over a read-only one-dimensional array of one-byte items.
+ */
+
+public class SimpleNIOBuffer extends BaseNIOBuffer {
+
+ /**
+ * The strides array for this type is always a single element array with a 1 in it.
+ */
+ protected static final int[] SIMPLE_STRIDES = {1}; // XXX Push up?
+
+ /**
+ * Provide an instance of <code>SimpleNIOBuffer</code> with navigation variables initialised,
+ * for sub-class use. The buffer ({@link #storage}, {@link #index0}), and the {@link #shape}
+ * array will be initialised from the arguments (which are checked for range). The
+ * {@link #strides} is set for (one-byte) unit stride. Only the call to
+ * {@link #checkRequestFlags(int)}, passing the consumer's request flags, really remains for the
+ * sub-class constructor to do.
+ *
+ * <pre>
+ * super(storage.duplicate(), index0, size);
+ * checkRequestFlags(flags); // Check request is compatible with type
+ * </pre>
+ *
+ * @param storage the <code>ByteBuffer</code> wrapping the exported object state. NOTE: this
+ * <code>PyBuffer</code> keeps a reference and may manipulate the position, mark and
+ * limit hereafter. Use {@link ByteBuffer#duplicate()} to give it an isolated copy.
+ * @param index0 offset where the data starts in that array (item[0])
+ * @param size the number of bytes occupied
+ * @throws NullPointerException if <code>storage</code> is null
+ * @throws ArrayIndexOutOfBoundsException if <code>index0</code> and <code>size</code> are
+ * inconsistent with <code>storage.capacity()</code>
+ */
+ protected SimpleNIOBuffer(ByteBuffer storage, int index0, int size) throws PyException,
+ ArrayIndexOutOfBoundsException {
+ super(storage, CONTIGUITY | SIMPLE);
+
+ // Initialise navigation
+ shape = new int[] {size}; // Number of items in exported data
+ strides = SIMPLE_STRIDES;
+ // suboffsets is always null for this type.
+
+ this.storage = storage; // Exported data
+ this.index0 = index0; // Index to be treated as item[0]
+
+ // Check arguments using the "all non-negative" trick
+ if ((index0 | size | storage.capacity() - (index0 + size)) < 0) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+ }
+
+ /**
+ * Provide an instance of <code>SimpleNIOBuffer</code>, on a slice of a {@link ByteBuffer},
+ * meeting the consumer's expectations as expressed in the <code>flags</code> argument, which is
+ * checked against the capabilities of the buffer type. No reference will be kept to the
+ * <code>ByteBuffer</code> passed in. (It is duplicated.)
+ *
+ * @param flags consumer requirements
+ * @param storage the <code>ByteBuffer</code> wrapping the exported object state
+ * @param index0 offset where the data starts in that buffer (item[0])
+ * @param size the number of bytes occupied
+ * @throws NullPointerException if <code>storage</code> is null
+ * @throws ArrayIndexOutOfBoundsException if <code>index0</code> and <code>size</code> are
+ * inconsistent with <code>storage.length</code>
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public SimpleNIOBuffer(int flags, ByteBuffer storage, int index0, int size) throws PyException,
+ ArrayIndexOutOfBoundsException, NullPointerException {
+ this(storage.duplicate(), index0, size); // Construct checked SimpleNIOBuffer
+ checkRequestFlags(flags); // Check request is compatible with type
+ }
+
+ /**
+ * Provide an instance of <code>SimpleNIOBuffer</code>, on the entirety of a {@link ByteBuffer},
+ * with navigation variables initialised, for sub-class use. The buffer ( {@link #storage},
+ * {@link #index0}), and the navigation ({@link #shape} array) will be initialised from the
+ * argument.
+ *
+ * @param storage the <code>ByteBuffer</code> wrapping the exported object state. NOTE: this
+ * <code>PyBuffer</code> keeps a reference and may manipulate the position, mark and
+ * limit hereafter. Use {@link ByteBuffer#duplicate()} to give it an isolated copy.
+ * @throws NullPointerException if <code>storage</code> is null
+ */
+ protected SimpleNIOBuffer(ByteBuffer storage) throws NullPointerException {
+ this(storage, 0, storage.capacity());
+ }
+
+ /**
+ * Provide an instance of <code>SimpleNIOBuffer</code>, on the entirety of a {@link ByteBuffer},
+ * meeting the consumer's expectations as expressed in the <code>flags</code> argument, which is
+ * checked against the capabilities of the buffer type. No reference will be kept to the
+ * <code>ByteBuffer</code> passed in. (It is duplicated.)
+ *
+ * @param flags consumer requirements
+ * @param storage the <code>ByteBuffer</code> wrapping the exported object state
+ * @throws NullPointerException if <code>storage</code> is null
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public SimpleNIOBuffer(int flags, ByteBuffer storage) throws PyException, NullPointerException {
+ this(storage.duplicate()); // Construct SimpleNIOBuffer on whole ByteBuffer
+ checkRequestFlags(flags); // Check request is compatible with type
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleNIOBuffer</code> provides an implementation optimised for contiguous bytes in
+ * one-dimension.
+ */
+ @Override
+ public int getLen() {
+ // Simplify for one-dimensional contiguous bytes
+ return shape[0];
+ }
+
+ @Override
+ protected int byteIndex(int index) throws IndexOutOfBoundsException {
+ return index0 + index;
+ }
+
+ // XXX Consider moving to clauses in getBufferSlice(int, int, int, int)
+ // to avoid delegation loop where that delegates to this but in BaseBuffer the reverse.
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int count) {
+ if (count > 0) {
+ // Translate relative to underlying buffer
+ int compIndex0 = index0 + start;
+ // Create the slice from the sub-range of the buffer
+ return new SimpleView(getRoot(), flags, storage, compIndex0, count);
+ } else {
+ // Special case for count==0 where above logic would fail. Efficient too.
+ return new ZeroByteBuffer.View(getRoot(), flags);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>SimpleNIOBuffer</code> provides an implementation for slicing contiguous bytes in one
+ * dimension. In that case, <i>x(i) = u(r+i)</i> for <i>i = 0..L-1</i> where u is the underlying
+ * buffer, and <i>r</i> and <i>L</i> are the start and count with which <i>x</i> was created
+ * from <i>u</i>. Thus <i>y(k) = u(r+s+km)</i>, that is, the composite offset is <i>r+s</i> and
+ * the stride is <i>m</i>.
+ */
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int count, int stride) {
+
+ if (stride == 1 || count < 2) {
+ // Unstrided slice of simple buffer is special case
+ return getBufferSlice(flags, start, count);
+
+ } else {
+ // Translate relative to underlying buffer
+ int compIndex0 = index0 + start;
+ // Construct a view, taking a lock on the root object (this or this.root)
+ return new Strided1DNIOBuffer.SlicedView(getRoot(), flags, storage, compIndex0, count,
+ stride);
+ }
+ }
+
+ /**
+ * A <code>SimpleNIOBuffer.SimpleView</code> represents a contiguous subsequence of another
+ * <code>SimpleNIOBuffer</code>.
+ */
+ static class SimpleView extends SimpleNIOBuffer {
+
+ /** The buffer on which this is a slice view */
+ PyBuffer root;
+
+ /**
+ * Construct a slice of a SimpleNIOBuffer.
+ *
+ * @param root buffer which will be acquired and must be released ultimately
+ * @param flags the request flags of the consumer that requested the slice
+ * @param storage <code>ByteBuffer</code> wrapping exported data (no reference kept)
+ * @param offset where the data starts in that buffer (item[0])
+ * @param size the number of bytes occupied
+ */
+ public SimpleView(PyBuffer root, int flags, ByteBuffer storage, int offset, int size) {
+ // Create a new SimpleNIOBuffer on the buffer passed in (part of the root)
+ super(flags, storage, offset, size);
+ // Get a lease on the root PyBuffer
+ this.root = root.getBuffer(FULL_RO);
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void releaseAction() {
+ // We have to release the root too if ours was final.
+ root.release();
+ }
+
+ }
+
+}
diff --git a/src/org/python/core/buffer/Strided1DNIOBuffer.java b/src/org/python/core/buffer/Strided1DNIOBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/Strided1DNIOBuffer.java
@@ -0,0 +1,219 @@
+package org.python.core.buffer;
+
+import java.nio.ByteBuffer;
+
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
+
+
+/**
+ * Read-only buffer API over a one-dimensional array of one-byte items, that are evenly-spaced in a
+ * storage array. The buffer has <code>storage</code>, <code>index0</code> and <code>length</code>
+ * properties in the usual way, designating a slice (or all) of a byte array, but also a
+ * <code>stride</code> property (equal to <code>getStrides()[0]</code>).
+ * <p>
+ * Let the underlying buffer be the byte array <i>u(i)</i> for <i>i=0..N-1</i>, let <i>x</i> be the
+ * <code>Strided1DNIOBuffer</code>, and let the stride be <i>p</i>. The storage works as follows.
+ * Designate by <i>x(j)</i>, for <i>j=0..L-1</i>, the byte at index <i>j</i>, that is, the byte
+ * retrieved by <code>x.byteAt(j)</code>. Thus, we store <i>x(j)</i> at <i>u(a+pj)</i>, that is,
+ * <i>x(0) = u(a)</i>. When we construct such a buffer, we have to supply <i>a</i> =
+ * <code>index0</code>, <i>L</i> = <code>count</code>, and <i>p</i> = <code>stride</code> as the
+ * constructor arguments. The last item in the slice <i>x(L-1)</i> is stored at <i>u(a+p(L-1))</i>.
+ * For the simple case of positive stride, constructor argument <code>index0</code> is the low index
+ * of the range occupied by the data. When the stride is negative, that is to say <i>p<0</i>, and
+ * <i>L>1</i>, this will be to the left of <i>u(a)</i>, and the constructor argument
+ * <code>index0</code> is not then the low index of the range occupied by the data. Clearly both
+ * these indexes must be in the range 0 to <i>N-1</i> inclusive, a rule enforced by the constructors
+ * (unless <i>L=0</i>, when it is assumed no array access will take place).
+ * <p>
+ * The class may be used by exporters to create a strided slice (e.g. to export the diagonal of a
+ * matrix) and in particular by other buffers to create strided slices of themselves, such as to
+ * create the <code>memoryview</code> that is returned as an extended slice of a
+ * <code>memoryview</code>.
+ */
+public class Strided1DNIOBuffer extends BaseNIOBuffer {
+
+ /**
+ * Step size in the underlying buffer essential to correct translation of an index (or indices)
+ * into an index into the storage. The value is returned by {@link #getStrides()} is an array
+ * with this as the only element.
+ */
+ protected int stride;
+
+
+ /**
+ * Provide an instance of <code>Strided1DNIOBuffer</code> with navigation variables initialised,
+ * for sub-class use. The buffer ({@link #storage}, {@link #index0}), and the navigation (
+ * {@link #shape} array and {@link #stride}) will be initialised from the arguments (which are
+ * checked for range).
+ * <p>
+ * The sub-class constructor should check that the intended access is compatible with this
+ * object by calling {@link #checkRequestFlags(int)}. (See the source of
+ * {@link Strided1DWritableBuffer#Strided1DWritableBuffer(int, ByteBuffer, int, int, int)}
+ * for an example of this use.)
+ *
+ * @param storage the <code>ByteBuffer</code> wrapping the exported object state. NOTE: this
+ * <code>PyBuffer</code> keeps a reference and may manipulate the position, mark and
+ * limit hereafter. Use {@link ByteBuffer#duplicate()} to give it an isolated copy.
+ * @param index0 index into storage of item[0]
+ * @param count number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws NullPointerException if <code>storage</code> is null
+ * @throws ArrayIndexOutOfBoundsException if <code>index0</code>, <code>count</code> and
+ * <code>stride</code> are inconsistent with <code>storage.length</code>
+ */
+ protected Strided1DNIOBuffer(ByteBuffer storage, int index0, int count, int stride)
+ throws ArrayIndexOutOfBoundsException, NullPointerException {
+ super(storage, STRIDES);
+ this.index0 = index0; // Index to be treated as item[0]
+ shape = new int[]{count}; // Number of items in exported data
+ this.stride = stride; // Between items
+ this.strides = new int[] {stride};
+
+ if (count == 0) {
+ // Nothing to check as we'll make no accesses
+ addFeatureFlags(CONTIGUITY);
+
+ } else {
+ // Need to check lowest and highest index against array
+ int lo, hi;
+
+ if (stride == 1) {
+ lo = index0; // First byte of item[0]
+ hi = index0 + count; // Last byte of item[L-1] + 1
+ addFeatureFlags(CONTIGUITY);
+
+ } else if (stride > 1) {
+ lo = index0; // First byte of item[0]
+ hi = index0 + (count - 1) * stride + 1; // Last byte of item[L-1] + 1
+
+ } else {
+ hi = index0 + 1; // Last byte of item[0] + 1
+ lo = index0 + (count - 1) * stride; // First byte of item[L-1]
+ }
+
+ // Check indices using "all non-negative" trick
+ int cap = storage.capacity();
+ if ((count | lo | (cap - lo) | hi | (cap - hi)) < 0) {
+ throw new ArrayIndexOutOfBoundsException();
+ }
+ }
+
+ // Deduce feature flags from the client's ByteBuffer
+ if (!storage.isReadOnly()) {
+ addFeatureFlags(WRITABLE);
+ }
+ if (storage.hasArray()) {
+ addFeatureFlags(AS_ARRAY);
+ }
+ }
+
+ /**
+ * Provide an instance of <code>Strided1DNIOBuffer</code> on a particular {@link ByteBuffer}
+ * specifying
+ * a starting index, the number of items in the result, and a byte-indexing stride. The result
+ * of <code>byteAt(i)</code> will be equal to <code>storage.get(index0+stride*i)</code>
+ * (whatever
+ * the sign of <code>stride</code>), valid for <code>0<=i<count</code>. The constructor
+ * checks that all these indices lie within the <code>storage</code> (unless
+ * <code>count=0</code>).
+ * No reference will be kept to the <code>ByteBuffer</code> passed in. (It is duplicated.)
+ * <p>
+ * The constructed <code>PyBuffer</code> meets the consumer's expectations as expressed in the
+ * <code>flags</code> argument, or an exception will be thrown if these are incompatible with
+ * the type (e.g. the consumer does not specify that it understands the strides array). Note
+ * that the actual range in the <code>storage</code> array, the lowest and highest index, is not
+ * explicitly passed, but is implicit in <code>index0</code>, <code>count</code> and
+ * <code>stride</code>. The constructor checks that these indices lie within the
+ * <code>storage</code> array (unless <code>count=0</code>).
+ *
+ * @param flags consumer requirements
+ * @param storage <code>ByteBuffer</code> wrapping exported data
+ * @param index0 index into storage of item[0]
+ * @param count number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws NullPointerException if <code>storage</code> is null
+ * @throws ArrayIndexOutOfBoundsException if <code>index0</code>, <code>count</code> and
+ * <code>stride</code> are inconsistent with <code>storage.length</code>
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public Strided1DNIOBuffer(int flags, ByteBuffer storage, int index0, int count, int stride)
+ throws ArrayIndexOutOfBoundsException, NullPointerException, PyException {
+ this(storage.duplicate(), index0, count, stride);
+ checkRequestFlags(flags); // Check request is compatible with type
+
+ }
+
+ @Override
+ protected int byteIndex(int index) throws IndexOutOfBoundsException {
+ return index0 + index * stride;
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * <code>Strided1DNIOBuffer</code> provides an implementation for slicing already-strided bytes in
+ * one dimension. In that case, <i>x(i) = u(r+ip)</i> for <i>i = 0..L-1</i> where u is the
+ * underlying buffer, and <i>r</i>, <i>p</i> and <i>L</i> are the start, stride and count with
+ * which <i>x</i> was created from <i>u</i>. Thus <i>y(k) = u(r+sp+kmp)</i>, that is, the
+ * composite <code>index0</code> is <i>r+sp</i> and the composite <code>stride</code> is
+ * <i>mp</i>.
+ */
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int count, int stride) {
+
+ if (count > 0) {
+ // Translate start relative to underlying buffer
+ int compStride = this.stride * stride;
+ int compIndex0 = index0 + start * this.stride;
+ // Construct a view, taking a lock on the root object (this or this.root)
+ return new SlicedView(getRoot(), flags, storage, compIndex0, count, compStride);
+
+ } else {
+ // Special case for count==0 where above logic would fail. Efficient too.
+ return new ZeroByteBuffer.View(getRoot(), flags);
+ }
+ }
+
+
+ /**
+ * A <code>Strided1DNIOBuffer.SlicedView</code> represents a non-contiguous subsequence of a simple
+ * buffer.
+ */
+ static class SlicedView extends Strided1DNIOBuffer {
+
+ /** The buffer on which this is a slice view */
+ PyBuffer root;
+
+ /**
+ * Construct a slice of a one-dimensional byte buffer.
+ *
+ * @param root on which release must be called when this is released
+ * @param flags consumer requirements
+ * @param storage <code>ByteBuffer</code> wrapping exported data (no reference kept)
+ * @param index0 index into storage of item[0]
+ * @param len number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public SlicedView(PyBuffer root, int flags, ByteBuffer storage, int index0, int len, int stride)
+ throws PyException {
+ // Create a new slice on the buffer passed in (part of the root)
+ super(flags, storage, index0, len, stride);
+ // Get a lease on the root PyBuffer (read-only)
+ this.root = root.getBuffer(FULL_RO);
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void releaseAction() {
+ // We have to release the root too if ours was final.
+ root.release();
+ }
+
+ }
+}
diff --git a/tests/java/org/python/core/PyBufferNIOTest.java b/tests/java/org/python/core/PyBufferNIOTest.java
new file mode 100644
--- /dev/null
+++ b/tests/java/org/python/core/PyBufferNIOTest.java
@@ -0,0 +1,316 @@
+package org.python.core;
+
+import java.lang.ref.WeakReference;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import org.junit.runners.Parameterized.Parameters;
+import org.python.core.ByteBufferTestSupport.ByteMaterial;
+import org.python.core.PyBufferTest.TestableExporter;
+import org.python.core.PyBufferTestSupport.ExporterFactory;
+import org.python.core.PyBufferTestSupport.TestSpec;
+import org.python.core.PyBufferTestSupport.WritableExporterFactory;
+import org.python.core.buffer.BaseBuffer;
+import org.python.core.buffer.SimpleNIOBuffer;
+
+public class PyBufferNIOTest extends PyBufferTest {
+
+ public PyBufferNIOTest(TestSpec spec) {
+ super(spec);
+ }
+
+ /**
+ * Generate test data to be held in the testing framework and used to construct tests. This
+ * method is called once by the test framework. Each element of the returned collection is a
+ * specification that becomes the arguments to the constructor when JUnit prepares to invoke a
+ * test.
+ * <p>
+ * Internally, this method creates a small number of instances of the object types whose
+ * <code>PyBuffer</code> export mechanism is to be tested. Each is paired with a reference value
+ * represented in several forms. The <code>PyBufferTestSupport</code> class then multiplies
+ * these by creating a selection of feasible sliced views, the whole collection of root and
+ * slice objects being returned.
+ *
+ * @return generated list of test data
+ */
+ @Parameters
+ public static Collection<TestSpec[]> genTestSpecs() {
+
+ PyBufferTestSupport s = new PyBufferTestSupport(sliceLengths, sliceSteps);
+
+ // Tests using local types of exporter
+
+ ExporterFactory rollYourOwnExporter = new WritableExporterFactory() {
+
+ @Override
+ public BufferProtocol make(ByteMaterial m) {
+ return new RollYourOwnExporter(m.getBuffer());
+ }
+
+ };
+ s.add(rollYourOwnExporter, byteMaterial);
+ s.add(rollYourOwnExporter, emptyMaterial);
+
+ // All combinations of heap/direct, writable and empty/small/large (I'm so thorough!)
+
+ ExporterFactory readonlyHeapNIOExporter = new TestNIOExporterFactory(false, false);
+ s.add(readonlyHeapNIOExporter, emptyMaterial);
+ s.add(readonlyHeapNIOExporter, byteMaterial);
+ s.add(readonlyHeapNIOExporter, longMaterial);
+
+ ExporterFactory writableHeapNIOExporter = new TestNIOExporterFactory(true, false);
+ s.add(writableHeapNIOExporter, emptyMaterial);
+ s.add(writableHeapNIOExporter, byteMaterial);
+ s.add(writableHeapNIOExporter, longMaterial);
+
+ ExporterFactory readonlyDirectNIOExporter = new TestNIOExporterFactory(false, true);
+ s.add(readonlyDirectNIOExporter, emptyMaterial);
+ s.add(readonlyDirectNIOExporter, byteMaterial);
+ s.add(readonlyDirectNIOExporter, longMaterial);
+
+ ExporterFactory writableDirectNIOExporter = new TestNIOExporterFactory(true, true);
+ s.add(writableDirectNIOExporter, emptyMaterial);
+ s.add(writableDirectNIOExporter, byteMaterial);
+ s.add(writableDirectNIOExporter, longMaterial);
+
+ // Return the generated test data
+
+ List<TestSpec[]> ret = s.getTestData();
+ if (PRINT_KEY) {
+ int key = 0;
+ for (TestSpec[] r : ret) {
+ TestSpec spec = r[0];
+ System.out.printf("%6d : %s\n", key++, spec.toString());
+ }
+ }
+ return ret;
+ }
+
+ /*
+ * --------------------------------------------------------------------------------------------
+ * A series of custom exporters that use a java.nio.ByteBuffer to store and export their
+ * implementation data.
+ * --------------------------------------------------------------------------------------------
+ */
+ /**
+ * A class to act as an exporter that uses the SimpleBuffer. The exporter shares a single
+ * exported buffer between all consumers and needs to take any action immediately when that
+ * buffer is finally released. You are most likely to use this approach with an exporting object
+ * type that modifies its behaviour while there are active exports, but where it is worth
+ * avoiding the cost of duplicate buffers. This is the case with PyByteArray, which prohibits
+ * operations that would resize it, while there are outstanding exports.
+ */
+ private static class TestNIOExporter extends TestableExporter {
+
+ protected ByteBuffer storage;
+
+ /**
+ * Construct a simple exporter from the bytes supplied.
+ *
+ * @param storage
+ */
+ public TestNIOExporter(ByteBuffer storage) {
+ this.storage = storage;
+ }
+
+ @Override
+ public PyBuffer getBuffer(int flags) {
+ // If we have already exported a buffer it may still be available for re-use
+ BaseBuffer pybuf = getExistingBuffer(flags);
+ if (pybuf == null) {
+ // No existing export we can re-use
+ pybuf = new SimpleNIOBuffer(flags, storage) {
+
+ @Override
+ protected void releaseAction() {
+ export = null; // Final release really is final (not reusable)
+ }
+ };
+
+ // Hold a reference for possible re-use
+ export = new WeakReference<BaseBuffer>(pybuf);
+ }
+ return pybuf;
+ }
+
+ }
+
+ /**
+ * A factory for exporting objects to be used in the tests. These objects use a
+ * <code>ByteBuffer</code> for their exported representation, and the factory is programmed on
+ * creation to whether these buffers should be writable or direct.
+ */
+ static class TestNIOExporterFactory implements ExporterFactory {
+
+ final boolean writable;
+ final boolean isDirect;
+
+ TestNIOExporterFactory(boolean writable, boolean isDirect) {
+ this.writable = writable;
+ this.isDirect = isDirect;
+ }
+
+ public boolean isWritable() {
+ return writable;
+ }
+
+ public boolean isDirect() {
+ return isDirect;
+ }
+
+ @Override
+ public BufferProtocol make(ByteMaterial m) {
+ ByteBuffer bb = m.getBuffer();
+ if (isDirect) {
+ // Replace bb with a direct buffer containing the same bytes
+ ByteBuffer direct = ByteBuffer.allocateDirect(bb.capacity());
+ direct.put(bb).flip();
+ bb = direct;
+ }
+ if (!writable) {
+ bb = bb.asReadOnlyBuffer();
+ }
+ return new TestNIOExporter(bb);
+ }
+
+ @Override
+ public boolean isReadonly() {
+ return !writable;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return !isDirect && writable;
+ }
+
+ }
+
+ /** A class to act as an exporter that uses the RollYourOwnNIOBuffer class. */
+ private static class RollYourOwnExporter extends TestableExporter {
+
+ protected ByteBuffer storage;
+
+ public RollYourOwnExporter(ByteBuffer storage) {
+ this.storage = storage;
+ }
+
+ @Override
+ public PyBuffer getBuffer(int flags) {
+ // If we have already exported a buffer it may still be available for re-use
+ BaseBuffer pybuf = getExistingBuffer(flags);
+ if (pybuf == null) {
+ // No existing export we can re-use
+ pybuf = new RollYourOwnNIOBuffer(flags, storage);
+ // Hold a reference for possible re-use
+ export = new WeakReference<BaseBuffer>(pybuf);
+ }
+ return pybuf;
+ }
+
+ }
+
+ /**
+ * Minimal extension of BaseBuffer in order to test the default implementations there. They're
+ * slow, so mostly we override them in the implementations BaseArrayBuffer and BaseNIOBuffer,
+ * but they still have to be correct. The class represents a one-dimensional, strided array of
+ * bytes, so it can represent a slice of itself.
+ */
+ private static class RollYourOwnNIOBuffer extends BaseBuffer {
+
+ final static int FEATURES = PyBUF.WRITABLE | PyBUF.AS_ARRAY;
+
+ final ByteBuffer storage;
+ final PyBuffer root;
+
+ /**
+ * Create a buffer view of a given <code>ByteBuffer</code> in which the data is the
+ * contiguous sequence of bytes from the position to the limit.
+ *
+ * @param flags consumer requirements
+ * @param storage buffer exported (from the position to the limit)
+ */
+ public RollYourOwnNIOBuffer(int flags, ByteBuffer storage) {
+ this(null /* =this */, flags, storage, storage.position(), storage.remaining(), 1);
+ }
+
+ /**
+ * Construct a slice of a one-dimensional byte buffer.
+ *
+ * @param root on which release must be called when this is released
+ * @param flags consumer requirements
+ * @param storage buffer containing exported data
+ * @param index0 index into storage of item[0]
+ * @param count number of items in the slice
+ * @param stride in between successive elements of the new PyBuffer
+ * @throws PyException (BufferError) when expectations do not correspond with the type
+ */
+ public RollYourOwnNIOBuffer(PyBuffer root, int flags, ByteBuffer storage, int index0,
+ int count, int stride) throws IndexOutOfBoundsException, NullPointerException,
+ PyException {
+ // Client will need to navigate using shape and strides if this is a slice
+ super(FEATURES | ((index0 == 0 && stride == 1) ? 0 : PyBUF.STRIDES));
+ this.storage = storage.duplicate();
+ this.index0 = index0;
+ shape = new int[] {count};
+ strides = new int[] {stride};
+ // Check the potential index range
+ if (count > 0) {
+ int end = index0 + (count - 1) * stride;
+ final int END = storage.capacity() - 1;
+ if (index0 < 0 || index0 > END || end < 0 || end > END) {
+ throw new IndexOutOfBoundsException();
+ }
+ }
+ // Check client is compatible
+ checkRequestFlags(flags);
+ // Get a lease on the root PyBuffer (read-only). Last in case a check above fails.
+ if (root == null) {
+ this.root = this;
+ } else {
+ this.root = root.getBuffer(FULL_RO);
+ }
+ }
+
+ @Override
+ protected PyBuffer getRoot() {
+ return root;
+ }
+
+ @Override
+ public void releaseAction() {
+ // XXX Consider making this automatic within BaseBuffer.release() when getRoot()!=this
+ /*
+ * ... so that {@link #release()} takes care of this: sub-classes should not propagate
+ * the release themselves when overriding {@link #releaseAction()}.
+ */
+ // We have to release the root too if ours was final and we are not that root.
+ if (root != this) {
+ root.release();
+ }
+ }
+
+ @Override
+ public PyBuffer getBufferSlice(int flags, int start, int count, int stride) {
+ int newStart = index0 + start * strides[0];
+ int newStride = strides[0] * stride;
+ return new RollYourOwnNIOBuffer(root, flags, storage, newStart, count, newStride);
+ }
+
+ @Override
+ public ByteBuffer getNIOByteBufferImpl() {
+ return storage.duplicate();
+ }
+
+ @Override
+ protected byte byteAtImpl(int byteIndex) {
+ return storage.get(byteIndex);
+ }
+
+ @Override
+ protected void storeAtImpl(byte value, int byteIndex) throws IndexOutOfBoundsException,
+ PyException {
+ storage.put(byteIndex, value);
+ }
+ }
+}
diff --git a/tests/java/org/python/core/PyBufferTest.java b/tests/java/org/python/core/PyBufferTest.java
--- a/tests/java/org/python/core/PyBufferTest.java
+++ b/tests/java/org/python/core/PyBufferTest.java
@@ -1242,7 +1242,7 @@
}
- /** A class to act as an exporter that uses the RollYourOwnBuffer. */
+ /** A class to act as an exporter that uses the RollYourOwnArrayBuffer class. */
private static class RollYourOwnExporter extends TestableExporter {
protected byte[] storage;
@@ -1290,28 +1290,28 @@
}
/**
- * Construct a slice of a one-dimensional byte buffer.
+ * Construct a slice of a one-dimensional byte array.
*
* @param root on which release must be called when this is released
* @param flags consumer requirements
* @param storage raw byte array containing exported data
* @param index0 index into storage of item[0]
- * @param length number of items in the slice
+ * @param count number of items in the slice
* @param stride in between successive elements of the new PyBuffer
* @throws PyException (BufferError) when expectations do not correspond with the type
*/
public RollYourOwnArrayBuffer(PyBuffer root, int flags, byte[] storage, int index0,
- int length, int stride) throws ArrayIndexOutOfBoundsException,
- NullPointerException, PyException {
+ int count, int stride) throws IndexOutOfBoundsException, NullPointerException,
+ PyException {
// Client will need to navigate using shape and strides if this is a slice
super(FEATURES | ((index0 == 0 && stride == 1) ? 0 : PyBUF.STRIDES));
this.storage = storage;
this.index0 = index0;
- shape = new int[] {length};
+ shape = new int[] {count};
strides = new int[] {stride};
// Check the potential index range
- if (length > 0) {
- int end = index0 + (length - 1) * stride;
+ if (count > 0) {
+ int end = index0 + (count - 1) * stride;
final int END = storage.length - 1;
if (index0 < 0 || index0 > END || end < 0 || end > END) {
throw new IndexOutOfBoundsException();
@@ -1367,6 +1367,5 @@
PyException {
storage[byteIndex] = value;
}
-
}
}
--
Repository URL: https://hg.python.org/jython
More information about the Jython-checkins
mailing list